Fix/yield pages extra request (#42)

* fix: prevent additional fetch when SerpResults.yield_pages is used to fetch only 1 page.

* chore: improve examples and test search requests

Author: adarshdigievo

README.md

@@ -196,7 +196,7 @@ import serpapi
 client = serpapi.Client(api_key=os.getenv("API_KEY"))
 results = client.search({
     'engine': 'home_depot',
-    'q': 'table',
+    'q': 'chair',
 })
 ```
 - API Documentation: [serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api)
@@ -363,7 +363,7 @@ import serpapi
 client = serpapi.Client(api_key=os.getenv("API_KEY"))
 results = client.search({
     'engine': 'google_jobs',
-    'q': 'coffee',
+    'q': 'software engineer',
 })
 ```
 - API Documentation: [serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api)

docs/examples/google-shopping-price-monitoring.md

@@ -1,11 +1,11 @@
 ---
 title: "Google Shopping Price Monitoring"
-description: "Filter Google Shopping listings by price and read selected product fields."
+description: "Compare US espresso-machine listings within a budget."
 ---
 
 # Google Shopping Price Monitoring
 
-Filter Google Shopping listings by price to compare offers from different merchants. The example searches for espresso machines priced between 100 and 800 in the search's currency.
+Search Google Shopping for a De'Longhi Stilosa espresso machine in Austin, Texas, and compare offers priced from $100 to $200. The example filters the returned prices in Python using the numeric `extracted_price` field.
 
 ## Search Filtered Products
 
@@ -20,17 +20,17 @@ client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20)
 
 results = client.search(
     engine="google_shopping",
-    q="espresso machine",
+    q="DeLonghi Stilosa espresso machine",
     location="Austin, Texas",
     gl="us",
     hl="en",
-    min_price=100,
-    max_price=800,
-    sort_by=1,
-    json_restrictor="shopping_results[].{title, price, source, rating, reviews, link}",
+    json_restrictor="error, shopping_results[].{title, price, extracted_price, source, rating, reviews, product_link}",
 )
 
-for product in results.get("shopping_results", [])[:5]:
+for product in results.get("shopping_results", []):
+    price = product.get("extracted_price")
+    if price is None or not 100 <= price <= 200:
+        continue
     print(product.get("title"))
     print(product.get("price"), product.get("source"))
     print(product.get("rating"), product.get("reviews"))
@@ -40,4 +40,4 @@ for product in results.get("shopping_results", [])[:5]:
 
 Read product listings from `shopping_results`. Useful fields include `title`, `product_id`, `price`, `extracted_price`, `source`, `rating`, `reviews`, `thumbnail`, `delivery`, `product_link`, and `serpapi_immersive_product_api`.
 
-Use `min_price`, `max_price`, `sort_by`, `free_shipping`, and `on_sale` to filter and order offers. To fetch more pages, use `serpapi_pagination.next` when it is present. The example limits response fields with `json_restrictor`. Include `serpapi_pagination` in that selector if you need pagination. See the [Google Shopping API documentation](https://serpapi.com/google-shopping-api) for the full parameter set.
+The example limits response fields with `json_restrictor` and keeps `error` so API errors remain visible. Use `extracted_price` for numeric comparisons and `price` for display. See the [Google Shopping API documentation](https://serpapi.com/google-shopping-api) for the full parameter set.

docs/examples/home-depot-product-search.md

@@ -20,7 +20,7 @@ client = serpapi.Client(api_key=os.environ["SERPAPI_KEY"], timeout=20)
 
 results = client.search(
     engine="home_depot",
-    q="table",
+    q="chair",
 )
 
 for product in results.get("products", [])[:5]:

serpapi/models.py

@@ -1,10 +1,8 @@
 import json
 
-from pprint import pformat
 from collections import UserDict
 
 from .textui import prettify_json
-from .exceptions import HTTPError
 
 
 class SerpResults(UserDict):
@@ -71,6 +69,8 @@ def yield_pages(self, max_pages=1_000):
         while current_page and current_page_count < max_pages:
             yield current_page
             current_page_count += 1
+            if current_page_count >= max_pages:
+                break
             if current_page.next_page_url:
                 current_page = current_page.next_page()
             else:

tests/example_search_google_jobs_test.py

@@ -6,7 +6,7 @@
 def test_search_google_jobs(client):
   data = client.search({
       'engine': 'google_jobs',
-      'q': 'coffee',
+      'q': 'software engineer',
   })
   assert data.get('error') is None
   assert data['jobs_results']

tests/example_search_home_depot_test.py

@@ -7,7 +7,7 @@ def test_search_home_depot(client):
 
   data = client.search({
       'engine': 'home_depot',
-      'q': 'table',
+      'q': 'chair',
   })
   assert data.get('error') is None
   assert data['products']

tests/test_pagination.py

@@ -0,0 +1,39 @@
+import json
+from unittest.mock import Mock
+
+import pytest
+import requests
+
+import serpapi
+
+
+@pytest.mark.parametrize(
+    ("max_pages", "available_pages", "expected_pages"),
+    [(1, 3, 1), (2, 3, 2), (3, 3, 3), (5, 3, 3), (5, 1, 1), (0, 3, 0)],
+)
+def test_yield_pages_does_not_request_unused_pages(
+    max_pages, available_pages, expected_pages
+):
+    responses = []
+    for page_number in range(1, available_pages + 1):
+        data = {"search_information": {"page_number": page_number}}
+        if page_number < available_pages:
+            data["serpapi_pagination"] = {
+                "next": f"https://serpapi.com/search?engine=google&q=Coffee&start={page_number * 10}"
+            }
+        response = requests.Response()
+        response.status_code = 200
+        response.headers["Content-Type"] = "application/json"
+        response._content = json.dumps(data).encode("utf-8")
+        responses.append(response)
+
+    client = serpapi.Client(api_key="test-api-key")
+    client.session.request = Mock(side_effect=responses)
+    results = client.search(engine="google", q="Coffee")
+
+    pages = list(results.yield_pages(max_pages=max_pages))
+
+    assert [page["search_information"]["page_number"] for page in pages] == list(
+        range(1, expected_pages + 1)
+    )
+    assert client.session.request.call_count == max(1, expected_pages)