Use pagination parameters from SerpApi instead of calculating on the client (#30)

* Use pagination parameters from SerpApi instead of calculating on the client

`start` and `num` parameters are not suitable for token-based
pagination. Such pagination is used on Google Maps, YouTube, Google
Scholar Authors, and other search engines.

This commit consumes URL query parameters for the next page. It stops
paginating when parameters not change.

Details: #22

Some tests are failing because `start` and `num` parameters are not supported
anymore. These tests will be fixed in the following commits.

* Add pagination tests for Bing, Baidu, and DuckDuckGo search API clients

* Fix typo in SerpApi name in documentation

* Add more pagination tests

All of the tests follow the same pattern. Limit number of pages,
iterate, and check for duplicates in the results. This is to make sure
that pagination actually changes pages.

* Test pagination for Naver and HomeDepot

* Stop pagination when SerpApi backend doesn't update parameters

* Fix flake8 linting errors

Example errors: https://github.com/serpapi/google-search-results-python/runs/6659757610?check_suite_focus=true#step:5:37

* Lint code via `make lint`

Currently linting script exists only in GitHub Action: `.github/workflows/python-package.yml`.

This commit wraps that script in Makefile and invokes in an Action.

* fix(tests): fix failing integration tests

DuckDuckGo tests are failing because DuckDuckGo pagination doesn't take
into account an offset of current results:
serpapi/public-roadmap#619

Co-authored-by: Dimitry <[email protected]>

* perf: run pytest in parallel

Sample output:

  platform linux -- Python 3.10.9, pytest-7.2.1, pluggy-1.0.0
  rootdir: /home/ilyazub/Workspace/google-search-results-python
  plugins: parallel-0.1.1
  collected 48

  pytest-parallel: 8 workers (processes), 6 tests per worker (threads)

`py` dependency is used because pytest-parallel depends on it but
doesn't require 😕
kevlened/pytest-parallel#118

Co-authored-by: Dimitry <[email protected]>

* style: don't lint vendor packages with Flake8

Co-authored-by: Dimitry <[email protected]>

* docs: fix minor typos in documentation

Co-authored-by: Dimitry <[email protected]>

* ci: cache pip dependencies

Support Python 3.7+ based on the readme:
https://github.com/serpapi/google-search-results-python/blob/35e51c94e7243c29650ed7b630db4e4e6d0c61aa/README.md#L18

Co-authored-by: "dimitryzub <[email protected]>"

---------

Co-authored-by: Dimitry <[email protected]>
Co-authored-by: Dimitry <[email protected]>

Author: 3 people

.flake8

@@ -0,0 +1,7 @@
+[flake8]
+exclude =
+    build
+    dist
+    .git
+    .env
+    .pytest_cache

.github/workflows/python-package.yml

@@ -11,30 +11,26 @@ on:
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [3.5, 3.6, 3.7, 3.8, 3.9]
+        python-version: [3.7, 3.8, 3.9, '3.10']
 
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: ${{ matrix.python-version }}
+        cache: pip
+
     - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
+      run: make install build_dep
+
     - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
+      run: make lint
+
     - name: Test with pytest
-      run: pytest
+      run: make test
       env:
         API_KEY: ${{secrets.API_KEY}}

.gitignore

@@ -13,4 +13,5 @@ build/
 google_search_results.egg-info/PKG-INFO
 google_search_results.egg-info/SOURCES.txt
 
-.env
+# Python virtual environment
+.env

Makefile

@@ -5,27 +5,38 @@ version=$(shell grep version setup.py | cut -d"'" -f2)
 
 .PHONY: build
 
-all: clean install test
+all: clean install lint test
 
 clean:
 	find . -name '*.pyc' -delete
 	find . -type d -name "__pycache__" -delete
 	pip3 uninstall google_search_results
 
+create_env:
+	python -m venv .env
+	source .env/bin/activate
+
 install:
-	pip3 install -r requirements.txt
+	python3 -m pip install --upgrade pip
+	if [ -f requirements.txt ]; then pip3 install -r requirements.txt; fi
+
+lint: build_dep
+	# stop the build if there are Python syntax errors or undefined names
+	flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
+	# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
+	flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
 
 # Test with Python 3
-test:
-	pytest
+test: build_dep
+	pytest --workers auto --tests-per-worker auto
 
 # run example only 
 #  and display output (-s)
 example:
 	pytest -s "tests/test_example.py::TestExample::test_async"
 
 build_dep:
-	pip3 install -U setuptools twine
+	pip3 install -U setuptools pytest py pytest-parallel flake8 twine
 
 # https://packaging.python.org/tutorials/packaging-projects/
 build: build_dep

README.md

@@ -79,7 +79,7 @@ See the [playground to generate your code.](https://serpapi.com/playground)
   - [Quick start](#quick-start)
   - [Summary](#summary)
     - [Google Search API capability](#google-search-api-capability)
-    - [How to set SERP API key](#how-to-set-serp-api-key)
+    - [How to set SerpApi key](#how-to-set-serp-api-key)
     - [Example by specification](#example-by-specification)
     - [Location API](#location-api)
     - [Search Archive API](#search-archive-api)
@@ -116,7 +116,7 @@ params = {
   "safe": "Safe Search Flag",
   "num": "Number of Results",
   "start": "Pagination Offset",
-  "api_key": "Your SERP API Key", 
+  "api_key": "Your SerpApi Key", 
   # To be match
   "tbm": "nws|isch|shop", 
   # To be search
@@ -145,7 +145,7 @@ object_result = search.get_object()
 
 See below for more hands-on examples.
 
-### How to set SERP API key
+### How to set SerpApi key
 
 You can get an API key here if you don't already have one: https://serpapi.com/users/sign_up
 
@@ -602,7 +602,7 @@ If it's a client error, then a SerpApiClientException is raised.
    - youtube
    - walmart
    - apple_app_store
-   - naver 
+   - naver
  - raise SerpApiClientException instead of raw string in order to follow Python guideline 3.5+
  - add more unit error tests for serp_api_client
 

README.rst

@@ -35,7 +35,7 @@ Quick start
 
 This example runs a search about "coffee" using your secret api key.
 
-The Serp API service (backend)
+The SerpApi service (backend)
 
 * searches on Google using the query: q = "coffee"
 * parses the messy HTML responses
@@ -44,7 +44,7 @@ The Serp API service (backend)
 The GoogleSearch class
 
 * Format the request
-* Execute GET http request against Serp API service
+* Execute GET http request against SerpApi service
 * Parse JSON response into a dictionary
 
 Et voila..

requirements.txt

@@ -1 +1,7 @@
+-i https://pypi.org/simple
+certifi==2022.5.18
+chardet==4.0.0
+-e .
+idna==2.10
 requests==2.25.1
+urllib3==1.26.9

serpapi/constant.py

@@ -1,8 +1,9 @@
 
 # Pagination constant
 DEFAULT_START = 0
-DEFAULT_END = 100
+DEFAULT_END = 1000000000
 DEFAULT_PAGE_SIZE = 10
+DEFAULT_LIMIT = 1000
 
 # Supported earch engine
 GOOGLE_ENGINE = 'google'
@@ -19,4 +20,4 @@
 NAVER_ENGINE = "naver"
 APPLE_APP_STORE_ENGINE = "apple_app_store"
 
-# from serpapi.constant import *
+# from serpapi.constant import *

serpapi/pagination.py

@@ -1,67 +1,66 @@
-from serpapi.serp_api_client_exception import SerpApiClientException
+from urllib import parse
+from serpapi import constant
 
-DEFAULT_START = 0 
-DEFAULT_END = 1000000000
-DEFAULT_num = 10
-
-# Paginate response in SearpApi
+# Paginate response in SerpApi
 class Pagination:
-  
-  def __init__(self, client, start = DEFAULT_START, end = DEFAULT_END, num = DEFAULT_num):
-    # serp api client
+
+  def __init__(self, client, start = constant.DEFAULT_START, end = constant.DEFAULT_END, num = constant.DEFAULT_PAGE_SIZE, limit = constant.DEFAULT_LIMIT):
+    # SerpApi client
     self.client = client
-    # range
-    self.start = start
-    self.end = end
-    self.num = num
-
-    # use value from the client
-    if self.start == DEFAULT_START:
-      if 'start' in self.client.params_dict:
-        self.start = self.client.params_dict['start']
-    if self.end == DEFAULT_END:
-      if 'end' in self.client.params_dict:
-        self.end = self.client.params_dict['end']
-    if self.num == DEFAULT_num:
-      if 'num' in self.client.params_dict:
-        self.num = self.client.params_dict['num']
-
-    # basic check
-    if self.start > self.end:
-        raise SerpApiClientException("start: {} must be less than end: {}".format(self.start, self.end))
-    if(self.start + self.num) > self.end:
-        raise SerpApiClientException("start + num: {} + {} must be less than end: {}".format(self.start, self.num, self.end))
+
+    self.limit = limit
+
+    """Backwards-compatible workaround.
+    `start`, `num`, and `end` parameters to `Pagination#__init__` are deprecated.
+
+    Set `start` and `num` search parameters.
+    It works for Google Search API only.
+    A correct way to set an offset, limit, and page size is in search parameters directly.
+    (A hash that is passed to `SerpApi#__init__`.)
+    """
+    if start != constant.DEFAULT_START:
+      self.client.params_dict['start'] = start
+
+    if end != constant.DEFAULT_END:
+      self.client.params_dict['end'] = end
+
+    if num != constant.DEFAULT_PAGE_SIZE:
+      self.client.params_dict['num'] = num
+
+
+    self.page_number = 0
 
   def __iter__(self):
-    self.update()
     return self
 
-  def update(self):
-    self.client.params_dict['start'] = self.start
-    self.client.params_dict['num'] = self.num
-    if self.start > 0:
-      self.client.params_dict['start'] += 1
-
   def __next__(self):
-    # update parameter
-    self.update()
+    if self.page_number >= self.limit:
+      raise StopIteration
 
     # execute request
     result = self.client.get_dict()
 
-    # stop if backend miss to return serpapi_pagination
-    if not 'serpapi_pagination' in result:
+    pagination = result.get('serpapi_pagination', result.get('pagination'))
+
+    # stop if backend miss to return `serpapi_pagination` or `pagination`
+    if not pagination:
       raise StopIteration
 
     # stop if no next page
-    if not 'next' in result['serpapi_pagination']:
-        raise StopIteration
-
-    # ends if no next page
-    if self.start + self.num > self.end:
-        raise StopIteration
-    
-    # increment start page
-    self.start += self.num
+    if not 'next' in pagination:
+      raise StopIteration
+
+    # Get actual parameters from next page of target website
+    params_from_target_website = dict(
+      parse.parse_qsl(parse.urlsplit(pagination['next']).query)
+    )
+
+    # stop if parameters from the target website were not changed
+    if params_from_target_website.items() <= self.client.params_dict.items():
+      raise StopIteration
+
+    self.client.params_dict.update(params_from_target_website)
+
+    self.page_number += 1
 
     return result

serpapi/serp_api_client.py

@@ -1,17 +1,9 @@
 import requests
 import json
+from serpapi.constant import *
 from serpapi.pagination import Pagination
 from serpapi.serp_api_client_exception import SerpApiClientException
 
-GOOGLE_ENGINE = 'google'
-BING_ENGINE = 'bing'
-BAIDU_ENGINE = 'baidu'
-GOOGLE_SCHOLAR_ENGINE = 'google_scholar'
-YANDEX_ENGINE = 'yandex'
-EBAY_ENGINE = 'ebay'
-YAHOO_ENGINE = 'yahoo'
-HOME_DEPOT_ENGINE = 'home_depot'
-YOUTUBE_ENGINE = 'youtube'
 
 class SerpApiClient(object):
     """SerpApiClient enables to query any search engines supported by SerpApi and parse the results.
@@ -71,7 +63,7 @@ def get_results(self, path='/search'):
 
     def get_html(self):
         """Returns:
-            Raw HTML search result from Gooogle
+            Raw HTML search result from Google
         """
         return self.get_results()
 
@@ -170,8 +162,8 @@ def get_location(self, q, limit = 5):
         buffer = self.get_results('/locations.json')
         return json.loads(buffer)
 
-    def pagination(self, start = 0, end = 1000000000, page_size = 10):
+    def pagination(self, start = DEFAULT_START, end = DEFAULT_END, page_size = DEFAULT_PAGE_SIZE, limit = DEFAULT_LIMIT):
         """Return:
             Generator to iterate the search results pagination
         """
-        return Pagination(self, start, end, page_size)
+        return Pagination(self, start, end, page_size, limit)

tests/test_apple_app_store_search.py

@@ -22,5 +22,22 @@ def test_get_json(self):
 				pp.pprint(data)
 				print(data.keys())
 
+		@unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided")
+		def test_paginate(self):
+				page_size = 20
+				search = AppleAppStoreSearch({"term": "Coffee", "page": 0, "num": page_size})
+
+				limit = 4
+				pages = search.pagination(limit=limit)
+
+				page_count = 0
+				result_count = 0
+
+				for page in pages:
+					page_count += 1
+					result_count += len(page["organic_results"])
+
+				self.assertEqual(page_count, limit)
+
 if __name__ == '__main__':
 		unittest.main()

tests/test_baidu_search.py

@@ -21,5 +21,22 @@ def test_get_json(self):
 				pp = pprint.PrettyPrinter(indent=2)
 				pp.pprint(data)
 
+		@unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided")
+		def test_paginate(self):
+				page_size = 30
+				search = BaiduSearch({"q": "Coffee", "pn": 10, "m": page_size})
+
+				limit = 3
+				pages = search.pagination(limit=limit)
+
+				page_count = 0
+				result_count = 0
+
+				for page in pages:
+					page_count += 1
+					result_count += len(page["organic_results"])
+
+				self.assertEqual(page_count, limit)
+
 if __name__ == '__main__':
 		unittest.main()