diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..289eb69 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: serpapi-python + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] + + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: pytest + env: + API_KEY: ${{secrets.API_KEY}} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8baaaec --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +script/ +.coverage +docs/build +dist/ +.pytest_cache/ +serpapi.egg-info/ + +serpapi/__pycache__ +tests/__pycache__ +*.pyc +.DS_Store diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e3e2233 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018-2022 SerpApi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/MIT-LICENSE.txt b/MIT-LICENSE.txt new file mode 100644 index 0000000..7aa3caf --- /dev/null +++ b/MIT-LICENSE.txt @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..0933b4c --- /dev/null +++ b/Makefile @@ -0,0 +1,69 @@ +# Automate pip package development +# +# Usage +# To release a package. +# - update version in serpapi/_version.py +# - review README version +# - run +# $ make release + +# current version +version=$(shell grep version setup.py | cut -d"'" -f2) +dist=dist/serpapi-$(version).tar.gz + +.PHONY: build + +all: clean install readme doc lint test build oobt check + +clean: + find . -name '*.pyc' -delete + find . -type d -name "__pycache__" -delete + python3 -m pip uninstall serpapi + +# lint check +lint: + python3 -m pylint serpapi + +# test with Python 3 +test: + python3 -mpytest --cov=serpapi --cov-report html tests/*.py + +# install dependencies +# +# pytest-cov - code coverage extension for pytest +# sphinx - documentation +# twine - release automation +install: + python3 -m pip install -U setuptools + python3 -m pip install -r requirements.txt + python3 -m pip install pylint + python3 -m pip install pytest-cov + python3 -m pip install twine + python3 -m pip install sphinx + +readme: + erb -T '-' README.md.erb > README.md + +doc: readme + $(MAKE) -C docs/ html + +# https://packaging.python.org/tutorials/packaging-projects/ +build: + python3 setup.py sdist + +# out of box testing / user acceptance before delivery +oobt: build + python3 -m pip install ./${dist} + python3 oobt/demo.py + + +check: oobt + python3 -m twine check ${dist} + +release: # check + python3 -m twine upload ${dist} + +# run example only +# and display output (-s) +example: + python3 -m pytest -s "tests/test_example.py::TestExample::test_async" diff --git a/README.md b/README.md index 4ef0d7c..71aa6ea 100644 --- a/README.md +++ b/README.md @@ -1 +1,747 @@ -master branch ready for pull request \ No newline at end of file + +
+

SerpApi Python Library

+ serpapi python library logo + + ![Package](https://badge.fury.io/py/serpapi.svg) + ![Downloads](https://static.pepy.tech/personalized-badge/serpapi?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=Downloads) + [![serpapi-python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) +
+ + +Integrate search data into your Ruby application. This library is the official wrapper for SerpApi (https://serpapi.com). + +SerpApi supports Google, Google Maps, Google Shopping, Baidu, Yandex, Yahoo, eBay, App Stores, and more. + +## Installation +Python3 must be installed. + +```sh +$ pip install serpapi +``` + +## Simple usage + +```python +import serpapi +client = serpapi.Client({ + 'api_key': "secret_api_key", # set personal API key from serpapi.com/dashboard + 'engine': "google", # set default search engine +}) +results = client.search({ + q: "coffee", # google query + location: "Austin,TX" # force the location [optional] +}) +print(results['organic_results']) +``` + +This example runs a search for "coffee" on Google. It then returns the results as a regular Ruby Hash. See the [playground](https://serpapi.com/playground) to generate your own code. + +## Advanced Usage +### Search API +```python +# load pip package +import serpapi + +# serpapi client created with default parameters +client = serpapi.Client({'api_key': 'secret_key', 'engine': 'google'}) + +# We recommend that you keep your keys safe. +# At least, don't commit them in plain text. +# More about configuration via environment variables: +# https://hackernoon.com/all-the-secrets-of-encrypting-api-keys-in-ruby-revealed-5qf3t5l + +# search query overview (more fields available depending on search engine) +params = { + # select the search engine (full list: https://serpapi.com/) + 'engine': "google", + # actual search query for google + 'q': "Coffee", + # then adds search engine specific options. + # for example: google specific parameters: https://serpapi.com/search-api + 'google_domain': "Google Domain", + 'location': "Location Requested", # example: Portland,Oregon,United States [see: Location API](#Location-API) + 'device': "desktop|mobile|tablet", + 'hl': "Google UI Language", + 'gl': "Google Country", + 'safe': "Safe Search Flag", + 'num': "Number of Results", + 'start': "Pagination Offset", + 'tbm': "nws|isch|shop", + 'tbs': "custom to be client criteria", + # tweak HTTP client behavior + 'async': False, # true when async call enabled. + 'timeout': 60, # HTTP timeout in seconds on the client side only. +} + +# formated search results as a Hash +# serpapi.com converts HTML -> JSON +results = client.search(params) + +# raw search engine html as a String +# serpapi.com acts a proxy to provive high throughputs, no search limit and more. +raw_html = client.html(params) +``` + +[Google search documentation](https://serpapi.com/search-api). More hands on examples are available below. + +### Documentation + * [API documentation](https://rubydoc.info/github/serpapi/serpapi-ruby/master) + * [Full documentation on SerpApi.com](https://serpapi.com) + * [Library Github page](https://github.com/serpapi/serpapi-ruby) + * [Library GEM page](https://rubygems.org/gems/serpapi/) + * [API health status](https://serpapi.com/status) + +### Location API + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +locations = client.location({'q':'Austin', 'limit': 3}) +print([loc['canonical_name'] for loc in locations]) +``` + +it prints the first 3 locations matching Austin: +```python +['Austin,TX,Texas,United States', 'Austin,Texas,United States', 'Rochester,MN-Mason City,IA-Austin,MN,United States'] +``` + +NOTE: api_key is not required for this endpoint. + +### Search Archive API + +This API allows retrieving previous search results. +To fetch earlier results from the search_id. + +First, you need to run a search and save the search id. +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key', 'engine': 'google'}) +results = client.search({'q': "Coffee"}) +search_id = results['search_metadata']['id'] +print("search_id: " + search_id) +``` + +Now let's retrieve the previous search results from the archive. + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +results = client.search_archive('search_id') +print(results) +``` + +This code prints the search results from the archive. :) + +### Account API + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +print(client.account()) +``` + +It prints your account information including plan, credit, montly + +## Basic example per search engines + +### Search bing +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'bing', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_bing_test.py] +see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api) + +### Search baidu +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'baidu', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_baidu_test.py] +see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api) + +### Search yahoo +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'yahoo', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'p': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_yahoo_test.py] +see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api) + +### Search youtube +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'youtube', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'search_query': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['video_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_youtube_test.py] +see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api) + +### Search walmart +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'walmart', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'query': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_walmart_test.py] +see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api) + +### Search ebay +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'ebay', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + '_nkw': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_ebay_test.py] +see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api) + +### Search naver +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'naver', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'query': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['ads_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_naver_test.py] +see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api) + +### Search home depot +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'home_depot', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'table', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['products']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_home_depot_test.py] +see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api) + +### Search apple app store +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'apple_app_store', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'term': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_apple_app_store_test.py] +see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store) + +### Search duckduckgo +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'duckduckgo', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_duckduckgo_test.py] +see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api) + +### Search google +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'coffee', + 'engine': 'google', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_test.py] +see: [https://serpapi.com/search-api](https://serpapi.com/search-api) + +### Search google scholar +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google_scholar', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_scholar_test.py] +see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api) + +### Search google autocomplete +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google_autocomplete', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['suggestions']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_autocomplete_test.py] +see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) + +### Search google product +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google_product', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'coffee', + 'product_id': '4172129135583325756', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['product_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_product_test.py] +see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api) + +### Search google reverse image +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google_reverse_image', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'image_url': 'https://i.imgur.com/5bGzZi7.jpg', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['image_sizes']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_reverse_image_test.py] +see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image) + +### Search google events +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google_events', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['events_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_events_test.py] +see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api) + +### Search google local services +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google_local_services', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'electrician', + 'data_cid': '6745062158417646970', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['local_ads']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_local_services_test.py] +see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api) + +### Search google maps +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google_maps', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'pizza', + 'll': '@40.7455096,-74.0083012,15.1z', + 'type': 'search', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['local_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_maps_test.py] +see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api) + +### Search google jobs +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google_jobs', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['jobs_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_jobs_test.py] +see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api) + +### Search google play +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google_play', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'q': 'kite', + 'store': 'apps', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_play_test.py] +see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api) + +### Search google images +```python +import serpapi +import pprint +import os + +client = serpapi.Client({ + 'engine': 'google_images', + 'api_key': os.getenv("API_KEY") + }) +data = client.search({ + 'engine': 'google_images', + 'tbm': 'isch', + 'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['images_results']) +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` +``` +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_images_test.py] +see: [https://serpapi.com/images-results](https://serpapi.com/images-results) + + +# Developer Guide +TODO update this section +### Key goals + - High code quality + - KISS principles (https://en.wikipedia.org/wiki/KISS_principle) + - Brand centric instead of search engine based + - No hard coded logic per search engine on the client side. + - Simple HTTP client (lightweight, reduced dependency) + - No magic default values + - Thread safe + - Leak free + - Easy to extends + - Defensive code style (raise custom exception) + - TDD - Test driven development (lint, ~100% code coverage) + - Follow best API coding pratice per platform + +### Inspiration +The API design was inpired by the most popular Python packages. + - urllib3 - https://github.com/urllib3/urllib3 + - Boto3 - https://github.com/boto/boto3 + - Numpy - + +### Quality expectation + - 0 lint issues using pylint `make lint` + - 99% code coverage running `make test` + - 100% test passing: `make test` + +# Developer Guide +## Design : UML diagram +### Client design: Class diagram +```mermaid +classDiagram + CustomClient *-- Client + HttpClient <-- Client + HttpClient *-- urllib3 + HttpClient *-- ObjectDecoder + + class Client { + 'engine': String + 'api_key': String + parameter: Hash + search() + html() + location() + search_archive() + account() + } + + class HttpClient { + start() + decode() + } + + class urllib3 { + request() + } +``` + +## JSON search() : Sequence diagram +```mermaid +sequenceDiagram + Client->>SerpApi.com: search() : http request + SerpApi.com-->>SerpApi.com: query search engine + SerpApi.com-->>SerpApi.com: parse HTML into JSON + SerpApi.com-->>Client: JSON payload + Client-->>Client: decode JSON into Dict +``` + +where: + - The end user implements the application. + - Client refers to SerpApi:Client. + - SerpApi.com is the backend HTTP / REST service. + - Engine refers to Google, Baidu, Bing, and more. + +The SerpApi.com service (backend) + - executes a scalable search on `'engine': "google"` using the search query: `q: "coffee"`. + - parses the messy HTML responses from Google on the backend. + - returns a standardized JSON response. +The class SerpApi::Client (client side / ruby): + - Format the request to SerpApi.com server. + - Execute HTTP Get request. + - Parse JSON into Ruby Hash using a standard JSON library. +Et voila! + +## Continuous integration +We love "true open source" and "continuous integration", and Test Drive Development (TDD). + We are using RSpec to test [our infrastructure around the clock]) using Github Action to achieve the best QoS (Quality Of Service). + +The directory spec/ includes specification which serves the dual purposes of examples and functional tests. + +Set your secret API key in your shell before running a test. +```bash +export API_KEY="your_secret_key" +``` +Install testing dependency +```bash +$ make install +``` + +Check code quality using Lint. +```bash +$ make lint +``` + +Run regression. +```bash +$ make test +``` + +To flush the flow. +```bash +$ make +``` + +Open coverage report generated by `rake test` +```sh +open coverage/index.html +``` + +Open ./Rakefile for more information. + +Contributions are welcome. Feel to submit a pull request! + +## Dependencies + +HTTP requests are executed using [URL LIB3 documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html). + +## TODO + - [] Release version 1.0.0 diff --git a/README.md.erb b/README.md.erb new file mode 100644 index 0000000..624159f --- /dev/null +++ b/README.md.erb @@ -0,0 +1,373 @@ +<%- +def snippet(format, path) + lines = File.new(path).readlines + stop = lines.size - 1 + slice = lines[9..stop] + slice.reject! { |l| l.match?(/self.assertIsNone\(/) } + buf = slice.map { |l| l.gsub(/(^\s{4})/, '').gsub(/^\s*$/, '') }.join + url = path.gsub(/^.*\/serpapi-python/, 'https://github.com/serpapi/serpapi-python') + buf.gsub!('self.assertIsNotNone(', "pp = pprint.PrettyPrinter(indent=2)\npp.pprint(") + %Q(```#{format}\nimport serpapi\nimport pprint\nimport os\n\n#{buf}```\ntest: [#{url}]) +end +-%> + +
+

SerpApi Python Library

+ serpapi python library logo + + ![Package](https://badge.fury.io/py/serpapi.svg) + ![Downloads](https://static.pepy.tech/personalized-badge/serpapi?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=Downloads) + [![serpapi-python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) +
+ + +Integrate search data into your Ruby application. This library is the official wrapper for SerpApi (https://serpapi.com). + +SerpApi supports Google, Google Maps, Google Shopping, Baidu, Yandex, Yahoo, eBay, App Stores, and more. + +## Installation +Python3 must be installed. + +```sh +$ pip install serpapi +``` + +## Simple usage + +```python +import serpapi +client = serpapi.Client({ + 'api_key': "secret_api_key", # set personal API key from serpapi.com/dashboard + 'engine': "google", # set default search engine +}) +results = client.search({ + q: "coffee", # google query + location: "Austin,TX" # force the location [optional] +}) +print(results['organic_results']) +``` + +This example runs a search for "coffee" on Google. It then returns the results as a regular Ruby Hash. See the [playground](https://serpapi.com/playground) to generate your own code. + +## Advanced Usage +### Search API +```python +# load pip package +import serpapi + +# serpapi client created with default parameters +client = serpapi.Client({'api_key': 'secret_key', 'engine': 'google'}) + +# We recommend that you keep your keys safe. +# At least, don't commit them in plain text. +# More about configuration via environment variables: +# https://hackernoon.com/all-the-secrets-of-encrypting-api-keys-in-ruby-revealed-5qf3t5l + +# search query overview (more fields available depending on search engine) +params = { + # select the search engine (full list: https://serpapi.com/) + 'engine': "google", + # actual search query for google + 'q': "Coffee", + # then adds search engine specific options. + # for example: google specific parameters: https://serpapi.com/search-api + 'google_domain': "Google Domain", + 'location': "Location Requested", # example: Portland,Oregon,United States [see: Location API](#Location-API) + 'device': "desktop|mobile|tablet", + 'hl': "Google UI Language", + 'gl': "Google Country", + 'safe': "Safe Search Flag", + 'num': "Number of Results", + 'start': "Pagination Offset", + 'tbm': "nws|isch|shop", + 'tbs': "custom to be client criteria", + # tweak HTTP client behavior + 'async': False, # true when async call enabled. + 'timeout': 60, # HTTP timeout in seconds on the client side only. +} + +# formated search results as a Hash +# serpapi.com converts HTML -> JSON +results = client.search(params) + +# raw search engine html as a String +# serpapi.com acts a proxy to provive high throughputs, no search limit and more. +raw_html = client.html(params) +``` + +[Google search documentation](https://serpapi.com/search-api). More hands on examples are available below. + +### Documentation + * [API documentation](https://rubydoc.info/github/serpapi/serpapi-ruby/master) + * [Full documentation on SerpApi.com](https://serpapi.com) + * [Library Github page](https://github.com/serpapi/serpapi-ruby) + * [Library GEM page](https://rubygems.org/gems/serpapi/) + * [API health status](https://serpapi.com/status) + +### Location API + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +locations = client.location({'q':'Austin', 'limit': 3}) +print([loc['canonical_name'] for loc in locations]) +``` + +it prints the first 3 locations matching Austin: +```python +['Austin,TX,Texas,United States', 'Austin,Texas,United States', 'Rochester,MN-Mason City,IA-Austin,MN,United States'] +``` + +NOTE: api_key is not required for this endpoint. + +### Search Archive API + +This API allows retrieving previous search results. +To fetch earlier results from the search_id. + +First, you need to run a search and save the search id. +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key', 'engine': 'google'}) +results = client.search({'q': "Coffee"}) +search_id = results['search_metadata']['id'] +print("search_id: " + search_id) +``` + +Now let's retrieve the previous search results from the archive. + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +results = client.search_archive('search_id') +print(results) +``` + +This code prints the search results from the archive. :) + +### Account API + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +print(client.account()) +``` + +It prints your account information including plan, credit, montly + +## Basic example per search engines + +### Search bing +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_bing_test.py') %> +see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api) + +### Search baidu +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_baidu_test.py') %> +see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api) + +### Search yahoo +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_yahoo_test.py') %> +see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api) + +### Search youtube +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_youtube_test.py') %> +see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api) + +### Search walmart +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_walmart_test.py') %> +see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api) + +### Search ebay +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_ebay_test.py') %> +see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api) + +### Search naver +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_naver_test.py') %> +see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api) + +### Search home depot +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_home_depot_test.py') %> +see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api) + +### Search apple app store +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_apple_app_store_test.py') %> +see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store) + +### Search duckduckgo +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_duckduckgo_test.py') %> +see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api) + +### Search google +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_test.py') %> +see: [https://serpapi.com/search-api](https://serpapi.com/search-api) + +### Search google scholar +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_scholar_test.py') %> +see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api) + +### Search google autocomplete +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_autocomplete_test.py') %> +see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) + +### Search google product +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_product_test.py') %> +see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api) + +### Search google reverse image +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_reverse_image_test.py') %> +see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image) + +### Search google events +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_events_test.py') %> +see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api) + +### Search google local services +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_local_services_test.py') %> +see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api) + +### Search google maps +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_maps_test.py') %> +see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api) + +### Search google jobs +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_jobs_test.py') %> +see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api) + +### Search google play +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_play_test.py') %> +see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api) + +### Search google images +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_images_test.py') %> +see: [https://serpapi.com/images-results](https://serpapi.com/images-results) + + +# Developer Guide +TODO update this section +### Key goals + - High code quality + - KISS principles (https://en.wikipedia.org/wiki/KISS_principle) + - Brand centric instead of search engine based + - No hard coded logic per search engine on the client side. + - Simple HTTP client (lightweight, reduced dependency) + - No magic default values + - Thread safe + - Leak free + - Easy to extends + - Defensive code style (raise custom exception) + - TDD - Test driven development (lint, ~100% code coverage) + - Follow best API coding pratice per platform + +### Inspiration +The API design was inpired by the most popular Python packages. + - urllib3 - https://github.com/urllib3/urllib3 + - Boto3 - https://github.com/boto/boto3 + - Numpy - + +### Quality expectation + - 0 lint issues using pylint `make lint` + - 99% code coverage running `make test` + - 100% test passing: `make test` + +# Developer Guide +## Design : UML diagram +### Client design: Class diagram +```mermaid +classDiagram + CustomClient *-- Client + HttpClient <-- Client + HttpClient *-- urllib3 + HttpClient *-- ObjectDecoder + + class Client { + 'engine': String + 'api_key': String + parameter: Hash + search() + html() + location() + search_archive() + account() + } + + class HttpClient { + start() + decode() + } + + class urllib3 { + request() + } +``` + +## JSON search() : Sequence diagram +```mermaid +sequenceDiagram + Client->>SerpApi.com: search() : http request + SerpApi.com-->>SerpApi.com: query search engine + SerpApi.com-->>SerpApi.com: parse HTML into JSON + SerpApi.com-->>Client: JSON payload + Client-->>Client: decode JSON into Dict +``` + +where: + - The end user implements the application. + - Client refers to SerpApi:Client. + - SerpApi.com is the backend HTTP / REST service. + - Engine refers to Google, Baidu, Bing, and more. + +The SerpApi.com service (backend) + - executes a scalable search on `'engine': "google"` using the search query: `q: "coffee"`. + - parses the messy HTML responses from Google on the backend. + - returns a standardized JSON response. +The class SerpApi::Client (client side / ruby): + - Format the request to SerpApi.com server. + - Execute HTTP Get request. + - Parse JSON into Ruby Hash using a standard JSON library. +Et voila! + +## Continuous integration +We love "true open source" and "continuous integration", and Test Drive Development (TDD). + We are using RSpec to test [our infrastructure around the clock]) using Github Action to achieve the best QoS (Quality Of Service). + +The directory spec/ includes specification which serves the dual purposes of examples and functional tests. + +Set your secret API key in your shell before running a test. +```bash +export API_KEY="your_secret_key" +``` +Install testing dependency +```bash +$ make install +``` + +Check code quality using Lint. +```bash +$ make lint +``` + +Run regression. +```bash +$ make test +``` + +To flush the flow. +```bash +$ make +``` + +Open coverage report generated by `rake test` +```sh +open coverage/index.html +``` + +Open ./Rakefile for more information. + +Contributions are welcome. Feel to submit a pull request! + +## Dependencies + +HTTP requests are executed using [URL LIB3 documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html). + +## TODO + - [] Release version 1.0.0 diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..32500da --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,59 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('../..')) + +# -- Project information ----------------------------------------------------- + +project = 'serpapi-python' +copyright = '© 2022 SerpApi, LLC' +author = 'Victor Benarbia' + +# The full version, including alpha/beta/rc tags +release = '1.0.0-beta' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.githubpages', + 'sphinx.ext.autodoc' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'classic' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Extension configuration ------------------------------------------------- \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..d20ad9b --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,24 @@ +.. serapi-python documentation master file, created by + sphinx-quickstart on Sun Apr 3 21:09:40 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to serapi-python's documentation! +========================================= + +.. automodule:: serpapi + :members: serpapi +.. automodule:: serpapi.serpapi + :members: +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/oobt/demo.py b/oobt/demo.py new file mode 100644 index 0000000..3815eae --- /dev/null +++ b/oobt/demo.py @@ -0,0 +1,26 @@ +# Out Of Box testing +# +# Run simple serparpi search +# +import serpapi +import sys, os, pprint + +print("initialize serpapi client") +client = serpapi.Client({ + "api_key": os.getenv("API_KEY", "demo") +}) +print("execute search") +result = client.search({ + "q": "coffee", + "location": "Austin,Texas", + }) +print("display result:") +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(result) +print("------") +if len(result) > 0: + print("OK: Out of box tests passed") + sys.exit(0) + +print("FAIL: Out of box tests failed: no result") +sys.exit(1) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..e069338 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +minversion = 7.0 +addopts = -ra -q +testpaths = tests +python_files = test_*.py example_*.py \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..121d69d --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +urllib3==1.26.9 diff --git a/serpapi/__init__.py b/serpapi/__init__.py new file mode 100644 index 0000000..72c4a4b --- /dev/null +++ b/serpapi/__init__.py @@ -0,0 +1,12 @@ +"""serpapi.com client implementation in python +This simple HTTP client allow to interact with SerpApi.com + +see: https://serpapi.com for more information +""" + +from ._version import __version__ +from .error import SerpApiException +from .serpapi import Client + +__author__ = "Victor Benarbia victor@serpapi.com" +__license__ = "MIT" diff --git a/serpapi/_version.py b/serpapi/_version.py new file mode 100644 index 0000000..d45db1f --- /dev/null +++ b/serpapi/_version.py @@ -0,0 +1,2 @@ +"""package current version""" +__version__ = "1.0.0" diff --git a/serpapi/error.py b/serpapi/error.py new file mode 100644 index 0000000..95af0c4 --- /dev/null +++ b/serpapi/error.py @@ -0,0 +1,3 @@ +"""wrap any serpapi related exception""" +class SerpApiException(Exception): + """custom exception for this module""" diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py new file mode 100644 index 0000000..c85d4e7 --- /dev/null +++ b/serpapi/serpapi.py @@ -0,0 +1,210 @@ +"""Client for SerpApi.com""" +import json +import urllib3 +from .error import SerpApiException +from ._version import __version__ + +class HttpClient: + """Simple HTTP client wrapper around urllib3""" + + BACKEND = "https://serpapi.com" + SUPPORTED_DECODER = ["json", "html"] + + def __init__(self, parameter: dict = None): + """Initialize a SerpApi Client with the default parameters provided. + An instance of urllib3 will be created + where + timeout is 60s by default + retries is disabled by default + both properties can be override by the parameter. + """ + # initialize the http client + self.http = urllib3.PoolManager() + + # initialize default client parameter + self.parameter = parameter + + # urllib3 configurations + # HTTP connect timeout + if "timeout" in parameter: + self.timeout = parameter["timeout"] + del parameter["timeout"] + else: + # 60s default + self.timeout = 60.0 + + # no HTTP retry + if "retries" in parameter: + self.retries = parameter["retries"] + del parameter["retries"] + else: + self.retries = False + + def start(self, path: str, parameter: dict = None, decoder: str = "json"): + """start HTTP request and decode response using urllib3. + The response is decoded using the selected decoder: + - html: raw HTML response + - json: deep dict contains search results + + Parameters: + --- + path: str + HTTP endpoint path under serpapi.com/ + decoder: str + define how to post process the HTTP response. + for example: json -> convert response to a dict + using the default JSON parser from Python + parameter: dict + search query + + Returns: + --- + dict|str + decoded HTTP response""" + # track client language + self.parameter["source"] = "serpapi-python:" + __version__ + self.parameter["output"] = decoder + + # merge parameter defaults and overrides + fields = self.parameter.copy() + fields.update(parameter) + + # execute HTTP get request + response = self.http.request("GET", + self.BACKEND + path, + fields=fields, + timeout=self.timeout, + retries=self.retries) + # decode response + return self.decode(response, decoder) + + def decode(self, response: any, decoder: str): + """Decode HTTP response using a given decoder""" + # handle HTTP error + if response.status != 200: + try: + raw = response.data.decode("utf-8") + payload = json.loads(raw) + raise SerpApiException(payload["error"]) + except Exception as ex: + raise SerpApiException(raw) from ex + + # HTTP success 200 + payload = response.data.decode("utf-8") + + # successful response decoding + if decoder == "json": + return json.loads(payload) + + if decoder == "html": + return payload + + raise SerpApiException(f"Invalid decoder: {decoder}, available: json, html") + + +class Client(HttpClient): + """ + Client performend http query to serpApi.com using urllib3 under the hood. + + The HTTP connection be tuned to allow + - retries : attempt to reconnect if the connection fail by default: False + - timeout : connection timeout by default 60s + for more details: https://urllib3.readthedocs.io/en/stable/user-guide.html + + """ + + def __init__(self, parameter: dict = None): + # define default parameter + if parameter is None: + parameter = {} + # initialize HTTP client + HttpClient.__init__(self, parameter) + + def search(self, parameter: dict = None, decoder: str = "json"): + """ + make search then decode the output + decoder supported "json", "html" + + Parameters + ---------- + parameter : dict + search query + decoder : str + set decoder to convert the datastructure received from + + Returns + ------- + dict|str + search results returns as : + dict if decoder = "json" + str if decoder = "html" + """ + return self.start(path="/search", parameter=parameter, decoder=decoder) + + def html(self, parameter: dict = None): + """ + html search + + Parameters + ---------- + parameter : dict + search query see: https://serpapi.com/search-api + + Returns + ------- + str + raw html search results directly from the search engine + """ + return self.start("/search", parameter, "html") + + def location(self, parameter: dict = None): + """ + Get location using Location API + + Parameters + ---------- + parameter : dict + location query like: {q: "Austin", limit: 5} + see: https://serpapi.com/locations-api + + Returns + ------- + array + list of matching locations + """ + return self.start("/locations.json", parameter, "json") + + def search_archive(self, search_id: str, decoder: str = "json"): + """ + Retrieve search results from the Search Archive API + + Parameters: + ----- + search_id: str + id from a previous client. in the JSON search response it is search_metadata.id + + """ + path = "/searches/" + str(search_id) + "." + if decoder in self.SUPPORTED_DECODER: + path += decoder + else: + raise SerpApiException(f"Invalid decoder: {decoder}, available: json, html. ") + return self.start(path, {}, decoder) + + def account(self, api_key: str = None): + """ + Get account information using Account API + + Parameters + --- + api_key: str + secret user key provided by serpapi.com + + Returns + --- + dict + user account information + """ + if api_key is not None: + self.parameter["api_key"] = api_key + return self.start("/account", self.parameter, "json") diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b2140f3 --- /dev/null +++ b/setup.py @@ -0,0 +1,32 @@ +from setuptools import setup, find_packages +from codecs import open +from os import path +from pathlib import Path +long_description = (Path(__file__).parent / "README.md").read_text() + +setup(name='serpapi', + version='1.0.0', + description='Scrape and search localized results from Google, Bing, Baidu, Yahoo, Yandex, Ebay, Homedepot, youtube at scale using SerpApi.com', + url='https://github.com/serpapi/serpapi-python', + author='vikoky', + author_email='victor@serpapi.com', + classifiers=[ + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Natural Language :: English', + 'Topic :: Utilities', + ], + python_requires='>=3.5', + zip_safe=False, + include_package_data=True, + license="MIT", + install_requires = ["urllib3"], + packages=find_packages(), + keywords='scrape,serp,api,json,search,localized,rank,google,bing,baidu,yandex,yahoo,ebay,scale,datamining,training,machine,ml,youtube,naver,walmart,apple,store,app,serpapi', + long_description=long_description, + long_description_content_type="text/markdown", +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..c739142 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +import serpapi \ No newline at end of file diff --git a/tests/example_search_apple_app_store_test.py b/tests/example_search_apple_app_store_test.py new file mode 100644 index 0000000..cd9e150 --- /dev/null +++ b/tests/example_search_apple_app_store_test.py @@ -0,0 +1,21 @@ +# Example: apple_app_store search engine +import unittest +import os +import serpapi + +class TestAppleAppStore(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_apple_app_store(self): + client = serpapi.Client({ + 'engine': 'apple_app_store', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'term': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_baidu_test.py b/tests/example_search_baidu_test.py new file mode 100644 index 0000000..0a98a9e --- /dev/null +++ b/tests/example_search_baidu_test.py @@ -0,0 +1,21 @@ +# Example: baidu search engine +import unittest +import os +import serpapi + +class TestBaidu(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_baidu(self): + client = serpapi.Client({ + 'engine': 'baidu', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_bing_test.py b/tests/example_search_bing_test.py new file mode 100644 index 0000000..ac246bb --- /dev/null +++ b/tests/example_search_bing_test.py @@ -0,0 +1,21 @@ +# Example: bing search engine +import unittest +import os +import serpapi + +class TestBing(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_bing(self): + client = serpapi.Client({ + 'engine': 'bing', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_duckduckgo_test.py b/tests/example_search_duckduckgo_test.py new file mode 100644 index 0000000..1c16fbb --- /dev/null +++ b/tests/example_search_duckduckgo_test.py @@ -0,0 +1,21 @@ +# Example: duckduckgo search engine +import unittest +import os +import serpapi + +class TestDuckduckgo(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_duckduckgo(self): + client = serpapi.Client({ + 'engine': 'duckduckgo', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_ebay_test.py b/tests/example_search_ebay_test.py new file mode 100644 index 0000000..c59efe1 --- /dev/null +++ b/tests/example_search_ebay_test.py @@ -0,0 +1,21 @@ +# Example: ebay search engine +import unittest +import os +import serpapi + +class TestEbay(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_ebay(self): + client = serpapi.Client({ + 'engine': 'ebay', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + '_nkw': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_autocomplete_test.py b/tests/example_search_google_autocomplete_test.py new file mode 100644 index 0000000..7e17695 --- /dev/null +++ b/tests/example_search_google_autocomplete_test.py @@ -0,0 +1,21 @@ +# Example: google_autocomplete search engine +import unittest +import os +import serpapi + +class TestGoogleAutocomplete(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_autocomplete(self): + client = serpapi.Client({ + 'engine': 'google_autocomplete', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['suggestions']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_events_test.py b/tests/example_search_google_events_test.py new file mode 100644 index 0000000..2c033aa --- /dev/null +++ b/tests/example_search_google_events_test.py @@ -0,0 +1,21 @@ +# Example: google_events search engine +import unittest +import os +import serpapi + +class TestGoogleEvents(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_events(self): + client = serpapi.Client({ + 'engine': 'google_events', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['events_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_images_test.py b/tests/example_search_google_images_test.py new file mode 100644 index 0000000..28876cc --- /dev/null +++ b/tests/example_search_google_images_test.py @@ -0,0 +1,23 @@ +# Example: google_images search engine +import unittest +import os +import serpapi + +class TestGoogleImages(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_images(self): + client = serpapi.Client({ + 'engine': 'google_images', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'engine': 'google_images', + 'tbm': 'isch', + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['images_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_jobs_test.py b/tests/example_search_google_jobs_test.py new file mode 100644 index 0000000..90fb9b3 --- /dev/null +++ b/tests/example_search_google_jobs_test.py @@ -0,0 +1,21 @@ +# Example: google_jobs search engine +import unittest +import os +import serpapi + +class TestGoogleJobs(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_jobs(self): + client = serpapi.Client({ + 'engine': 'google_jobs', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['jobs_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_local_services_test.py b/tests/example_search_google_local_services_test.py new file mode 100644 index 0000000..c4a9ef3 --- /dev/null +++ b/tests/example_search_google_local_services_test.py @@ -0,0 +1,22 @@ +# Example: google_local_services search engine +import unittest +import os +import serpapi + +class TestGoogleLocalServices(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_local_services(self): + client = serpapi.Client({ + 'engine': 'google_local_services', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'electrician', + 'data_cid': '6745062158417646970', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['local_ads']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_maps_test.py b/tests/example_search_google_maps_test.py new file mode 100644 index 0000000..1a96ddf --- /dev/null +++ b/tests/example_search_google_maps_test.py @@ -0,0 +1,23 @@ +# Example: google_maps search engine +import unittest +import os +import serpapi + +class TestGoogleMaps(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_maps(self): + client = serpapi.Client({ + 'engine': 'google_maps', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'pizza', + 'll': '@40.7455096,-74.0083012,15.1z', + 'type': 'search', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['local_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_play_test.py b/tests/example_search_google_play_test.py new file mode 100644 index 0000000..bb2adc5 --- /dev/null +++ b/tests/example_search_google_play_test.py @@ -0,0 +1,22 @@ +# Example: google_play search engine +import unittest +import os +import serpapi + +class TestGooglePlay(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_play(self): + client = serpapi.Client({ + 'engine': 'google_play', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'kite', + 'store': 'apps', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_product_test.py b/tests/example_search_google_product_test.py new file mode 100644 index 0000000..fcc7094 --- /dev/null +++ b/tests/example_search_google_product_test.py @@ -0,0 +1,22 @@ +# Example: google_product search engine +import unittest +import os +import serpapi + +class TestGoogleProduct(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_product(self): + client = serpapi.Client({ + 'engine': 'google_product', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + 'product_id': '4172129135583325756', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['product_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_reverse_image_test.py b/tests/example_search_google_reverse_image_test.py new file mode 100644 index 0000000..72caace --- /dev/null +++ b/tests/example_search_google_reverse_image_test.py @@ -0,0 +1,21 @@ +# Example: google_reverse_image search engine +import unittest +import os +import serpapi + +class TestGoogleReverseImage(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_reverse_image(self): + client = serpapi.Client({ + 'engine': 'google_reverse_image', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'image_url': 'https://i.imgur.com/5bGzZi7.jpg', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['image_sizes']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_scholar_test.py b/tests/example_search_google_scholar_test.py new file mode 100644 index 0000000..598fc3f --- /dev/null +++ b/tests/example_search_google_scholar_test.py @@ -0,0 +1,21 @@ +# Example: google_scholar search engine +import unittest +import os +import serpapi + +class TestGoogleScholar(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_scholar(self): + client = serpapi.Client({ + 'engine': 'google_scholar', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_test.py b/tests/example_search_google_test.py new file mode 100644 index 0000000..dff3585 --- /dev/null +++ b/tests/example_search_google_test.py @@ -0,0 +1,22 @@ +# Example: google search engine +import unittest +import os +import serpapi + +class TestGoogle(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google(self): + client = serpapi.Client({ + 'engine': 'google', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + 'engine': 'google', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_home_depot_test.py b/tests/example_search_home_depot_test.py new file mode 100644 index 0000000..4f97412 --- /dev/null +++ b/tests/example_search_home_depot_test.py @@ -0,0 +1,21 @@ +# Example: home_depot search engine +import unittest +import os +import serpapi + +class TestHomeDepot(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_home_depot(self): + client = serpapi.Client({ + 'engine': 'home_depot', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'table', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['products']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_naver_test.py b/tests/example_search_naver_test.py new file mode 100644 index 0000000..b780973 --- /dev/null +++ b/tests/example_search_naver_test.py @@ -0,0 +1,21 @@ +# Example: naver search engine +import unittest +import os +import serpapi + +class TestNaver(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_naver(self): + client = serpapi.Client({ + 'engine': 'naver', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'query': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['ads_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_walmart_test.py b/tests/example_search_walmart_test.py new file mode 100644 index 0000000..f9cd916 --- /dev/null +++ b/tests/example_search_walmart_test.py @@ -0,0 +1,21 @@ +# Example: walmart search engine +import unittest +import os +import serpapi + +class TestWalmart(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_walmart(self): + client = serpapi.Client({ + 'engine': 'walmart', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'query': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_yahoo_test.py b/tests/example_search_yahoo_test.py new file mode 100644 index 0000000..5a54ad4 --- /dev/null +++ b/tests/example_search_yahoo_test.py @@ -0,0 +1,21 @@ +# Example: yahoo search engine +import unittest +import os +import serpapi + +class TestYahoo(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_yahoo(self): + client = serpapi.Client({ + 'engine': 'yahoo', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'p': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_youtube_test.py b/tests/example_search_youtube_test.py new file mode 100644 index 0000000..186811b --- /dev/null +++ b/tests/example_search_youtube_test.py @@ -0,0 +1,21 @@ +# Example: youtube search engine +import unittest +import os +import serpapi + +class TestYoutube(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_youtube(self): + client = serpapi.Client({ + 'engine': 'youtube', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'search_query': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['video_results']) + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/test_account_api.py b/tests/test_account_api.py new file mode 100644 index 0000000..4d1e911 --- /dev/null +++ b/tests/test_account_api.py @@ -0,0 +1,12 @@ +import random +import unittest +import os +import serpapi + +class TestAccountApi(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_get_account(self): + client = serpapi.Client() + account = client.account(os.getenv('API_KEY')) + self.assertIsNotNone(account.get("account_id")) diff --git a/tests/test_example.py b/tests/test_example.py new file mode 100644 index 0000000..b5ae32a --- /dev/null +++ b/tests/test_example.py @@ -0,0 +1,152 @@ +# # Unit testing +# import unittest + +# # Operating system +# import os + +# # regular expression library +# import re + +# # safe queue +# import sys +# if (sys.version_info > (3, 0)): +# from queue import Queue +# else: +# from Queue import Queue + +# # Time utility +# import time + +# # Serp API search +# from serpapi import GoogleSearch + +# # download file with wget +# #import wget + +# class TestExample(unittest.TestCase): + +# def setUp(self): +# GoogleSearch.SERP_API_KEY = os.getenv("API_KEY","demo") + +# @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") +# def test_get_json(self): +# client = GoogleSearch({"q": "Coffee", "engine": "google_scholar"}) +# data = client.get_json() +# print(data['search_metadata']) +# search_id = data['search_metadata']['id'] +# # retrieve search from the archive - blocker +# print(search_id + ": get search from archive") +# raw_html = client.get_search_archive(search_id, 'html') +# # print(search_id + ": status = " + search_archived['search_metadata']['status']) +# print(raw_html) +# #print(search_html) + +# @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") +# def test_search_google_images(self): +# client = GoogleSearch({"q": "coffe", "tbm": "isch"}) +# for image_result in client.get_json()['images_results']: +# try: +# link = image_result["original"] +# print("link is found: " + link) +# # uncomment the line below to down the original image +# # wget.download(link, '.') +# except: +# print("link is not found.") +# pass +# # https://github.com/serpapi/showcase-serpapi-tensorflow-keras-image-training/blob/master/fetch.py + +# @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") +# def test_async(self): +# # store searches +# search_queue = Queue() + +# # Serp API search +# client = GoogleSearch({ +# "location": "Austin,Texas", +# "async": True +# }) + +# # loop through companies +# for company in ['amd','nvidia','intel']: +# print("execute async search: q = " + company) +# client.params_dict["q"] = company +# data = client.get_dict() +# if data is not None: +# print("oops data is empty for: " + company) +# continue +# print("add search to the queue where id: " + data['search_metadata']['id']) +# # add search to the search_queue +# search_queue.put(data) + +# print("wait until all search statuses are cached or success") + +# # Create regular search +# client = GoogleSearch({"async": True}) +# while not search_queue.empty(): +# data = search_queue.get() +# search_id = data['search_metadata']['id'] + +# # retrieve search from the archive - blocker +# print(search_id + ": get search from archive") +# search_archived = client.get_search_archive(search_id) +# print(search_id + ": status = " + search_archived['search_metadata']['status']) + +# # check status +# if re.search('Cached|Success', search_archived['search_metadata']['status']): +# print(search_id + ": search done with q = " + search_archived['search_parameters']['q']) +# else: +# # requeue search_queue +# print(search_id + ": requeue search") +# search_queue.put(search) +# # wait 1s +# time.sleep(1) +# # search is over. +# print('all searches completed') + +# @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") +# def test_search_google_news(self): +# client = GoogleSearch({ +# "q": "coffe", # search search +# "tbm": "nws", # news +# "tbs": "qdr:d", # last 24h +# "num": 10 +# }) +# for offset in [0,1,2]: +# client.params_dict["start"] = offset * 10 +# data = client.get_json() +# for news_result in data['news_results']: +# print(str(news_result['position'] + offset * 10) + " - " + news_result['title']) + +# @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") +# def test_search_google_shopping(self): +# client = GoogleSearch({ +# "q": "coffe", # search search +# "tbm": "shop", # news +# "tbs": "p_ord:rv", # last 24h +# "num": 100 +# }) +# data = client.get_json() +# if 'shopping_results' in data: +# for shopping_result in data['shopping_results']: +# print(str(shopping_result['position']) + " - " + shopping_result['title']) +# else: +# print("WARNING: oops shopping_results is missing from search result with tbm=shop") + +# @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") +# def test_search_by_location(self): +# for city in ["new york", "paris", "berlin"]: +# location = GoogleSearch({}).get_location(city, 1)[0]["canonical_name"] +# client = GoogleSearch({ +# "q": "best coffee shop", # search search +# "location": location, +# "num": 10, +# "start": 0 +# }) +# data = client.get_json() +# self.assertIsNone(data.get("error")) +# top_result = data['organic_results'][0]["title"] +# print("top coffee result for " + location + " is: " + top_result) + + +# if __name__ == '__main__': +# unittest.main() diff --git a/tests/test_location_api.py b/tests/test_location_api.py new file mode 100644 index 0000000..f64b2ca --- /dev/null +++ b/tests/test_location_api.py @@ -0,0 +1,16 @@ +import unittest +import os +import serpapi + +class TestLocationApi(unittest.TestCase): + + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_get_account(self): + client = serpapi.Client({'api_key': os.getenv('API_KEY')}) + locations = client.location({'q':'Austin', 'limit': 3}) + self.assertGreater(len(locations), 1) + self.assertLessEqual(len(locations), 3) + self.assertTrue('id' in locations[0]) + self.assertTrue('name' in locations[0]) + self.assertTrue('canonical_name' in locations[0]) diff --git a/tests/test_search_archive.py b/tests/test_search_archive.py new file mode 100644 index 0000000..c549fe0 --- /dev/null +++ b/tests/test_search_archive.py @@ -0,0 +1,37 @@ +import unittest +import os +import serpapi +import pytest + +class TestSearchArchive(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_archive(self): + client = serpapi.Client({ + "engine": "google", + "api_key": os.getenv("API_KEY") + }) + data = client.search({ + "q": "Coffee", + "location": "Austin,Texas" + }) + self.assertEqual(data.get("error"), None) + self.assertIsNotNone(data["organic_results"][0]["title"]) + # search unique search identifier + search_id = data['search_metadata']['id'] + # fetch results from the archive (free of charge) + data_archive = client.search_archive(search_id) + self.assertEqual(data_archive['organic_results'][0], data["organic_results"][0]) + + # fetch results from the archive again (code coverage) + object_archive = client.search_archive(search_id, 'json') + self.assertIsNotNone(object_archive) + self.assertEqual(object_archive['organic_results'][0]['title'], data["organic_results"][0]["title"]) + + def test_bad_decoder(self): + client = serpapi.Client({ + "engine": "google", + "api_key": os.getenv("API_KEY") + }) + with pytest.raises(serpapi.SerpApiException, match=r'Invalid decoder'): + client.search_archive('007', 'bad') diff --git a/tests/test_serpapi.py b/tests/test_serpapi.py new file mode 100644 index 0000000..60981a0 --- /dev/null +++ b/tests/test_serpapi.py @@ -0,0 +1,102 @@ +import unittest +import os +import pprint +import serpapi +import pytest + +# This test shows how to extends serpapi.Client +# without using client engine wrapper. +# +class TestSerpApi(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search(self): + client = serpapi.Client({ + "engine": "google", + "api_key": os.getenv("API_KEY") + }) + data = client.search({ + "q": "Coffee", + "location": "Austin,Texas" + }) + assert data.get("error") == None + self.assertIsNotNone(data["organic_results"][0]["title"]) + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_html(self): + client = serpapi.Client({ + "engine": "google", + "api_key": os.getenv("API_KEY") + }) + data = client.html({ + "q": "Coffee", + "location": "Austin,Texas", + }) + self.assertRegex(data, r'$') + + def test_invalid_api_key(self): + client = serpapi.Client({ + "engine": "google", + "api_key": "invalid_api_key" + }) + with pytest.raises(serpapi.SerpApiException, match=r'Invalid API key'): + client.search({ + "q": "Coffee", + "location": "USA", + }) + + def test_invalid_decoder(self): + client = serpapi.Client({ + "engine": "google", + "api_key": os.getenv("API_KEY") + }) + mockResponse = MockResponse() + self.assertEqual(mockResponse.status, 200) + with pytest.raises(serpapi.SerpApiException, match=r'Invalid decoder'): + client.decode(mockResponse, 'bad') + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_error_missing_engine(self): + client = serpapi.Client({ + "api_key": os.getenv("API_KEY"), + "engine": "" + }) + with pytest.raises(serpapi.SerpApiException, match=r'Unsupported.*search engine.'): + client.search({"q": "Coffee"}) + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_missing_q(self): + client = serpapi.Client({ + "api_key": os.getenv("API_KEY") + }) + with pytest.raises(serpapi.SerpApiException, match=r'Missing query'): + client.search({"engine": "google"}) + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_no_parameter(self): + client = serpapi.Client() + with pytest.raises(serpapi.SerpApiException, match=r'Missing query'): + client.search({"engine": "google", 'api_key': os.getenv('API_KEY')}) + + + def debug(self, payload): + pp = pprint.PrettyPrinter(indent=2) + pp.pprint(payload) + +# Mock object to enable higher code coverage +# +class MockResponse: + '''Mock HTTP response in order to test serpapi.decode''' + def __init__(self, status=200): + self.status = 200 + self.data = MockString("{}") + +class MockString: + def __init__(self, data: str): + self.data = data + + def decode(self, encoding) -> str: + return self.data + +if __name__ == '__main__': + unittest.main()