From b415d0fd7ce6fdd16d100afcff46f789fc5bbb67 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 20 Mar 2022 21:47:19 -0500 Subject: [PATCH 01/25] basic client implementation and tests passing --- .gitignore | 2 + LICENSE | 21 ++ Makefile | 49 +++++ requirements.txt | 1 + serpapi/__init__.py | 12 ++ serpapi/__pycache__/__init__.cpython-39.pyc | Bin 0 -> 528 bytes serpapi/__pycache__/_version.cpython-39.pyc | Bin 0 -> 216 bytes serpapi/__pycache__/error.cpython-39.pyc | Bin 0 -> 404 bytes .../__pycache__/object_decoder.cpython-39.pyc | Bin 0 -> 1396 bytes serpapi/__pycache__/serpapi.cpython-39.pyc | Bin 0 -> 5090 bytes serpapi/_version.py | 2 + serpapi/error.py | 3 + serpapi/object_decoder.py | 42 ++++ serpapi/serpapi.py | 196 ++++++++++++++++++ setup.py | 32 +++ tests/__init__.py | 1 + tests/__pycache__/__init__.cpython-39.pyc | Bin 0 -> 172 bytes .../test_serpapi.cpython-39-pytest-7.1.0.pyc | Bin 0 -> 2714 bytes tests/test_serpapi.py | 57 +++++ 19 files changed, 418 insertions(+) create mode 100644 .gitignore create mode 100644 LICENSE create mode 100644 Makefile create mode 100644 requirements.txt create mode 100644 serpapi/__init__.py create mode 100644 serpapi/__pycache__/__init__.cpython-39.pyc create mode 100644 serpapi/__pycache__/_version.cpython-39.pyc create mode 100644 serpapi/__pycache__/error.cpython-39.pyc create mode 100644 serpapi/__pycache__/object_decoder.cpython-39.pyc create mode 100644 serpapi/__pycache__/serpapi.cpython-39.pyc create mode 100644 serpapi/_version.py create mode 100644 serpapi/error.py create mode 100644 serpapi/object_decoder.py create mode 100644 serpapi/serpapi.py create mode 100644 setup.py create mode 100644 tests/__init__.py create mode 100644 tests/__pycache__/__init__.cpython-39.pyc create mode 100644 tests/__pycache__/test_serpapi.cpython-39-pytest-7.1.0.pyc create mode 100644 tests/test_serpapi.py diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5996dfd --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +.coverage + diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..e3e2233 --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018-2022 SerpApi + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d8eae0c --- /dev/null +++ b/Makefile @@ -0,0 +1,49 @@ +# Automate pip package development +# +# current version +version=$(shell grep version setup.py | cut -d"'" -f2) + +.PHONY: build + +all: clean install test test2 + +clean: + find . -name '*.pyc' -delete + find . -type d -name "__pycache__" -delete + pip3 uninstall google_search_results + +install: + pip3 install -r requirements.txt + +lint: + pylint serpapi + +# Test with Python 3 +test: lint + pytest --cov=serpapi tests/ + +# run example only +# and display output (-s) +example: + pytest -s "tests/test_example.py::TestExample::test_async" + +install: + pip3 install -U setuptools + pip install pytest-cov + +doc: + pydoc + +# https://packaging.python.org/tutorials/packaging-projects/ +build: doc + python3 setup.py sdist + +oobt: build + pip3 install ./dist/google_search_results-$(version).tar.gz + python3 oobt/oobt.py + +check: oobt + twine check dist/google_search_results-$(version).tar.gz + +release: # check + twine upload dist/google_search_results-$(version).tar.gz diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..121d69d --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +urllib3==1.26.9 diff --git a/serpapi/__init__.py b/serpapi/__init__.py new file mode 100644 index 0000000..72c4a4b --- /dev/null +++ b/serpapi/__init__.py @@ -0,0 +1,12 @@ +"""serpapi.com client implementation in python +This simple HTTP client allow to interact with SerpApi.com + +see: https://serpapi.com for more information +""" + +from ._version import __version__ +from .error import SerpApiException +from .serpapi import Client + +__author__ = "Victor Benarbia victor@serpapi.com" +__license__ = "MIT" diff --git a/serpapi/__pycache__/__init__.cpython-39.pyc b/serpapi/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f0484f33ff69f768e61a9f6ca2dc995807e375d4 GIT binary patch literal 528 zcmYjOyH3L}6wRwi2(82?SZp0=79^w!p#nm{0IJl&ki~KnqZrw7WV?m-Ls<9%ekl?Y z6TiU3b=pXE6z3e@oO2&0gF)iJ`usjy+`5kQW6A#VfK1@kr?3@H<`iz>aaVYm=UVJ@ zUj$ju;(+%=n1z6QSyY5sTtqWx7;io_RW+&ED3=1|oYBf4CTdOvAYz!T5UWtVHl?hR zdC4@=Z5Z9n=hIF=IG1n8Nch!IMRJ4Qm?_Z=tS`6LNuntoqtcjKkJEHN>5EiINJT*m zP_#LdU+C8vy3HZRE2=ba7fRbW|yFPi1yYI<%njK9sW)(9SkLq<3p9@(mIlZv`YkU1c*mvRa0{0(N C8=<2B literal 0 HcmV?d00001 diff --git a/serpapi/__pycache__/_version.cpython-39.pyc b/serpapi/__pycache__/_version.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..be98ee12455e2d5eeefc6ebce41ad1341c590a4a GIT binary patch literal 216 zcmYe~<>g`kg44InlT3j0V-N=!FakLaKwQiNBvKfn7*ZIc7*m*n88n%z#0wIWvlG)( z6_QJfic<4R6v|SIiZk=`s#p#64D<~AG?{L($H%ASC&$O%;*O7p%Erg9WGG?=ng%9* zIp~KL1GVXwWhR&87wHES PPO2Tqt;I}0f`t(P*Zw=6 literal 0 HcmV?d00001 diff --git a/serpapi/__pycache__/error.cpython-39.pyc b/serpapi/__pycache__/error.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..88d8f66123013f5d78b06d6cd892bfd71468efe7 GIT binary patch literal 404 zcmY*UJx{|h5Vf5^Lq(-7NQ_=HKt>ioWk75dQpJ+RGWMaSN@8O>h<1fP!(R#`OMe3s zmxS_h(!INPw%_^5$z)70w(n>6E36+K`5Q5k8Kyr*&?F}}L{rUPiDp1wqMYeyLDG10 z1kM@99h5SF=Bkw5gID(Gt5JIVR*+sjU=SBdEt%>8B? literal 0 HcmV?d00001 diff --git a/serpapi/__pycache__/object_decoder.cpython-39.pyc b/serpapi/__pycache__/object_decoder.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b3dac8779deb281d2232fa40367cdb4d9c930a5d GIT binary patch literal 1396 zcmZuxPfHvz6i+glKd!4-+e#Iof%dXcUFfACrC2I06sm&sva}52B9OCUr@l9{ZdYg^Z}O7--rrv)T3Bci7(af!I(+65@*9=e;efIY)2x7y zq(?pwNhP}>l8so8O6P!d_~3EeH#R>P`QfRGt&l3trP47#?hOXoHcTTxC<1WOW0Lk9 z$)s~bdR%hp!q=4^nDwiLz1rSRV{Wk15-{GTX?`VaF5*19RN9J<2Yb6BP0o~%k&TSi z{n+-k5=kcZukA^mO~Tr&;W1`(Hv(ArFyFy6-#}PWQcI6$LPpNeDV-4+a?46)oe|Uf zmY2Mw$8bhT$#%%s^`UEB+#9i5uOzpANlSOk#*~ta&5|FG8XEw%EZObcm;h$w3mbYv zA8xlw53v3itX&*t?Z$e^YU_aQ(81475u{bjW+9F=6j~;+t<%AZ>_<8}S5|2=-P+jL zn65K8+!9ci=?;R5NUO|>qeP`r;1w*Xu@MyXDoW3UJy9ZqE>1zHY~6Q^c1+&aF_azx zLLZab&$K7WKxv3EwbhxgxVEG0sjYn0O#U82jZ%?7J`vsB-lNipV#n+$lO{|I{u3J}(@KtH{ORGX61smaN$ zgL^&YoP0vgwd_TW^Z?mwkdqtExF#Ubv9& lIKY3G!E6G`onTet-fo9!>}sw24_O7T8_#d>x5<$^;#c^B(SBv3 zT`-Hel5qV+kEb$BVvJ2c-rnaud7t+Z#zl|l$FBxn?&8k2(QpHMG_XIj(_mJ|-o@_z zV0N195^FN&bF<^H%dEvF&~w=p=CMiiTI?#DV$1gU%it+C?fJO$yPtA_bM<-5{5>gC|5?t(kuMWJ)2CnVC4EKj=UEu< z`niaraQlaT9^+iT+~fXUlCWh@(QM)MkR`orIZRe~ypqXaJK`%jgtN4p zhm0@p$$q5x4K9-WXgNtPNJEm`LZh(vvm`ETqAQ$4Sw*{NLG9H=eY)@A&bH7DVGX$< z=0am=bdA^Ox0FlamP%M~iitE3L7z)5#01C$fvs>$QFJCZk|g>hh=Bt^hmq-*R$*s6+7Xi= z{1V&6Gk?OgmQB=KF&*k)}Xmz_`9LjF@GT3F8&=^+B zY*|jju^jX~6Sp;()z-UQQ40rzJwjD=QNM{>NtO0CH15Ae_r z4mgZH5WPJzycl*t*i}XdyGnCela^D$&+;@)goJV5BrV@u^ly@$>8Uk+Q{#g0wo!>! zn;#7=)mP2BRs!spL5TGP(U-eZ5UnTlHU;Q2De6&8F6*5FT7bnN7eE<6$^sdHJnzXI z2mt7Kc!*Q%2!Orj@q}YGer>z;Jne7z`bU)n11?HzM{4+n4F~-B zYVGRAd{rr)CAhB9$N!%VjH_P+&%w7o0d%izfQ2wH867F}Qs!40GLLt|xOCvqkhn=ivjlu&cmU&Fr`E! zJ0BT{!V8|-TP@f{1Bfc-H=k~7JbtpZe!qKv{odpI>rVv5GjSEof~`DMXF`Lh8#3*u zC{~_BOEFW@klVu%x**92RfaoPMp6$kb#VEN1M8O5Z->)$a6lL}b5mTPbWxkSaE#A5o+po;g)+Ev{p(&rTqm?ibDD=CHZnVy%OQ_|xD9`V%J} zn>aO(9XWY2HFVH(<@C^H-Zg`u_o)#t+lHJu`lD$aw@hPj1+&gebJ+UK`qcQ9!6uJe zziq`RXXts$xMhs-bOCM~;QqUuRovZExPVOqqf;3D*U(jc!2kEAm>H)zo`4d%cn;4djxfn<>LROd{{<4~|5 zrz)dp>9@|T(e6Z*LPSf>{46ykWP#6);-DY)O7AmrWTPNA+U_{^D1AB)JXQChj7*ad z7$=IdMjS)k%556IQ*X7B$+^@O1s?JO6qe7aBuo?N1TgOLEK>qkU#g_OTr)!kK{XwJ z5%kjt5mmX^r8{-rTdh!q=SQ9)rt760`IF{KNp*HZOT z);dwI=}TAhW|3CO^iZp=UZh3DDht)-SkD%YE~Su`Cb{VGQOH59sfw1O@o;?$DX+5= z@`z=l+V0ig;5G)$T<$D=R5VfbC8Dj;tD;p&YA_$hhd~sMjMpE$zvu^tsF$e@)-qmH zR=ik+uzBU|3cHt}oOALNr3M$%_!bqL3Sfo%v$cD_TyNhWZ3g?7fy}eQrplynO2Z&0 zSTv&~z{V*678CVvETBaBfgd{1{-BJWp7yOO5~|#tN>lM&YHrf%27p$$X>ddw3yZ%H zOEgE-p>8O{S3&l7=&0Y3jB8$_WlmXM!!xPZGA+|K?f*Eghu#$2^PD+j`qmWYW)Mql z%ftJWWt-N2E!+dIbly~3nC%6ttwRF3^k2VNs@GnarV1FKv{s+K92n+xK literal 0 HcmV?d00001 diff --git a/serpapi/_version.py b/serpapi/_version.py new file mode 100644 index 0000000..d45db1f --- /dev/null +++ b/serpapi/_version.py @@ -0,0 +1,2 @@ +"""package current version""" +__version__ = "1.0.0" diff --git a/serpapi/error.py b/serpapi/error.py new file mode 100644 index 0000000..bd4e3fb --- /dev/null +++ b/serpapi/error.py @@ -0,0 +1,3 @@ +"""error related classes""" +class SerpApiException(Exception): + """custom exception for this module""" diff --git a/serpapi/object_decoder.py b/serpapi/object_decoder.py new file mode 100644 index 0000000..e7a48ad --- /dev/null +++ b/serpapi/object_decoder.py @@ -0,0 +1,42 @@ +"""custom object decoder""" +class ObjectDecoder: + """ + Allow to convert JSON like datastructure in Python object. + """ + def dict2object(self, name, node): + """Make python object from dict + Parameters + --- + name: str + parent field name to start walk the node + node: dict + source dict structure to convert into object + """ + pytype = type(name, (object, ), {}) + pyobj = pytype() + + if isinstance(node, list): + setattr(pyobj, name, []) + for item in node: + getattr(pyobj, name).append(self.dict2object(name, item)) + return pyobj + if isinstance(node, dict): + for child_name, child in node.items(): + self.add_node(child_name, pyobj, child) + else: + setattr(pyobj, name, node) + + return pyobj + + def add_node(self, name, pyobj, child): + """ + add node in object + """ + if isinstance(child, list): + setattr(pyobj, name, []) + for item in child: + getattr(pyobj, name).append(self.dict2object(name, item)) + elif isinstance(child, dict): + setattr(pyobj, name, self.dict2object(name, child)) + else: + setattr(pyobj, name, child) diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py new file mode 100644 index 0000000..aadfdd6 --- /dev/null +++ b/serpapi/serpapi.py @@ -0,0 +1,196 @@ +"""SerpApi package for python""" +import json +import urllib3 +from .error import SerpApiException +from .object_decoder import ObjectDecoder + +class Client(ObjectDecoder): + """ + Client performend http query to serpApi.com + using urllib3 under the hood. + + The HTTP connection be tuned to allow + - retries : attempt to reconnect if the connection fail by default: False + - timeout : connection timeout by default 60s + for more details: https://urllib3.readthedocs.io/en/stable/user-guide.html + """ + + BACKEND = 'https://serpapi.com' + SUPPORTED_DECODER = ['json', 'html', 'object'] + + def __init__(self, parameter=None): + # define default parameter + if parameter is None: + self.parameter = {} + else: + self.parameter = parameter + # urllib3 options + # 60s default + self.timeout = 60.0 + # no HTTP retry + self.retries = False + # override default + if 'timeout' in parameter: + self.timeout = parameter['timeout'] + if 'retries' in parameter: + self.retries = parameter['retries'] + # initialize the http client + self.http = urllib3.PoolManager() + + def search(self, parameter=None, decoder='json'): + """ + make search then decode the output + decoder supported 'json', 'html', 'object' + + Parameters + ---------- + parameter : dict + search query + decoder : str + set decoder to convert the datastructure received from + + Returns + ------- + dict|str + search results returns as : + dict if decoder = 'json' + str if decoder = 'html' + object if decoder = 'object' + """ + return self.run(path='/search', decoder=decoder, parameter=parameter) + + def html(self, parameter=None): + """ + html search + + Parameters + ---------- + parameter : dict + search query see: https://serpapi.com/search-api + + Returns + ------- + str + raw html search results directly from the search engine + """ + return self.run('/search', 'html', parameter) + + def location(self, parameter=None): + """ + Get location using Location API + + Parameters + ---------- + parameter : dict + location query like: {q: "Austin", limit: 5} + see: https://serpapi.com/locations-api + + Returns + ------- + array + list of matching locations + """ + return self.run('/locations.json', 'json', parameter) + + def search_archive(self, search_id, decoder='json'): + """ + Retrieve search results from the Search Archive API + + Parameters: + + """ + path = "/searches/" + str(search_id) + "." + if decoder in self.SUPPORTED_DECODER: + if decoder == "object": + path += "json" + else: + path += decoder + else: + raise SerpApiException('decoder must be json or html or object') + return self.run(path, decoder, {}) + + def account(self, api_key=None): + """ + Get account information using Account API + + Parameters + --- + api_key: str + secret user key provided by serpapi.com + Returns + --- + dict + user account information + """ + if api_key is not None: + self.parameter['api_key'] = api_key + return self.run('/account', 'json', self.parameter) + + def run(self, path, decoder='json', parameter=None): + """ + run HTTP request and decode response using urllib3 + the response is decoded using the selected decoder: + - html: raw HTML response + - json: deep dict contains search results + - object: containing search results as a dynamic object + + Parameters: + --- + path: str + HTTP endpoint path under serpapi.com/ + decoder: str + define how to post process the HTTP response. + for example: json -> convert response to a dict + using the default JSON parser from Python + parameter: dict + search query + + Returns: + --- + dict|str|object + decoded HTTP response + """ + # set client language + self.parameter['source'] = 'python' + + # set output type + if decoder == 'object': + self.parameter['output'] = 'json' + else: + self.parameter['output'] = decoder + + # merge parameter defaults and overrides + fields = self.parameter.copy() + fields.update(parameter) + + # execute HTTP get request + response = self.http.request('GET', + self.BACKEND + path, + fields=fields, + timeout=self.timeout, + retries=self.retries) + + # handle HTTP error + if response.status != 200: + try: + raw = response.data.decode('utf-8') + payload = json.loads(raw) + raise SerpApiException(payload['error']) + except Exception as ex: + raise SerpApiException(raw) from ex + + payload = response.data.decode('utf-8') + + # successful response decoding + if decoder == 'json': + return dict(json.loads(payload)) + + if decoder == 'html': + return payload + + if decoder == 'object': + data = dict(json.loads(payload)) + return self.dict2object(data, 'response') + + raise SerpApiException("invalid decoder: " + + decoder + ", available: json, html, object") diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..b2140f3 --- /dev/null +++ b/setup.py @@ -0,0 +1,32 @@ +from setuptools import setup, find_packages +from codecs import open +from os import path +from pathlib import Path +long_description = (Path(__file__).parent / "README.md").read_text() + +setup(name='serpapi', + version='1.0.0', + description='Scrape and search localized results from Google, Bing, Baidu, Yahoo, Yandex, Ebay, Homedepot, youtube at scale using SerpApi.com', + url='https://github.com/serpapi/serpapi-python', + author='vikoky', + author_email='victor@serpapi.com', + classifiers=[ + 'License :: OSI Approved :: MIT License', + 'Programming Language :: Python :: 3.5', + 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', + 'Programming Language :: Python :: 3.8', + 'Programming Language :: Python :: 3.9', + 'Natural Language :: English', + 'Topic :: Utilities', + ], + python_requires='>=3.5', + zip_safe=False, + include_package_data=True, + license="MIT", + install_requires = ["urllib3"], + packages=find_packages(), + keywords='scrape,serp,api,json,search,localized,rank,google,bing,baidu,yandex,yahoo,ebay,scale,datamining,training,machine,ml,youtube,naver,walmart,apple,store,app,serpapi', + long_description=long_description, + long_description_content_type="text/markdown", +) diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..c739142 --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +import serpapi \ No newline at end of file diff --git a/tests/__pycache__/__init__.cpython-39.pyc b/tests/__pycache__/__init__.cpython-39.pyc new file mode 100644 index 0000000000000000000000000000000000000000..675c02b4fa8d319ba10c5c4f9d5615b0ac00226d GIT binary patch literal 172 zcmYe~<>g`kf}QuxllXx2V-N=!FabFZKwQiNBvKes7;_k+fMhU(Ceurx2!o#{<1O~$ z)S`mKg3Of+Ma)1UF!9SyKNKijtY4OyT#{d;A5fH^m6}|l4^;}Ibqgv>GV=5EOHzwV iiuL2;GxIV_;^XxSDsOSv$W z)BeE9=x0LZ9we~^f@+k+nom@&?;5&Bv|ZD;NYP{aw(4)TGuE?T~;aHg%q8V2Mjhke6tgRv<6aDjkEoLUmYgTvi@3k*_hHEoG5( zmR47SZ|^^DbS@-mEzF`|hqY@NPxm9rXp@YXg!EiU%aFuhAabowD9QD{LG=NNz@{GQ zlDW{qe3&$7eJ;0cmnZ8xTBmAUD;J&JYkcVEfWuPB%6f z>r|IoB9D@f9eAv=vjO8f?btdKd>uxXWZI5a9;H8RYnqvK%<_oMA{uGlNE7Cjr72jvA?>CzoL6vP6||xNF+W=Q*dOWnWVG_>XyuNt zwoxoUKle&);Yj1}!Wu8WoP7k)#O!|5%u_zQ%F}Ju%x8xnkMo;AYb#A=5t&dj7;4nB zwyY?U1>P!5DlUT1bd!|-vkjBz`2XWtzmd+R5x=gFjC#e1br^iElTI5CE*XW&>p zaOZ6!*S1Y`;ivkqHKKt_gELcbQPd#X58w_a{D%h7lF4~Zd^HEb4FAWFl&h2)a5Iz} z_!CjGAI1?4hOXo39M|U_P(k0~FU$cqou3zk%Wu z$QcDv1+-u{5+X`AgJL0EM&PwYnl8~5&Cazm9M2p3dYIqCeCItr`7ug(fZ*;PqlDZ0 z8kFChLFqIKf*9aCW8W27r$@^q9p5suDNIOa>L}*V3C&_pOv3hKk zvpjFFVuit*q?|%JS=f%#kV+lYPMxVm*?Md97Osm|N+VU$nHsfP^?Oz61}aPkfh-5X zZc1D5_64#U1beM8#sCgAx0OUWQt@rE6J;wKBbV$wn4EuzqK;w~#cdQX4N@g^S&kb- zleV89o2)j?1oVi1n^$D$4?Bu1OwzuP{fOR5%VEF)H(Z!M>#w&Cr#2B zw(&P`wa-u}cUZuZiaqEP(nS! Date: Sun, 20 Mar 2022 21:49:15 -0500 Subject: [PATCH 02/25] add git action workflow --- .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..bb3bd22 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,40 @@ +# This workflow will install Python dependencies, run tests and lint with a variety of Python versions +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: SerpApi Python + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install flake8 pytest + if [ -f requirements.txt ]; then pip install -r requirements.txt; fi + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics + - name: Test with pytest + run: pytest + env: + API_KEY: ${{secrets.API_KEY}} From 2d59f01435b9ac40c063cbfae8175bfb5439ea54 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 3 Apr 2022 21:57:00 -0500 Subject: [PATCH 03/25] remove cached files pushed by mistake --- .gitignore | 7 +++++++ serpapi/__pycache__/__init__.cpython-39.pyc | Bin 528 -> 0 bytes serpapi/__pycache__/_version.cpython-39.pyc | Bin 216 -> 0 bytes serpapi/__pycache__/error.cpython-39.pyc | Bin 404 -> 0 bytes .../__pycache__/object_decoder.cpython-39.pyc | Bin 1396 -> 0 bytes serpapi/__pycache__/serpapi.cpython-39.pyc | Bin 5090 -> 0 bytes serpapi/object_decoder.py | 2 +- serpapi/serpapi.py | 2 +- tests/__pycache__/__init__.cpython-39.pyc | Bin 172 -> 0 bytes .../test_serpapi.cpython-39-pytest-7.1.0.pyc | Bin 2714 -> 0 bytes 10 files changed, 9 insertions(+), 2 deletions(-) delete mode 100644 serpapi/__pycache__/__init__.cpython-39.pyc delete mode 100644 serpapi/__pycache__/_version.cpython-39.pyc delete mode 100644 serpapi/__pycache__/error.cpython-39.pyc delete mode 100644 serpapi/__pycache__/object_decoder.cpython-39.pyc delete mode 100644 serpapi/__pycache__/serpapi.cpython-39.pyc delete mode 100644 tests/__pycache__/__init__.cpython-39.pyc delete mode 100644 tests/__pycache__/test_serpapi.cpython-39-pytest-7.1.0.pyc diff --git a/.gitignore b/.gitignore index 5996dfd..15f133b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,9 @@ .coverage +docs/build +dist/ +.pytest_cache/ +serpapi.egg-info/ +serpapi/__pycache__ +tests/__pycache__ +*.pyc \ No newline at end of file diff --git a/serpapi/__pycache__/__init__.cpython-39.pyc b/serpapi/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index f0484f33ff69f768e61a9f6ca2dc995807e375d4..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 528 zcmYjOyH3L}6wRwi2(82?SZp0=79^w!p#nm{0IJl&ki~KnqZrw7WV?m-Ls<9%ekl?Y z6TiU3b=pXE6z3e@oO2&0gF)iJ`usjy+`5kQW6A#VfK1@kr?3@H<`iz>aaVYm=UVJ@ zUj$ju;(+%=n1z6QSyY5sTtqWx7;io_RW+&ED3=1|oYBf4CTdOvAYz!T5UWtVHl?hR zdC4@=Z5Z9n=hIF=IG1n8Nch!IMRJ4Qm?_Z=tS`6LNuntoqtcjKkJEHN>5EiINJT*m zP_#LdU+C8vy3HZRE2=ba7fRbW|yFPi1yYI<%njK9sW)(9SkLq<3p9@(mIlZv`YkU1c*mvRa0{0(N C8=<2B diff --git a/serpapi/__pycache__/_version.cpython-39.pyc b/serpapi/__pycache__/_version.cpython-39.pyc deleted file mode 100644 index be98ee12455e2d5eeefc6ebce41ad1341c590a4a..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 216 zcmYe~<>g`kg44InlT3j0V-N=!FakLaKwQiNBvKfn7*ZIc7*m*n88n%z#0wIWvlG)( z6_QJfic<4R6v|SIiZk=`s#p#64D<~AG?{L($H%ASC&$O%;*O7p%Erg9WGG?=ng%9* zIp~KL1GVXwWhR&87wHES PPO2Tqt;I}0f`t(P*Zw=6 diff --git a/serpapi/__pycache__/error.cpython-39.pyc b/serpapi/__pycache__/error.cpython-39.pyc deleted file mode 100644 index 88d8f66123013f5d78b06d6cd892bfd71468efe7..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 404 zcmY*UJx{|h5Vf5^Lq(-7NQ_=HKt>ioWk75dQpJ+RGWMaSN@8O>h<1fP!(R#`OMe3s zmxS_h(!INPw%_^5$z)70w(n>6E36+K`5Q5k8Kyr*&?F}}L{rUPiDp1wqMYeyLDG10 z1kM@99h5SF=Bkw5gID(Gt5JIVR*+sjU=SBdEt%>8B? diff --git a/serpapi/__pycache__/object_decoder.cpython-39.pyc b/serpapi/__pycache__/object_decoder.cpython-39.pyc deleted file mode 100644 index b3dac8779deb281d2232fa40367cdb4d9c930a5d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 1396 zcmZuxPfHvz6i+glKd!4-+e#Iof%dXcUFfACrC2I06sm&sva}52B9OCUr@l9{ZdYg^Z}O7--rrv)T3Bci7(af!I(+65@*9=e;efIY)2x7y zq(?pwNhP}>l8so8O6P!d_~3EeH#R>P`QfRGt&l3trP47#?hOXoHcTTxC<1WOW0Lk9 z$)s~bdR%hp!q=4^nDwiLz1rSRV{Wk15-{GTX?`VaF5*19RN9J<2Yb6BP0o~%k&TSi z{n+-k5=kcZukA^mO~Tr&;W1`(Hv(ArFyFy6-#}PWQcI6$LPpNeDV-4+a?46)oe|Uf zmY2Mw$8bhT$#%%s^`UEB+#9i5uOzpANlSOk#*~ta&5|FG8XEw%EZObcm;h$w3mbYv zA8xlw53v3itX&*t?Z$e^YU_aQ(81475u{bjW+9F=6j~;+t<%AZ>_<8}S5|2=-P+jL zn65K8+!9ci=?;R5NUO|>qeP`r;1w*Xu@MyXDoW3UJy9ZqE>1zHY~6Q^c1+&aF_azx zLLZab&$K7WKxv3EwbhxgxVEG0sjYn0O#U82jZ%?7J`vsB-lNipV#n+$lO{|I{u3J}(@KtH{ORGX61smaN$ zgL^&YoP0vgwd_TW^Z?mwkdqtExF#Ubv9& lIKY3G!E6G`onTet-fo9!>}sw24_O7T8_#d>x5<$^;#c^B(SBv3 zT`-Hel5qV+kEb$BVvJ2c-rnaud7t+Z#zl|l$FBxn?&8k2(QpHMG_XIj(_mJ|-o@_z zV0N195^FN&bF<^H%dEvF&~w=p=CMiiTI?#DV$1gU%it+C?fJO$yPtA_bM<-5{5>gC|5?t(kuMWJ)2CnVC4EKj=UEu< z`niaraQlaT9^+iT+~fXUlCWh@(QM)MkR`orIZRe~ypqXaJK`%jgtN4p zhm0@p$$q5x4K9-WXgNtPNJEm`LZh(vvm`ETqAQ$4Sw*{NLG9H=eY)@A&bH7DVGX$< z=0am=bdA^Ox0FlamP%M~iitE3L7z)5#01C$fvs>$QFJCZk|g>hh=Bt^hmq-*R$*s6+7Xi= z{1V&6Gk?OgmQB=KF&*k)}Xmz_`9LjF@GT3F8&=^+B zY*|jju^jX~6Sp;()z-UQQ40rzJwjD=QNM{>NtO0CH15Ae_r z4mgZH5WPJzycl*t*i}XdyGnCela^D$&+;@)goJV5BrV@u^ly@$>8Uk+Q{#g0wo!>! zn;#7=)mP2BRs!spL5TGP(U-eZ5UnTlHU;Q2De6&8F6*5FT7bnN7eE<6$^sdHJnzXI z2mt7Kc!*Q%2!Orj@q}YGer>z;Jne7z`bU)n11?HzM{4+n4F~-B zYVGRAd{rr)CAhB9$N!%VjH_P+&%w7o0d%izfQ2wH867F}Qs!40GLLt|xOCvqkhn=ivjlu&cmU&Fr`E! zJ0BT{!V8|-TP@f{1Bfc-H=k~7JbtpZe!qKv{odpI>rVv5GjSEof~`DMXF`Lh8#3*u zC{~_BOEFW@klVu%x**92RfaoPMp6$kb#VEN1M8O5Z->)$a6lL}b5mTPbWxkSaE#A5o+po;g)+Ev{p(&rTqm?ibDD=CHZnVy%OQ_|xD9`V%J} zn>aO(9XWY2HFVH(<@C^H-Zg`u_o)#t+lHJu`lD$aw@hPj1+&gebJ+UK`qcQ9!6uJe zziq`RXXts$xMhs-bOCM~;QqUuRovZExPVOqqf;3D*U(jc!2kEAm>H)zo`4d%cn;4djxfn<>LROd{{<4~|5 zrz)dp>9@|T(e6Z*LPSf>{46ykWP#6);-DY)O7AmrWTPNA+U_{^D1AB)JXQChj7*ad z7$=IdMjS)k%556IQ*X7B$+^@O1s?JO6qe7aBuo?N1TgOLEK>qkU#g_OTr)!kK{XwJ z5%kjt5mmX^r8{-rTdh!q=SQ9)rt760`IF{KNp*HZOT z);dwI=}TAhW|3CO^iZp=UZh3DDht)-SkD%YE~Su`Cb{VGQOH59sfw1O@o;?$DX+5= z@`z=l+V0ig;5G)$T<$D=R5VfbC8Dj;tD;p&YA_$hhd~sMjMpE$zvu^tsF$e@)-qmH zR=ik+uzBU|3cHt}oOALNr3M$%_!bqL3Sfo%v$cD_TyNhWZ3g?7fy}eQrplynO2Z&0 zSTv&~z{V*678CVvETBaBfgd{1{-BJWp7yOO5~|#tN>lM&YHrf%27p$$X>ddw3yZ%H zOEgE-p>8O{S3&l7=&0Y3jB8$_WlmXM!!xPZGA+|K?f*Eghu#$2^PD+j`qmWYW)Mql z%ftJWWt-N2E!+dIbly~3nC%6ttwRF3^k2VNs@GnarV1FKv{s+K92n+xK diff --git a/serpapi/object_decoder.py b/serpapi/object_decoder.py index e7a48ad..11d4d9d 100644 --- a/serpapi/object_decoder.py +++ b/serpapi/object_decoder.py @@ -1,4 +1,4 @@ -"""custom object decoder""" +"""custom object decoder to convert a JSON into a python object""" class ObjectDecoder: """ Allow to convert JSON like datastructure in Python object. diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py index aadfdd6..55c95c6 100644 --- a/serpapi/serpapi.py +++ b/serpapi/serpapi.py @@ -1,4 +1,4 @@ -"""SerpApi package for python""" +"""SerpApi client library for python""" import json import urllib3 from .error import SerpApiException diff --git a/tests/__pycache__/__init__.cpython-39.pyc b/tests/__pycache__/__init__.cpython-39.pyc deleted file mode 100644 index 675c02b4fa8d319ba10c5c4f9d5615b0ac00226d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 172 zcmYe~<>g`kf}QuxllXx2V-N=!FabFZKwQiNBvKes7;_k+fMhU(Ceurx2!o#{<1O~$ z)S`mKg3Of+Ma)1UF!9SyKNKijtY4OyT#{d;A5fH^m6}|l4^;}Ibqgv>GV=5EOHzwV iiuL2;GxIV_;^XxSDsOSv$W z)BeE9=x0LZ9we~^f@+k+nom@&?;5&Bv|ZD;NYP{aw(4)TGuE?T~;aHg%q8V2Mjhke6tgRv<6aDjkEoLUmYgTvi@3k*_hHEoG5( zmR47SZ|^^DbS@-mEzF`|hqY@NPxm9rXp@YXg!EiU%aFuhAabowD9QD{LG=NNz@{GQ zlDW{qe3&$7eJ;0cmnZ8xTBmAUD;J&JYkcVEfWuPB%6f z>r|IoB9D@f9eAv=vjO8f?btdKd>uxXWZI5a9;H8RYnqvK%<_oMA{uGlNE7Cjr72jvA?>CzoL6vP6||xNF+W=Q*dOWnWVG_>XyuNt zwoxoUKle&);Yj1}!Wu8WoP7k)#O!|5%u_zQ%F}Ju%x8xnkMo;AYb#A=5t&dj7;4nB zwyY?U1>P!5DlUT1bd!|-vkjBz`2XWtzmd+R5x=gFjC#e1br^iElTI5CE*XW&>p zaOZ6!*S1Y`;ivkqHKKt_gELcbQPd#X58w_a{D%h7lF4~Zd^HEb4FAWFl&h2)a5Iz} z_!CjGAI1?4hOXo39M|U_P(k0~FU$cqou3zk%Wu z$QcDv1+-u{5+X`AgJL0EM&PwYnl8~5&Cazm9M2p3dYIqCeCItr`7ug(fZ*;PqlDZ0 z8kFChLFqIKf*9aCW8W27r$@^q9p5suDNIOa>L}*V3C&_pOv3hKk zvpjFFVuit*q?|%JS=f%#kV+lYPMxVm*?Md97Osm|N+VU$nHsfP^?Oz61}aPkfh-5X zZc1D5_64#U1beM8#sCgAx0OUWQt@rE6J;wKBbV$wn4EuzqK;w~#cdQX4N@g^S&kb- zleV89o2)j?1oVi1n^$D$4?Bu1OwzuP{fOR5%VEF)H(Z!M>#w&Cr#2B zw(&P`wa-u}cUZuZiaqEP(nS! Date: Sun, 3 Apr 2022 21:58:42 -0500 Subject: [PATCH 04/25] add better documentation system --- .gitignore | 3 +- Makefile | 45 ++++++++++++---------- README.md | 86 +++++++++++++++++++++++++++++++++++++++++- README.md.erb | 88 +++++++++++++++++++++++++++++++++++++++++++ docs/Makefile | 20 ++++++++++ docs/source/conf.py | 60 +++++++++++++++++++++++++++++ docs/source/index.rst | 24 ++++++++++++ 7 files changed, 305 insertions(+), 21 deletions(-) create mode 100644 README.md.erb create mode 100644 docs/Makefile create mode 100644 docs/source/conf.py create mode 100644 docs/source/index.rst diff --git a/.gitignore b/.gitignore index 15f133b..3c9d107 100644 --- a/.gitignore +++ b/.gitignore @@ -6,4 +6,5 @@ serpapi.egg-info/ serpapi/__pycache__ tests/__pycache__ -*.pyc \ No newline at end of file +*.pyc +.DS_Store diff --git a/Makefile b/Makefile index d8eae0c..b6f80ef 100644 --- a/Makefile +++ b/Makefile @@ -2,48 +2,55 @@ # # current version version=$(shell grep version setup.py | cut -d"'" -f2) +dist=dist/serpapi-$(version).tar.gz .PHONY: build -all: clean install test test2 +all: clean install readme doc lint test build clean: find . -name '*.pyc' -delete find . -type d -name "__pycache__" -delete - pip3 uninstall google_search_results - -install: - pip3 install -r requirements.txt + pip3 uninstall serpapi +# lint check lint: pylint serpapi -# Test with Python 3 -test: lint +# test with Python 3 +test: pytest --cov=serpapi tests/ -# run example only -# and display output (-s) -example: - pytest -s "tests/test_example.py::TestExample::test_async" - +# pytest-cov - code coverage extension for pytest +# sphinx - documentation install: pip3 install -U setuptools - pip install pytest-cov + pip3 install -r requirements.txt + pip3 install pytest-cov + pip3 install sphinx + +readme: + erb -T '-' README.md.erb > README.md -doc: - pydoc +doc: readme + $(MAKE) -C docs/ html # https://packaging.python.org/tutorials/packaging-projects/ -build: doc +build: doc test python3 setup.py sdist +# out of box testing / user acceptance before delivery oobt: build - pip3 install ./dist/google_search_results-$(version).tar.gz + pip3 install ./${dist} python3 oobt/oobt.py check: oobt - twine check dist/google_search_results-$(version).tar.gz + twine check ${dist} release: # check - twine upload dist/google_search_results-$(version).tar.gz + twine upload ${dist} + +# run example only +# and display output (-s) +example: + pytest -s "tests/test_example.py::TestExample::test_async" diff --git a/README.md b/README.md index 4ef0d7c..5a9201b 100644 --- a/README.md +++ b/README.md @@ -1 +1,85 @@ -master branch ready for pull request \ No newline at end of file +# User guide +Scrape Google and other search engines from our fast, easy, and complete API using SerpApi.com +This ruby library is meant to scrape and parse results from all major search engine available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Apple and more using [SerpApi](https://serpapi.com). +SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly. + +## Installation +serpapi can be installed with pip. + +```sh +$ python -m pip install serpapi +``` + +## Quick start +First things first, import the serpapi module: + +```python +>>> import serpapi +``` +You’ll need a Client instance to make search. This object handles all of the details of connection pooling and thread safety so that you don’t have to: + +```python +>>> client = serpapi.Client() +``` +To make a search using SerpApi.com client. + +```python +>>> parameter = { + api_key: "secret_api_key", # from serpapi.com + engine: "google", # search engine + q: "coffee", # search topic + location: "Austin,TX" # location + } + results = searpapi.search(parameter) +``` +Putting everything together. +```python +import serpapi + +parameter = { + api_key: "secret_api_key", # from serpapi.com + engine: "google", # search engine + q: "coffee", # search topic + location: "Austin,TX" # location +} +results = searpapi.search(parameter) +print(results) +``` + +### Advanced settings +SerpApi Client uses urllib3 under the hood. +The HTTP connection be tuned by setting the following client specific setting. + - retries : attempt to reconnect if the connection failed by default: False + - timeout : connection timeout by default 60s +for more details: https://urllib3.readthedocs.io/en/stable/user-guide.html + +For example: +parameter = { + retries: 5, + timeout: 4.0 +} + +## Developer's note +### Key goals + - Brand centric instead of search engine based + - No hard coded logic per search engine + - Simple HTTP client (lightweight, reduced dependency) + - No magic default values + - Thread safe + - Easy to extends + - Defensive code style (raise cutsom exception) + - TDD + - Best API coding pratice per platform + +### Design +The API design was inpired by the most popular Python packages. + - urllib3 - https://github.com/urllib3/urllib3 + - Boto3 - https://github.com/boto/boto3 + +### Quality expectation + - 0 lint issues using pylint `make lint` + - 99% code coverage running `make test` + +### TODO + - Add more test + - Release flow \ No newline at end of file diff --git a/README.md.erb b/README.md.erb new file mode 100644 index 0000000..88454fd --- /dev/null +++ b/README.md.erb @@ -0,0 +1,88 @@ +<%- +def snippet(format, path, start, stop) + slice = File.new(path).readlines[start..stop] + slice.reject! { |l| l =~ /expect\(/ } + %Q(```#{format}\n#{slice.join}```\n see: #{path}) +end +-%> +# User guide +Scrape Google and other search engines from our fast, easy, and complete API using SerpApi.com +This ruby library is meant to scrape and parse results from all major search engine available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Apple and more using [SerpApi](https://serpapi.com). +SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly. + +## Installation +serpapi can be installed with pip. + +```sh +$ python -m pip install serpapi +``` + +## Quick start +First things first, import the serpapi module: + +```python +>>> import serpapi +``` +You’ll need a Client instance to make search. This object handles all of the details of connection pooling and thread safety so that you don’t have to: + +```python +>>> client = serpapi.Client() +``` +To make a search using SerpApi.com client. + +```python +>>> parameter = { + api_key: "secret_api_key", # from serpapi.com + engine: "google", # search engine + q: "coffee", # search topic + location: "Austin,TX" # location + } + results = searpapi.search(parameter) +``` +Putting everything together. +```python +import serpapi + +parameter = { + api_key: "secret_api_key", # from serpapi.com + engine: "google", # search engine + q: "coffee", # search topic + location: "Austin,TX" # location +} +results = searpapi.search(parameter) +print(results) +``` + +### Advanced settings +SerpApi Client uses urllib3 under the hood. +The HTTP connection be tuned by setting the following client specific setting. + - retries : attempt to reconnect if the connection failed by default: False + - timeout : connection timeout by default 60s +for more details: https://urllib3.readthedocs.io/en/stable/user-guide.html + +For example: +parameter = { + retries: 5, + timeout: 4.0 +} + +## Developer's note +### Key goals + - Brand centric instead of search engine based + - No hard coded logic per search engine + - Simple HTTP client (lightweight, reduced dependency) + - No magic default values + - Thread safe + - Easy to extends + - Defensive code style (raise cutsom exception) + - TDD + - Best API coding pratice per platform + +### Design +The API design was inpired by the most popular Python packages. + - urllib3 - https://github.com/urllib3/urllib3 + - Boto3 - https://github.com/boto/boto3 + +### Quality expectation + - 0 lint issues using pylint `make lint` + - 99% code coverage running `make test` \ No newline at end of file diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 0000000..d0c3cbf --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,20 @@ +# Minimal makefile for Sphinx documentation +# + +# You can set these variables from the command line, and also +# from the environment for the first two. +SPHINXOPTS ?= +SPHINXBUILD ?= sphinx-build +SOURCEDIR = source +BUILDDIR = build + +# Put it first so that "make" without argument is like "make help". +help: + @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) + +.PHONY: help Makefile + +# Catch-all target: route all unknown targets to Sphinx using the new +# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). +%: Makefile + @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/docs/source/conf.py b/docs/source/conf.py new file mode 100644 index 0000000..5a8b25c --- /dev/null +++ b/docs/source/conf.py @@ -0,0 +1,60 @@ +# Configuration file for the Sphinx documentation builder. +# +# This file only contains a selection of the most common options. For a full +# list see the documentation: +# https://www.sphinx-doc.org/en/master/usage/configuration.html + +# -- Path setup -------------------------------------------------------------- + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +# +import os +import sys +sys.path.insert(0, os.path.abspath('..')) + + +# -- Project information ----------------------------------------------------- + +project = 'serapi-python' +copyright = '2022, Victor Benarbia' +author = 'Victor Benarbia' + +# The full version, including alpha/beta/rc tags +release = '1.0.0' + + +# -- General configuration --------------------------------------------------- + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. +extensions = [ + 'sphinx.ext.githubpages', + 'sphinx.ext.autodoc' +] + +# Add any paths that contain templates here, relative to this directory. +templates_path = ['_templates'] + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = [] + + +# -- Options for HTML output ------------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +# +html_theme = 'classic' + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +html_static_path = ['_static'] + + +# -- Extension configuration ------------------------------------------------- \ No newline at end of file diff --git a/docs/source/index.rst b/docs/source/index.rst new file mode 100644 index 0000000..d20ad9b --- /dev/null +++ b/docs/source/index.rst @@ -0,0 +1,24 @@ +.. serapi-python documentation master file, created by + sphinx-quickstart on Sun Apr 3 21:09:40 2022. + You can adapt this file completely to your liking, but it should at least + contain the root `toctree` directive. + +Welcome to serapi-python's documentation! +========================================= + +.. automodule:: serpapi + :members: serpapi +.. automodule:: serpapi.serpapi + :members: +.. toctree:: + :maxdepth: 2 + :caption: Contents: + + + +Indices and tables +================== + +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` From 7a33a48a94893fb321a070938b7ede02e4e2be56 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 3 Apr 2022 21:59:12 -0500 Subject: [PATCH 05/25] add oobt / acceptable testing for package release --- oobt/oobt.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 oobt/oobt.py diff --git a/oobt/oobt.py b/oobt/oobt.py new file mode 100644 index 0000000..3815eae --- /dev/null +++ b/oobt/oobt.py @@ -0,0 +1,26 @@ +# Out Of Box testing +# +# Run simple serparpi search +# +import serpapi +import sys, os, pprint + +print("initialize serpapi client") +client = serpapi.Client({ + "api_key": os.getenv("API_KEY", "demo") +}) +print("execute search") +result = client.search({ + "q": "coffee", + "location": "Austin,Texas", + }) +print("display result:") +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(result) +print("------") +if len(result) > 0: + print("OK: Out of box tests passed") + sys.exit(0) + +print("FAIL: Out of box tests failed: no result") +sys.exit(1) From 0988c41c211d0c097efb3ed2c0aa380991437ea6 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Wed, 13 Apr 2022 21:29:15 -0500 Subject: [PATCH 06/25] fix object decoder test and fix location and account API --- MIT-LICENSE.txt | 19 ++ Makefile | 2 +- README.md | 405 ++++++++++++++++++++++++++++++++++- README.md.erb | 85 +++++++- serpapi/object_decoder.py | 16 +- serpapi/serpapi.py | 28 ++- tests/test_account_api.py | 12 ++ tests/test_location_api.py | 15 ++ tests/test_object_decoder.py | 24 +++ tests/test_serpapi.py | 33 ++- 10 files changed, 611 insertions(+), 28 deletions(-) create mode 100644 MIT-LICENSE.txt create mode 100644 tests/test_account_api.py create mode 100644 tests/test_location_api.py create mode 100644 tests/test_object_decoder.py diff --git a/MIT-LICENSE.txt b/MIT-LICENSE.txt new file mode 100644 index 0000000..7aa3caf --- /dev/null +++ b/MIT-LICENSE.txt @@ -0,0 +1,19 @@ +The MIT License (MIT) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/Makefile b/Makefile index b6f80ef..6286d92 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ lint: # test with Python 3 test: - pytest --cov=serpapi tests/ + pytest --cov=serpapi --cov-report html tests/ # pytest-cov - code coverage extension for pytest # sphinx - documentation diff --git a/README.md b/README.md index 5a9201b..20d9546 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ First things first, import the serpapi module: ```python >>> import serpapi ``` -You’ll need a Client instance to make search. This object handles all of the details of connection pooling and thread safety so that you don’t have to: +You'll need a client instance to make a search. This object handles all of the details of connection pooling and thread safety so that you don't have to: ```python >>> client = serpapi.Client() @@ -59,6 +59,403 @@ parameter = { timeout: 4.0 } +## Basic example per search engine +### Search bing +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'bing', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_bing.py +(see https://serpapi.com/bing)[https://serpapi.com/bing] + +### Search baidu +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'baidu', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_baidu.py +(see https://serpapi.com/baidu)[https://serpapi.com/baidu] + +### Search yahoo +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'yahoo', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'p': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_yahoo.py +(see https://serpapi.com/yahoo)[https://serpapi.com/yahoo] + +### Search youtube +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'youtube', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'search_query': 'coffee', +'results': 'video_results', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_youtube.py +(see https://serpapi.com/youtube)[https://serpapi.com/youtube] + +### Search walmart +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'walmart', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'query': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_walmart.py +(see https://serpapi.com/walmart)[https://serpapi.com/walmart] + +### Search ebay +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'ebay', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'_nkw': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_ebay.py +(see https://serpapi.com/ebay)[https://serpapi.com/ebay] + +### Search naver +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'naver', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'query': 'coffee', +'results': 'ads_results', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_naver.py +(see https://serpapi.com/naver)[https://serpapi.com/naver] + +### Search home depot +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'home_depot', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'table', +'results': 'products', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_home_depot.py +(see https://serpapi.com/home_depot)[https://serpapi.com/home_depot] + +### Search apple app store +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'apple_app_store', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'term': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_apple_app_store.py +(see https://serpapi.com/apple_app_store)[https://serpapi.com/apple_app_store] + +### Search duckduckgo +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'duckduckgo', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_duckduckgo.py +(see https://serpapi.com/duckduckgo)[https://serpapi.com/duckduckgo] + +### Search google scholar +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'google_scholar', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_google_scholar.py +(see https://serpapi.com/google_scholar)[https://serpapi.com/google_scholar] + +### Search google product +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'google_product', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'coffee', +'product_id': '4172129135583325756', +'results': 'product_results', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_google_product.py +(see https://serpapi.com/google_product)[https://serpapi.com/google_product] + +### Search google reverse image +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'google_reverse_image', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'image_url': 'https://i.imgur.com/5bGzZi7.jpg', +'results': 'image_sizes', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_google_reverse_image.py +(see https://serpapi.com/google_reverse_image)[https://serpapi.com/google_reverse_image] + +### Search google events +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'google_events', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_google_events.py +(see https://serpapi.com/google_events)[https://serpapi.com/google_events] + +### Search google local services +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'google_local_services', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'Electrician', +'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g', +'results': 'local_ads', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_google_local_services.py +(see https://serpapi.com/google_local_services)[https://serpapi.com/google_local_services] + +### Search google maps +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'google_maps', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'pizza', +'ll': '@40.7455096,-74.0083012,15.1z', +'type': 'search', +'results': 'local_results', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_google_maps.py +(see https://serpapi.com/google_maps)[https://serpapi.com/google_maps] + +### Search google jobs +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'google_jobs', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'coffee', +'results': 'jobs_results', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_google_jobs.py +(see https://serpapi.com/google_jobs)[https://serpapi.com/google_jobs] + +### Search google play +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'google_play', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'coffee', +'store': 'apps', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_google_play.py +(see https://serpapi.com/google_play)[https://serpapi.com/google_play] + +### Search google images +```python +import 'serpapi' +import 'pprint' +import 'os' + +client = serpapi.Client({ +'engine': 'google_images', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'engine': 'google', +'tbm': 'isch', +'q': 'coffee', +'results': 'images_results', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + see: tests/example_search_google_images.py +(see https://serpapi.com/google_images)[https://serpapi.com/google_images] + ## Developer's note ### Key goals - Brand centric instead of search engine based @@ -78,8 +475,4 @@ The API design was inpired by the most popular Python packages. ### Quality expectation - 0 lint issues using pylint `make lint` - - 99% code coverage running `make test` - -### TODO - - Add more test - - Release flow \ No newline at end of file + - 99% code coverage running `make test` \ No newline at end of file diff --git a/README.md.erb b/README.md.erb index 88454fd..6dd4916 100644 --- a/README.md.erb +++ b/README.md.erb @@ -1,8 +1,10 @@ <%- def snippet(format, path, start, stop) slice = File.new(path).readlines[start..stop] - slice.reject! { |l| l =~ /expect\(/ } - %Q(```#{format}\n#{slice.join}```\n see: #{path}) + slice.reject! { |l| l =~ /self.assertIsNone\(/ } + buf = slice.map { |l| l.gsub(/(^\s+)/, '')}.join + buf.gsub!('self.assertIsNotNone(', "pp = pprint.PrettyPrinter(indent=2)\npp.pprint(") + %Q(```#{format}\nimport 'serpapi'\nimport 'pprint'\nimport 'os'\n\n#{buf}```\n see: #{path}) end -%> # User guide @@ -23,7 +25,7 @@ First things first, import the serpapi module: ```python >>> import serpapi ``` -You’ll need a Client instance to make search. This object handles all of the details of connection pooling and thread safety so that you don’t have to: +You'll need a client instance to make a search. This object handles all of the details of connection pooling and thread safety so that you don't have to: ```python >>> client = serpapi.Client() @@ -66,6 +68,83 @@ parameter = { timeout: 4.0 } +## Basic example per search engine +### Search bing +<%= snippet('python', 'tests/example_search_bing.py', 9, 25) %> +(see https://serpapi.com/bing)[https://serpapi.com/bing] + +### Search baidu +<%= snippet('python', 'tests/example_search_baidu.py', 9, 25) %> +(see https://serpapi.com/baidu)[https://serpapi.com/baidu] + +### Search yahoo +<%= snippet('python', 'tests/example_search_yahoo.py', 9, 25) %> +(see https://serpapi.com/yahoo)[https://serpapi.com/yahoo] + +### Search youtube +<%= snippet('python', 'tests/example_search_youtube.py', 9, 25) %> +(see https://serpapi.com/youtube)[https://serpapi.com/youtube] + +### Search walmart +<%= snippet('python', 'tests/example_search_walmart.py', 9, 25) %> +(see https://serpapi.com/walmart)[https://serpapi.com/walmart] + +### Search ebay +<%= snippet('python', 'tests/example_search_ebay.py', 9, 25) %> +(see https://serpapi.com/ebay)[https://serpapi.com/ebay] + +### Search naver +<%= snippet('python', 'tests/example_search_naver.py', 9, 25) %> +(see https://serpapi.com/naver)[https://serpapi.com/naver] + +### Search home depot +<%= snippet('python', 'tests/example_search_home_depot.py', 9, 25) %> +(see https://serpapi.com/home_depot)[https://serpapi.com/home_depot] + +### Search apple app store +<%= snippet('python', 'tests/example_search_apple_app_store.py', 9, 25) %> +(see https://serpapi.com/apple_app_store)[https://serpapi.com/apple_app_store] + +### Search duckduckgo +<%= snippet('python', 'tests/example_search_duckduckgo.py', 9, 25) %> +(see https://serpapi.com/duckduckgo)[https://serpapi.com/duckduckgo] + +### Search google scholar +<%= snippet('python', 'tests/example_search_google_scholar.py', 9, 25) %> +(see https://serpapi.com/google_scholar)[https://serpapi.com/google_scholar] + +### Search google product +<%= snippet('python', 'tests/example_search_google_product.py', 9, 25) %> +(see https://serpapi.com/google_product)[https://serpapi.com/google_product] + +### Search google reverse image +<%= snippet('python', 'tests/example_search_google_reverse_image.py', 9, 25) %> +(see https://serpapi.com/google_reverse_image)[https://serpapi.com/google_reverse_image] + +### Search google events +<%= snippet('python', 'tests/example_search_google_events.py', 9, 25) %> +(see https://serpapi.com/google_events)[https://serpapi.com/google_events] + +### Search google local services +<%= snippet('python', 'tests/example_search_google_local_services.py', 9, 25) %> +(see https://serpapi.com/google_local_services)[https://serpapi.com/google_local_services] + +### Search google maps +<%= snippet('python', 'tests/example_search_google_maps.py', 9, 25) %> +(see https://serpapi.com/google_maps)[https://serpapi.com/google_maps] + +### Search google jobs +<%= snippet('python', 'tests/example_search_google_jobs.py', 9, 25) %> +(see https://serpapi.com/google_jobs)[https://serpapi.com/google_jobs] + +### Search google play +<%= snippet('python', 'tests/example_search_google_play.py', 9, 25) %> +(see https://serpapi.com/google_play)[https://serpapi.com/google_play] + +### Search google images +<%= snippet('python', 'tests/example_search_google_images.py', 9, 25) %> +(see https://serpapi.com/google_images)[https://serpapi.com/google_images] + ## Developer's note ### Key goals - Brand centric instead of search engine based diff --git a/serpapi/object_decoder.py b/serpapi/object_decoder.py index 11d4d9d..b6465e1 100644 --- a/serpapi/object_decoder.py +++ b/serpapi/object_decoder.py @@ -3,7 +3,7 @@ class ObjectDecoder: """ Allow to convert JSON like datastructure in Python object. """ - def dict2object(self, name, node): + def child2object(self, name, node): """Make python object from dict Parameters --- @@ -18,14 +18,13 @@ def dict2object(self, name, node): if isinstance(node, list): setattr(pyobj, name, []) for item in node: - getattr(pyobj, name).append(self.dict2object(name, item)) + getattr(pyobj, name).append(self.child2object(name, item)) return pyobj if isinstance(node, dict): for child_name, child in node.items(): self.add_node(child_name, pyobj, child) else: setattr(pyobj, name, node) - return pyobj def add_node(self, name, pyobj, child): @@ -35,8 +34,15 @@ def add_node(self, name, pyobj, child): if isinstance(child, list): setattr(pyobj, name, []) for item in child: - getattr(pyobj, name).append(self.dict2object(name, item)) + getattr(pyobj, name).append(self.child2object(name, item)) elif isinstance(child, dict): - setattr(pyobj, name, self.dict2object(name, child)) + setattr(pyobj, name, self.child2object(name, child)) else: setattr(pyobj, name, child) + + def dict2object(self, node): + pytype = type('response', (object, ), {}) + pyobj = pytype() + for child_key, child_node in node.items(): + self.add_node(child_key, pyobj, child_node) + return pyobj \ No newline at end of file diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py index 55c95c6..bc3f25b 100644 --- a/serpapi/serpapi.py +++ b/serpapi/serpapi.py @@ -57,7 +57,7 @@ def search(self, parameter=None, decoder='json'): str if decoder = 'html' object if decoder = 'object' """ - return self.run(path='/search', decoder=decoder, parameter=parameter) + return self.start(path='/search', parameter=parameter, decoder=decoder) def html(self, parameter=None): """ @@ -73,7 +73,7 @@ def html(self, parameter=None): str raw html search results directly from the search engine """ - return self.run('/search', 'html', parameter) + return self.start('/search', parameter, 'html') def location(self, parameter=None): """ @@ -90,7 +90,7 @@ def location(self, parameter=None): array list of matching locations """ - return self.run('/locations.json', 'json', parameter) + return self.start('/locations.json', parameter, 'json') def search_archive(self, search_id, decoder='json'): """ @@ -107,7 +107,7 @@ def search_archive(self, search_id, decoder='json'): path += decoder else: raise SerpApiException('decoder must be json or html or object') - return self.run(path, decoder, {}) + return self.start(path, {}, decoder) def account(self, api_key=None): """ @@ -117,6 +117,7 @@ def account(self, api_key=None): --- api_key: str secret user key provided by serpapi.com + Returns --- dict @@ -124,11 +125,11 @@ def account(self, api_key=None): """ if api_key is not None: self.parameter['api_key'] = api_key - return self.run('/account', 'json', self.parameter) + return self.start('/account', self.parameter, 'json') - def run(self, path, decoder='json', parameter=None): + def start(self, path, parameter=None, decoder='json'): """ - run HTTP request and decode response using urllib3 + start HTTP request and decode response using urllib3 the response is decoded using the selected decoder: - html: raw HTML response - json: deep dict contains search results @@ -169,7 +170,11 @@ def run(self, path, decoder='json', parameter=None): fields=fields, timeout=self.timeout, retries=self.retries) - + # decode response + return self.decode(response, decoder) + + def decode(self, response, decoder): + """decode HTTP response using the given decoder""" # handle HTTP error if response.status != 200: try: @@ -179,18 +184,19 @@ def run(self, path, decoder='json', parameter=None): except Exception as ex: raise SerpApiException(raw) from ex + # HTTP success 200 payload = response.data.decode('utf-8') # successful response decoding if decoder == 'json': - return dict(json.loads(payload)) + return json.loads(payload) if decoder == 'html': return payload if decoder == 'object': - data = dict(json.loads(payload)) - return self.dict2object(data, 'response') + data = json.loads(payload) + return self.dict2object(data) raise SerpApiException("invalid decoder: " + decoder + ", available: json, html, object") diff --git a/tests/test_account_api.py b/tests/test_account_api.py new file mode 100644 index 0000000..01360db --- /dev/null +++ b/tests/test_account_api.py @@ -0,0 +1,12 @@ +import random +import unittest +import os +import serpapi + +class TestAccountApi(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_get_account(self): + client = serpapi.Client({'api_key': os.getenv('API_KEY')}) + account = client.account() + self.assertIsNotNone(account.get("account_id")) diff --git a/tests/test_location_api.py b/tests/test_location_api.py new file mode 100644 index 0000000..5e9e94f --- /dev/null +++ b/tests/test_location_api.py @@ -0,0 +1,15 @@ +import unittest +import os +import serpapi + +class TestLocationApi(unittest.TestCase): + + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_get_account(self): + client = serpapi.Client({'api_key': os.getenv('API_KEY')}) + locations = client.location({'q':'Austin', 'limit': 3}) + self.assertGreater(len(locations), 1) + self.assertTrue('id' in locations[0]) + + diff --git a/tests/test_object_decoder.py b/tests/test_object_decoder.py new file mode 100644 index 0000000..7beb7e7 --- /dev/null +++ b/tests/test_object_decoder.py @@ -0,0 +1,24 @@ +import unittest +import os +import pprint +from serpapi.object_decoder import ObjectDecoder +import pytest + +class TestObjectDecoder(unittest.TestCase): + + def test_basic(self): + decoder = ObjectDecoder() + data = { + "organic_results": [ + { + 'name': 'ok' + }, + { + 'name': 'good' + } + ] + } + obj = decoder.dict2object(data) + self.assertEqual(len(obj.organic_results), 2) + self.assertEqual(obj.organic_results[0].name, 'ok') + self.assertEqual(obj.organic_results[1].name, 'good') diff --git a/tests/test_serpapi.py b/tests/test_serpapi.py index 4cce675..bfbecb7 100644 --- a/tests/test_serpapi.py +++ b/tests/test_serpapi.py @@ -12,16 +12,45 @@ class TestSerpApi(unittest.TestCase): @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") def test_search(self): client = serpapi.Client({ + "engine": "google", "api_key": os.getenv("API_KEY") }) data = client.search({ "q": "Coffee", - "location": "Austin,Texas", - "engine": "google_scholar", + "location": "Austin,Texas" }) assert data.get("error") == None self.assertIsNotNone(data["organic_results"][0]["title"]) + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_html(self): + client = serpapi.Client({ + "engine": "google", + "api_key": os.getenv("API_KEY") + }) + data = client.html({ + "q": "Coffee", + "location": "Austin,Texas", + }) + self.assertRegex(data, r'$') + + # test ObjectDecoder + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_object(self): + client = serpapi.Client({ + "engine": "google", + "api_key": os.getenv("API_KEY") + }) + data = client.search({ + "q": "Coffee", + "location": "Austin,Texas" + }, decoder="object") + self.assertIsInstance(data, object) + self.assertIsNotNone(data.organic_results) + self.assertIsInstance(data.organic_results, list) + self.assertGreater(len(data.organic_results), 3) + self.assertIsInstance(data.organic_results[0].link, str) + def test_invalid_api_key(self): client = serpapi.Client({ "engine": "google", From e911313523fb153311162015ab138697dc5436d7 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Thu, 14 Apr 2022 21:43:09 -0500 Subject: [PATCH 07/25] fix unit level test and add pytest a configuration file --- pytest.ini | 5 + tests/example_search_apple_app_store.py | 20 +++ tests/example_search_baidu.py | 20 +++ tests/example_search_bing.py | 20 +++ tests/example_search_duckduckgo.py | 20 +++ tests/example_search_ebay.py | 20 +++ tests/example_search_google_autocomplete.py | 20 +++ tests/example_search_google_events.py | 20 +++ tests/example_search_google_images.py | 22 +++ tests/example_search_google_jobs.py | 20 +++ tests/example_search_google_local_services.py | 21 +++ tests/example_search_google_maps.py | 22 +++ tests/example_search_google_play.py | 21 +++ tests/example_search_google_product.py | 21 +++ tests/example_search_google_reverse_image.py | 20 +++ tests/example_search_google_scholar.py | 20 +++ tests/example_search_home_depot.py | 20 +++ tests/example_search_naver.py | 20 +++ tests/example_search_walmart.py | 20 +++ tests/example_search_yahoo.py | 20 +++ tests/example_search_youtube.py | 20 +++ tests/test_example.py | 152 ++++++++++++++++++ 22 files changed, 564 insertions(+) create mode 100644 pytest.ini create mode 100644 tests/example_search_apple_app_store.py create mode 100644 tests/example_search_baidu.py create mode 100644 tests/example_search_bing.py create mode 100644 tests/example_search_duckduckgo.py create mode 100644 tests/example_search_ebay.py create mode 100644 tests/example_search_google_autocomplete.py create mode 100644 tests/example_search_google_events.py create mode 100644 tests/example_search_google_images.py create mode 100644 tests/example_search_google_jobs.py create mode 100644 tests/example_search_google_local_services.py create mode 100644 tests/example_search_google_maps.py create mode 100644 tests/example_search_google_play.py create mode 100644 tests/example_search_google_product.py create mode 100644 tests/example_search_google_reverse_image.py create mode 100644 tests/example_search_google_scholar.py create mode 100644 tests/example_search_home_depot.py create mode 100644 tests/example_search_naver.py create mode 100644 tests/example_search_walmart.py create mode 100644 tests/example_search_yahoo.py create mode 100644 tests/example_search_youtube.py create mode 100644 tests/test_example.py diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..e069338 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +minversion = 7.0 +addopts = -ra -q +testpaths = tests +python_files = test_*.py example_*.py \ No newline at end of file diff --git a/tests/example_search_apple_app_store.py b/tests/example_search_apple_app_store.py new file mode 100644 index 0000000..5798091 --- /dev/null +++ b/tests/example_search_apple_app_store.py @@ -0,0 +1,20 @@ +# test apple app store +import unittest +import os +import serpapi + +class TestAppleAppStore(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_apple_app_store(self): + client = serpapi.Client({ + 'engine': 'apple_app_store', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'term': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_baidu.py b/tests/example_search_baidu.py new file mode 100644 index 0000000..9949cf4 --- /dev/null +++ b/tests/example_search_baidu.py @@ -0,0 +1,20 @@ +# test baidu +import unittest +import os +import serpapi + +class TestBaidu(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_baidu(self): + client = serpapi.Client({ + 'engine': 'baidu', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_bing.py b/tests/example_search_bing.py new file mode 100644 index 0000000..f0b6cb6 --- /dev/null +++ b/tests/example_search_bing.py @@ -0,0 +1,20 @@ +# test bing +import unittest +import os +import serpapi + +class TestBing(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_bing(self): + client = serpapi.Client({ + 'engine': 'bing', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_duckduckgo.py b/tests/example_search_duckduckgo.py new file mode 100644 index 0000000..189c8f6 --- /dev/null +++ b/tests/example_search_duckduckgo.py @@ -0,0 +1,20 @@ +# test duckduckgo +import unittest +import os +import serpapi + +class TestDuckduckgo(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_duckduckgo(self): + client = serpapi.Client({ + 'engine': 'duckduckgo', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_ebay.py b/tests/example_search_ebay.py new file mode 100644 index 0000000..addcdc7 --- /dev/null +++ b/tests/example_search_ebay.py @@ -0,0 +1,20 @@ +# test ebay +import unittest +import os +import serpapi + +class TestEbay(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_ebay(self): + client = serpapi.Client({ + 'engine': 'ebay', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + '_nkw': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_google_autocomplete.py b/tests/example_search_google_autocomplete.py new file mode 100644 index 0000000..3411b6a --- /dev/null +++ b/tests/example_search_google_autocomplete.py @@ -0,0 +1,20 @@ +# test google autocomplete +import unittest +import os +import serpapi + +class TestGoogleAutocomplete(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_autocomplete(self): + client = serpapi.Client({ + 'engine': 'google_autocomplete', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['suggestions']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_google_events.py b/tests/example_search_google_events.py new file mode 100644 index 0000000..8fa2863 --- /dev/null +++ b/tests/example_search_google_events.py @@ -0,0 +1,20 @@ +# test google events +import unittest +import os +import serpapi + +class TestGoogleEvents(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_events(self): + client = serpapi.Client({ + 'engine': 'google_events', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['events_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_google_images.py b/tests/example_search_google_images.py new file mode 100644 index 0000000..f31fc9b --- /dev/null +++ b/tests/example_search_google_images.py @@ -0,0 +1,22 @@ +# test google images +import unittest +import os +import serpapi + +class TestGoogleImages(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_images(self): + client = serpapi.Client({ + 'engine': 'google_images', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'engine': 'google', + 'tbm': 'isch', + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['images_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_google_jobs.py b/tests/example_search_google_jobs.py new file mode 100644 index 0000000..a217cf1 --- /dev/null +++ b/tests/example_search_google_jobs.py @@ -0,0 +1,20 @@ +# test google jobs +import unittest +import os +import serpapi + +class TestGoogleJobs(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_jobs(self): + client = serpapi.Client({ + 'engine': 'google_jobs', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['jobs_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_google_local_services.py b/tests/example_search_google_local_services.py new file mode 100644 index 0000000..48adc8f --- /dev/null +++ b/tests/example_search_google_local_services.py @@ -0,0 +1,21 @@ +# test google local services +import unittest +import os +import serpapi + +class TestGoogleLocalServices(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_local_services(self): + client = serpapi.Client({ + 'engine': 'google_local_services', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'Electrician', + 'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['local_ads']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_google_maps.py b/tests/example_search_google_maps.py new file mode 100644 index 0000000..e096aa2 --- /dev/null +++ b/tests/example_search_google_maps.py @@ -0,0 +1,22 @@ +# test google maps +import unittest +import os +import serpapi + +class TestGoogleMaps(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_maps(self): + client = serpapi.Client({ + 'engine': 'google_maps', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'pizza', + 'll': '@40.7455096,-74.0083012,15.1z', + 'type': 'search', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['local_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_google_play.py b/tests/example_search_google_play.py new file mode 100644 index 0000000..35d4836 --- /dev/null +++ b/tests/example_search_google_play.py @@ -0,0 +1,21 @@ +# test google play +import unittest +import os +import serpapi + +class TestGooglePlay(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_play(self): + client = serpapi.Client({ + 'engine': 'google_play', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + 'store': 'apps', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_google_product.py b/tests/example_search_google_product.py new file mode 100644 index 0000000..f5652fc --- /dev/null +++ b/tests/example_search_google_product.py @@ -0,0 +1,21 @@ +# test google product +import unittest +import os +import serpapi + +class TestGoogleProduct(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_product(self): + client = serpapi.Client({ + 'engine': 'google_product', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + 'product_id': '4172129135583325756', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['product_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_google_reverse_image.py b/tests/example_search_google_reverse_image.py new file mode 100644 index 0000000..9da6e0f --- /dev/null +++ b/tests/example_search_google_reverse_image.py @@ -0,0 +1,20 @@ +# test google reverse image +import unittest +import os +import serpapi + +class TestGoogleReverseImage(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_reverse_image(self): + client = serpapi.Client({ + 'engine': 'google_reverse_image', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'image_url': 'https://i.imgur.com/5bGzZi7.jpg', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['image_sizes']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_google_scholar.py b/tests/example_search_google_scholar.py new file mode 100644 index 0000000..ec221d9 --- /dev/null +++ b/tests/example_search_google_scholar.py @@ -0,0 +1,20 @@ +# test google scholar +import unittest +import os +import serpapi + +class TestGoogleScholar(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_scholar(self): + client = serpapi.Client({ + 'engine': 'google_scholar', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_home_depot.py b/tests/example_search_home_depot.py new file mode 100644 index 0000000..4673218 --- /dev/null +++ b/tests/example_search_home_depot.py @@ -0,0 +1,20 @@ +# test home depot +import unittest +import os +import serpapi + +class TestHomeDepot(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_home_depot(self): + client = serpapi.Client({ + 'engine': 'home_depot', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'table', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['products']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_naver.py b/tests/example_search_naver.py new file mode 100644 index 0000000..bbb2990 --- /dev/null +++ b/tests/example_search_naver.py @@ -0,0 +1,20 @@ +# test naver +import unittest +import os +import serpapi + +class TestNaver(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_naver(self): + client = serpapi.Client({ + 'engine': 'naver', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'query': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['ads_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_walmart.py b/tests/example_search_walmart.py new file mode 100644 index 0000000..804c91c --- /dev/null +++ b/tests/example_search_walmart.py @@ -0,0 +1,20 @@ +# test walmart +import unittest +import os +import serpapi + +class TestWalmart(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_walmart(self): + client = serpapi.Client({ + 'engine': 'walmart', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'query': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_yahoo.py b/tests/example_search_yahoo.py new file mode 100644 index 0000000..4033b34 --- /dev/null +++ b/tests/example_search_yahoo.py @@ -0,0 +1,20 @@ +# test yahoo +import unittest +import os +import serpapi + +class TestYahoo(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_yahoo(self): + client = serpapi.Client({ + 'engine': 'yahoo', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'p': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_youtube.py b/tests/example_search_youtube.py new file mode 100644 index 0000000..b5877b5 --- /dev/null +++ b/tests/example_search_youtube.py @@ -0,0 +1,20 @@ +# test youtube +import unittest +import os +import serpapi + +class TestYoutube(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_youtube(self): + client = serpapi.Client({ + 'engine': 'youtube', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'search_query': 'coffee', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['video_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/test_example.py b/tests/test_example.py new file mode 100644 index 0000000..c9765b7 --- /dev/null +++ b/tests/test_example.py @@ -0,0 +1,152 @@ +# # Unit testing +# import unittest + +# # Operating system +# import os + +# # regular expression library +# import re + +# # safe queue +# import sys +# if (sys.version_info > (3, 0)): +# from queue import Queue +# else: +# from Queue import Queue + +# # Time utility +# import time + +# # Serp API search +# from serpapi import GoogleSearch + +# # download file with wget +# #import wget + +# class TestExample(unittest.TestCase): + +# def setUp(self): +# GoogleSearch.SERP_API_KEY = os.getenv("API_KEY","demo") + +# @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") +# def test_get_json(self): +# search = GoogleSearch({"q": "Coffee", "engine": "google_scholar"}) +# data = search.get_json() +# print(data['search_metadata']) +# search_id = data['search_metadata']['id'] +# # retrieve search from the archive - blocker +# print(search_id + ": get search from archive") +# raw_html = search.get_search_archive(search_id, 'html') +# # print(search_id + ": status = " + search_archived['search_metadata']['status']) +# print(raw_html) +# #print(search_html) + +# @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") +# def test_search_google_images(self): +# search = GoogleSearch({"q": "coffe", "tbm": "isch"}) +# for image_result in search.get_json()['images_results']: +# try: +# link = image_result["original"] +# print("link is found: " + link) +# # uncomment the line below to down the original image +# # wget.download(link, '.') +# except: +# print("link is not found.") +# pass +# # https://github.com/serpapi/showcase-serpapi-tensorflow-keras-image-training/blob/master/fetch.py + +# @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") +# def test_async(self): +# # store searches +# search_queue = Queue() + +# # Serp API search +# search = GoogleSearch({ +# "location": "Austin,Texas", +# "async": True +# }) + +# # loop through companies +# for company in ['amd','nvidia','intel']: +# print("execute async search: q = " + company) +# search.params_dict["q"] = company +# data = search.get_dict() +# if data is not None: +# print("oops data is empty for: " + company) +# continue +# print("add search to the queue where id: " + data['search_metadata']['id']) +# # add search to the search_queue +# search_queue.put(data) + +# print("wait until all search statuses are cached or success") + +# # Create regular search +# search = GoogleSearch({"async": True}) +# while not search_queue.empty(): +# data = search_queue.get() +# search_id = data['search_metadata']['id'] + +# # retrieve search from the archive - blocker +# print(search_id + ": get search from archive") +# search_archived = search.get_search_archive(search_id) +# print(search_id + ": status = " + search_archived['search_metadata']['status']) + +# # check status +# if re.search('Cached|Success', search_archived['search_metadata']['status']): +# print(search_id + ": search done with q = " + search_archived['search_parameters']['q']) +# else: +# # requeue search_queue +# print(search_id + ": requeue search") +# search_queue.put(search) +# # wait 1s +# time.sleep(1) +# # search is over. +# print('all searches completed') + +# @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") +# def test_search_google_news(self): +# search = GoogleSearch({ +# "q": "coffe", # search search +# "tbm": "nws", # news +# "tbs": "qdr:d", # last 24h +# "num": 10 +# }) +# for offset in [0,1,2]: +# search.params_dict["start"] = offset * 10 +# data = search.get_json() +# for news_result in data['news_results']: +# print(str(news_result['position'] + offset * 10) + " - " + news_result['title']) + +# @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") +# def test_search_google_shopping(self): +# search = GoogleSearch({ +# "q": "coffe", # search search +# "tbm": "shop", # news +# "tbs": "p_ord:rv", # last 24h +# "num": 100 +# }) +# data = search.get_json() +# if 'shopping_results' in data: +# for shopping_result in data['shopping_results']: +# print(str(shopping_result['position']) + " - " + shopping_result['title']) +# else: +# print("WARNING: oops shopping_results is missing from search result with tbm=shop") + +# @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") +# def test_search_by_location(self): +# for city in ["new york", "paris", "berlin"]: +# location = GoogleSearch({}).get_location(city, 1)[0]["canonical_name"] +# search = GoogleSearch({ +# "q": "best coffee shop", # search search +# "location": location, +# "num": 10, +# "start": 0 +# }) +# data = search.get_json() +# self.assertIsNone(data.get("error")) +# top_result = data['organic_results'][0]["title"] +# print("top coffee result for " + location + " is: " + top_result) + + +# if __name__ == '__main__': +# unittest.main() From 05682b56403dc3a9425e0a82164c9dfba5004db9 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 5 Jun 2022 14:54:47 -0500 Subject: [PATCH 08/25] ~100% code coverage --- Makefile | 2 +- serpapi/object_decoder.py | 5 +++- serpapi/serpapi.py | 30 +++++++++++---------- tests/example_search_google_play.py | 11 +++++--- tests/test_account_api.py | 4 +-- tests/test_object_decoder.py | 14 +++++++++- tests/test_search_archive.py | 36 +++++++++++++++++++++++++ tests/test_serpapi.py | 41 ++++++++++++++++++++++++++--- 8 files changed, 117 insertions(+), 26 deletions(-) create mode 100644 tests/test_search_archive.py diff --git a/Makefile b/Makefile index 6286d92..9ae5aa7 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ lint: # test with Python 3 test: - pytest --cov=serpapi --cov-report html tests/ + pytest --cov=serpapi --cov-report html tests/*.py # pytest-cov - code coverage extension for pytest # sphinx - documentation diff --git a/serpapi/object_decoder.py b/serpapi/object_decoder.py index b6465e1..bf2bdf6 100644 --- a/serpapi/object_decoder.py +++ b/serpapi/object_decoder.py @@ -15,6 +15,9 @@ def child2object(self, name, node): pytype = type(name, (object, ), {}) pyobj = pytype() + if isinstance(node, int): + pyobj = node + return pyobj if isinstance(node, list): setattr(pyobj, name, []) for item in node: @@ -29,7 +32,7 @@ def child2object(self, name, node): def add_node(self, name, pyobj, child): """ - add node in object + Add node in generic Python object """ if isinstance(child, list): setattr(pyobj, name, []) diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py index bc3f25b..0224c6f 100644 --- a/serpapi/serpapi.py +++ b/serpapi/serpapi.py @@ -19,24 +19,26 @@ class Client(ObjectDecoder): SUPPORTED_DECODER = ['json', 'html', 'object'] def __init__(self, parameter=None): - # define default parameter - if parameter is None: - self.parameter = {} - else: - self.parameter = parameter - # urllib3 options + # urllib3 configuration # 60s default self.timeout = 60.0 # no HTTP retry self.retries = False - # override default - if 'timeout' in parameter: - self.timeout = parameter['timeout'] - if 'retries' in parameter: - self.retries = parameter['retries'] # initialize the http client self.http = urllib3.PoolManager() + # define default parameter + if parameter is None: + self.parameter = {} + else: + # assign user parameter + self.parameter = parameter + # override default + if 'timeout' in parameter: + self.timeout = parameter['timeout'] + if 'retries' in parameter: + self.retries = parameter['retries'] + def search(self, parameter=None, decoder='json'): """ make search then decode the output @@ -106,7 +108,7 @@ def search_archive(self, search_id, decoder='json'): else: path += decoder else: - raise SerpApiException('decoder must be json or html or object') + raise SerpApiException('Decoder must be json or html or object') return self.start(path, {}, decoder) def account(self, api_key=None): @@ -174,7 +176,7 @@ def start(self, path, parameter=None, decoder='json'): return self.decode(response, decoder) def decode(self, response, decoder): - """decode HTTP response using the given decoder""" + """Decode HTTP response using a given decoder""" # handle HTTP error if response.status != 200: try: @@ -198,5 +200,5 @@ def decode(self, response, decoder): data = json.loads(payload) return self.dict2object(data) - raise SerpApiException("invalid decoder: " + + raise SerpApiException("Invalid decoder: " + decoder + ", available: json, html, object") diff --git a/tests/example_search_google_play.py b/tests/example_search_google_play.py index 35d4836..235bdf5 100644 --- a/tests/example_search_google_play.py +++ b/tests/example_search_google_play.py @@ -12,10 +12,13 @@ def test_search_google_play(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', - 'store': 'apps', + "engine": "google_play", + "q": "maps", + "hl": "en", + "gl": "us", + "store": "apps" }) - self.assertIsNone(data.get('error')) - self.assertIsNotNone(data['organic_results']) + # self.assertIsNone(data.get('error')) + # self.assertIsNotNone(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com \ No newline at end of file diff --git a/tests/test_account_api.py b/tests/test_account_api.py index 01360db..4d1e911 100644 --- a/tests/test_account_api.py +++ b/tests/test_account_api.py @@ -7,6 +7,6 @@ class TestAccountApi(unittest.TestCase): @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") def test_get_account(self): - client = serpapi.Client({'api_key': os.getenv('API_KEY')}) - account = client.account() + client = serpapi.Client() + account = client.account(os.getenv('API_KEY')) self.assertIsNotNone(account.get("account_id")) diff --git a/tests/test_object_decoder.py b/tests/test_object_decoder.py index 7beb7e7..d55ebd2 100644 --- a/tests/test_object_decoder.py +++ b/tests/test_object_decoder.py @@ -6,7 +6,7 @@ class TestObjectDecoder(unittest.TestCase): - def test_basic(self): + def test_decode_basic(self): decoder = ObjectDecoder() data = { "organic_results": [ @@ -22,3 +22,15 @@ def test_basic(self): self.assertEqual(len(obj.organic_results), 2) self.assertEqual(obj.organic_results[0].name, 'ok') self.assertEqual(obj.organic_results[1].name, 'good') + + def test_decode_list(self): + decoder = ObjectDecoder() + data = { + "organic_results": { + 'list': [0,1,2,3,4] + } + } + obj = decoder.dict2object(data) + print(obj.organic_results.list[0]) + self.assertEqual(len(obj.organic_results.list), 5) + self.assertEqual(obj.organic_results.list, [0,1,2,3,4]) diff --git a/tests/test_search_archive.py b/tests/test_search_archive.py new file mode 100644 index 0000000..06201a9 --- /dev/null +++ b/tests/test_search_archive.py @@ -0,0 +1,36 @@ +import unittest +import os +import pprint +import serpapi +import pytest + +# This test shows how to extends serpapi.Client +# without using client engine wrapper. +# + + +class TestSearchArchive(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_archive(self): + client = serpapi.Client({ + "engine": "google", + "api_key": os.getenv("API_KEY") + }) + data = client.search({ + "q": "Coffee", + "location": "Austin,Texas" + }) + self.assertEqual(data.get("error"), None) + self.assertIsNotNone(data["organic_results"][0]["title"]) + search_id = data['search_metadata']['id'] + data_archive = client.search_archive(search_id) + self.assertEqual(data_archive['organic_results'][0], data["organic_results"][0]) + + # code coverage + object_archive = client.search_archive(search_id, 'object') + self.assertIsNotNone(object_archive) + self.assertEqual(object_archive.organic_results[0].title, data["organic_results"][0]["title"]) + + with pytest.raises(serpapi.SerpApiException, match=r'Decoder must be json or html'): + client.search_archive(search_id, 'bad') diff --git a/tests/test_serpapi.py b/tests/test_serpapi.py index bfbecb7..bb56b7d 100644 --- a/tests/test_serpapi.py +++ b/tests/test_serpapi.py @@ -39,8 +39,10 @@ def test_html(self): def test_object(self): client = serpapi.Client({ "engine": "google", - "api_key": os.getenv("API_KEY") - }) + "api_key": os.getenv("API_KEY"), + 'timeout': 120, + 'retries': True + }) data = client.search({ "q": "Coffee", "location": "Austin,Texas" @@ -62,7 +64,17 @@ def test_invalid_api_key(self): "location": "USA", }) - # TODO file a ticket default search engine is google + def test_invalid_decoder(self): + client = serpapi.Client({ + "engine": "google", + "api_key": os.getenv("API_KEY"), + }) + mockResponse = MockResponse() + self.assertEqual(mockResponse.status, 200) + with pytest.raises(serpapi.SerpApiException, match=r'Invalid decoder'): + client.decode(mockResponse, 'bad') + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") def test_error_missing_engine(self): client = serpapi.Client({ "api_key": os.getenv("API_KEY"), @@ -71,6 +83,7 @@ def test_error_missing_engine(self): with pytest.raises(serpapi.SerpApiException, match=r'Unsupported.*search engine.'): client.search({"q": "Coffee"}) + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") def test_missing_q(self): client = serpapi.Client({ "api_key": os.getenv("API_KEY") @@ -78,9 +91,31 @@ def test_missing_q(self): with pytest.raises(serpapi.SerpApiException, match=r'Missing query'): client.search({"engine": "google"}) + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_no_parameter(self): + client = serpapi.Client() + with pytest.raises(serpapi.SerpApiException, match=r'Missing query'): + client.search({"engine": "google", 'api_key': os.getenv('API_KEY')}) + + def debug(self, payload): pp = pprint.PrettyPrinter(indent=2) pp.pprint(payload) +# Mock object to enable higher code coverage +# +class MockResponse: + '''Mock HTTP response in order to test serpapi.decode''' + def __init__(self, status=200): + self.status = 200 + self.data = MockString("{}") + +class MockString: + def __init__(self, data: str): + self.data = data + + def decode(self, encoding) -> str: + return self.data + if __name__ == '__main__': unittest.main() From 391cdeb2fb35dd9f88f0a01aaafd7ffb776fbc26 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Wed, 8 Jun 2022 21:54:14 -0500 Subject: [PATCH 09/25] improve python coding style --- serpapi/serpapi.py | 203 +++++++++++++++++++++++---------------------- 1 file changed, 106 insertions(+), 97 deletions(-) diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py index 0224c6f..cdad6b5 100644 --- a/serpapi/serpapi.py +++ b/serpapi/serpapi.py @@ -4,42 +4,125 @@ from .error import SerpApiException from .object_decoder import ObjectDecoder -class Client(ObjectDecoder): +class HttpClient: + """Simple HTTP client wrapper around urllib3""" + + def __init__(self, parameter: dict = {}): + # initialize the http client + self.http = urllib3.PoolManager() + + # urllib3 configurations + # HTTP connect timeout + if 'timeout' in parameter: + self.timeout = parameter['timeout'] + else: + # 60s default + self.timeout = 60.0 + + # no HTTP retry + if 'retries' in parameter: + self.retries = parameter['retries'] + else: + self.retries = False + + def start(self, path : str, parameter: dict = None, decoder : str = 'json'): + """ + start HTTP request and decode response using urllib3 + the response is decoded using the selected decoder: + - html: raw HTML response + - json: deep dict contains search results + - object: containing search results as a dynamic object + + Parameters: + --- + path: str + HTTP endpoint path under serpapi.com/ + decoder: str + define how to post process the HTTP response. + for example: json -> convert response to a dict + using the default JSON parser from Python + parameter: dict + search query + + Returns: + --- + dict|str|object + decoded HTTP response + """ + # set client language + self.parameter['source'] = 'python' + + # set output type + if decoder == 'object': + self.parameter['output'] = 'json' + else: + self.parameter['output'] = decoder + + # merge parameter defaults and overrides + fields = self.parameter.copy() + fields.update(parameter) + + # execute HTTP get request + response = self.http.request('GET', + self.BACKEND + path, + fields=fields, + timeout=self.timeout, + retries=self.retries) + # decode response + return self.decode(response, decoder) + + def decode(self, response: any, decoder: str): + """Decode HTTP response using a given decoder""" + # handle HTTP error + if response.status != 200: + try: + raw = response.data.decode('utf-8') + payload = json.loads(raw) + raise SerpApiException(payload['error']) + except Exception as ex: + raise SerpApiException(raw) from ex + + # HTTP success 200 + payload = response.data.decode('utf-8') + + # successful response decoding + if decoder == 'json': + return json.loads(payload) + + if decoder == 'html': + return payload + + if decoder == 'object': + data = json.loads(payload) + return self.dict2object(data) + + raise SerpApiException("Invalid decoder: " + + decoder + ", available: json, html, object") + +class Client(ObjectDecoder, HttpClient): """ - Client performend http query to serpApi.com - using urllib3 under the hood. + Client performend http query to serpApi.com using urllib3 under the hood. The HTTP connection be tuned to allow - retries : attempt to reconnect if the connection fail by default: False - timeout : connection timeout by default 60s for more details: https://urllib3.readthedocs.io/en/stable/user-guide.html + """ BACKEND = 'https://serpapi.com' SUPPORTED_DECODER = ['json', 'html', 'object'] - def __init__(self, parameter=None): - # urllib3 configuration - # 60s default - self.timeout = 60.0 - # no HTTP retry - self.retries = False - # initialize the http client - self.http = urllib3.PoolManager() - + def __init__(self, parameter: dict = None): # define default parameter if parameter is None: self.parameter = {} else: # assign user parameter self.parameter = parameter - # override default - if 'timeout' in parameter: - self.timeout = parameter['timeout'] - if 'retries' in parameter: - self.retries = parameter['retries'] + HttpClient.__init__(self, self.parameter) - def search(self, parameter=None, decoder='json'): + def search(self, parameter: dict = None, decoder: str = 'json'): """ make search then decode the output decoder supported 'json', 'html', 'object' @@ -61,7 +144,7 @@ def search(self, parameter=None, decoder='json'): """ return self.start(path='/search', parameter=parameter, decoder=decoder) - def html(self, parameter=None): + def html(self, parameter: dict = None): """ html search @@ -77,7 +160,7 @@ def html(self, parameter=None): """ return self.start('/search', parameter, 'html') - def location(self, parameter=None): + def location(self, parameter: dict = None): """ Get location using Location API @@ -94,7 +177,7 @@ def location(self, parameter=None): """ return self.start('/locations.json', parameter, 'json') - def search_archive(self, search_id, decoder='json'): + def search_archive(self, search_id : str, decoder : str ='json'): """ Retrieve search results from the Search Archive API @@ -111,7 +194,7 @@ def search_archive(self, search_id, decoder='json'): raise SerpApiException('Decoder must be json or html or object') return self.start(path, {}, decoder) - def account(self, api_key=None): + def account(self, api_key: str = None): """ Get account information using Account API @@ -119,7 +202,7 @@ def account(self, api_key=None): --- api_key: str secret user key provided by serpapi.com - + Returns --- dict @@ -128,77 +211,3 @@ def account(self, api_key=None): if api_key is not None: self.parameter['api_key'] = api_key return self.start('/account', self.parameter, 'json') - - def start(self, path, parameter=None, decoder='json'): - """ - start HTTP request and decode response using urllib3 - the response is decoded using the selected decoder: - - html: raw HTML response - - json: deep dict contains search results - - object: containing search results as a dynamic object - - Parameters: - --- - path: str - HTTP endpoint path under serpapi.com/ - decoder: str - define how to post process the HTTP response. - for example: json -> convert response to a dict - using the default JSON parser from Python - parameter: dict - search query - - Returns: - --- - dict|str|object - decoded HTTP response - """ - # set client language - self.parameter['source'] = 'python' - - # set output type - if decoder == 'object': - self.parameter['output'] = 'json' - else: - self.parameter['output'] = decoder - - # merge parameter defaults and overrides - fields = self.parameter.copy() - fields.update(parameter) - - # execute HTTP get request - response = self.http.request('GET', - self.BACKEND + path, - fields=fields, - timeout=self.timeout, - retries=self.retries) - # decode response - return self.decode(response, decoder) - - def decode(self, response, decoder): - """Decode HTTP response using a given decoder""" - # handle HTTP error - if response.status != 200: - try: - raw = response.data.decode('utf-8') - payload = json.loads(raw) - raise SerpApiException(payload['error']) - except Exception as ex: - raise SerpApiException(raw) from ex - - # HTTP success 200 - payload = response.data.decode('utf-8') - - # successful response decoding - if decoder == 'json': - return json.loads(payload) - - if decoder == 'html': - return payload - - if decoder == 'object': - data = json.loads(payload) - return self.dict2object(data) - - raise SerpApiException("Invalid decoder: " + - decoder + ", available: json, html, object") From 978be0cc364369aa79dca9d77d3246dc9957c838 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Wed, 8 Jun 2022 21:54:45 -0500 Subject: [PATCH 10/25] fix sphinx documentation generation improve README --- README.md | 149 +++++++++++++++++++++++++------------------- README.md.erb | 60 ++++++++++-------- docs/source/conf.py | 9 ++- 3 files changed, 123 insertions(+), 95 deletions(-) diff --git a/README.md b/README.md index 20d9546..9efe68c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ # User guide Scrape Google and other search engines from our fast, easy, and complete API using SerpApi.com -This ruby library is meant to scrape and parse results from all major search engine available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Apple and more using [SerpApi](https://serpapi.com). +This python library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi](https://serpapi.com). +This is an open source project hosted under https://github.com/serpapi/serpapi-python. + SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly. ## Installation @@ -51,13 +53,17 @@ SerpApi Client uses urllib3 under the hood. The HTTP connection be tuned by setting the following client specific setting. - retries : attempt to reconnect if the connection failed by default: False - timeout : connection timeout by default 60s -for more details: https://urllib3.readthedocs.io/en/stable/user-guide.html -For example: + ```python parameter = { - retries: 5, - timeout: 4.0 + retries: 5, + timeout: 4.0 + # regular parameters } +``` + +for more details: [URL LIB3 documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html) + ## Basic example per search engine ### Search bing @@ -77,8 +83,8 @@ pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_bing.py -(see https://serpapi.com/bing)[https://serpapi.com/bing] + test: tests/example_search_bing.py +[see https://serpapi.com/bing](https://serpapi.com/bing) ### Search baidu ```python @@ -97,8 +103,8 @@ pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_baidu.py -(see https://serpapi.com/baidu)[https://serpapi.com/baidu] + test: tests/example_search_baidu.py +[see https://serpapi.com/baidu](https://serpapi.com/baidu) ### Search yahoo ```python @@ -117,8 +123,8 @@ pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_yahoo.py -(see https://serpapi.com/yahoo)[https://serpapi.com/yahoo] + test: tests/example_search_yahoo.py +[see https://serpapi.com/yahoo](https://serpapi.com/yahoo) ### Search youtube ```python @@ -132,14 +138,13 @@ client = serpapi.Client({ }) data = client.search({ 'search_query': 'coffee', -'results': 'video_results', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) +pp.pprint(data['video_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_youtube.py -(see https://serpapi.com/youtube)[https://serpapi.com/youtube] + test: tests/example_search_youtube.py +[see https://serpapi.com/youtube](https://serpapi.com/youtube) ### Search walmart ```python @@ -158,8 +163,8 @@ pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_walmart.py -(see https://serpapi.com/walmart)[https://serpapi.com/walmart] + test: tests/example_search_walmart.py +[see https://serpapi.com/walmart](https://serpapi.com/walmart) ### Search ebay ```python @@ -178,8 +183,8 @@ pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_ebay.py -(see https://serpapi.com/ebay)[https://serpapi.com/ebay] + test: tests/example_search_ebay.py +[see https://serpapi.com/ebay](https://serpapi.com/ebay) ### Search naver ```python @@ -193,14 +198,13 @@ client = serpapi.Client({ }) data = client.search({ 'query': 'coffee', -'results': 'ads_results', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) +pp.pprint(data['ads_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_naver.py -(see https://serpapi.com/naver)[https://serpapi.com/naver] + test: tests/example_search_naver.py +[see https://serpapi.com/naver](https://serpapi.com/naver) ### Search home depot ```python @@ -214,14 +218,13 @@ client = serpapi.Client({ }) data = client.search({ 'q': 'table', -'results': 'products', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) +pp.pprint(data['products']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_home_depot.py -(see https://serpapi.com/home_depot)[https://serpapi.com/home_depot] + test: tests/example_search_home_depot.py +[see https://serpapi.com/home_depot](https://serpapi.com/home_depot) ### Search apple app store ```python @@ -240,8 +243,8 @@ pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_apple_app_store.py -(see https://serpapi.com/apple_app_store)[https://serpapi.com/apple_app_store] + test: tests/example_search_apple_app_store.py +[see https://serpapi.com/apple_app_store](https://serpapi.com/apple_app_store) ### Search duckduckgo ```python @@ -260,8 +263,8 @@ pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_duckduckgo.py -(see https://serpapi.com/duckduckgo)[https://serpapi.com/duckduckgo] + test: tests/example_search_duckduckgo.py +[see https://serpapi.com/duckduckgo](https://serpapi.com/duckduckgo) ### Search google scholar ```python @@ -280,8 +283,27 @@ pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_google_scholar.py -(see https://serpapi.com/google_scholar)[https://serpapi.com/google_scholar] + test: tests/example_search_google_scholar.py +[see https://serpapi.com/google_scholar](https://serpapi.com/google_scholar) + +### Search google autocomplete +```ruby +import 'serpapi' +import 'pprint' +import 'os' + +'engine': 'google_autocomplete', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['suggestions']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + test: tests/example_search_google_autocomplete.py +[see https://serpapi.com/google_autocomplete](https://serpapi.com/google_autocomplete) ### Search google product ```python @@ -296,14 +318,13 @@ client = serpapi.Client({ data = client.search({ 'q': 'coffee', 'product_id': '4172129135583325756', -'results': 'product_results', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) +pp.pprint(data['product_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_google_product.py -(see https://serpapi.com/google_product)[https://serpapi.com/google_product] + test: tests/example_search_google_product.py +[see https://serpapi.com/google_product](https://serpapi.com/google_product) ### Search google reverse image ```python @@ -317,14 +338,13 @@ client = serpapi.Client({ }) data = client.search({ 'image_url': 'https://i.imgur.com/5bGzZi7.jpg', -'results': 'image_sizes', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) +pp.pprint(data['image_sizes']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_google_reverse_image.py -(see https://serpapi.com/google_reverse_image)[https://serpapi.com/google_reverse_image] + test: tests/example_search_google_reverse_image.py +[see https://serpapi.com/google_reverse_image](https://serpapi.com/google_reverse_image) ### Search google events ```python @@ -340,11 +360,11 @@ data = client.search({ 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) +pp.pprint(data['events_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_google_events.py -(see https://serpapi.com/google_events)[https://serpapi.com/google_events] + test: tests/example_search_google_events.py +[see https://serpapi.com/google_events](https://serpapi.com/google_events) ### Search google local services ```python @@ -359,14 +379,13 @@ client = serpapi.Client({ data = client.search({ 'q': 'Electrician', 'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g', -'results': 'local_ads', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) +pp.pprint(data['local_ads']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_google_local_services.py -(see https://serpapi.com/google_local_services)[https://serpapi.com/google_local_services] + test: tests/example_search_google_local_services.py +[see https://serpapi.com/google_local_services](https://serpapi.com/google_local_services) ### Search google maps ```python @@ -382,14 +401,13 @@ data = client.search({ 'q': 'pizza', 'll': '@40.7455096,-74.0083012,15.1z', 'type': 'search', -'results': 'local_results', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) +pp.pprint(data['local_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_google_maps.py -(see https://serpapi.com/google_maps)[https://serpapi.com/google_maps] + test: tests/example_search_google_maps.py +[see https://serpapi.com/google_maps](https://serpapi.com/google_maps) ### Search google jobs ```python @@ -403,14 +421,13 @@ client = serpapi.Client({ }) data = client.search({ 'q': 'coffee', -'results': 'jobs_results', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) +pp.pprint(data['jobs_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_google_jobs.py -(see https://serpapi.com/google_jobs)[https://serpapi.com/google_jobs] + test: tests/example_search_google_jobs.py +[see https://serpapi.com/google_jobs](https://serpapi.com/google_jobs) ### Search google play ```python @@ -423,15 +440,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', -'store': 'apps', +"engine": "google_play", +"q": "maps", +"hl": "en", +"gl": "us", +"store": "apps" }) -pp = pprint.PrettyPrinter(indent=2) +# pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_google_play.py -(see https://serpapi.com/google_play)[https://serpapi.com/google_play] + test: tests/example_search_google_play.py +[see https://serpapi.com/google_play](https://serpapi.com/google_play) ### Search google images ```python @@ -447,14 +467,13 @@ data = client.search({ 'engine': 'google', 'tbm': 'isch', 'q': 'coffee', -'results': 'images_results', }) pp = pprint.PrettyPrinter(indent=2) -pp.pprint(data['organic_results']) +pp.pprint(data['images_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` - see: tests/example_search_google_images.py -(see https://serpapi.com/google_images)[https://serpapi.com/google_images] + test: tests/example_search_google_images.py +[see https://serpapi.com/google_images](https://serpapi.com/google_images) ## Developer's note ### Key goals diff --git a/README.md.erb b/README.md.erb index 6dd4916..3a9afa6 100644 --- a/README.md.erb +++ b/README.md.erb @@ -4,12 +4,14 @@ def snippet(format, path, start, stop) slice.reject! { |l| l =~ /self.assertIsNone\(/ } buf = slice.map { |l| l.gsub(/(^\s+)/, '')}.join buf.gsub!('self.assertIsNotNone(', "pp = pprint.PrettyPrinter(indent=2)\npp.pprint(") - %Q(```#{format}\nimport 'serpapi'\nimport 'pprint'\nimport 'os'\n\n#{buf}```\n see: #{path}) + %Q(```#{format}\nimport 'serpapi'\nimport 'pprint'\nimport 'os'\n\n#{buf}```\n test: #{path}) end -%> # User guide Scrape Google and other search engines from our fast, easy, and complete API using SerpApi.com -This ruby library is meant to scrape and parse results from all major search engine available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Apple and more using [SerpApi](https://serpapi.com). +This python library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi](https://serpapi.com). +This is an open source project hosted under https://github.com/serpapi/serpapi-python. + SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly. ## Installation @@ -60,90 +62,98 @@ SerpApi Client uses urllib3 under the hood. The HTTP connection be tuned by setting the following client specific setting. - retries : attempt to reconnect if the connection failed by default: False - timeout : connection timeout by default 60s -for more details: https://urllib3.readthedocs.io/en/stable/user-guide.html -For example: + ```python parameter = { - retries: 5, - timeout: 4.0 + retries: 5, + timeout: 4.0 + # regular parameters } +``` + +for more details: [URL LIB3 documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html) + ## Basic example per search engine ### Search bing <%= snippet('python', 'tests/example_search_bing.py', 9, 25) %> -(see https://serpapi.com/bing)[https://serpapi.com/bing] +[see https://serpapi.com/bing](https://serpapi.com/bing) ### Search baidu <%= snippet('python', 'tests/example_search_baidu.py', 9, 25) %> -(see https://serpapi.com/baidu)[https://serpapi.com/baidu] +[see https://serpapi.com/baidu](https://serpapi.com/baidu) ### Search yahoo <%= snippet('python', 'tests/example_search_yahoo.py', 9, 25) %> -(see https://serpapi.com/yahoo)[https://serpapi.com/yahoo] +[see https://serpapi.com/yahoo](https://serpapi.com/yahoo) ### Search youtube <%= snippet('python', 'tests/example_search_youtube.py', 9, 25) %> -(see https://serpapi.com/youtube)[https://serpapi.com/youtube] +[see https://serpapi.com/youtube](https://serpapi.com/youtube) ### Search walmart <%= snippet('python', 'tests/example_search_walmart.py', 9, 25) %> -(see https://serpapi.com/walmart)[https://serpapi.com/walmart] +[see https://serpapi.com/walmart](https://serpapi.com/walmart) ### Search ebay <%= snippet('python', 'tests/example_search_ebay.py', 9, 25) %> -(see https://serpapi.com/ebay)[https://serpapi.com/ebay] +[see https://serpapi.com/ebay](https://serpapi.com/ebay) ### Search naver <%= snippet('python', 'tests/example_search_naver.py', 9, 25) %> -(see https://serpapi.com/naver)[https://serpapi.com/naver] +[see https://serpapi.com/naver](https://serpapi.com/naver) ### Search home depot <%= snippet('python', 'tests/example_search_home_depot.py', 9, 25) %> -(see https://serpapi.com/home_depot)[https://serpapi.com/home_depot] +[see https://serpapi.com/home_depot](https://serpapi.com/home_depot) ### Search apple app store <%= snippet('python', 'tests/example_search_apple_app_store.py', 9, 25) %> -(see https://serpapi.com/apple_app_store)[https://serpapi.com/apple_app_store] +[see https://serpapi.com/apple_app_store](https://serpapi.com/apple_app_store) ### Search duckduckgo <%= snippet('python', 'tests/example_search_duckduckgo.py', 9, 25) %> -(see https://serpapi.com/duckduckgo)[https://serpapi.com/duckduckgo] +[see https://serpapi.com/duckduckgo](https://serpapi.com/duckduckgo) ### Search google scholar <%= snippet('python', 'tests/example_search_google_scholar.py', 9, 25) %> -(see https://serpapi.com/google_scholar)[https://serpapi.com/google_scholar] +[see https://serpapi.com/google_scholar](https://serpapi.com/google_scholar) + +### Search google autocomplete +<%= snippet('ruby', 'tests/example_search_google_autocomplete.py', 10, 25) %> +[see https://serpapi.com/google_autocomplete](https://serpapi.com/google_autocomplete) ### Search google product <%= snippet('python', 'tests/example_search_google_product.py', 9, 25) %> -(see https://serpapi.com/google_product)[https://serpapi.com/google_product] +[see https://serpapi.com/google_product](https://serpapi.com/google_product) ### Search google reverse image <%= snippet('python', 'tests/example_search_google_reverse_image.py', 9, 25) %> -(see https://serpapi.com/google_reverse_image)[https://serpapi.com/google_reverse_image] +[see https://serpapi.com/google_reverse_image](https://serpapi.com/google_reverse_image) ### Search google events <%= snippet('python', 'tests/example_search_google_events.py', 9, 25) %> -(see https://serpapi.com/google_events)[https://serpapi.com/google_events] +[see https://serpapi.com/google_events](https://serpapi.com/google_events) ### Search google local services <%= snippet('python', 'tests/example_search_google_local_services.py', 9, 25) %> -(see https://serpapi.com/google_local_services)[https://serpapi.com/google_local_services] +[see https://serpapi.com/google_local_services](https://serpapi.com/google_local_services) ### Search google maps <%= snippet('python', 'tests/example_search_google_maps.py', 9, 25) %> -(see https://serpapi.com/google_maps)[https://serpapi.com/google_maps] +[see https://serpapi.com/google_maps](https://serpapi.com/google_maps) ### Search google jobs <%= snippet('python', 'tests/example_search_google_jobs.py', 9, 25) %> -(see https://serpapi.com/google_jobs)[https://serpapi.com/google_jobs] +[see https://serpapi.com/google_jobs](https://serpapi.com/google_jobs) ### Search google play <%= snippet('python', 'tests/example_search_google_play.py', 9, 25) %> -(see https://serpapi.com/google_play)[https://serpapi.com/google_play] +[see https://serpapi.com/google_play](https://serpapi.com/google_play) ### Search google images <%= snippet('python', 'tests/example_search_google_images.py', 9, 25) %> -(see https://serpapi.com/google_images)[https://serpapi.com/google_images] +[see https://serpapi.com/google_images](https://serpapi.com/google_images) ## Developer's note ### Key goals diff --git a/docs/source/conf.py b/docs/source/conf.py index 5a8b25c..32500da 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -12,17 +12,16 @@ # import os import sys -sys.path.insert(0, os.path.abspath('..')) - +sys.path.insert(0, os.path.abspath('../..')) # -- Project information ----------------------------------------------------- -project = 'serapi-python' -copyright = '2022, Victor Benarbia' +project = 'serpapi-python' +copyright = '© 2022 SerpApi, LLC' author = 'Victor Benarbia' # The full version, including alpha/beta/rc tags -release = '1.0.0' +release = '1.0.0-beta' # -- General configuration --------------------------------------------------- From 65f0d8218901ad8f8484f1250146d6caedecf0c4 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Tue, 14 Jun 2022 19:44:41 -0500 Subject: [PATCH 11/25] Quality improvement: documentation, code style, lint, coverage Refractor Object decoder --- Makefile | 15 +- README.md | 197 +++++++++--------- README.md.erb | 166 ++++++++------- serpapi/object_decoder.py | 25 ++- serpapi/serpapi.py | 47 +++-- tests/example_search_apple_app_store.py | 2 +- tests/example_search_baidu.py | 2 +- tests/example_search_bing.py | 2 +- tests/example_search_duckduckgo.py | 2 +- tests/example_search_ebay.py | 2 +- tests/example_search_google_autocomplete.py | 2 +- tests/example_search_google_events.py | 2 +- tests/example_search_google_images.py | 2 +- tests/example_search_google_jobs.py | 2 +- tests/example_search_google_local_services.py | 2 +- tests/example_search_google_maps.py | 2 +- tests/example_search_google_play.py | 14 +- tests/example_search_google_product.py | 2 +- tests/example_search_google_reverse_image.py | 2 +- tests/example_search_google_scholar.py | 2 +- tests/example_search_google_search.py | 21 ++ tests/example_search_home_depot.py | 2 +- tests/example_search_naver.py | 2 +- tests/example_search_walmart.py | 2 +- tests/example_search_yahoo.py | 2 +- tests/example_search_youtube.py | 2 +- tests/test_object_decoder.py | 6 +- 27 files changed, 290 insertions(+), 239 deletions(-) create mode 100644 tests/example_search_google_search.py diff --git a/Makefile b/Makefile index 9ae5aa7..20b4823 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,12 @@ # Automate pip package development # +# Usage +# To release a package. +# - update version in serpapi/_version.py +# - review README version +# - run +# $ make release + # current version version=$(shell grep version setup.py | cut -d"'" -f2) dist=dist/serpapi-$(version).tar.gz @@ -21,12 +28,17 @@ lint: test: pytest --cov=serpapi --cov-report html tests/*.py +# install dependencies +# # pytest-cov - code coverage extension for pytest # sphinx - documentation +# twine - release automation install: pip3 install -U setuptools pip3 install -r requirements.txt + pip3 install pylint pip3 install pytest-cov + pip3 install twine pip3 install sphinx readme: @@ -36,7 +48,7 @@ doc: readme $(MAKE) -C docs/ html # https://packaging.python.org/tutorials/packaging-projects/ -build: doc test +build: doc lint test python3 setup.py sdist # out of box testing / user acceptance before delivery @@ -44,6 +56,7 @@ oobt: build pip3 install ./${dist} python3 oobt/oobt.py + check: oobt twine check ${dist} diff --git a/README.md b/README.md index 9efe68c..6e02b96 100644 --- a/README.md +++ b/README.md @@ -16,23 +16,23 @@ $ python -m pip install serpapi First things first, import the serpapi module: ```python ->>> import serpapi +import serpapi ``` You'll need a client instance to make a search. This object handles all of the details of connection pooling and thread safety so that you don't have to: ```python ->>> client = serpapi.Client() +client = serpapi.Client() ``` -To make a search using SerpApi.com client. +To make a search using SerpApi.com: ```python ->>> parameter = { - api_key: "secret_api_key", # from serpapi.com - engine: "google", # search engine - q: "coffee", # search topic - location: "Austin,TX" # location - } - results = searpapi.search(parameter) +parameter = { + api_key: "secret_api_key", # from serpapi.com + engine: "google", # search engine + q: "coffee", # search topic + location: "Austin,TX" # location +} +results = searpapi.search(parameter) ``` Putting everything together. ```python @@ -50,29 +50,28 @@ print(results) ### Advanced settings SerpApi Client uses urllib3 under the hood. -The HTTP connection be tuned by setting the following client specific setting. - - retries : attempt to reconnect if the connection failed by default: False +Optionally, rhe HTTP connection can be tuned: - timeout : connection timeout by default 60s + - retries : attempt to reconnect if the connection failed by default: False. + serpapi is reliable at 99.99% but your company network might not be as stable. ```python parameter = { retries: 5, - timeout: 4.0 - # regular parameters + timeout: 4.0, + # extra user parameters } ``` for more details: [URL LIB3 documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html) - -## Basic example per search engine -### Search bing -```python +## Basic example per search engines +### Search Bing +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'bing', 'api_key': os.getenv("API_KEY") }) @@ -84,15 +83,14 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_bing.py -[see https://serpapi.com/bing](https://serpapi.com/bing) +(see https://serpapi.com/bing)[https://serpapi.com/bing] -### Search baidu -```python +### Search Baidu +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'baidu', 'api_key': os.getenv("API_KEY") }) @@ -104,15 +102,14 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_baidu.py -[see https://serpapi.com/baidu](https://serpapi.com/baidu) +(see https://serpapi.com/baidu)[https://serpapi.com/baidu] -### Search yahoo -```python +### Search Yahoo +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'yahoo', 'api_key': os.getenv("API_KEY") }) @@ -124,15 +121,14 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_yahoo.py -[see https://serpapi.com/yahoo](https://serpapi.com/yahoo) +(see https://serpapi.com/yahoo)[https://serpapi.com/yahoo] -### Search youtube -```python +### Search Youtube +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'youtube', 'api_key': os.getenv("API_KEY") }) @@ -144,15 +140,14 @@ pp.pprint(data['video_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_youtube.py -[see https://serpapi.com/youtube](https://serpapi.com/youtube) +(see https://serpapi.com/youtube)[https://serpapi.com/youtube] -### Search walmart -```python +### Search Walmart +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'walmart', 'api_key': os.getenv("API_KEY") }) @@ -164,15 +159,14 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_walmart.py -[see https://serpapi.com/walmart](https://serpapi.com/walmart) +(see https://serpapi.com/walmart)[https://serpapi.com/walmart] -### Search ebay -```python +### Search Ebay +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'ebay', 'api_key': os.getenv("API_KEY") }) @@ -184,15 +178,14 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_ebay.py -[see https://serpapi.com/ebay](https://serpapi.com/ebay) +(see https://serpapi.com/ebay)[https://serpapi.com/ebay] -### Search naver -```python +### Search Naver +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'naver', 'api_key': os.getenv("API_KEY") }) @@ -204,15 +197,14 @@ pp.pprint(data['ads_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_naver.py -[see https://serpapi.com/naver](https://serpapi.com/naver) +(see https://serpapi.com/naver)[https://serpapi.com/naver] -### Search home depot -```python +### Search Home Depot +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'home_depot', 'api_key': os.getenv("API_KEY") }) @@ -224,15 +216,14 @@ pp.pprint(data['products']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_home_depot.py -[see https://serpapi.com/home_depot](https://serpapi.com/home_depot) +(see https://serpapi.com/home_depot)[https://serpapi.com/home_depot] -### Search apple app store -```python +### Search Apple App Store +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'apple_app_store', 'api_key': os.getenv("API_KEY") }) @@ -244,15 +235,14 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_apple_app_store.py -[see https://serpapi.com/apple_app_store](https://serpapi.com/apple_app_store) +(see https://serpapi.com/apple_app_store)[https://serpapi.com/apple_app_store] -### Search duckduckgo -```python +### Search Duckduckgo +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'duckduckgo', 'api_key': os.getenv("API_KEY") }) @@ -264,15 +254,33 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_duckduckgo.py -[see https://serpapi.com/duckduckgo](https://serpapi.com/duckduckgo) +(see https://serpapi.com/duckduckgo)[https://serpapi.com/duckduckgo] -### Search google scholar -```python +### Search Google Search +```ruby +import 'serpapi' +import 'pprint' +import 'os' + +'engine': 'google_search', +'api_key': os.getenv("API_KEY") +}) +data = client.search({ +'q': 'coffee', +}) +pp = pprint.PrettyPrinter(indent=2) +pp.pprint(data['organic_results']) +# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +``` + test: tests/example_search_google_search.py +(see https://serpapi.com/google_search)[https://serpapi.com/google_search] + +### Search Google Scholar +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'google_scholar', 'api_key': os.getenv("API_KEY") }) @@ -284,9 +292,9 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_scholar.py -[see https://serpapi.com/google_scholar](https://serpapi.com/google_scholar) +(see https://serpapi.com/google_scholar)[https://serpapi.com/google_scholar] -### Search google autocomplete +### Search Google Autocomplete ```ruby import 'serpapi' import 'pprint' @@ -303,15 +311,14 @@ pp.pprint(data['suggestions']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_autocomplete.py -[see https://serpapi.com/google_autocomplete](https://serpapi.com/google_autocomplete) +(see https://serpapi.com/google_autocomplete)[https://serpapi.com/google_autocomplete] -### Search google product -```python +### Search Google Product +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'google_product', 'api_key': os.getenv("API_KEY") }) @@ -324,15 +331,14 @@ pp.pprint(data['product_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_product.py -[see https://serpapi.com/google_product](https://serpapi.com/google_product) +(see https://serpapi.com/google_product)[https://serpapi.com/google_product] -### Search google reverse image -```python +### Search Google Reverse Image +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'google_reverse_image', 'api_key': os.getenv("API_KEY") }) @@ -344,15 +350,14 @@ pp.pprint(data['image_sizes']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_reverse_image.py -[see https://serpapi.com/google_reverse_image](https://serpapi.com/google_reverse_image) +(see https://serpapi.com/google_reverse_image)[https://serpapi.com/google_reverse_image] -### Search google events -```python +### Search Google Events +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'google_events', 'api_key': os.getenv("API_KEY") }) @@ -364,15 +369,14 @@ pp.pprint(data['events_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_events.py -[see https://serpapi.com/google_events](https://serpapi.com/google_events) +(see https://serpapi.com/google_events)[https://serpapi.com/google_events] -### Search google local services -```python +### Search Google Local Services +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'google_local_services', 'api_key': os.getenv("API_KEY") }) @@ -385,15 +389,14 @@ pp.pprint(data['local_ads']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_local_services.py -[see https://serpapi.com/google_local_services](https://serpapi.com/google_local_services) +(see https://serpapi.com/google_local_services)[https://serpapi.com/google_local_services] -### Search google maps -```python +### Search Google Maps +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'google_maps', 'api_key': os.getenv("API_KEY") }) @@ -407,15 +410,14 @@ pp.pprint(data['local_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_maps.py -[see https://serpapi.com/google_maps](https://serpapi.com/google_maps) +(see https://serpapi.com/google_maps)[https://serpapi.com/google_maps] -### Search google jobs -```python +### Search Google Jobs +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'google_jobs', 'api_key': os.getenv("API_KEY") }) @@ -427,39 +429,34 @@ pp.pprint(data['jobs_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_jobs.py -[see https://serpapi.com/google_jobs](https://serpapi.com/google_jobs) +(see https://serpapi.com/google_jobs)[https://serpapi.com/google_jobs] -### Search google play -```python +### Search Google Play +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'google_play', 'api_key': os.getenv("API_KEY") }) data = client.search({ -"engine": "google_play", -"q": "maps", -"hl": "en", -"gl": "us", -"store": "apps" +'q': 'coffee', +'store': 'apps', }) -# pp = pprint.PrettyPrinter(indent=2) +pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_play.py -[see https://serpapi.com/google_play](https://serpapi.com/google_play) +(see https://serpapi.com/google_play)[https://serpapi.com/google_play] -### Search google images -```python +### Search Google Images +```ruby import 'serpapi' import 'pprint' import 'os' -client = serpapi.Client({ 'engine': 'google_images', 'api_key': os.getenv("API_KEY") }) @@ -473,7 +470,7 @@ pp.pprint(data['images_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_images.py -[see https://serpapi.com/google_images](https://serpapi.com/google_images) +(see https://serpapi.com/google_images)[https://serpapi.com/google_images] ## Developer's note ### Key goals diff --git a/README.md.erb b/README.md.erb index 3a9afa6..3d091f2 100644 --- a/README.md.erb +++ b/README.md.erb @@ -8,14 +8,16 @@ def snippet(format, path, start, stop) end -%> # User guide -Scrape Google and other search engines from our fast, easy, and complete API using SerpApi.com -This python library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi](https://serpapi.com). +[SerpApi](https://serpapi.com) allows to scrape any search engine results. + It's easy, fast, easy, feature rich, cost effective, scalable and reliable. + +This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi](https://serpapi.com). This is an open source project hosted under https://github.com/serpapi/serpapi-python. SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly. ## Installation -serpapi can be installed with pip. +SerpApi can be installed with pip. ```sh $ python -m pip install serpapi @@ -25,23 +27,23 @@ $ python -m pip install serpapi First things first, import the serpapi module: ```python ->>> import serpapi +import serpapi ``` You'll need a client instance to make a search. This object handles all of the details of connection pooling and thread safety so that you don't have to: ```python ->>> client = serpapi.Client() +client = serpapi.Client() ``` -To make a search using SerpApi.com client. +To make a search using SerpApi.com: ```python ->>> parameter = { - api_key: "secret_api_key", # from serpapi.com - engine: "google", # search engine - q: "coffee", # search topic - location: "Austin,TX" # location - } - results = searpapi.search(parameter) +parameter = { + api_key: "secret_api_key", # from serpapi.com + engine: "google", # search engine + q: "coffee", # search topic + location: "Austin,TX" # location +} +results = searpapi.search(parameter) ``` Putting everything together. ```python @@ -59,111 +61,117 @@ print(results) ### Advanced settings SerpApi Client uses urllib3 under the hood. -The HTTP connection be tuned by setting the following client specific setting. - - retries : attempt to reconnect if the connection failed by default: False +Optionally, rhe HTTP connection can be tuned: - timeout : connection timeout by default 60s + - retries : attempt to reconnect if the connection failed by default: False. + serpapi is reliable at 99.99% but your company network might not be as stable. ```python parameter = { retries: 5, - timeout: 4.0 - # regular parameters + timeout: 4.0, + # extra user parameters } ``` for more details: [URL LIB3 documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html) +## Basic example per search engines +### Search Bing +<%= snippet('ruby', 'tests/example_search_bing.py', 10, 25) %> +(see https://serpapi.com/bing)[https://serpapi.com/bing] -## Basic example per search engine -### Search bing -<%= snippet('python', 'tests/example_search_bing.py', 9, 25) %> -[see https://serpapi.com/bing](https://serpapi.com/bing) +### Search Baidu +<%= snippet('ruby', 'tests/example_search_baidu.py', 10, 25) %> +(see https://serpapi.com/baidu)[https://serpapi.com/baidu] -### Search baidu -<%= snippet('python', 'tests/example_search_baidu.py', 9, 25) %> -[see https://serpapi.com/baidu](https://serpapi.com/baidu) +### Search Yahoo +<%= snippet('ruby', 'tests/example_search_yahoo.py', 10, 25) %> +(see https://serpapi.com/yahoo)[https://serpapi.com/yahoo] -### Search yahoo -<%= snippet('python', 'tests/example_search_yahoo.py', 9, 25) %> -[see https://serpapi.com/yahoo](https://serpapi.com/yahoo) +### Search Youtube +<%= snippet('ruby', 'tests/example_search_youtube.py', 10, 25) %> +(see https://serpapi.com/youtube)[https://serpapi.com/youtube] -### Search youtube -<%= snippet('python', 'tests/example_search_youtube.py', 9, 25) %> -[see https://serpapi.com/youtube](https://serpapi.com/youtube) +### Search Walmart +<%= snippet('ruby', 'tests/example_search_walmart.py', 10, 25) %> +(see https://serpapi.com/walmart)[https://serpapi.com/walmart] -### Search walmart -<%= snippet('python', 'tests/example_search_walmart.py', 9, 25) %> -[see https://serpapi.com/walmart](https://serpapi.com/walmart) +### Search Ebay +<%= snippet('ruby', 'tests/example_search_ebay.py', 10, 25) %> +(see https://serpapi.com/ebay)[https://serpapi.com/ebay] -### Search ebay -<%= snippet('python', 'tests/example_search_ebay.py', 9, 25) %> -[see https://serpapi.com/ebay](https://serpapi.com/ebay) +### Search Naver +<%= snippet('ruby', 'tests/example_search_naver.py', 10, 25) %> +(see https://serpapi.com/naver)[https://serpapi.com/naver] -### Search naver -<%= snippet('python', 'tests/example_search_naver.py', 9, 25) %> -[see https://serpapi.com/naver](https://serpapi.com/naver) +### Search Home Depot +<%= snippet('ruby', 'tests/example_search_home_depot.py', 10, 25) %> +(see https://serpapi.com/home_depot)[https://serpapi.com/home_depot] -### Search home depot -<%= snippet('python', 'tests/example_search_home_depot.py', 9, 25) %> -[see https://serpapi.com/home_depot](https://serpapi.com/home_depot) +### Search Apple App Store +<%= snippet('ruby', 'tests/example_search_apple_app_store.py', 10, 25) %> +(see https://serpapi.com/apple_app_store)[https://serpapi.com/apple_app_store] -### Search apple app store -<%= snippet('python', 'tests/example_search_apple_app_store.py', 9, 25) %> -[see https://serpapi.com/apple_app_store](https://serpapi.com/apple_app_store) +### Search Duckduckgo +<%= snippet('ruby', 'tests/example_search_duckduckgo.py', 10, 25) %> +(see https://serpapi.com/duckduckgo)[https://serpapi.com/duckduckgo] -### Search duckduckgo -<%= snippet('python', 'tests/example_search_duckduckgo.py', 9, 25) %> -[see https://serpapi.com/duckduckgo](https://serpapi.com/duckduckgo) +### Search Google Search +<%= snippet('ruby', 'tests/example_search_google_search.py', 10, 25) %> +(see https://serpapi.com/google_search)[https://serpapi.com/google_search] -### Search google scholar -<%= snippet('python', 'tests/example_search_google_scholar.py', 9, 25) %> -[see https://serpapi.com/google_scholar](https://serpapi.com/google_scholar) +### Search Google Scholar +<%= snippet('ruby', 'tests/example_search_google_scholar.py', 10, 25) %> +(see https://serpapi.com/google_scholar)[https://serpapi.com/google_scholar] -### Search google autocomplete +### Search Google Autocomplete <%= snippet('ruby', 'tests/example_search_google_autocomplete.py', 10, 25) %> -[see https://serpapi.com/google_autocomplete](https://serpapi.com/google_autocomplete) +(see https://serpapi.com/google_autocomplete)[https://serpapi.com/google_autocomplete] -### Search google product -<%= snippet('python', 'tests/example_search_google_product.py', 9, 25) %> -[see https://serpapi.com/google_product](https://serpapi.com/google_product) +### Search Google Product +<%= snippet('ruby', 'tests/example_search_google_product.py', 10, 25) %> +(see https://serpapi.com/google_product)[https://serpapi.com/google_product] -### Search google reverse image -<%= snippet('python', 'tests/example_search_google_reverse_image.py', 9, 25) %> -[see https://serpapi.com/google_reverse_image](https://serpapi.com/google_reverse_image) +### Search Google Reverse Image +<%= snippet('ruby', 'tests/example_search_google_reverse_image.py', 10, 25) %> +(see https://serpapi.com/google_reverse_image)[https://serpapi.com/google_reverse_image] -### Search google events -<%= snippet('python', 'tests/example_search_google_events.py', 9, 25) %> -[see https://serpapi.com/google_events](https://serpapi.com/google_events) +### Search Google Events +<%= snippet('ruby', 'tests/example_search_google_events.py', 10, 25) %> +(see https://serpapi.com/google_events)[https://serpapi.com/google_events] -### Search google local services -<%= snippet('python', 'tests/example_search_google_local_services.py', 9, 25) %> -[see https://serpapi.com/google_local_services](https://serpapi.com/google_local_services) +### Search Google Local Services +<%= snippet('ruby', 'tests/example_search_google_local_services.py', 10, 25) %> +(see https://serpapi.com/google_local_services)[https://serpapi.com/google_local_services] -### Search google maps -<%= snippet('python', 'tests/example_search_google_maps.py', 9, 25) %> -[see https://serpapi.com/google_maps](https://serpapi.com/google_maps) +### Search Google Maps +<%= snippet('ruby', 'tests/example_search_google_maps.py', 10, 25) %> +(see https://serpapi.com/google_maps)[https://serpapi.com/google_maps] -### Search google jobs -<%= snippet('python', 'tests/example_search_google_jobs.py', 9, 25) %> -[see https://serpapi.com/google_jobs](https://serpapi.com/google_jobs) +### Search Google Jobs +<%= snippet('ruby', 'tests/example_search_google_jobs.py', 10, 25) %> +(see https://serpapi.com/google_jobs)[https://serpapi.com/google_jobs] -### Search google play -<%= snippet('python', 'tests/example_search_google_play.py', 9, 25) %> -[see https://serpapi.com/google_play](https://serpapi.com/google_play) +### Search Google Play +<%= snippet('ruby', 'tests/example_search_google_play.py', 10, 25) %> +(see https://serpapi.com/google_play)[https://serpapi.com/google_play] -### Search google images -<%= snippet('python', 'tests/example_search_google_images.py', 9, 25) %> -[see https://serpapi.com/google_images](https://serpapi.com/google_images) +### Search Google Images +<%= snippet('ruby', 'tests/example_search_google_images.py', 10, 25) %> +(see https://serpapi.com/google_images)[https://serpapi.com/google_images] ## Developer's note ### Key goals + - High code quality + - KISS principles - Brand centric instead of search engine based - No hard coded logic per search engine - Simple HTTP client (lightweight, reduced dependency) - No magic default values - Thread safe - Easy to extends - - Defensive code style (raise cutsom exception) + - Defensive code style (raise custom exception) - TDD - Best API coding pratice per platform @@ -174,4 +182,4 @@ The API design was inpired by the most popular Python packages. ### Quality expectation - 0 lint issues using pylint `make lint` - - 99% code coverage running `make test` \ No newline at end of file + - 99% code coverage running `make test` diff --git a/serpapi/object_decoder.py b/serpapi/object_decoder.py index bf2bdf6..056b077 100644 --- a/serpapi/object_decoder.py +++ b/serpapi/object_decoder.py @@ -1,8 +1,26 @@ """custom object decoder to convert a JSON into a python object""" + + class ObjectDecoder: """ Allow to convert JSON like datastructure in Python object. """ + + def __init__(self, node): + self.node = node + + def create(self): + """Create Python Object from a Dict or an Array + Returns + ---- + pyobj: python Object + """ + pytype = type('response', (object, ), {}) + pyobj = pytype() + for child_key, child_node in self.node.items(): + self.add_node(child_key, pyobj, child_node) + return pyobj + def child2object(self, name, node): """Make python object from dict Parameters @@ -42,10 +60,3 @@ def add_node(self, name, pyobj, child): setattr(pyobj, name, self.child2object(name, child)) else: setattr(pyobj, name, child) - - def dict2object(self, node): - pytype = type('response', (object, ), {}) - pyobj = pytype() - for child_key, child_node in node.items(): - self.add_node(child_key, pyobj, child_node) - return pyobj \ No newline at end of file diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py index cdad6b5..eb1d9b3 100644 --- a/serpapi/serpapi.py +++ b/serpapi/serpapi.py @@ -7,28 +7,35 @@ class HttpClient: """Simple HTTP client wrapper around urllib3""" - def __init__(self, parameter: dict = {}): + BACKEND = 'https://serpapi.com' + SUPPORTED_DECODER = ['json', 'html', 'object'] + + def __init__(self, parameter: dict = None): # initialize the http client self.http = urllib3.PoolManager() + # initialize parameter + if parameter is None: + parameter = {} + self.parameter = parameter + # urllib3 configurations - # HTTP connect timeout + # HTTP connect timeout if 'timeout' in parameter: self.timeout = parameter['timeout'] else: # 60s default self.timeout = 60.0 - + # no HTTP retry if 'retries' in parameter: self.retries = parameter['retries'] else: self.retries = False - def start(self, path : str, parameter: dict = None, decoder : str = 'json'): - """ - start HTTP request and decode response using urllib3 - the response is decoded using the selected decoder: + def start(self, path: str, parameter: dict = None, decoder: str = 'json'): + """start HTTP request and decode response using urllib3. + The response is decoded using the selected decoder: - html: raw HTML response - json: deep dict contains search results - object: containing search results as a dynamic object @@ -47,8 +54,7 @@ def start(self, path : str, parameter: dict = None, decoder : str = 'json'): Returns: --- dict|str|object - decoded HTTP response - """ + decoded HTTP response""" # set client language self.parameter['source'] = 'python' @@ -94,12 +100,13 @@ def decode(self, response: any, decoder: str): if decoder == 'object': data = json.loads(payload) - return self.dict2object(data) + return ObjectDecoder(data).create() raise SerpApiException("Invalid decoder: " + decoder + ", available: json, html, object") -class Client(ObjectDecoder, HttpClient): + +class Client(HttpClient): """ Client performend http query to serpApi.com using urllib3 under the hood. @@ -110,17 +117,12 @@ class Client(ObjectDecoder, HttpClient): """ - BACKEND = 'https://serpapi.com' - SUPPORTED_DECODER = ['json', 'html', 'object'] - def __init__(self, parameter: dict = None): # define default parameter if parameter is None: - self.parameter = {} - else: - # assign user parameter - self.parameter = parameter - HttpClient.__init__(self, self.parameter) + parameter = {} + # initialize HTTP client + HttpClient.__init__(self, parameter) def search(self, parameter: dict = None, decoder: str = 'json'): """ @@ -177,11 +179,14 @@ def location(self, parameter: dict = None): """ return self.start('/locations.json', parameter, 'json') - def search_archive(self, search_id : str, decoder : str ='json'): + def search_archive(self, search_id: str, decoder: str = 'json'): """ Retrieve search results from the Search Archive API Parameters: + ----- + search_id: str + id from a previous search. in the JSON search response it is search_metadata.id """ path = "/searches/" + str(search_id) + "." @@ -201,7 +206,7 @@ def account(self, api_key: str = None): Parameters --- api_key: str - secret user key provided by serpapi.com + secret user key provided by serpapi.com Returns --- diff --git a/tests/example_search_apple_app_store.py b/tests/example_search_apple_app_store.py index 5798091..708350c 100644 --- a/tests/example_search_apple_app_store.py +++ b/tests/example_search_apple_app_store.py @@ -1,4 +1,4 @@ -# test apple app store +# test Apple App Store search engine import unittest import os import serpapi diff --git a/tests/example_search_baidu.py b/tests/example_search_baidu.py index 9949cf4..9eca07f 100644 --- a/tests/example_search_baidu.py +++ b/tests/example_search_baidu.py @@ -1,4 +1,4 @@ -# test baidu +# test Baidu search engine import unittest import os import serpapi diff --git a/tests/example_search_bing.py b/tests/example_search_bing.py index f0b6cb6..340eaee 100644 --- a/tests/example_search_bing.py +++ b/tests/example_search_bing.py @@ -1,4 +1,4 @@ -# test bing +# test Bing search engine import unittest import os import serpapi diff --git a/tests/example_search_duckduckgo.py b/tests/example_search_duckduckgo.py index 189c8f6..3c0b81c 100644 --- a/tests/example_search_duckduckgo.py +++ b/tests/example_search_duckduckgo.py @@ -1,4 +1,4 @@ -# test duckduckgo +# test Duckduckgo search engine import unittest import os import serpapi diff --git a/tests/example_search_ebay.py b/tests/example_search_ebay.py index addcdc7..3a6cb26 100644 --- a/tests/example_search_ebay.py +++ b/tests/example_search_ebay.py @@ -1,4 +1,4 @@ -# test ebay +# test Ebay search engine import unittest import os import serpapi diff --git a/tests/example_search_google_autocomplete.py b/tests/example_search_google_autocomplete.py index 3411b6a..3c0c24b 100644 --- a/tests/example_search_google_autocomplete.py +++ b/tests/example_search_google_autocomplete.py @@ -1,4 +1,4 @@ -# test google autocomplete +# test Google Autocomplete search engine import unittest import os import serpapi diff --git a/tests/example_search_google_events.py b/tests/example_search_google_events.py index 8fa2863..156a372 100644 --- a/tests/example_search_google_events.py +++ b/tests/example_search_google_events.py @@ -1,4 +1,4 @@ -# test google events +# test Google Events search engine import unittest import os import serpapi diff --git a/tests/example_search_google_images.py b/tests/example_search_google_images.py index f31fc9b..1512edd 100644 --- a/tests/example_search_google_images.py +++ b/tests/example_search_google_images.py @@ -1,4 +1,4 @@ -# test google images +# test Google Images search engine import unittest import os import serpapi diff --git a/tests/example_search_google_jobs.py b/tests/example_search_google_jobs.py index a217cf1..9249de3 100644 --- a/tests/example_search_google_jobs.py +++ b/tests/example_search_google_jobs.py @@ -1,4 +1,4 @@ -# test google jobs +# test Google Jobs search engine import unittest import os import serpapi diff --git a/tests/example_search_google_local_services.py b/tests/example_search_google_local_services.py index 48adc8f..121b4ed 100644 --- a/tests/example_search_google_local_services.py +++ b/tests/example_search_google_local_services.py @@ -1,4 +1,4 @@ -# test google local services +# test Google Local Services search engine import unittest import os import serpapi diff --git a/tests/example_search_google_maps.py b/tests/example_search_google_maps.py index e096aa2..32a3350 100644 --- a/tests/example_search_google_maps.py +++ b/tests/example_search_google_maps.py @@ -1,4 +1,4 @@ -# test google maps +# test Google Maps search engine import unittest import os import serpapi diff --git a/tests/example_search_google_play.py b/tests/example_search_google_play.py index 235bdf5..a8f32eb 100644 --- a/tests/example_search_google_play.py +++ b/tests/example_search_google_play.py @@ -1,10 +1,11 @@ -# test google play +# test Google Play search engine import unittest import os import serpapi class TestGooglePlay(unittest.TestCase): + @unittest.skip @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") def test_search_google_play(self): client = serpapi.Client({ @@ -12,13 +13,10 @@ def test_search_google_play(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - "engine": "google_play", - "q": "maps", - "hl": "en", - "gl": "us", - "store": "apps" + 'q': 'coffee', + 'store': 'apps', }) - # self.assertIsNone(data.get('error')) - # self.assertIsNotNone(data['organic_results']) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com \ No newline at end of file diff --git a/tests/example_search_google_product.py b/tests/example_search_google_product.py index f5652fc..89fbb72 100644 --- a/tests/example_search_google_product.py +++ b/tests/example_search_google_product.py @@ -1,4 +1,4 @@ -# test google product +# test Google Product search engine import unittest import os import serpapi diff --git a/tests/example_search_google_reverse_image.py b/tests/example_search_google_reverse_image.py index 9da6e0f..b6cc8ee 100644 --- a/tests/example_search_google_reverse_image.py +++ b/tests/example_search_google_reverse_image.py @@ -1,4 +1,4 @@ -# test google reverse image +# test Google Reverse Image search engine import unittest import os import serpapi diff --git a/tests/example_search_google_scholar.py b/tests/example_search_google_scholar.py index ec221d9..0868e50 100644 --- a/tests/example_search_google_scholar.py +++ b/tests/example_search_google_scholar.py @@ -1,4 +1,4 @@ -# test google scholar +# test Google Scholar search engine import unittest import os import serpapi diff --git a/tests/example_search_google_search.py b/tests/example_search_google_search.py new file mode 100644 index 0000000..b3a2b1a --- /dev/null +++ b/tests/example_search_google_search.py @@ -0,0 +1,21 @@ +# test Google Search search engine +import unittest +import os +import serpapi + +class TestGoogleSearch(unittest.TestCase): + + @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") + def test_search_google_search(self): + client = serpapi.Client({ + 'engine': 'google_search', + 'api_key': os.getenv("API_KEY") + }) + data = client.search({ + 'q': 'coffee', + 'engine': 'google', + }) + self.assertIsNone(data.get('error')) + self.assertIsNotNone(data['organic_results']) + # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com + \ No newline at end of file diff --git a/tests/example_search_home_depot.py b/tests/example_search_home_depot.py index 4673218..30bf890 100644 --- a/tests/example_search_home_depot.py +++ b/tests/example_search_home_depot.py @@ -1,4 +1,4 @@ -# test home depot +# test Home Depot search engine import unittest import os import serpapi diff --git a/tests/example_search_naver.py b/tests/example_search_naver.py index bbb2990..62c2ee8 100644 --- a/tests/example_search_naver.py +++ b/tests/example_search_naver.py @@ -1,4 +1,4 @@ -# test naver +# test Naver search engine import unittest import os import serpapi diff --git a/tests/example_search_walmart.py b/tests/example_search_walmart.py index 804c91c..1989ab7 100644 --- a/tests/example_search_walmart.py +++ b/tests/example_search_walmart.py @@ -1,4 +1,4 @@ -# test walmart +# test Walmart search engine import unittest import os import serpapi diff --git a/tests/example_search_yahoo.py b/tests/example_search_yahoo.py index 4033b34..e39e0c4 100644 --- a/tests/example_search_yahoo.py +++ b/tests/example_search_yahoo.py @@ -1,4 +1,4 @@ -# test yahoo +# test Yahoo search engine import unittest import os import serpapi diff --git a/tests/example_search_youtube.py b/tests/example_search_youtube.py index b5877b5..a03ce73 100644 --- a/tests/example_search_youtube.py +++ b/tests/example_search_youtube.py @@ -1,4 +1,4 @@ -# test youtube +# test Youtube search engine import unittest import os import serpapi diff --git a/tests/test_object_decoder.py b/tests/test_object_decoder.py index d55ebd2..f9d2821 100644 --- a/tests/test_object_decoder.py +++ b/tests/test_object_decoder.py @@ -7,7 +7,6 @@ class TestObjectDecoder(unittest.TestCase): def test_decode_basic(self): - decoder = ObjectDecoder() data = { "organic_results": [ { @@ -18,19 +17,18 @@ def test_decode_basic(self): } ] } - obj = decoder.dict2object(data) + obj = ObjectDecoder(data).create() self.assertEqual(len(obj.organic_results), 2) self.assertEqual(obj.organic_results[0].name, 'ok') self.assertEqual(obj.organic_results[1].name, 'good') def test_decode_list(self): - decoder = ObjectDecoder() data = { "organic_results": { 'list': [0,1,2,3,4] } } - obj = decoder.dict2object(data) + obj = ObjectDecoder(data).create() print(obj.organic_results.list[0]) self.assertEqual(len(obj.organic_results.list), 5) self.assertEqual(obj.organic_results.list, [0,1,2,3,4]) From aa1803291e4235ef5ea90dba9409e5462122ed3b Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Mon, 20 Jun 2022 00:47:15 -0500 Subject: [PATCH 12/25] improve documentation and tests --- README.md | 309 ++++++++++++++++---------- README.md.erb | 146 ++++++++---- tests/example_search_google_images.py | 2 +- tests/example_search_google_play.py | 2 +- tests/example_search_google_search.py | 2 +- 5 files changed, 292 insertions(+), 169 deletions(-) diff --git a/README.md b/README.md index 6e02b96..add4128 100644 --- a/README.md +++ b/README.md @@ -1,12 +1,14 @@ # User guide -Scrape Google and other search engines from our fast, easy, and complete API using SerpApi.com -This python library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi](https://serpapi.com). +[SerpApi]](https://serpapi.com) allows to scrape any search engine results. + It's easy, fast, easy, feature rich, cost effective, scalable and reliable. + +This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi]](https://serpapi.com). This is an open source project hosted under https://github.com/serpapi/serpapi-python. -SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly. +SerpApi.com provides a [script builder]](https://serpapi.com/demo) to get you started quickly. ## Installation -serpapi can be installed with pip. +SerpApi can be installed with pip. ```sh $ python -m pip install serpapi @@ -63,15 +65,16 @@ parameter = { } ``` -for more details: [URL LIB3 documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html) +for more details: [URL LIB3 documentation]](https://urllib3.readthedocs.io/en/stable/user-guide.html) ## Basic example per search engines ### Search Bing -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'bing', 'api_key': os.getenv("API_KEY") }) @@ -83,14 +86,15 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_bing.py -(see https://serpapi.com/bing)[https://serpapi.com/bing] +see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api) ### Search Baidu -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'baidu', 'api_key': os.getenv("API_KEY") }) @@ -102,14 +106,15 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_baidu.py -(see https://serpapi.com/baidu)[https://serpapi.com/baidu] +see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api) ### Search Yahoo -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'yahoo', 'api_key': os.getenv("API_KEY") }) @@ -121,14 +126,15 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_yahoo.py -(see https://serpapi.com/yahoo)[https://serpapi.com/yahoo] +see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api) ### Search Youtube -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'youtube', 'api_key': os.getenv("API_KEY") }) @@ -140,14 +146,15 @@ pp.pprint(data['video_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_youtube.py -(see https://serpapi.com/youtube)[https://serpapi.com/youtube] +see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api) ### Search Walmart -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'walmart', 'api_key': os.getenv("API_KEY") }) @@ -159,14 +166,15 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_walmart.py -(see https://serpapi.com/walmart)[https://serpapi.com/walmart] +see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api) ### Search Ebay -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'ebay', 'api_key': os.getenv("API_KEY") }) @@ -178,14 +186,15 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_ebay.py -(see https://serpapi.com/ebay)[https://serpapi.com/ebay] +see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api) ### Search Naver -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'naver', 'api_key': os.getenv("API_KEY") }) @@ -197,14 +206,15 @@ pp.pprint(data['ads_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_naver.py -(see https://serpapi.com/naver)[https://serpapi.com/naver] +see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api) ### Search Home Depot -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'home_depot', 'api_key': os.getenv("API_KEY") }) @@ -216,14 +226,15 @@ pp.pprint(data['products']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_home_depot.py -(see https://serpapi.com/home_depot)[https://serpapi.com/home_depot] +see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api) ### Search Apple App Store -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'apple_app_store', 'api_key': os.getenv("API_KEY") }) @@ -235,14 +246,15 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_apple_app_store.py -(see https://serpapi.com/apple_app_store)[https://serpapi.com/apple_app_store] +see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store) ### Search Duckduckgo -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'duckduckgo', 'api_key': os.getenv("API_KEY") }) @@ -254,33 +266,36 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_duckduckgo.py -(see https://serpapi.com/duckduckgo)[https://serpapi.com/duckduckgo] +see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api) ### Search Google Search -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os -'engine': 'google_search', +client = serpapi.Client({ +'engine': 'google', 'api_key': os.getenv("API_KEY") }) data = client.search({ 'q': 'coffee', +'engine': 'google', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_search.py -(see https://serpapi.com/google_search)[https://serpapi.com/google_search] +see: [https://serpapi.com/search-api](https://serpapi.com/search-api) ### Search Google Scholar -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'google_scholar', 'api_key': os.getenv("API_KEY") }) @@ -292,14 +307,15 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_scholar.py -(see https://serpapi.com/google_scholar)[https://serpapi.com/google_scholar] +see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api) ### Search Google Autocomplete -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'google_autocomplete', 'api_key': os.getenv("API_KEY") }) @@ -311,14 +327,15 @@ pp.pprint(data['suggestions']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_autocomplete.py -(see https://serpapi.com/google_autocomplete)[https://serpapi.com/google_autocomplete] +see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) ### Search Google Product -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'google_product', 'api_key': os.getenv("API_KEY") }) @@ -331,14 +348,15 @@ pp.pprint(data['product_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_product.py -(see https://serpapi.com/google_product)[https://serpapi.com/google_product] +see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api) ### Search Google Reverse Image -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'google_reverse_image', 'api_key': os.getenv("API_KEY") }) @@ -350,14 +368,15 @@ pp.pprint(data['image_sizes']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_reverse_image.py -(see https://serpapi.com/google_reverse_image)[https://serpapi.com/google_reverse_image] +see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image) ### Search Google Events -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'google_events', 'api_key': os.getenv("API_KEY") }) @@ -369,14 +388,15 @@ pp.pprint(data['events_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_events.py -(see https://serpapi.com/google_events)[https://serpapi.com/google_events] +see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api) ### Search Google Local Services -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'google_local_services', 'api_key': os.getenv("API_KEY") }) @@ -389,14 +409,15 @@ pp.pprint(data['local_ads']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_local_services.py -(see https://serpapi.com/google_local_services)[https://serpapi.com/google_local_services] +see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api) ### Search Google Maps -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'google_maps', 'api_key': os.getenv("API_KEY") }) @@ -410,14 +431,15 @@ pp.pprint(data['local_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_maps.py -(see https://serpapi.com/google_maps)[https://serpapi.com/google_maps] +see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api) ### Search Google Jobs -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +client = serpapi.Client({ 'engine': 'google_jobs', 'api_key': os.getenv("API_KEY") }) @@ -429,19 +451,21 @@ pp.pprint(data['jobs_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_jobs.py -(see https://serpapi.com/google_jobs)[https://serpapi.com/google_jobs] +see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api) ### Search Google Play -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os +def test_search_google_play(self): +client = serpapi.Client({ 'engine': 'google_play', 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', +'q': 'kite', 'store': 'apps', }) pp = pprint.PrettyPrinter(indent=2) @@ -449,15 +473,16 @@ pp.pprint(data['organic_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_play.py -(see https://serpapi.com/google_play)[https://serpapi.com/google_play] +see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api) ### Search Google Images -```ruby -import 'serpapi' -import 'pprint' -import 'os' +```python +import serpapi +import pprint +import os -'engine': 'google_images', +client = serpapi.Client({ +'engine': 'google', 'api_key': os.getenv("API_KEY") }) data = client.search({ @@ -470,25 +495,75 @@ pp.pprint(data['images_results']) # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com ``` test: tests/example_search_google_images.py -(see https://serpapi.com/google_images)[https://serpapi.com/google_images] +see: [https://serpapi.com/images-results](https://serpapi.com/images-results) -## Developer's note +# Developer Guide ### Key goals + - High code quality + - KISS principles - Brand centric instead of search engine based - No hard coded logic per search engine - Simple HTTP client (lightweight, reduced dependency) - No magic default values - Thread safe - Easy to extends - - Defensive code style (raise cutsom exception) + - Defensive code style (raise custom exception) - TDD - Best API coding pratice per platform -### Design +### Inspiration The API design was inpired by the most popular Python packages. - urllib3 - https://github.com/urllib3/urllib3 - Boto3 - https://github.com/boto/boto3 ### Quality expectation - 0 lint issues using pylint `make lint` - - 99% code coverage running `make test` \ No newline at end of file + - 99% code coverage running `make test` + - 100% test passing: `make test` + +### Client design: Class diagram +```mermaid +classDiagram + CustomClient *-- Client + HttpClient <-- Client + HttpClient *-- urllib3 + HttpClient *-- ObjectDecoder + + class Client { + engine: String + api_key: String + parameter: Hash + search() + html() + location() + search_archive() + account() + } + + class HttpClient { + start() + decode() + } + + class urllib3 { + request() + } +``` + +## JSON search() : Sequence diagram +```mermaid +sequenceDiagram + Client->>SerpApi.com: search() : http request + SerpApi.com-->>SerpApi.com: query search engine + SerpApi.com-->>SerpApi.com: parse HTML into JSON + SerpApi.com-->>Client: JSON payload + Client-->>Client: decode JSON into Dict +``` + +## Build flow + +This project is automated using a good old Makefile. +To pipe-clean the project run this: +`make` + +Open Makefile for more details. diff --git a/README.md.erb b/README.md.erb index 3d091f2..ccceab9 100644 --- a/README.md.erb +++ b/README.md.erb @@ -4,17 +4,17 @@ def snippet(format, path, start, stop) slice.reject! { |l| l =~ /self.assertIsNone\(/ } buf = slice.map { |l| l.gsub(/(^\s+)/, '')}.join buf.gsub!('self.assertIsNotNone(', "pp = pprint.PrettyPrinter(indent=2)\npp.pprint(") - %Q(```#{format}\nimport 'serpapi'\nimport 'pprint'\nimport 'os'\n\n#{buf}```\n test: #{path}) + %Q(```#{format}\nimport serpapi\nimport pprint\nimport os\n\n#{buf}```\n test: #{path}) end -%> # User guide -[SerpApi](https://serpapi.com) allows to scrape any search engine results. +[SerpApi]](https://serpapi.com) allows to scrape any search engine results. It's easy, fast, easy, feature rich, cost effective, scalable and reliable. -This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi](https://serpapi.com). +This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi]](https://serpapi.com). This is an open source project hosted under https://github.com/serpapi/serpapi-python. -SerpApi.com provides a [script builder](https://serpapi.com/demo) to get you started quickly. +SerpApi.com provides a [script builder]](https://serpapi.com/demo) to get you started quickly. ## Installation SerpApi can be installed with pip. @@ -74,94 +74,94 @@ parameter = { } ``` -for more details: [URL LIB3 documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html) +for more details: [URL LIB3 documentation]](https://urllib3.readthedocs.io/en/stable/user-guide.html) ## Basic example per search engines ### Search Bing -<%= snippet('ruby', 'tests/example_search_bing.py', 10, 25) %> -(see https://serpapi.com/bing)[https://serpapi.com/bing] +<%= snippet('python', 'tests/example_search_bing.py', 9, 25) %> +see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api) ### Search Baidu -<%= snippet('ruby', 'tests/example_search_baidu.py', 10, 25) %> -(see https://serpapi.com/baidu)[https://serpapi.com/baidu] +<%= snippet('python', 'tests/example_search_baidu.py', 9, 25) %> +see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api) ### Search Yahoo -<%= snippet('ruby', 'tests/example_search_yahoo.py', 10, 25) %> -(see https://serpapi.com/yahoo)[https://serpapi.com/yahoo] +<%= snippet('python', 'tests/example_search_yahoo.py', 9, 25) %> +see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api) ### Search Youtube -<%= snippet('ruby', 'tests/example_search_youtube.py', 10, 25) %> -(see https://serpapi.com/youtube)[https://serpapi.com/youtube] +<%= snippet('python', 'tests/example_search_youtube.py', 9, 25) %> +see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api) ### Search Walmart -<%= snippet('ruby', 'tests/example_search_walmart.py', 10, 25) %> -(see https://serpapi.com/walmart)[https://serpapi.com/walmart] +<%= snippet('python', 'tests/example_search_walmart.py', 9, 25) %> +see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api) ### Search Ebay -<%= snippet('ruby', 'tests/example_search_ebay.py', 10, 25) %> -(see https://serpapi.com/ebay)[https://serpapi.com/ebay] +<%= snippet('python', 'tests/example_search_ebay.py', 9, 25) %> +see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api) ### Search Naver -<%= snippet('ruby', 'tests/example_search_naver.py', 10, 25) %> -(see https://serpapi.com/naver)[https://serpapi.com/naver] +<%= snippet('python', 'tests/example_search_naver.py', 9, 25) %> +see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api) ### Search Home Depot -<%= snippet('ruby', 'tests/example_search_home_depot.py', 10, 25) %> -(see https://serpapi.com/home_depot)[https://serpapi.com/home_depot] +<%= snippet('python', 'tests/example_search_home_depot.py', 9, 25) %> +see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api) ### Search Apple App Store -<%= snippet('ruby', 'tests/example_search_apple_app_store.py', 10, 25) %> -(see https://serpapi.com/apple_app_store)[https://serpapi.com/apple_app_store] +<%= snippet('python', 'tests/example_search_apple_app_store.py', 9, 25) %> +see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store) ### Search Duckduckgo -<%= snippet('ruby', 'tests/example_search_duckduckgo.py', 10, 25) %> -(see https://serpapi.com/duckduckgo)[https://serpapi.com/duckduckgo] +<%= snippet('python', 'tests/example_search_duckduckgo.py', 9, 25) %> +see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api) ### Search Google Search -<%= snippet('ruby', 'tests/example_search_google_search.py', 10, 25) %> -(see https://serpapi.com/google_search)[https://serpapi.com/google_search] +<%= snippet('python', 'tests/example_search_google_search.py', 9, 25) %> +see: [https://serpapi.com/search-api](https://serpapi.com/search-api) ### Search Google Scholar -<%= snippet('ruby', 'tests/example_search_google_scholar.py', 10, 25) %> -(see https://serpapi.com/google_scholar)[https://serpapi.com/google_scholar] +<%= snippet('python', 'tests/example_search_google_scholar.py', 9, 25) %> +see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api) ### Search Google Autocomplete -<%= snippet('ruby', 'tests/example_search_google_autocomplete.py', 10, 25) %> -(see https://serpapi.com/google_autocomplete)[https://serpapi.com/google_autocomplete] +<%= snippet('python', 'tests/example_search_google_autocomplete.py', 9, 25) %> +see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) ### Search Google Product -<%= snippet('ruby', 'tests/example_search_google_product.py', 10, 25) %> -(see https://serpapi.com/google_product)[https://serpapi.com/google_product] +<%= snippet('python', 'tests/example_search_google_product.py', 9, 25) %> +see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api) ### Search Google Reverse Image -<%= snippet('ruby', 'tests/example_search_google_reverse_image.py', 10, 25) %> -(see https://serpapi.com/google_reverse_image)[https://serpapi.com/google_reverse_image] +<%= snippet('python', 'tests/example_search_google_reverse_image.py', 9, 25) %> +see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image) ### Search Google Events -<%= snippet('ruby', 'tests/example_search_google_events.py', 10, 25) %> -(see https://serpapi.com/google_events)[https://serpapi.com/google_events] +<%= snippet('python', 'tests/example_search_google_events.py', 9, 25) %> +see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api) ### Search Google Local Services -<%= snippet('ruby', 'tests/example_search_google_local_services.py', 10, 25) %> -(see https://serpapi.com/google_local_services)[https://serpapi.com/google_local_services] +<%= snippet('python', 'tests/example_search_google_local_services.py', 9, 25) %> +see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api) ### Search Google Maps -<%= snippet('ruby', 'tests/example_search_google_maps.py', 10, 25) %> -(see https://serpapi.com/google_maps)[https://serpapi.com/google_maps] +<%= snippet('python', 'tests/example_search_google_maps.py', 9, 25) %> +see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api) ### Search Google Jobs -<%= snippet('ruby', 'tests/example_search_google_jobs.py', 10, 25) %> -(see https://serpapi.com/google_jobs)[https://serpapi.com/google_jobs] +<%= snippet('python', 'tests/example_search_google_jobs.py', 9, 25) %> +see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api) ### Search Google Play -<%= snippet('ruby', 'tests/example_search_google_play.py', 10, 25) %> -(see https://serpapi.com/google_play)[https://serpapi.com/google_play] +<%= snippet('python', 'tests/example_search_google_play.py', 9, 25) %> +see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api) ### Search Google Images -<%= snippet('ruby', 'tests/example_search_google_images.py', 10, 25) %> -(see https://serpapi.com/google_images)[https://serpapi.com/google_images] +<%= snippet('python', 'tests/example_search_google_images.py', 9, 25) %> +see: [https://serpapi.com/images-results](https://serpapi.com/images-results) -## Developer's note +# Developer Guide ### Key goals - High code quality - KISS principles @@ -175,7 +175,7 @@ for more details: [URL LIB3 documentation](https://urllib3.readthedocs.io/en/sta - TDD - Best API coding pratice per platform -### Design +### Inspiration The API design was inpired by the most popular Python packages. - urllib3 - https://github.com/urllib3/urllib3 - Boto3 - https://github.com/boto/boto3 @@ -183,3 +183,51 @@ The API design was inpired by the most popular Python packages. ### Quality expectation - 0 lint issues using pylint `make lint` - 99% code coverage running `make test` + - 100% test passing: `make test` + +### Client design: Class diagram +```mermaid +classDiagram + CustomClient *-- Client + HttpClient <-- Client + HttpClient *-- urllib3 + HttpClient *-- ObjectDecoder + + class Client { + engine: String + api_key: String + parameter: Hash + search() + html() + location() + search_archive() + account() + } + + class HttpClient { + start() + decode() + } + + class urllib3 { + request() + } +``` + +## JSON search() : Sequence diagram +```mermaid +sequenceDiagram + Client->>SerpApi.com: search() : http request + SerpApi.com-->>SerpApi.com: query search engine + SerpApi.com-->>SerpApi.com: parse HTML into JSON + SerpApi.com-->>Client: JSON payload + Client-->>Client: decode JSON into Dict +``` + +## Build flow + +This project is automated using a good old Makefile. +To pipe-clean the project run this: +`make` + +Open Makefile for more details. diff --git a/tests/example_search_google_images.py b/tests/example_search_google_images.py index 1512edd..81c1f2f 100644 --- a/tests/example_search_google_images.py +++ b/tests/example_search_google_images.py @@ -8,7 +8,7 @@ class TestGoogleImages(unittest.TestCase): @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") def test_search_google_images(self): client = serpapi.Client({ - 'engine': 'google_images', + 'engine': 'google', 'api_key': os.getenv("API_KEY") }) data = client.search({ diff --git a/tests/example_search_google_play.py b/tests/example_search_google_play.py index a8f32eb..2d393ad 100644 --- a/tests/example_search_google_play.py +++ b/tests/example_search_google_play.py @@ -13,7 +13,7 @@ def test_search_google_play(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', + 'q': 'kite', 'store': 'apps', }) self.assertIsNone(data.get('error')) diff --git a/tests/example_search_google_search.py b/tests/example_search_google_search.py index b3a2b1a..9e3b48f 100644 --- a/tests/example_search_google_search.py +++ b/tests/example_search_google_search.py @@ -8,7 +8,7 @@ class TestGoogleSearch(unittest.TestCase): @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") def test_search_google_search(self): client = serpapi.Client({ - 'engine': 'google_search', + 'engine': 'google', 'api_key': os.getenv("API_KEY") }) data = client.search({ From ee5514f8f781552052990134ce284e839ee3b681 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Mon, 20 Jun 2022 00:51:32 -0500 Subject: [PATCH 13/25] drop python 3.5 from gitaction --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bb3bd22..43d70a4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.5, 3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9] steps: - uses: actions/checkout@v2 From 6513098bb130a3422a0ea96d447acf274d614aa7 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Mon, 20 Jun 2022 00:51:46 -0500 Subject: [PATCH 14/25] ignore local script directory --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 3c9d107..8baaaec 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +script/ .coverage docs/build dist/ From 3d0e114bd791613b3efdb3bfbdf777b54f72265d Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Mon, 20 Jun 2022 00:54:47 -0500 Subject: [PATCH 15/25] add CI tests badge --- README.md | 2 ++ README.md.erb | 2 ++ 2 files changed, 4 insertions(+) diff --git a/README.md b/README.md index add4128..bcba299 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,6 @@ # User guide +[![SerpApi-Python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) + [SerpApi]](https://serpapi.com) allows to scrape any search engine results. It's easy, fast, easy, feature rich, cost effective, scalable and reliable. diff --git a/README.md.erb b/README.md.erb index ccceab9..9e1fb9f 100644 --- a/README.md.erb +++ b/README.md.erb @@ -8,6 +8,8 @@ def snippet(format, path, start, stop) end -%> # User guide +[![SerpApi-Python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) + [SerpApi]](https://serpapi.com) allows to scrape any search engine results. It's easy, fast, easy, feature rich, cost effective, scalable and reliable. From 2723efd026eee1a0a69fbbd86b74e31766047508 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 7 May 2023 09:15:23 -0500 Subject: [PATCH 16/25] update README and more... --- Makefile | 2 +- README.md | 297 ++++++++++-------- README.md.erb | 160 +++++----- oobt/{oobt.py => demo.py} | 0 serpapi/error.py | 2 +- serpapi/object_decoder.py | 62 ---- serpapi/serpapi.py | 27 +- ...=> example_search_apple_app_store_test.py} | 9 +- ..._baidu.py => example_search_baidu_test.py} | 9 +- ...ch_bing.py => example_search_bing_test.py} | 9 +- ...o.py => example_search_duckduckgo_test.py} | 9 +- ...ch_ebay.py => example_search_ebay_test.py} | 9 +- ...xample_search_google_autocomplete_test.py} | 9 +- ...y => example_search_google_events_test.py} | 9 +- ...y => example_search_google_images_test.py} | 15 +- ....py => example_search_google_jobs_test.py} | 9 +- ...mple_search_google_local_services_test.py} | 11 +- ....py => example_search_google_maps_test.py} | 13 +- ....py => example_search_google_play_test.py} | 12 +- ... => example_search_google_product_test.py} | 11 +- ...ample_search_google_reverse_image_test.py} | 9 +- ... => example_search_google_scholar_test.py} | 9 +- ...earch.py => example_search_google_test.py} | 15 +- ...t.py => example_search_home_depot_test.py} | 9 +- ..._naver.py => example_search_naver_test.py} | 9 +- ...mart.py => example_search_walmart_test.py} | 9 +- ..._yahoo.py => example_search_yahoo_test.py} | 9 +- ...tube.py => example_search_youtube_test.py} | 9 +- tests/test_example.py | 34 +- tests/test_object_decoder.py | 34 -- 30 files changed, 392 insertions(+), 438 deletions(-) rename oobt/{oobt.py => demo.py} (100%) delete mode 100644 serpapi/object_decoder.py rename tests/{example_search_apple_app_store.py => example_search_apple_app_store_test.py} (65%) rename tests/{example_search_baidu.py => example_search_baidu_test.py} (65%) rename tests/{example_search_bing.py => example_search_bing_test.py} (65%) rename tests/{example_search_duckduckgo.py => example_search_duckduckgo_test.py} (65%) rename tests/{example_search_ebay.py => example_search_ebay_test.py} (65%) rename tests/{example_search_google_autocomplete.py => example_search_google_autocomplete_test.py} (65%) rename tests/{example_search_google_events.py => example_search_google_events_test.py} (65%) rename tests/{example_search_google_images.py => example_search_google_images_test.py} (55%) rename tests/{example_search_google_jobs.py => example_search_google_jobs_test.py} (65%) rename tests/{example_search_google_local_services.py => example_search_google_local_services_test.py} (60%) rename tests/{example_search_google_maps.py => example_search_google_maps_test.py} (59%) rename tests/{example_search_google_play.py => example_search_google_play_test.py} (63%) rename tests/{example_search_google_product.py => example_search_google_product_test.py} (61%) rename tests/{example_search_google_reverse_image.py => example_search_google_reverse_image_test.py} (62%) rename tests/{example_search_google_scholar.py => example_search_google_scholar_test.py} (65%) rename tests/{example_search_google_search.py => example_search_google_test.py} (52%) rename tests/{example_search_home_depot.py => example_search_home_depot_test.py} (65%) rename tests/{example_search_naver.py => example_search_naver_test.py} (64%) rename tests/{example_search_walmart.py => example_search_walmart_test.py} (65%) rename tests/{example_search_yahoo.py => example_search_yahoo_test.py} (65%) rename tests/{example_search_youtube.py => example_search_youtube_test.py} (64%) delete mode 100644 tests/test_object_decoder.py diff --git a/Makefile b/Makefile index 20b4823..e717742 100644 --- a/Makefile +++ b/Makefile @@ -54,7 +54,7 @@ build: doc lint test # out of box testing / user acceptance before delivery oobt: build pip3 install ./${dist} - python3 oobt/oobt.py + python3 oobt/demo.py check: oobt diff --git a/README.md b/README.md index bcba299..4ee99a2 100644 --- a/README.md +++ b/README.md @@ -1,58 +1,47 @@ -# User guide -[![SerpApi-Python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) -[SerpApi]](https://serpapi.com) allows to scrape any search engine results. - It's easy, fast, easy, feature rich, cost effective, scalable and reliable. +
+

SerpApi Python Library

+ serpapi python library logo -This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi]](https://serpapi.com). -This is an open source project hosted under https://github.com/serpapi/serpapi-python. + ![Package](https://badge.fury.io/py/serpapi.svg) + ![Downloads](https://static.pepy.tech/personalized-badge/serpapi?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=Downloads) + ![Buiild](https://github.com/serpapi/serpapi-python/actions/workflows/python-package.yml/badge.svg) +
-SerpApi.com provides a [script builder]](https://serpapi.com/demo) to get you started quickly. + +Integrate search data into your Ruby application. This library is the official wrapper for SerpApi (https://serpapi.com). + +SerpApi supports Google, Google Maps, Google Shopping, Baidu, Yandex, Yahoo, eBay, App Stores, and more. ## Installation -SerpApi can be installed with pip. +Python3 must be installed. ```sh -$ python -m pip install serpapi +$ pip install serpapi ``` -## Quick start -First things first, import the serpapi module: +## Simple usage ```python import serpapi +client = serpapi.Client({ + 'api_key': "secret_api_key", # set personal API key from serpapi.com/dashboard + 'engine': "google", # set default search engine +}) +results = client.search({ + q: "coffee", # google query + location: "Austin,TX" # force the location [optional] +}) +print(results['organic_results']) ``` -You'll need a client instance to make a search. This object handles all of the details of connection pooling and thread safety so that you don't have to: -```python -client = serpapi.Client() -``` -To make a search using SerpApi.com: +This example runs a search for "coffee" on Google. It then returns the results as a regular Ruby Hash. See the [playground](https://serpapi.com/playground) to generate your own code. -```python -parameter = { - api_key: "secret_api_key", # from serpapi.com - engine: "google", # search engine - q: "coffee", # search topic - location: "Austin,TX" # location -} -results = searpapi.search(parameter) -``` -Putting everything together. -```python -import serpapi +## Advanced Usage +### Search API -parameter = { - api_key: "secret_api_key", # from serpapi.com - engine: "google", # search engine - q: "coffee", # search topic - location: "Austin,TX" # location -} -results = searpapi.search(parameter) -print(results) -``` +TODO update this specification -### Advanced settings SerpApi Client uses urllib3 under the hood. Optionally, rhe HTTP connection can be tuned: - timeout : connection timeout by default 60s @@ -70,7 +59,8 @@ parameter = { for more details: [URL LIB3 documentation]](https://urllib3.readthedocs.io/en/stable/user-guide.html) ## Basic example per search engines -### Search Bing + +### Search bing ```python import serpapi import pprint @@ -81,16 +71,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', +'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_bing.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_bing_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_bing_test.py) see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api) -### Search Baidu +### Search baidu ```python import serpapi import pprint @@ -101,16 +93,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', +'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_baidu.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_baidu_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_baidu_test.py) see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api) -### Search Yahoo +### Search yahoo ```python import serpapi import pprint @@ -121,16 +115,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'p': 'coffee', +'p': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_yahoo.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_yahoo_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_yahoo_test.py) see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api) -### Search Youtube +### Search youtube ```python import serpapi import pprint @@ -141,16 +137,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'search_query': 'coffee', +'search_query': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['video_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_youtube.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_youtube_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_youtube_test.py) see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api) -### Search Walmart +### Search walmart ```python import serpapi import pprint @@ -161,16 +159,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'query': 'coffee', +'query': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_walmart.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_walmart_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_walmart_test.py) see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api) -### Search Ebay +### Search ebay ```python import serpapi import pprint @@ -181,16 +181,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'_nkw': 'coffee', +'_nkw': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_ebay.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_ebay_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_ebay_test.py) see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api) -### Search Naver +### Search naver ```python import serpapi import pprint @@ -201,16 +203,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'query': 'coffee', +'query': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['ads_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_naver.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_naver_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_naver_test.py) see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api) -### Search Home Depot +### Search home depot ```python import serpapi import pprint @@ -221,16 +225,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'table', +'q': 'table', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['products']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_home_depot.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_home_depot_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_home_depot_test.py) see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api) -### Search Apple App Store +### Search apple app store ```python import serpapi import pprint @@ -241,16 +247,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'term': 'coffee', +'term': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_apple_app_store.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_apple_app_store_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_apple_app_store_test.py) see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store) -### Search Duckduckgo +### Search duckduckgo ```python import serpapi import pprint @@ -261,16 +269,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', +'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_duckduckgo.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_duckduckgo_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_duckduckgo_test.py) see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api) -### Search Google Search +### Search google ```python import serpapi import pprint @@ -281,17 +291,19 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', -'engine': 'google', +'q': 'coffee', +'engine': 'google', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_search.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_test.py) see: [https://serpapi.com/search-api](https://serpapi.com/search-api) -### Search Google Scholar +### Search google scholar ```python import serpapi import pprint @@ -302,16 +314,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', +'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_scholar.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_scholar_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_scholar_test.py) see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api) -### Search Google Autocomplete +### Search google autocomplete ```python import serpapi import pprint @@ -322,16 +336,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', +'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['suggestions']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_autocomplete.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_autocomplete_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_autocomplete_test.py) see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) -### Search Google Product +### Search google product ```python import serpapi import pprint @@ -342,17 +358,19 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', -'product_id': '4172129135583325756', +'q': 'coffee', +'product_id': '4172129135583325756', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['product_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_product.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_product_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_product_test.py) see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api) -### Search Google Reverse Image +### Search google reverse image ```python import serpapi import pprint @@ -363,16 +381,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'image_url': 'https://i.imgur.com/5bGzZi7.jpg', +'image_url': 'https://i.imgur.com/5bGzZi7.jpg', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['image_sizes']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_reverse_image.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_reverse_image_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_reverse_image_test.py) see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image) -### Search Google Events +### Search google events ```python import serpapi import pprint @@ -383,16 +403,18 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', +'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['events_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_events.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_events_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_events_test.py) see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api) -### Search Google Local Services +### Search google local services ```python import serpapi import pprint @@ -403,17 +425,19 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'Electrician', -'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g', +'q': 'Electrician', +'data_cid': 'ChIJOwg_06VPwokRYv534QaPC8g', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['local_ads']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_local_services.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_local_services_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_local_services_test.py) see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api) -### Search Google Maps +### Search google maps ```python import serpapi import pprint @@ -424,18 +448,20 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'pizza', -'ll': '@40.7455096,-74.0083012,15.1z', -'type': 'search', +'q': 'pizza', +'ll': '@40.7455096,-74.0083012,15.1z', +'type': 'search', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['local_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_maps.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_maps_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_maps_test.py) see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api) -### Search Google Jobs +### Search google jobs ```python import serpapi import pprint @@ -446,60 +472,67 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'coffee', +'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['jobs_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_jobs.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_jobs_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_jobs_test.py) see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api) -### Search Google Play +### Search google play ```python import serpapi import pprint import os -def test_search_google_play(self): client = serpapi.Client({ 'engine': 'google_play', 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'kite', -'store': 'apps', +'q': 'kite', +'store': 'apps', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_play.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_play_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_play_test.py) see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api) -### Search Google Images +### Search google images ```python import serpapi import pprint import os client = serpapi.Client({ -'engine': 'google', +'engine': 'google_images', 'api_key': os.getenv("API_KEY") }) data = client.search({ -'engine': 'google', -'tbm': 'isch', -'q': 'coffee', +'engine': 'google_images', +'tbm': 'isch', +'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['images_results']) -# os.getenv("API_KEY") captures the secret user API available from http://serpapi.com +# os.getenv("API_KEY") is your secret API Key +# copy/paste from [http://serpapi.com/dashboard] to your bash +# ```export API_KEY="your_secure_api_key"``` ``` - test: tests/example_search_google_images.py + * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_images_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_images_test.py) see: [https://serpapi.com/images-results](https://serpapi.com/images-results) + # Developer Guide +TODO update this section ### Key goals - High code quality - KISS principles diff --git a/README.md.erb b/README.md.erb index 9e1fb9f..23457bf 100644 --- a/README.md.erb +++ b/README.md.erb @@ -1,67 +1,58 @@ <%- -def snippet(format, path, start, stop) - slice = File.new(path).readlines[start..stop] +def snippet(format, path) + lines = File.new(path).readlines + stop = lines.size - 1 + slice = lines[9..stop] slice.reject! { |l| l =~ /self.assertIsNone\(/ } buf = slice.map { |l| l.gsub(/(^\s+)/, '')}.join buf.gsub!('self.assertIsNotNone(', "pp = pprint.PrettyPrinter(indent=2)\npp.pprint(") - %Q(```#{format}\nimport serpapi\nimport pprint\nimport os\n\n#{buf}```\n test: #{path}) + %Q(```#{format}\nimport serpapi\nimport pprint\nimport os\n\n#{buf}```\n * test: [#{path}](#{path})) end -%> -# User guide -[![SerpApi-Python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) -[SerpApi]](https://serpapi.com) allows to scrape any search engine results. - It's easy, fast, easy, feature rich, cost effective, scalable and reliable. +
+

SerpApi Python Library

+ serpapi python library logo -This Python 3 library is meant to scrape and parse results from all major search engines available world wide including Google, Bing, Baidu, Yandex, Yahoo, Ebay, Home depot, Apple and more using [SerpApi]](https://serpapi.com). -This is an open source project hosted under https://github.com/serpapi/serpapi-python. + ![Package](https://badge.fury.io/py/serpapi.svg) + ![Downloads](https://static.pepy.tech/personalized-badge/serpapi?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=Downloads) + ![Buiild](https://github.com/serpapi/serpapi-python/actions/workflows/python-package.yml/badge.svg) +
-SerpApi.com provides a [script builder]](https://serpapi.com/demo) to get you started quickly. + +Integrate search data into your Ruby application. This library is the official wrapper for SerpApi (https://serpapi.com). + +SerpApi supports Google, Google Maps, Google Shopping, Baidu, Yandex, Yahoo, eBay, App Stores, and more. ## Installation -SerpApi can be installed with pip. +Python3 must be installed. ```sh -$ python -m pip install serpapi +$ pip install serpapi ``` -## Quick start -First things first, import the serpapi module: +## Simple usage ```python import serpapi +client = serpapi.Client({ + 'api_key': "secret_api_key", # set personal API key from serpapi.com/dashboard + 'engine': "google", # set default search engine +}) +results = client.search({ + q: "coffee", # google query + location: "Austin,TX" # force the location [optional] +}) +print(results['organic_results']) ``` -You'll need a client instance to make a search. This object handles all of the details of connection pooling and thread safety so that you don't have to: -```python -client = serpapi.Client() -``` -To make a search using SerpApi.com: +This example runs a search for "coffee" on Google. It then returns the results as a regular Ruby Hash. See the [playground](https://serpapi.com/playground) to generate your own code. -```python -parameter = { - api_key: "secret_api_key", # from serpapi.com - engine: "google", # search engine - q: "coffee", # search topic - location: "Austin,TX" # location -} -results = searpapi.search(parameter) -``` -Putting everything together. -```python -import serpapi +## Advanced Usage +### Search API -parameter = { - api_key: "secret_api_key", # from serpapi.com - engine: "google", # search engine - q: "coffee", # search topic - location: "Austin,TX" # location -} -results = searpapi.search(parameter) -print(results) -``` +TODO update this specification -### Advanced settings SerpApi Client uses urllib3 under the hood. Optionally, rhe HTTP connection can be tuned: - timeout : connection timeout by default 60s @@ -79,91 +70,94 @@ parameter = { for more details: [URL LIB3 documentation]](https://urllib3.readthedocs.io/en/stable/user-guide.html) ## Basic example per search engines -### Search Bing -<%= snippet('python', 'tests/example_search_bing.py', 9, 25) %> + +### Search bing +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_bing_test.py') %> see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api) -### Search Baidu -<%= snippet('python', 'tests/example_search_baidu.py', 9, 25) %> +### Search baidu +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_baidu_test.py') %> see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api) -### Search Yahoo -<%= snippet('python', 'tests/example_search_yahoo.py', 9, 25) %> +### Search yahoo +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_yahoo_test.py') %> see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api) -### Search Youtube -<%= snippet('python', 'tests/example_search_youtube.py', 9, 25) %> +### Search youtube +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_youtube_test.py') %> see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api) -### Search Walmart -<%= snippet('python', 'tests/example_search_walmart.py', 9, 25) %> +### Search walmart +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_walmart_test.py') %> see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api) -### Search Ebay -<%= snippet('python', 'tests/example_search_ebay.py', 9, 25) %> +### Search ebay +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_ebay_test.py') %> see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api) -### Search Naver -<%= snippet('python', 'tests/example_search_naver.py', 9, 25) %> +### Search naver +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_naver_test.py') %> see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api) -### Search Home Depot -<%= snippet('python', 'tests/example_search_home_depot.py', 9, 25) %> +### Search home depot +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_home_depot_test.py') %> see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api) -### Search Apple App Store -<%= snippet('python', 'tests/example_search_apple_app_store.py', 9, 25) %> +### Search apple app store +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_apple_app_store_test.py') %> see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store) -### Search Duckduckgo -<%= snippet('python', 'tests/example_search_duckduckgo.py', 9, 25) %> +### Search duckduckgo +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_duckduckgo_test.py') %> see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api) -### Search Google Search -<%= snippet('python', 'tests/example_search_google_search.py', 9, 25) %> +### Search google +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_test.py') %> see: [https://serpapi.com/search-api](https://serpapi.com/search-api) -### Search Google Scholar -<%= snippet('python', 'tests/example_search_google_scholar.py', 9, 25) %> +### Search google scholar +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_scholar_test.py') %> see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api) -### Search Google Autocomplete -<%= snippet('python', 'tests/example_search_google_autocomplete.py', 9, 25) %> +### Search google autocomplete +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_autocomplete_test.py') %> see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) -### Search Google Product -<%= snippet('python', 'tests/example_search_google_product.py', 9, 25) %> +### Search google product +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_product_test.py') %> see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api) -### Search Google Reverse Image -<%= snippet('python', 'tests/example_search_google_reverse_image.py', 9, 25) %> +### Search google reverse image +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_reverse_image_test.py') %> see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image) -### Search Google Events -<%= snippet('python', 'tests/example_search_google_events.py', 9, 25) %> +### Search google events +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_events_test.py') %> see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api) -### Search Google Local Services -<%= snippet('python', 'tests/example_search_google_local_services.py', 9, 25) %> +### Search google local services +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_local_services_test.py') %> see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api) -### Search Google Maps -<%= snippet('python', 'tests/example_search_google_maps.py', 9, 25) %> +### Search google maps +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_maps_test.py') %> see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api) -### Search Google Jobs -<%= snippet('python', 'tests/example_search_google_jobs.py', 9, 25) %> +### Search google jobs +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_jobs_test.py') %> see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api) -### Search Google Play -<%= snippet('python', 'tests/example_search_google_play.py', 9, 25) %> +### Search google play +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_play_test.py') %> see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api) -### Search Google Images -<%= snippet('python', 'tests/example_search_google_images.py', 9, 25) %> +### Search google images +<%= snippet('python', '/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_images_test.py') %> see: [https://serpapi.com/images-results](https://serpapi.com/images-results) + # Developer Guide +TODO update this section ### Key goals - High code quality - KISS principles diff --git a/oobt/oobt.py b/oobt/demo.py similarity index 100% rename from oobt/oobt.py rename to oobt/demo.py diff --git a/serpapi/error.py b/serpapi/error.py index bd4e3fb..95af0c4 100644 --- a/serpapi/error.py +++ b/serpapi/error.py @@ -1,3 +1,3 @@ -"""error related classes""" +"""wrap any serpapi related exception""" class SerpApiException(Exception): """custom exception for this module""" diff --git a/serpapi/object_decoder.py b/serpapi/object_decoder.py deleted file mode 100644 index 056b077..0000000 --- a/serpapi/object_decoder.py +++ /dev/null @@ -1,62 +0,0 @@ -"""custom object decoder to convert a JSON into a python object""" - - -class ObjectDecoder: - """ - Allow to convert JSON like datastructure in Python object. - """ - - def __init__(self, node): - self.node = node - - def create(self): - """Create Python Object from a Dict or an Array - Returns - ---- - pyobj: python Object - """ - pytype = type('response', (object, ), {}) - pyobj = pytype() - for child_key, child_node in self.node.items(): - self.add_node(child_key, pyobj, child_node) - return pyobj - - def child2object(self, name, node): - """Make python object from dict - Parameters - --- - name: str - parent field name to start walk the node - node: dict - source dict structure to convert into object - """ - pytype = type(name, (object, ), {}) - pyobj = pytype() - - if isinstance(node, int): - pyobj = node - return pyobj - if isinstance(node, list): - setattr(pyobj, name, []) - for item in node: - getattr(pyobj, name).append(self.child2object(name, item)) - return pyobj - if isinstance(node, dict): - for child_name, child in node.items(): - self.add_node(child_name, pyobj, child) - else: - setattr(pyobj, name, node) - return pyobj - - def add_node(self, name, pyobj, child): - """ - Add node in generic Python object - """ - if isinstance(child, list): - setattr(pyobj, name, []) - for item in child: - getattr(pyobj, name).append(self.child2object(name, item)) - elif isinstance(child, dict): - setattr(pyobj, name, self.child2object(name, child)) - else: - setattr(pyobj, name, child) diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py index eb1d9b3..7f9025c 100644 --- a/serpapi/serpapi.py +++ b/serpapi/serpapi.py @@ -1,8 +1,8 @@ -"""SerpApi client library for python""" +"""Client for SerpApi.com""" import json import urllib3 from .error import SerpApiException -from .object_decoder import ObjectDecoder +from ._version import __version__ class HttpClient: """Simple HTTP client wrapper around urllib3""" @@ -11,18 +11,24 @@ class HttpClient: SUPPORTED_DECODER = ['json', 'html', 'object'] def __init__(self, parameter: dict = None): + """Initialize a SerpApi Client with the default parameters provided. + An instance of urllib3 will be created + where + timeout is 60s by default + retries is disabled by default + both properties can be override by the parameter. + """ # initialize the http client self.http = urllib3.PoolManager() - # initialize parameter - if parameter is None: - parameter = {} + # initialize default client parameter self.parameter = parameter # urllib3 configurations # HTTP connect timeout if 'timeout' in parameter: self.timeout = parameter['timeout'] + del parameter['timeout'] else: # 60s default self.timeout = 60.0 @@ -30,6 +36,7 @@ def __init__(self, parameter: dict = None): # no HTTP retry if 'retries' in parameter: self.retries = parameter['retries'] + del parameter['retries'] else: self.retries = False @@ -55,8 +62,8 @@ def start(self, path: str, parameter: dict = None, decoder: str = 'json'): --- dict|str|object decoded HTTP response""" - # set client language - self.parameter['source'] = 'python' + # track client language + self.parameter['source'] = 'serpapi-python:' + __version__ # set output type if decoder == 'object': @@ -98,10 +105,6 @@ def decode(self, response: any, decoder: str): if decoder == 'html': return payload - if decoder == 'object': - data = json.loads(payload) - return ObjectDecoder(data).create() - raise SerpApiException("Invalid decoder: " + decoder + ", available: json, html, object") @@ -186,7 +189,7 @@ def search_archive(self, search_id: str, decoder: str = 'json'): Parameters: ----- search_id: str - id from a previous search. in the JSON search response it is search_metadata.id + id from a previous client. in the JSON search response it is search_metadata.id """ path = "/searches/" + str(search_id) + "." diff --git a/tests/example_search_apple_app_store.py b/tests/example_search_apple_app_store_test.py similarity index 65% rename from tests/example_search_apple_app_store.py rename to tests/example_search_apple_app_store_test.py index 708350c..cd9e150 100644 --- a/tests/example_search_apple_app_store.py +++ b/tests/example_search_apple_app_store_test.py @@ -1,4 +1,4 @@ -# test Apple App Store search engine +# Example: apple_app_store search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_apple_app_store(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'term': 'coffee', + 'term': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['organic_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_baidu.py b/tests/example_search_baidu_test.py similarity index 65% rename from tests/example_search_baidu.py rename to tests/example_search_baidu_test.py index 9eca07f..0a98a9e 100644 --- a/tests/example_search_baidu.py +++ b/tests/example_search_baidu_test.py @@ -1,4 +1,4 @@ -# test Baidu search engine +# Example: baidu search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_baidu(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', + 'q': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['organic_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_bing.py b/tests/example_search_bing_test.py similarity index 65% rename from tests/example_search_bing.py rename to tests/example_search_bing_test.py index 340eaee..ac246bb 100644 --- a/tests/example_search_bing.py +++ b/tests/example_search_bing_test.py @@ -1,4 +1,4 @@ -# test Bing search engine +# Example: bing search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_bing(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', + 'q': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['organic_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_duckduckgo.py b/tests/example_search_duckduckgo_test.py similarity index 65% rename from tests/example_search_duckduckgo.py rename to tests/example_search_duckduckgo_test.py index 3c0b81c..1c16fbb 100644 --- a/tests/example_search_duckduckgo.py +++ b/tests/example_search_duckduckgo_test.py @@ -1,4 +1,4 @@ -# test Duckduckgo search engine +# Example: duckduckgo search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_duckduckgo(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', + 'q': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['organic_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_ebay.py b/tests/example_search_ebay_test.py similarity index 65% rename from tests/example_search_ebay.py rename to tests/example_search_ebay_test.py index 3a6cb26..c59efe1 100644 --- a/tests/example_search_ebay.py +++ b/tests/example_search_ebay_test.py @@ -1,4 +1,4 @@ -# test Ebay search engine +# Example: ebay search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_ebay(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - '_nkw': 'coffee', + '_nkw': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['organic_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_autocomplete.py b/tests/example_search_google_autocomplete_test.py similarity index 65% rename from tests/example_search_google_autocomplete.py rename to tests/example_search_google_autocomplete_test.py index 3c0c24b..7e17695 100644 --- a/tests/example_search_google_autocomplete.py +++ b/tests/example_search_google_autocomplete_test.py @@ -1,4 +1,4 @@ -# test Google Autocomplete search engine +# Example: google_autocomplete search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_google_autocomplete(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', + 'q': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['suggestions']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_events.py b/tests/example_search_google_events_test.py similarity index 65% rename from tests/example_search_google_events.py rename to tests/example_search_google_events_test.py index 156a372..2c033aa 100644 --- a/tests/example_search_google_events.py +++ b/tests/example_search_google_events_test.py @@ -1,4 +1,4 @@ -# test Google Events search engine +# Example: google_events search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_google_events(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', + 'q': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['events_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_images.py b/tests/example_search_google_images_test.py similarity index 55% rename from tests/example_search_google_images.py rename to tests/example_search_google_images_test.py index 81c1f2f..28876cc 100644 --- a/tests/example_search_google_images.py +++ b/tests/example_search_google_images_test.py @@ -1,4 +1,4 @@ -# test Google Images search engine +# Example: google_images search engine import unittest import os import serpapi @@ -8,15 +8,16 @@ class TestGoogleImages(unittest.TestCase): @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") def test_search_google_images(self): client = serpapi.Client({ - 'engine': 'google', + 'engine': 'google_images', 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'engine': 'google', - 'tbm': 'isch', - 'q': 'coffee', + 'engine': 'google_images', + 'tbm': 'isch', + 'q': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['images_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_jobs.py b/tests/example_search_google_jobs_test.py similarity index 65% rename from tests/example_search_google_jobs.py rename to tests/example_search_google_jobs_test.py index 9249de3..90fb9b3 100644 --- a/tests/example_search_google_jobs.py +++ b/tests/example_search_google_jobs_test.py @@ -1,4 +1,4 @@ -# test Google Jobs search engine +# Example: google_jobs search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_google_jobs(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', + 'q': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['jobs_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_local_services.py b/tests/example_search_google_local_services_test.py similarity index 60% rename from tests/example_search_google_local_services.py rename to tests/example_search_google_local_services_test.py index 121b4ed..b2de7d3 100644 --- a/tests/example_search_google_local_services.py +++ b/tests/example_search_google_local_services_test.py @@ -1,4 +1,4 @@ -# test Google Local Services search engine +# Example: google_local_services search engine import unittest import os import serpapi @@ -12,10 +12,11 @@ def test_search_google_local_services(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'Electrician', - 'place_id': 'ChIJOwg_06VPwokRYv534QaPC8g', + 'q': 'Electrician', + 'data_cid': 'ChIJOwg_06VPwokRYv534QaPC8g', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['local_ads']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_maps.py b/tests/example_search_google_maps_test.py similarity index 59% rename from tests/example_search_google_maps.py rename to tests/example_search_google_maps_test.py index 32a3350..1a96ddf 100644 --- a/tests/example_search_google_maps.py +++ b/tests/example_search_google_maps_test.py @@ -1,4 +1,4 @@ -# test Google Maps search engine +# Example: google_maps search engine import unittest import os import serpapi @@ -12,11 +12,12 @@ def test_search_google_maps(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'pizza', - 'll': '@40.7455096,-74.0083012,15.1z', - 'type': 'search', + 'q': 'pizza', + 'll': '@40.7455096,-74.0083012,15.1z', + 'type': 'search', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['local_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_play.py b/tests/example_search_google_play_test.py similarity index 63% rename from tests/example_search_google_play.py rename to tests/example_search_google_play_test.py index 2d393ad..bb2adc5 100644 --- a/tests/example_search_google_play.py +++ b/tests/example_search_google_play_test.py @@ -1,11 +1,10 @@ -# test Google Play search engine +# Example: google_play search engine import unittest import os import serpapi class TestGooglePlay(unittest.TestCase): - @unittest.skip @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") def test_search_google_play(self): client = serpapi.Client({ @@ -13,10 +12,11 @@ def test_search_google_play(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'kite', - 'store': 'apps', + 'q': 'kite', + 'store': 'apps', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['organic_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_product.py b/tests/example_search_google_product_test.py similarity index 61% rename from tests/example_search_google_product.py rename to tests/example_search_google_product_test.py index 89fbb72..fcc7094 100644 --- a/tests/example_search_google_product.py +++ b/tests/example_search_google_product_test.py @@ -1,4 +1,4 @@ -# test Google Product search engine +# Example: google_product search engine import unittest import os import serpapi @@ -12,10 +12,11 @@ def test_search_google_product(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', - 'product_id': '4172129135583325756', + 'q': 'coffee', + 'product_id': '4172129135583325756', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['product_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_reverse_image.py b/tests/example_search_google_reverse_image_test.py similarity index 62% rename from tests/example_search_google_reverse_image.py rename to tests/example_search_google_reverse_image_test.py index b6cc8ee..72caace 100644 --- a/tests/example_search_google_reverse_image.py +++ b/tests/example_search_google_reverse_image_test.py @@ -1,4 +1,4 @@ -# test Google Reverse Image search engine +# Example: google_reverse_image search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_google_reverse_image(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'image_url': 'https://i.imgur.com/5bGzZi7.jpg', + 'image_url': 'https://i.imgur.com/5bGzZi7.jpg', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['image_sizes']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_scholar.py b/tests/example_search_google_scholar_test.py similarity index 65% rename from tests/example_search_google_scholar.py rename to tests/example_search_google_scholar_test.py index 0868e50..598fc3f 100644 --- a/tests/example_search_google_scholar.py +++ b/tests/example_search_google_scholar_test.py @@ -1,4 +1,4 @@ -# test Google Scholar search engine +# Example: google_scholar search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_google_scholar(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', + 'q': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['organic_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_google_search.py b/tests/example_search_google_test.py similarity index 52% rename from tests/example_search_google_search.py rename to tests/example_search_google_test.py index 9e3b48f..dff3585 100644 --- a/tests/example_search_google_search.py +++ b/tests/example_search_google_test.py @@ -1,21 +1,22 @@ -# test Google Search search engine +# Example: google search engine import unittest import os import serpapi -class TestGoogleSearch(unittest.TestCase): +class TestGoogle(unittest.TestCase): @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") - def test_search_google_search(self): + def test_search_google(self): client = serpapi.Client({ 'engine': 'google', 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'coffee', - 'engine': 'google', + 'q': 'coffee', + 'engine': 'google', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['organic_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_home_depot.py b/tests/example_search_home_depot_test.py similarity index 65% rename from tests/example_search_home_depot.py rename to tests/example_search_home_depot_test.py index 30bf890..4f97412 100644 --- a/tests/example_search_home_depot.py +++ b/tests/example_search_home_depot_test.py @@ -1,4 +1,4 @@ -# test Home Depot search engine +# Example: home_depot search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_home_depot(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'table', + 'q': 'table', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['products']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_naver.py b/tests/example_search_naver_test.py similarity index 64% rename from tests/example_search_naver.py rename to tests/example_search_naver_test.py index 62c2ee8..b780973 100644 --- a/tests/example_search_naver.py +++ b/tests/example_search_naver_test.py @@ -1,4 +1,4 @@ -# test Naver search engine +# Example: naver search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_naver(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'query': 'coffee', + 'query': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['ads_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_walmart.py b/tests/example_search_walmart_test.py similarity index 65% rename from tests/example_search_walmart.py rename to tests/example_search_walmart_test.py index 1989ab7..f9cd916 100644 --- a/tests/example_search_walmart.py +++ b/tests/example_search_walmart_test.py @@ -1,4 +1,4 @@ -# test Walmart search engine +# Example: walmart search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_walmart(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'query': 'coffee', + 'query': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['organic_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_yahoo.py b/tests/example_search_yahoo_test.py similarity index 65% rename from tests/example_search_yahoo.py rename to tests/example_search_yahoo_test.py index e39e0c4..5a54ad4 100644 --- a/tests/example_search_yahoo.py +++ b/tests/example_search_yahoo_test.py @@ -1,4 +1,4 @@ -# test Yahoo search engine +# Example: yahoo search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_yahoo(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'p': 'coffee', + 'p': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['organic_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/example_search_youtube.py b/tests/example_search_youtube_test.py similarity index 64% rename from tests/example_search_youtube.py rename to tests/example_search_youtube_test.py index a03ce73..186811b 100644 --- a/tests/example_search_youtube.py +++ b/tests/example_search_youtube_test.py @@ -1,4 +1,4 @@ -# test Youtube search engine +# Example: youtube search engine import unittest import os import serpapi @@ -12,9 +12,10 @@ def test_search_youtube(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'search_query': 'coffee', + 'search_query': 'coffee', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['video_results']) - # os.getenv("API_KEY") captures the secret user API available from http://serpapi.com - \ No newline at end of file + # os.getenv("API_KEY") is your secret API Key + # copy/paste from [http://serpapi.com/dashboard] to your bash + # ```export API_KEY="your_secure_api_key"``` diff --git a/tests/test_example.py b/tests/test_example.py index c9765b7..b5ae32a 100644 --- a/tests/test_example.py +++ b/tests/test_example.py @@ -30,21 +30,21 @@ # @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") # def test_get_json(self): -# search = GoogleSearch({"q": "Coffee", "engine": "google_scholar"}) -# data = search.get_json() +# client = GoogleSearch({"q": "Coffee", "engine": "google_scholar"}) +# data = client.get_json() # print(data['search_metadata']) # search_id = data['search_metadata']['id'] # # retrieve search from the archive - blocker # print(search_id + ": get search from archive") -# raw_html = search.get_search_archive(search_id, 'html') +# raw_html = client.get_search_archive(search_id, 'html') # # print(search_id + ": status = " + search_archived['search_metadata']['status']) # print(raw_html) # #print(search_html) # @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") # def test_search_google_images(self): -# search = GoogleSearch({"q": "coffe", "tbm": "isch"}) -# for image_result in search.get_json()['images_results']: +# client = GoogleSearch({"q": "coffe", "tbm": "isch"}) +# for image_result in client.get_json()['images_results']: # try: # link = image_result["original"] # print("link is found: " + link) @@ -61,7 +61,7 @@ # search_queue = Queue() # # Serp API search -# search = GoogleSearch({ +# client = GoogleSearch({ # "location": "Austin,Texas", # "async": True # }) @@ -69,8 +69,8 @@ # # loop through companies # for company in ['amd','nvidia','intel']: # print("execute async search: q = " + company) -# search.params_dict["q"] = company -# data = search.get_dict() +# client.params_dict["q"] = company +# data = client.get_dict() # if data is not None: # print("oops data is empty for: " + company) # continue @@ -81,14 +81,14 @@ # print("wait until all search statuses are cached or success") # # Create regular search -# search = GoogleSearch({"async": True}) +# client = GoogleSearch({"async": True}) # while not search_queue.empty(): # data = search_queue.get() # search_id = data['search_metadata']['id'] # # retrieve search from the archive - blocker # print(search_id + ": get search from archive") -# search_archived = search.get_search_archive(search_id) +# search_archived = client.get_search_archive(search_id) # print(search_id + ": status = " + search_archived['search_metadata']['status']) # # check status @@ -105,27 +105,27 @@ # @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") # def test_search_google_news(self): -# search = GoogleSearch({ +# client = GoogleSearch({ # "q": "coffe", # search search # "tbm": "nws", # news # "tbs": "qdr:d", # last 24h # "num": 10 # }) # for offset in [0,1,2]: -# search.params_dict["start"] = offset * 10 -# data = search.get_json() +# client.params_dict["start"] = offset * 10 +# data = client.get_json() # for news_result in data['news_results']: # print(str(news_result['position'] + offset * 10) + " - " + news_result['title']) # @unittest.skipIf((os.getenv("DEBUGAPI_KEY") == None), "no api_key provided") # def test_search_google_shopping(self): -# search = GoogleSearch({ +# client = GoogleSearch({ # "q": "coffe", # search search # "tbm": "shop", # news # "tbs": "p_ord:rv", # last 24h # "num": 100 # }) -# data = search.get_json() +# data = client.get_json() # if 'shopping_results' in data: # for shopping_result in data['shopping_results']: # print(str(shopping_result['position']) + " - " + shopping_result['title']) @@ -136,13 +136,13 @@ # def test_search_by_location(self): # for city in ["new york", "paris", "berlin"]: # location = GoogleSearch({}).get_location(city, 1)[0]["canonical_name"] -# search = GoogleSearch({ +# client = GoogleSearch({ # "q": "best coffee shop", # search search # "location": location, # "num": 10, # "start": 0 # }) -# data = search.get_json() +# data = client.get_json() # self.assertIsNone(data.get("error")) # top_result = data['organic_results'][0]["title"] # print("top coffee result for " + location + " is: " + top_result) diff --git a/tests/test_object_decoder.py b/tests/test_object_decoder.py deleted file mode 100644 index f9d2821..0000000 --- a/tests/test_object_decoder.py +++ /dev/null @@ -1,34 +0,0 @@ -import unittest -import os -import pprint -from serpapi.object_decoder import ObjectDecoder -import pytest - -class TestObjectDecoder(unittest.TestCase): - - def test_decode_basic(self): - data = { - "organic_results": [ - { - 'name': 'ok' - }, - { - 'name': 'good' - } - ] - } - obj = ObjectDecoder(data).create() - self.assertEqual(len(obj.organic_results), 2) - self.assertEqual(obj.organic_results[0].name, 'ok') - self.assertEqual(obj.organic_results[1].name, 'good') - - def test_decode_list(self): - data = { - "organic_results": { - 'list': [0,1,2,3,4] - } - } - obj = ObjectDecoder(data).create() - print(obj.organic_results.list[0]) - self.assertEqual(len(obj.organic_results.list), 5) - self.assertEqual(obj.organic_results.list, [0,1,2,3,4]) From dc2ae8bacfc9047c9d8974f53123ddfde50325ba Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 7 May 2023 12:33:35 -0500 Subject: [PATCH 17/25] remove decoder object --- serpapi/serpapi.py | 24 +++++++----------------- tests/test_search_archive.py | 10 +++------- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py index 7f9025c..323fdcd 100644 --- a/serpapi/serpapi.py +++ b/serpapi/serpapi.py @@ -8,7 +8,7 @@ class HttpClient: """Simple HTTP client wrapper around urllib3""" BACKEND = 'https://serpapi.com' - SUPPORTED_DECODER = ['json', 'html', 'object'] + SUPPORTED_DECODER = ['json', 'html'] def __init__(self, parameter: dict = None): """Initialize a SerpApi Client with the default parameters provided. @@ -45,7 +45,6 @@ def start(self, path: str, parameter: dict = None, decoder: str = 'json'): The response is decoded using the selected decoder: - html: raw HTML response - json: deep dict contains search results - - object: containing search results as a dynamic object Parameters: --- @@ -60,16 +59,11 @@ def start(self, path: str, parameter: dict = None, decoder: str = 'json'): Returns: --- - dict|str|object + dict|str decoded HTTP response""" # track client language self.parameter['source'] = 'serpapi-python:' + __version__ - - # set output type - if decoder == 'object': - self.parameter['output'] = 'json' - else: - self.parameter['output'] = decoder + self.parameter['output'] = decoder # merge parameter defaults and overrides fields = self.parameter.copy() @@ -106,7 +100,7 @@ def decode(self, response: any, decoder: str): return payload raise SerpApiException("Invalid decoder: " + - decoder + ", available: json, html, object") + decoder + ", available: json, html") class Client(HttpClient): @@ -130,7 +124,7 @@ def __init__(self, parameter: dict = None): def search(self, parameter: dict = None, decoder: str = 'json'): """ make search then decode the output - decoder supported 'json', 'html', 'object' + decoder supported 'json', 'html' Parameters ---------- @@ -145,7 +139,6 @@ def search(self, parameter: dict = None, decoder: str = 'json'): search results returns as : dict if decoder = 'json' str if decoder = 'html' - object if decoder = 'object' """ return self.start(path='/search', parameter=parameter, decoder=decoder) @@ -194,12 +187,9 @@ def search_archive(self, search_id: str, decoder: str = 'json'): """ path = "/searches/" + str(search_id) + "." if decoder in self.SUPPORTED_DECODER: - if decoder == "object": - path += "json" - else: - path += decoder + path += decoder else: - raise SerpApiException('Decoder must be json or html or object') + raise SerpApiException('Decoder must be json or html') return self.start(path, {}, decoder) def account(self, api_key: str = None): diff --git a/tests/test_search_archive.py b/tests/test_search_archive.py index 06201a9..a7e7d5c 100644 --- a/tests/test_search_archive.py +++ b/tests/test_search_archive.py @@ -1,14 +1,8 @@ import unittest import os -import pprint import serpapi import pytest -# This test shows how to extends serpapi.Client -# without using client engine wrapper. -# - - class TestSearchArchive(unittest.TestCase): @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") @@ -23,11 +17,13 @@ def test_search_archive(self): }) self.assertEqual(data.get("error"), None) self.assertIsNotNone(data["organic_results"][0]["title"]) + # search unique search identifier search_id = data['search_metadata']['id'] + # fetch results from the archive (free of charge) data_archive = client.search_archive(search_id) self.assertEqual(data_archive['organic_results'][0], data["organic_results"][0]) - # code coverage + # fetch results from the archive again (code coverage) object_archive = client.search_archive(search_id, 'object') self.assertIsNotNone(object_archive) self.assertEqual(object_archive.organic_results[0].title, data["organic_results"][0]["title"]) From 1ce9794ccc6043b14623a3512da74be11298024f Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 7 May 2023 13:06:28 -0500 Subject: [PATCH 18/25] fix tests --- serpapi/serpapi.py | 2 +- ...ample_search_google_local_services_test.py | 4 ++-- tests/test_location_api.py | 5 +++-- tests/test_search_archive.py | 11 +++++++--- tests/test_serpapi.py | 21 +------------------ 5 files changed, 15 insertions(+), 28 deletions(-) diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py index 323fdcd..626bd8c 100644 --- a/serpapi/serpapi.py +++ b/serpapi/serpapi.py @@ -189,7 +189,7 @@ def search_archive(self, search_id: str, decoder: str = 'json'): if decoder in self.SUPPORTED_DECODER: path += decoder else: - raise SerpApiException('Decoder must be json or html') + raise SerpApiException(f'Invalid decoder: {decoder} must be json or html. ') return self.start(path, {}, decoder) def account(self, api_key: str = None): diff --git a/tests/example_search_google_local_services_test.py b/tests/example_search_google_local_services_test.py index b2de7d3..c4a9ef3 100644 --- a/tests/example_search_google_local_services_test.py +++ b/tests/example_search_google_local_services_test.py @@ -12,8 +12,8 @@ def test_search_google_local_services(self): 'api_key': os.getenv("API_KEY") }) data = client.search({ - 'q': 'Electrician', - 'data_cid': 'ChIJOwg_06VPwokRYv534QaPC8g', + 'q': 'electrician', + 'data_cid': '6745062158417646970', }) self.assertIsNone(data.get('error')) self.assertIsNotNone(data['local_ads']) diff --git a/tests/test_location_api.py b/tests/test_location_api.py index 5e9e94f..f64b2ca 100644 --- a/tests/test_location_api.py +++ b/tests/test_location_api.py @@ -10,6 +10,7 @@ def test_get_account(self): client = serpapi.Client({'api_key': os.getenv('API_KEY')}) locations = client.location({'q':'Austin', 'limit': 3}) self.assertGreater(len(locations), 1) + self.assertLessEqual(len(locations), 3) self.assertTrue('id' in locations[0]) - - + self.assertTrue('name' in locations[0]) + self.assertTrue('canonical_name' in locations[0]) diff --git a/tests/test_search_archive.py b/tests/test_search_archive.py index a7e7d5c..24f43f0 100644 --- a/tests/test_search_archive.py +++ b/tests/test_search_archive.py @@ -24,9 +24,14 @@ def test_search_archive(self): self.assertEqual(data_archive['organic_results'][0], data["organic_results"][0]) # fetch results from the archive again (code coverage) - object_archive = client.search_archive(search_id, 'object') + object_archive = client.search_archive(search_id, 'json') self.assertIsNotNone(object_archive) - self.assertEqual(object_archive.organic_results[0].title, data["organic_results"][0]["title"]) + self.assertEqual(object_archive['organic_results'][0]['title'], data["organic_results"][0]["title"]) + def test_bad_decoder(self): + client = serpapi.Client({ + "engine": "google", + "api_key": os.getenv("API_KEY") + }) with pytest.raises(serpapi.SerpApiException, match=r'Decoder must be json or html'): - client.search_archive(search_id, 'bad') + client.search_archive('007', 'bad') diff --git a/tests/test_serpapi.py b/tests/test_serpapi.py index bb56b7d..60981a0 100644 --- a/tests/test_serpapi.py +++ b/tests/test_serpapi.py @@ -34,25 +34,6 @@ def test_html(self): }) self.assertRegex(data, r'$') - # test ObjectDecoder - @unittest.skipIf((os.getenv("API_KEY") == None), "no api_key provided") - def test_object(self): - client = serpapi.Client({ - "engine": "google", - "api_key": os.getenv("API_KEY"), - 'timeout': 120, - 'retries': True - }) - data = client.search({ - "q": "Coffee", - "location": "Austin,Texas" - }, decoder="object") - self.assertIsInstance(data, object) - self.assertIsNotNone(data.organic_results) - self.assertIsInstance(data.organic_results, list) - self.assertGreater(len(data.organic_results), 3) - self.assertIsInstance(data.organic_results[0].link, str) - def test_invalid_api_key(self): client = serpapi.Client({ "engine": "google", @@ -67,7 +48,7 @@ def test_invalid_api_key(self): def test_invalid_decoder(self): client = serpapi.Client({ "engine": "google", - "api_key": os.getenv("API_KEY"), + "api_key": os.getenv("API_KEY") }) mockResponse = MockResponse() self.assertEqual(mockResponse.status, 200) From cfe86b609851d952fdbb68d1069f514852cb6e8b Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 7 May 2023 21:02:02 -0500 Subject: [PATCH 19/25] add 3.11 version of python in the matrix --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 43d70a4..3b3a48b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,7 +1,7 @@ # This workflow will install Python dependencies, run tests and lint with a variety of Python versions # For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions -name: SerpApi Python +name: serpapi-python on: push: @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.6, 3.7, 3.8, 3.9, 3.11] steps: - uses: actions/checkout@v2 @@ -35,6 +35,6 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest - run: pytest + run: make test env: API_KEY: ${{secrets.API_KEY}} From 02c2aaa1f63391905754e3e4d2196e306575dbb2 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 7 May 2023 21:04:07 -0500 Subject: [PATCH 20/25] improve README --- Makefile | 4 +- README.md | 197 +++++++++++++++++++++++++++++++++++++++++++------- README.md.erb | 193 ++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 340 insertions(+), 54 deletions(-) diff --git a/Makefile b/Makefile index e717742..e263111 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ dist=dist/serpapi-$(version).tar.gz .PHONY: build -all: clean install readme doc lint test build +all: clean install readme doc lint test build oobt check clean: find . -name '*.pyc' -delete @@ -48,7 +48,7 @@ doc: readme $(MAKE) -C docs/ html # https://packaging.python.org/tutorials/packaging-projects/ -build: doc lint test +build: python3 setup.py sdist # out of box testing / user acceptance before delivery diff --git a/README.md b/README.md index 4ee99a2..1f81aba 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![Package](https://badge.fury.io/py/serpapi.svg) ![Downloads](https://static.pepy.tech/personalized-badge/serpapi?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=Downloads) - ![Buiild](https://github.com/serpapi/serpapi-python/actions/workflows/python-package.yml/badge.svg) + ![Build](https://github.com/serpapi/serpapi-python/actions/workflows/python-package.yml/badge.svg) @@ -39,24 +39,109 @@ This example runs a search for "coffee" on Google. It then returns the results a ## Advanced Usage ### Search API +```python +# load pip package +import serpapi + +# serpapi client created with default parameters +client = serpapi.Client({'api_key': 'secret_key', 'engine': 'google'}) + +# We recommend that you keep your keys safe. +# At least, don't commit them in plain text. +# More about configuration via environment variables: +# https://hackernoon.com/all-the-secrets-of-encrypting-api-keys-in-ruby-revealed-5qf3t5l + +# search query overview (more fields available depending on search engine) +params = { + # select the search engine (full list: https://serpapi.com/) + 'engine': "google", + # actual search query for google + 'q': "Coffee", + # then adds search engine specific options. + # for example: google specific parameters: https://serpapi.com/search-api + 'google_domain': "Google Domain", + 'location': "Location Requested", # example: Portland,Oregon,United States [see: Location API](#Location-API) + 'device': "desktop|mobile|tablet", + 'hl': "Google UI Language", + 'gl': "Google Country", + 'safe': "Safe Search Flag", + 'num': "Number of Results", + 'start': "Pagination Offset", + 'tbm': "nws|isch|shop", + 'tbs': "custom to be client criteria", + # tweak HTTP client behavior + 'async': False, # true when async call enabled. + 'timeout': 60, # HTTP timeout in seconds on the client side only. +} -TODO update this specification +# formated search results as a Hash +# serpapi.com converts HTML -> JSON +results = client.search(params) -SerpApi Client uses urllib3 under the hood. -Optionally, rhe HTTP connection can be tuned: - - timeout : connection timeout by default 60s - - retries : attempt to reconnect if the connection failed by default: False. - serpapi is reliable at 99.99% but your company network might not be as stable. +# raw search engine html as a String +# serpapi.com acts a proxy to provive high throughputs, no search limit and more. +raw_html = client.html(params) +``` - ```python -parameter = { - retries: 5, - timeout: 4.0, - # extra user parameters -} +[Google search documentation](https://serpapi.com/search-api). More hands on examples are available below. + +### Documentation + * [API documentation](https://rubydoc.info/github/serpapi/serpapi-ruby/master) + * [Full documentation on SerpApi.com](https://serpapi.com) + * [Library Github page](https://github.com/serpapi/serpapi-ruby) + * [Library GEM page](https://rubygems.org/gems/serpapi/) + * [API health status](https://serpapi.com/status) + +### Location API + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +locations = client.location({'q':'Austin', 'limit': 3}) +print([loc['canonical_name'] for loc in locations]) +``` + +it prints the first 3 locations matching Austin: +```python +['Austin,TX,Texas,United States', 'Austin,Texas,United States', 'Rochester,MN-Mason City,IA-Austin,MN,United States'] +``` + +NOTE: api_key is not required for this endpoint. + +### Search Archive API + +This API allows retrieving previous search results. +To fetch earlier results from the search_id. + +First, you need to run a search and save the search id. +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key', 'engine': 'google'}) +results = client.search({'q': "Coffee"}) +search_id = results['search_metadata']['id'] +print("search_id: " + search_id) ``` -for more details: [URL LIB3 documentation]](https://urllib3.readthedocs.io/en/stable/user-guide.html) +Now let's retrieve the previous search results from the archive. + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +results = client.search_archive('search_id') +print(results) +``` + +This code prints the search results from the archive. :) + +### Account API + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +print(client.account()) +``` + +It prints your account information including plan, credit, montly ## Basic example per search engines @@ -425,8 +510,8 @@ client = serpapi.Client({ 'api_key': os.getenv("API_KEY") }) data = client.search({ -'q': 'Electrician', -'data_cid': 'ChIJOwg_06VPwokRYv534QaPC8g', +'q': 'electrician', +'data_cid': '6745062158417646970', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['local_ads']) @@ -535,27 +620,31 @@ see: [https://serpapi.com/images-results](https://serpapi.com/images-results) TODO update this section ### Key goals - High code quality - - KISS principles + - KISS principles (https://en.wikipedia.org/wiki/KISS_principle) - Brand centric instead of search engine based - - No hard coded logic per search engine + - No hard coded logic per search engine on the client side. - Simple HTTP client (lightweight, reduced dependency) - No magic default values - Thread safe + - Leak free - Easy to extends - Defensive code style (raise custom exception) - - TDD - - Best API coding pratice per platform + - TDD - Test driven development (lint, ~100% code coverage) + - Follow best API coding pratice per platform ### Inspiration The API design was inpired by the most popular Python packages. - urllib3 - https://github.com/urllib3/urllib3 - Boto3 - https://github.com/boto/boto3 + - Numpy - ### Quality expectation - 0 lint issues using pylint `make lint` - 99% code coverage running `make test` - 100% test passing: `make test` +# Developer Guide +## Design : UML diagram ### Client design: Class diagram ```mermaid classDiagram @@ -565,8 +654,8 @@ classDiagram HttpClient *-- ObjectDecoder class Client { - engine: String - api_key: String + 'engine': String + 'api_key': String parameter: Hash search() html() @@ -595,10 +684,64 @@ sequenceDiagram Client-->>Client: decode JSON into Dict ``` -## Build flow +where: + - The end user implements the application. + - Client refers to SerpApi:Client. + - SerpApi.com is the backend HTTP / REST service. + - Engine refers to Google, Baidu, Bing, and more. + +The SerpApi.com service (backend) + - executes a scalable search on `'engine': "google"` using the search query: `q: "coffee"`. + - parses the messy HTML responses from Google on the backend. + - returns a standardized JSON response. +The class SerpApi::Client (client side / ruby): + - Format the request to SerpApi.com server. + - Execute HTTP Get request. + - Parse JSON into Ruby Hash using a standard JSON library. +Et voila! + +## Continuous integration +We love "true open source" and "continuous integration", and Test Drive Development (TDD). + We are using RSpec to test [our infrastructure around the clock]) using Github Action to achieve the best QoS (Quality Of Service). + +The directory spec/ includes specification which serves the dual purposes of examples and functional tests. + +Set your secret API key in your shell before running a test. +```bash +export API_KEY="your_secret_key" +``` +Install testing dependency +```bash +$ make install +``` + +Check code quality using Lint. +```bash +$ make lint +``` + +Run regression. +```bash +$ make test +``` + +To flush the flow. +```bash +$ make +``` + +Open coverage report generated by `rake test` +```sh +open coverage/index.html +``` + +Open ./Rakefile for more information. + +Contributions are welcome. Feel to submit a pull request! + +## Dependencies -This project is automated using a good old Makefile. -To pipe-clean the project run this: -`make` +HTTP requests are executed using [URL LIB3 documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html). -Open Makefile for more details. +## TODO + - [] Release version 1.0.0 diff --git a/README.md.erb b/README.md.erb index 23457bf..8359c6d 100644 --- a/README.md.erb +++ b/README.md.erb @@ -16,7 +16,7 @@ end ![Package](https://badge.fury.io/py/serpapi.svg) ![Downloads](https://static.pepy.tech/personalized-badge/serpapi?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=Downloads) - ![Buiild](https://github.com/serpapi/serpapi-python/actions/workflows/python-package.yml/badge.svg) + ![Build](https://github.com/serpapi/serpapi-python/actions/workflows/python-package.yml/badge.svg) @@ -50,24 +50,109 @@ This example runs a search for "coffee" on Google. It then returns the results a ## Advanced Usage ### Search API +```python +# load pip package +import serpapi + +# serpapi client created with default parameters +client = serpapi.Client({'api_key': 'secret_key', 'engine': 'google'}) + +# We recommend that you keep your keys safe. +# At least, don't commit them in plain text. +# More about configuration via environment variables: +# https://hackernoon.com/all-the-secrets-of-encrypting-api-keys-in-ruby-revealed-5qf3t5l + +# search query overview (more fields available depending on search engine) +params = { + # select the search engine (full list: https://serpapi.com/) + 'engine': "google", + # actual search query for google + 'q': "Coffee", + # then adds search engine specific options. + # for example: google specific parameters: https://serpapi.com/search-api + 'google_domain': "Google Domain", + 'location': "Location Requested", # example: Portland,Oregon,United States [see: Location API](#Location-API) + 'device': "desktop|mobile|tablet", + 'hl': "Google UI Language", + 'gl': "Google Country", + 'safe': "Safe Search Flag", + 'num': "Number of Results", + 'start': "Pagination Offset", + 'tbm': "nws|isch|shop", + 'tbs': "custom to be client criteria", + # tweak HTTP client behavior + 'async': False, # true when async call enabled. + 'timeout': 60, # HTTP timeout in seconds on the client side only. +} -TODO update this specification +# formated search results as a Hash +# serpapi.com converts HTML -> JSON +results = client.search(params) -SerpApi Client uses urllib3 under the hood. -Optionally, rhe HTTP connection can be tuned: - - timeout : connection timeout by default 60s - - retries : attempt to reconnect if the connection failed by default: False. - serpapi is reliable at 99.99% but your company network might not be as stable. +# raw search engine html as a String +# serpapi.com acts a proxy to provive high throughputs, no search limit and more. +raw_html = client.html(params) +``` - ```python -parameter = { - retries: 5, - timeout: 4.0, - # extra user parameters -} +[Google search documentation](https://serpapi.com/search-api). More hands on examples are available below. + +### Documentation + * [API documentation](https://rubydoc.info/github/serpapi/serpapi-ruby/master) + * [Full documentation on SerpApi.com](https://serpapi.com) + * [Library Github page](https://github.com/serpapi/serpapi-ruby) + * [Library GEM page](https://rubygems.org/gems/serpapi/) + * [API health status](https://serpapi.com/status) + +### Location API + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +locations = client.location({'q':'Austin', 'limit': 3}) +print([loc['canonical_name'] for loc in locations]) +``` + +it prints the first 3 locations matching Austin: +```python +['Austin,TX,Texas,United States', 'Austin,Texas,United States', 'Rochester,MN-Mason City,IA-Austin,MN,United States'] +``` + +NOTE: api_key is not required for this endpoint. + +### Search Archive API + +This API allows retrieving previous search results. +To fetch earlier results from the search_id. + +First, you need to run a search and save the search id. +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key', 'engine': 'google'}) +results = client.search({'q': "Coffee"}) +search_id = results['search_metadata']['id'] +print("search_id: " + search_id) ``` -for more details: [URL LIB3 documentation]](https://urllib3.readthedocs.io/en/stable/user-guide.html) +Now let's retrieve the previous search results from the archive. + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +results = client.search_archive('search_id') +print(results) +``` + +This code prints the search results from the archive. :) + +### Account API + +```python +import serpapi +client = serpapi.Client({'api_key': 'secret_api_key'}) +print(client.account()) +``` + +It prints your account information including plan, credit, montly ## Basic example per search engines @@ -160,27 +245,31 @@ see: [https://serpapi.com/images-results](https://serpapi.com/images-results) TODO update this section ### Key goals - High code quality - - KISS principles + - KISS principles (https://en.wikipedia.org/wiki/KISS_principle) - Brand centric instead of search engine based - - No hard coded logic per search engine + - No hard coded logic per search engine on the client side. - Simple HTTP client (lightweight, reduced dependency) - No magic default values - Thread safe + - Leak free - Easy to extends - Defensive code style (raise custom exception) - - TDD - - Best API coding pratice per platform + - TDD - Test driven development (lint, ~100% code coverage) + - Follow best API coding pratice per platform ### Inspiration The API design was inpired by the most popular Python packages. - urllib3 - https://github.com/urllib3/urllib3 - Boto3 - https://github.com/boto/boto3 + - Numpy - ### Quality expectation - 0 lint issues using pylint `make lint` - 99% code coverage running `make test` - 100% test passing: `make test` +# Developer Guide +## Design : UML diagram ### Client design: Class diagram ```mermaid classDiagram @@ -190,8 +279,8 @@ classDiagram HttpClient *-- ObjectDecoder class Client { - engine: String - api_key: String + 'engine': String + 'api_key': String parameter: Hash search() html() @@ -220,10 +309,64 @@ sequenceDiagram Client-->>Client: decode JSON into Dict ``` -## Build flow +where: + - The end user implements the application. + - Client refers to SerpApi:Client. + - SerpApi.com is the backend HTTP / REST service. + - Engine refers to Google, Baidu, Bing, and more. + +The SerpApi.com service (backend) + - executes a scalable search on `'engine': "google"` using the search query: `q: "coffee"`. + - parses the messy HTML responses from Google on the backend. + - returns a standardized JSON response. +The class SerpApi::Client (client side / ruby): + - Format the request to SerpApi.com server. + - Execute HTTP Get request. + - Parse JSON into Ruby Hash using a standard JSON library. +Et voila! + +## Continuous integration +We love "true open source" and "continuous integration", and Test Drive Development (TDD). + We are using RSpec to test [our infrastructure around the clock]) using Github Action to achieve the best QoS (Quality Of Service). + +The directory spec/ includes specification which serves the dual purposes of examples and functional tests. + +Set your secret API key in your shell before running a test. +```bash +export API_KEY="your_secret_key" +``` +Install testing dependency +```bash +$ make install +``` + +Check code quality using Lint. +```bash +$ make lint +``` + +Run regression. +```bash +$ make test +``` + +To flush the flow. +```bash +$ make +``` + +Open coverage report generated by `rake test` +```sh +open coverage/index.html +``` + +Open ./Rakefile for more information. + +Contributions are welcome. Feel to submit a pull request! + +## Dependencies -This project is automated using a good old Makefile. -To pipe-clean the project run this: -`make` +HTTP requests are executed using [URL LIB3 documentation](https://urllib3.readthedocs.io/en/stable/user-guide.html). -Open Makefile for more details. +## TODO + - [] Release version 1.0.0 From 3f9863dfab5ebbec9308c21cdb1dee91afb4a170 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 7 May 2023 21:23:13 -0500 Subject: [PATCH 21/25] upgrade gitaction configuration file --- .github/workflows/ci.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b3a48b..60d695a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,12 +15,12 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9, 3.11] + python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v2 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies From 906dca019039ecf92e250a3f95aff398d2a1128d Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 7 May 2023 21:24:26 -0500 Subject: [PATCH 22/25] run pytest directly in gitaction --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 60d695a..289eb69 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,6 @@ jobs: # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - name: Test with pytest - run: make test + run: pytest env: API_KEY: ${{secrets.API_KEY}} From b108a2ab3d03886bf2e61cae5f0823b141bb1745 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 7 May 2023 21:26:05 -0500 Subject: [PATCH 23/25] fix gitaction link in README file --- README.md | 2 +- README.md.erb | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1f81aba..9516f01 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ ![Package](https://badge.fury.io/py/serpapi.svg) ![Downloads](https://static.pepy.tech/personalized-badge/serpapi?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=Downloads) - ![Build](https://github.com/serpapi/serpapi-python/actions/workflows/python-package.yml/badge.svg) + [![serpapi-python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) diff --git a/README.md.erb b/README.md.erb index 8359c6d..7fb7ab5 100644 --- a/README.md.erb +++ b/README.md.erb @@ -16,7 +16,7 @@ end ![Package](https://badge.fury.io/py/serpapi.svg) ![Downloads](https://static.pepy.tech/personalized-badge/serpapi?period=month&units=international_system&left_color=grey&right_color=brightgreen&left_text=Downloads) - ![Build](https://github.com/serpapi/serpapi-python/actions/workflows/python-package.yml/badge.svg) + [![serpapi-python](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml/badge.svg)](https://github.com/serpapi/serpapi-python/actions/workflows/ci.yml) From 77ede28bf8ddd01ea5b09667cdb7fde8cd497489 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 7 May 2023 21:28:00 -0500 Subject: [PATCH 24/25] fix broken test --- tests/test_search_archive.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_search_archive.py b/tests/test_search_archive.py index 24f43f0..c549fe0 100644 --- a/tests/test_search_archive.py +++ b/tests/test_search_archive.py @@ -33,5 +33,5 @@ def test_bad_decoder(self): "engine": "google", "api_key": os.getenv("API_KEY") }) - with pytest.raises(serpapi.SerpApiException, match=r'Decoder must be json or html'): + with pytest.raises(serpapi.SerpApiException, match=r'Invalid decoder'): client.search_archive('007', 'bad') From d4a2aed2c607a68776eb7eb4b2a15d20652387d7 Mon Sep 17 00:00:00 2001 From: Victor Benarbia Date: Sun, 7 May 2023 21:57:55 -0500 Subject: [PATCH 25/25] cleanup - run python using "python3 -m" - convert single quote into double quote - polish README file --- Makefile | 26 +++--- README.md | 226 ++++++++++++++++++++++----------------------- README.md.erb | 7 +- serpapi/serpapi.py | 59 ++++++------ 4 files changed, 159 insertions(+), 159 deletions(-) diff --git a/Makefile b/Makefile index e263111..0933b4c 100644 --- a/Makefile +++ b/Makefile @@ -18,15 +18,15 @@ all: clean install readme doc lint test build oobt check clean: find . -name '*.pyc' -delete find . -type d -name "__pycache__" -delete - pip3 uninstall serpapi + python3 -m pip uninstall serpapi # lint check lint: - pylint serpapi + python3 -m pylint serpapi # test with Python 3 test: - pytest --cov=serpapi --cov-report html tests/*.py + python3 -mpytest --cov=serpapi --cov-report html tests/*.py # install dependencies # @@ -34,12 +34,12 @@ test: # sphinx - documentation # twine - release automation install: - pip3 install -U setuptools - pip3 install -r requirements.txt - pip3 install pylint - pip3 install pytest-cov - pip3 install twine - pip3 install sphinx + python3 -m pip install -U setuptools + python3 -m pip install -r requirements.txt + python3 -m pip install pylint + python3 -m pip install pytest-cov + python3 -m pip install twine + python3 -m pip install sphinx readme: erb -T '-' README.md.erb > README.md @@ -53,17 +53,17 @@ build: # out of box testing / user acceptance before delivery oobt: build - pip3 install ./${dist} + python3 -m pip install ./${dist} python3 oobt/demo.py check: oobt - twine check ${dist} + python3 -m twine check ${dist} release: # check - twine upload ${dist} + python3 -m twine upload ${dist} # run example only # and display output (-s) example: - pytest -s "tests/test_example.py::TestExample::test_async" + python3 -m pytest -s "tests/test_example.py::TestExample::test_async" diff --git a/README.md b/README.md index 9516f01..71aa6ea 100644 --- a/README.md +++ b/README.md @@ -152,11 +152,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'bing', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'bing', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) @@ -164,7 +164,7 @@ pp.pprint(data['organic_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_bing_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_bing_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_bing_test.py] see: [https://serpapi.com/bing-search-api](https://serpapi.com/bing-search-api) ### Search baidu @@ -174,11 +174,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'baidu', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'baidu', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) @@ -186,7 +186,7 @@ pp.pprint(data['organic_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_baidu_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_baidu_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_baidu_test.py] see: [https://serpapi.com/baidu-search-api](https://serpapi.com/baidu-search-api) ### Search yahoo @@ -196,11 +196,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'yahoo', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'yahoo', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'p': 'coffee', + 'p': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) @@ -208,7 +208,7 @@ pp.pprint(data['organic_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_yahoo_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_yahoo_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_yahoo_test.py] see: [https://serpapi.com/yahoo-search-api](https://serpapi.com/yahoo-search-api) ### Search youtube @@ -218,11 +218,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'youtube', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'youtube', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'search_query': 'coffee', + 'search_query': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['video_results']) @@ -230,7 +230,7 @@ pp.pprint(data['video_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_youtube_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_youtube_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_youtube_test.py] see: [https://serpapi.com/youtube-search-api](https://serpapi.com/youtube-search-api) ### Search walmart @@ -240,11 +240,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'walmart', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'walmart', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'query': 'coffee', + 'query': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) @@ -252,7 +252,7 @@ pp.pprint(data['organic_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_walmart_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_walmart_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_walmart_test.py] see: [https://serpapi.com/walmart-search-api](https://serpapi.com/walmart-search-api) ### Search ebay @@ -262,11 +262,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'ebay', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'ebay', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'_nkw': 'coffee', + '_nkw': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) @@ -274,7 +274,7 @@ pp.pprint(data['organic_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_ebay_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_ebay_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_ebay_test.py] see: [https://serpapi.com/ebay-search-api](https://serpapi.com/ebay-search-api) ### Search naver @@ -284,11 +284,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'naver', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'naver', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'query': 'coffee', + 'query': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['ads_results']) @@ -296,7 +296,7 @@ pp.pprint(data['ads_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_naver_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_naver_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_naver_test.py] see: [https://serpapi.com/naver-search-api](https://serpapi.com/naver-search-api) ### Search home depot @@ -306,11 +306,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'home_depot', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'home_depot', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'table', + 'q': 'table', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['products']) @@ -318,7 +318,7 @@ pp.pprint(data['products']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_home_depot_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_home_depot_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_home_depot_test.py] see: [https://serpapi.com/home-depot-search-api](https://serpapi.com/home-depot-search-api) ### Search apple app store @@ -328,11 +328,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'apple_app_store', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'apple_app_store', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'term': 'coffee', + 'term': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) @@ -340,7 +340,7 @@ pp.pprint(data['organic_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_apple_app_store_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_apple_app_store_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_apple_app_store_test.py] see: [https://serpapi.com/apple-app-store](https://serpapi.com/apple-app-store) ### Search duckduckgo @@ -350,11 +350,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'duckduckgo', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'duckduckgo', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) @@ -362,7 +362,7 @@ pp.pprint(data['organic_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_duckduckgo_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_duckduckgo_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_duckduckgo_test.py] see: [https://serpapi.com/duckduckgo-search-api](https://serpapi.com/duckduckgo-search-api) ### Search google @@ -372,12 +372,12 @@ import pprint import os client = serpapi.Client({ -'engine': 'google', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', -'engine': 'google', + 'q': 'coffee', + 'engine': 'google', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) @@ -385,7 +385,7 @@ pp.pprint(data['organic_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_test.py] see: [https://serpapi.com/search-api](https://serpapi.com/search-api) ### Search google scholar @@ -395,11 +395,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'google_scholar', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_scholar', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) @@ -407,7 +407,7 @@ pp.pprint(data['organic_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_scholar_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_scholar_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_scholar_test.py] see: [https://serpapi.com/google-scholar-api](https://serpapi.com/google-scholar-api) ### Search google autocomplete @@ -417,11 +417,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'google_autocomplete', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_autocomplete', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['suggestions']) @@ -429,7 +429,7 @@ pp.pprint(data['suggestions']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_autocomplete_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_autocomplete_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_autocomplete_test.py] see: [https://serpapi.com/google-autocomplete-api](https://serpapi.com/google-autocomplete-api) ### Search google product @@ -439,12 +439,12 @@ import pprint import os client = serpapi.Client({ -'engine': 'google_product', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_product', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', -'product_id': '4172129135583325756', + 'q': 'coffee', + 'product_id': '4172129135583325756', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['product_results']) @@ -452,7 +452,7 @@ pp.pprint(data['product_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_product_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_product_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_product_test.py] see: [https://serpapi.com/google-product-api](https://serpapi.com/google-product-api) ### Search google reverse image @@ -462,11 +462,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'google_reverse_image', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_reverse_image', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'image_url': 'https://i.imgur.com/5bGzZi7.jpg', + 'image_url': 'https://i.imgur.com/5bGzZi7.jpg', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['image_sizes']) @@ -474,7 +474,7 @@ pp.pprint(data['image_sizes']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_reverse_image_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_reverse_image_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_reverse_image_test.py] see: [https://serpapi.com/google-reverse-image](https://serpapi.com/google-reverse-image) ### Search google events @@ -484,11 +484,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'google_events', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_events', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['events_results']) @@ -496,7 +496,7 @@ pp.pprint(data['events_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_events_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_events_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_events_test.py] see: [https://serpapi.com/google-events-api](https://serpapi.com/google-events-api) ### Search google local services @@ -506,12 +506,12 @@ import pprint import os client = serpapi.Client({ -'engine': 'google_local_services', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_local_services', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'electrician', -'data_cid': '6745062158417646970', + 'q': 'electrician', + 'data_cid': '6745062158417646970', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['local_ads']) @@ -519,7 +519,7 @@ pp.pprint(data['local_ads']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_local_services_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_local_services_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_local_services_test.py] see: [https://serpapi.com/google-local-services-api](https://serpapi.com/google-local-services-api) ### Search google maps @@ -529,13 +529,13 @@ import pprint import os client = serpapi.Client({ -'engine': 'google_maps', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_maps', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'pizza', -'ll': '@40.7455096,-74.0083012,15.1z', -'type': 'search', + 'q': 'pizza', + 'll': '@40.7455096,-74.0083012,15.1z', + 'type': 'search', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['local_results']) @@ -543,7 +543,7 @@ pp.pprint(data['local_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_maps_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_maps_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_maps_test.py] see: [https://serpapi.com/google-maps-api](https://serpapi.com/google-maps-api) ### Search google jobs @@ -553,11 +553,11 @@ import pprint import os client = serpapi.Client({ -'engine': 'google_jobs', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_jobs', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'coffee', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['jobs_results']) @@ -565,7 +565,7 @@ pp.pprint(data['jobs_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_jobs_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_jobs_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_jobs_test.py] see: [https://serpapi.com/google-jobs-api](https://serpapi.com/google-jobs-api) ### Search google play @@ -575,12 +575,12 @@ import pprint import os client = serpapi.Client({ -'engine': 'google_play', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_play', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'q': 'kite', -'store': 'apps', + 'q': 'kite', + 'store': 'apps', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['organic_results']) @@ -588,7 +588,7 @@ pp.pprint(data['organic_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_play_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_play_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_play_test.py] see: [https://serpapi.com/google-play-api](https://serpapi.com/google-play-api) ### Search google images @@ -598,13 +598,13 @@ import pprint import os client = serpapi.Client({ -'engine': 'google_images', -'api_key': os.getenv("API_KEY") -}) + 'engine': 'google_images', + 'api_key': os.getenv("API_KEY") + }) data = client.search({ -'engine': 'google_images', -'tbm': 'isch', -'q': 'coffee', + 'engine': 'google_images', + 'tbm': 'isch', + 'q': 'coffee', }) pp = pprint.PrettyPrinter(indent=2) pp.pprint(data['images_results']) @@ -612,7 +612,7 @@ pp.pprint(data['images_results']) # copy/paste from [http://serpapi.com/dashboard] to your bash # ```export API_KEY="your_secure_api_key"``` ``` - * test: [/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_images_test.py](/Users/victor/Project/serpapi/serpapi-python/tests/example_search_google_images_test.py) +test: [https://github.com/serpapi/serpapi-python/tests/example_search_google_images_test.py] see: [https://serpapi.com/images-results](https://serpapi.com/images-results) diff --git a/README.md.erb b/README.md.erb index 7fb7ab5..624159f 100644 --- a/README.md.erb +++ b/README.md.erb @@ -3,10 +3,11 @@ def snippet(format, path) lines = File.new(path).readlines stop = lines.size - 1 slice = lines[9..stop] - slice.reject! { |l| l =~ /self.assertIsNone\(/ } - buf = slice.map { |l| l.gsub(/(^\s+)/, '')}.join + slice.reject! { |l| l.match?(/self.assertIsNone\(/) } + buf = slice.map { |l| l.gsub(/(^\s{4})/, '').gsub(/^\s*$/, '') }.join + url = path.gsub(/^.*\/serpapi-python/, 'https://github.com/serpapi/serpapi-python') buf.gsub!('self.assertIsNotNone(', "pp = pprint.PrettyPrinter(indent=2)\npp.pprint(") - %Q(```#{format}\nimport serpapi\nimport pprint\nimport os\n\n#{buf}```\n * test: [#{path}](#{path})) + %Q(```#{format}\nimport serpapi\nimport pprint\nimport os\n\n#{buf}```\ntest: [#{url}]) end -%> diff --git a/serpapi/serpapi.py b/serpapi/serpapi.py index 626bd8c..c85d4e7 100644 --- a/serpapi/serpapi.py +++ b/serpapi/serpapi.py @@ -7,8 +7,8 @@ class HttpClient: """Simple HTTP client wrapper around urllib3""" - BACKEND = 'https://serpapi.com' - SUPPORTED_DECODER = ['json', 'html'] + BACKEND = "https://serpapi.com" + SUPPORTED_DECODER = ["json", "html"] def __init__(self, parameter: dict = None): """Initialize a SerpApi Client with the default parameters provided. @@ -26,21 +26,21 @@ def __init__(self, parameter: dict = None): # urllib3 configurations # HTTP connect timeout - if 'timeout' in parameter: - self.timeout = parameter['timeout'] - del parameter['timeout'] + if "timeout" in parameter: + self.timeout = parameter["timeout"] + del parameter["timeout"] else: # 60s default self.timeout = 60.0 # no HTTP retry - if 'retries' in parameter: - self.retries = parameter['retries'] - del parameter['retries'] + if "retries" in parameter: + self.retries = parameter["retries"] + del parameter["retries"] else: self.retries = False - def start(self, path: str, parameter: dict = None, decoder: str = 'json'): + def start(self, path: str, parameter: dict = None, decoder: str = "json"): """start HTTP request and decode response using urllib3. The response is decoded using the selected decoder: - html: raw HTML response @@ -62,15 +62,15 @@ def start(self, path: str, parameter: dict = None, decoder: str = 'json'): dict|str decoded HTTP response""" # track client language - self.parameter['source'] = 'serpapi-python:' + __version__ - self.parameter['output'] = decoder + self.parameter["source"] = "serpapi-python:" + __version__ + self.parameter["output"] = decoder # merge parameter defaults and overrides fields = self.parameter.copy() fields.update(parameter) # execute HTTP get request - response = self.http.request('GET', + response = self.http.request("GET", self.BACKEND + path, fields=fields, timeout=self.timeout, @@ -83,24 +83,23 @@ def decode(self, response: any, decoder: str): # handle HTTP error if response.status != 200: try: - raw = response.data.decode('utf-8') + raw = response.data.decode("utf-8") payload = json.loads(raw) - raise SerpApiException(payload['error']) + raise SerpApiException(payload["error"]) except Exception as ex: raise SerpApiException(raw) from ex # HTTP success 200 - payload = response.data.decode('utf-8') + payload = response.data.decode("utf-8") # successful response decoding - if decoder == 'json': + if decoder == "json": return json.loads(payload) - if decoder == 'html': + if decoder == "html": return payload - raise SerpApiException("Invalid decoder: " + - decoder + ", available: json, html") + raise SerpApiException(f"Invalid decoder: {decoder}, available: json, html") class Client(HttpClient): @@ -121,10 +120,10 @@ def __init__(self, parameter: dict = None): # initialize HTTP client HttpClient.__init__(self, parameter) - def search(self, parameter: dict = None, decoder: str = 'json'): + def search(self, parameter: dict = None, decoder: str = "json"): """ make search then decode the output - decoder supported 'json', 'html' + decoder supported "json", "html" Parameters ---------- @@ -137,10 +136,10 @@ def search(self, parameter: dict = None, decoder: str = 'json'): ------- dict|str search results returns as : - dict if decoder = 'json' - str if decoder = 'html' + dict if decoder = "json" + str if decoder = "html" """ - return self.start(path='/search', parameter=parameter, decoder=decoder) + return self.start(path="/search", parameter=parameter, decoder=decoder) def html(self, parameter: dict = None): """ @@ -156,7 +155,7 @@ def html(self, parameter: dict = None): str raw html search results directly from the search engine """ - return self.start('/search', parameter, 'html') + return self.start("/search", parameter, "html") def location(self, parameter: dict = None): """ @@ -173,9 +172,9 @@ def location(self, parameter: dict = None): array list of matching locations """ - return self.start('/locations.json', parameter, 'json') + return self.start("/locations.json", parameter, "json") - def search_archive(self, search_id: str, decoder: str = 'json'): + def search_archive(self, search_id: str, decoder: str = "json"): """ Retrieve search results from the Search Archive API @@ -189,7 +188,7 @@ def search_archive(self, search_id: str, decoder: str = 'json'): if decoder in self.SUPPORTED_DECODER: path += decoder else: - raise SerpApiException(f'Invalid decoder: {decoder} must be json or html. ') + raise SerpApiException(f"Invalid decoder: {decoder}, available: json, html. ") return self.start(path, {}, decoder) def account(self, api_key: str = None): @@ -207,5 +206,5 @@ def account(self, api_key: str = None): user account information """ if api_key is not None: - self.parameter['api_key'] = api_key - return self.start('/account', self.parameter, 'json') + self.parameter["api_key"] = api_key + return self.start("/account", self.parameter, "json")