From 39e2707f71a676563f63b0ac217076e489716cac Mon Sep 17 00:00:00 2001 From: olegkulyk Date: Mon, 20 Sep 2021 22:40:29 +0300 Subject: [PATCH 1/3] #18: residential proxies support --- README.md | 3 ++- scrapingant_client/__init__.py | 4 +++- scrapingant_client/client.py | 4 ++++ scrapingant_client/proxy_type.py | 6 ++++++ 4 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 scrapingant_client/proxy_type.py diff --git a/README.md b/README.md index 8c016e8..58f7456 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ https://docs.scrapingant.com/request-response-format#available-parameters | url | string | | | cookies | List[Cookie] | None | | js_snippet | string | None | +| proxy_type | ProxyType | datacenter | | proxy_country | str | None | | return_text | boolean | False | | wait_for_selector | str | None | @@ -61,7 +62,7 @@ https://docs.scrapingant.com/request-response-format#available-parameters * * * #### Cookie -Class defining cookie. Curently supports only name and value +Class defining cookie. Currently it supports only name and value | Param | Type | | --- | --- | diff --git a/scrapingant_client/__init__.py b/scrapingant_client/__init__.py index 4257100..32c2864 100644 --- a/scrapingant_client/__init__.py +++ b/scrapingant_client/__init__.py @@ -1,4 +1,4 @@ -__version__ = "0.3.6" +__version__ = "0.3.7" from scrapingant_client.client import ScrapingAntClient from scrapingant_client.cookie import Cookie @@ -10,11 +10,13 @@ ScrapingantSiteNotReachableException, ScrapingantDetectedException, ) +from scrapingant_client.proxy_type import ProxyType from scrapingant_client.response import Response __all__ = [ 'ScrapingAntClient', 'Cookie', + 'ProxyType', 'ScrapingantClientException', 'ScrapingantInvalidTokenException', 'ScrapingantInvalidInputException', diff --git a/scrapingant_client/client.py b/scrapingant_client/client.py index ac2309e..267df1d 100644 --- a/scrapingant_client/client.py +++ b/scrapingant_client/client.py @@ -14,6 +14,7 @@ ScrapingantSiteNotReachableException, ScrapingantDetectedException, ) +from scrapingant_client.proxy_type import ProxyType from scrapingant_client.response import Response from scrapingant_client.utils import base64_encode_string @@ -34,6 +35,7 @@ def general_request( url: str, cookies: Optional[List[Cookie]] = None, js_snippet: Optional[str] = None, + proxy_type: Optional[ProxyType] = None, proxy_country: Optional[str] = None, return_text: bool = False, wait_for_selector: Optional[str] = None, @@ -45,6 +47,8 @@ def general_request( if js_snippet is not None: encoded_js_snippet = base64_encode_string(js_snippet) request_data['js_snippet'] = encoded_js_snippet + if proxy_type is not None: + request_data['proxy_type'] = proxy_type if proxy_country is not None: request_data['proxy_country'] = proxy_country.lower() if wait_for_selector is not None: diff --git a/scrapingant_client/proxy_type.py b/scrapingant_client/proxy_type.py new file mode 100644 index 0000000..2a66c6f --- /dev/null +++ b/scrapingant_client/proxy_type.py @@ -0,0 +1,6 @@ +from enum import Enum + + +class ProxyType(str, Enum): + datacenter = 'datacenter' + residential = 'residential' From f2dda37ec74957594f8b50ba49ce23c69746545a Mon Sep 17 00:00:00 2001 From: olegkulyk Date: Mon, 20 Sep 2021 22:43:08 +0300 Subject: [PATCH 2/3] #18: explicit datacenter proxy type --- scrapingant_client/client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scrapingant_client/client.py b/scrapingant_client/client.py index 267df1d..29d1a48 100644 --- a/scrapingant_client/client.py +++ b/scrapingant_client/client.py @@ -35,7 +35,7 @@ def general_request( url: str, cookies: Optional[List[Cookie]] = None, js_snippet: Optional[str] = None, - proxy_type: Optional[ProxyType] = None, + proxy_type: Optional[ProxyType] = ProxyType.datacenter, proxy_country: Optional[str] = None, return_text: bool = False, wait_for_selector: Optional[str] = None, From 343018e7b4c8c471bff80f77171f8046974bc282 Mon Sep 17 00:00:00 2001 From: "andrii.kovalenko" Date: Mon, 20 Sep 2021 23:09:49 +0300 Subject: [PATCH 3/3] feature/residential_proxy_support: minor typing fixes --- scrapingant_client/client.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scrapingant_client/client.py b/scrapingant_client/client.py index 29d1a48..8aae10a 100644 --- a/scrapingant_client/client.py +++ b/scrapingant_client/client.py @@ -35,7 +35,7 @@ def general_request( url: str, cookies: Optional[List[Cookie]] = None, js_snippet: Optional[str] = None, - proxy_type: Optional[ProxyType] = ProxyType.datacenter, + proxy_type: ProxyType = ProxyType.datacenter, proxy_country: Optional[str] = None, return_text: bool = False, wait_for_selector: Optional[str] = None, @@ -47,8 +47,7 @@ def general_request( if js_snippet is not None: encoded_js_snippet = base64_encode_string(js_snippet) request_data['js_snippet'] = encoded_js_snippet - if proxy_type is not None: - request_data['proxy_type'] = proxy_type + request_data['proxy_type'] = proxy_type if proxy_country is not None: request_data['proxy_country'] = proxy_country.lower() if wait_for_selector is not None: