diff --git a/README.md b/README.md index 954e17e..eadb562 100644 --- a/README.md +++ b/README.md @@ -62,19 +62,20 @@ Main class of this library. https://docs.scrapingant.com/request-response-format#available-parameters -| Param | Type | Default | -|-------------------|----------------------------------------------------------------------------------------------------------------------------|------------| -| url | string | | -| method | string | GET | -| cookies | List[Cookie] | None | -| headers | List[Dict[str, str]] | None | -| js_snippet | string | None | -| proxy_type | ProxyType | datacenter | -| proxy_country | str | None | -| wait_for_selector | str | None | -| browser | boolean | True | -| data | same as [requests param 'data'](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) | None | -| json | same as [requests param 'json'](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) | None | +| Param | Type | Default | +|---------------------|----------------------------------------------------------------------------------------------------------------------------|------------| +| url | string | | +| method | string | GET | +| cookies | List[Cookie] | None | +| headers | List[Dict[str, str]] | None | +| js_snippet | string | None | +| proxy_type | ProxyType | datacenter | +| proxy_country | str | None | +| wait_for_selector | str | None | +| browser | boolean | True | +| return_page_source | boolean | False | +| data | same as [requests param 'data'](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) | None | +| json | same as [requests param 'json'](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) | None | **IMPORTANT NOTE:** js_snippet will be encoded to Base64 automatically by the ScrapingAnt client library. diff --git a/scrapingant_client/__init__.py b/scrapingant_client/__init__.py index 70920d3..337a075 100644 --- a/scrapingant_client/__init__.py +++ b/scrapingant_client/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.0.0" +__version__ = "2.0.1" from scrapingant_client.client import ScrapingAntClient from scrapingant_client.cookie import Cookie diff --git a/scrapingant_client/client.py b/scrapingant_client/client.py index 6e12e03..1c9536e 100644 --- a/scrapingant_client/client.py +++ b/scrapingant_client/client.py @@ -41,6 +41,7 @@ def _form_payload( proxy_country: Optional[str] = None, wait_for_selector: Optional[str] = None, browser: bool = True, + return_page_source: Optional[bool] = None, ) -> Dict: request_data = {'url': url} if cookies is not None: @@ -54,6 +55,9 @@ def _form_payload( if wait_for_selector is not None: request_data['wait_for_selector'] = wait_for_selector request_data['browser'] = browser + if return_page_source: + assert browser, 'return_page_source can only be used with browser=True' + request_data['return_page_source'] = return_page_source return request_data def _parse_response(self, response_status_code: int, response_data: Dict, url: str) -> Response: @@ -90,6 +94,7 @@ def general_request( proxy_country: Optional[str] = None, wait_for_selector: Optional[str] = None, browser: bool = True, + return_page_source: Optional[bool] = None, data=None, json=None, ) -> Response: @@ -101,6 +106,7 @@ def general_request( proxy_country=proxy_country, wait_for_selector=wait_for_selector, browser=browser, + return_page_source=return_page_source, ) try: response = self.requests_session.request( @@ -129,6 +135,7 @@ async def general_request_async( proxy_country: Optional[str] = None, wait_for_selector: Optional[str] = None, browser: bool = True, + return_page_source: Optional[bool] = None, data=None, json=None, ) -> Response: @@ -142,6 +149,7 @@ async def general_request_async( proxy_country=proxy_country, wait_for_selector=wait_for_selector, browser=browser, + return_page_source=return_page_source, ) async with httpx.AsyncClient( headers={ diff --git a/tests/test_integration.py b/tests/test_integration.py index ace4b6c..b870e8e 100644 --- a/tests/test_integration.py +++ b/tests/test_integration.py @@ -21,7 +21,8 @@ def test_integration(): '&proxy_type=datacenter' '&proxy_country=test_country' '&wait_for_selector=test_selector' - '&browser=True', + '&browser=True' + '&return_page_source=True', json={ "html": "test_content", "cookies": "test_key1=test_value1;test_key2=test_value2", @@ -40,6 +41,7 @@ def test_integration(): proxy_country='test_country', wait_for_selector='test_selector', browser=True, + return_page_source=True, ) expected = { 'content': 'test_content', @@ -68,7 +70,8 @@ async def test_integration_async(httpx_mock: HTTPXMock): '&proxy_type=datacenter' '&proxy_country=test_country' '&wait_for_selector=test_selector' - '&browser=true', + '&browser=true' + '&return_page_source=true', json={ "html": "test_content", "cookies": "test_key1=test_value1;test_key2=test_value2", @@ -87,6 +90,7 @@ async def test_integration_async(httpx_mock: HTTPXMock): proxy_country='test_country', wait_for_selector='test_selector', browser=True, + return_page_source=True, ) expected = { 'content': 'test_content',