diff --git a/README.md b/README.md
index 954e17e..eadb562 100644
--- a/README.md
+++ b/README.md
@@ -62,19 +62,20 @@ Main class of this library.
https://docs.scrapingant.com/request-response-format#available-parameters
-| Param | Type | Default |
-|-------------------|----------------------------------------------------------------------------------------------------------------------------|------------|
-| url | string
| |
-| method | string
| GET |
-| cookies | List[Cookie]
| None |
-| headers | List[Dict[str, str]]
| None |
-| js_snippet | string
| None |
-| proxy_type | ProxyType
| datacenter |
-| proxy_country | str
| None |
-| wait_for_selector | str
| None |
-| browser | boolean
| True |
-| data | same as [requests param 'data'](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) | None |
-| json | same as [requests param 'json'](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) | None |
+| Param | Type | Default |
+|---------------------|----------------------------------------------------------------------------------------------------------------------------|------------|
+| url | string
| |
+| method | string
| GET |
+| cookies | List[Cookie]
| None |
+| headers | List[Dict[str, str]]
| None |
+| js_snippet | string
| None |
+| proxy_type | ProxyType
| datacenter |
+| proxy_country | str
| None |
+| wait_for_selector | str
| None |
+| browser | boolean
| True |
+| return_page_source | boolean
| False |
+| data | same as [requests param 'data'](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) | None |
+| json | same as [requests param 'json'](https://requests.readthedocs.io/en/latest/user/quickstart/#more-complicated-post-requests) | None |
**IMPORTANT NOTE:** js_snippet
will be encoded to Base64 automatically by the ScrapingAnt client library.
diff --git a/scrapingant_client/__init__.py b/scrapingant_client/__init__.py
index 70920d3..337a075 100644
--- a/scrapingant_client/__init__.py
+++ b/scrapingant_client/__init__.py
@@ -1,4 +1,4 @@
-__version__ = "2.0.0"
+__version__ = "2.0.1"
from scrapingant_client.client import ScrapingAntClient
from scrapingant_client.cookie import Cookie
diff --git a/scrapingant_client/client.py b/scrapingant_client/client.py
index 6e12e03..1c9536e 100644
--- a/scrapingant_client/client.py
+++ b/scrapingant_client/client.py
@@ -41,6 +41,7 @@ def _form_payload(
proxy_country: Optional[str] = None,
wait_for_selector: Optional[str] = None,
browser: bool = True,
+ return_page_source: Optional[bool] = None,
) -> Dict:
request_data = {'url': url}
if cookies is not None:
@@ -54,6 +55,9 @@ def _form_payload(
if wait_for_selector is not None:
request_data['wait_for_selector'] = wait_for_selector
request_data['browser'] = browser
+ if return_page_source:
+ assert browser, 'return_page_source can only be used with browser=True'
+ request_data['return_page_source'] = return_page_source
return request_data
def _parse_response(self, response_status_code: int, response_data: Dict, url: str) -> Response:
@@ -90,6 +94,7 @@ def general_request(
proxy_country: Optional[str] = None,
wait_for_selector: Optional[str] = None,
browser: bool = True,
+ return_page_source: Optional[bool] = None,
data=None,
json=None,
) -> Response:
@@ -101,6 +106,7 @@ def general_request(
proxy_country=proxy_country,
wait_for_selector=wait_for_selector,
browser=browser,
+ return_page_source=return_page_source,
)
try:
response = self.requests_session.request(
@@ -129,6 +135,7 @@ async def general_request_async(
proxy_country: Optional[str] = None,
wait_for_selector: Optional[str] = None,
browser: bool = True,
+ return_page_source: Optional[bool] = None,
data=None,
json=None,
) -> Response:
@@ -142,6 +149,7 @@ async def general_request_async(
proxy_country=proxy_country,
wait_for_selector=wait_for_selector,
browser=browser,
+ return_page_source=return_page_source,
)
async with httpx.AsyncClient(
headers={
diff --git a/tests/test_integration.py b/tests/test_integration.py
index ace4b6c..b870e8e 100644
--- a/tests/test_integration.py
+++ b/tests/test_integration.py
@@ -21,7 +21,8 @@ def test_integration():
'&proxy_type=datacenter'
'&proxy_country=test_country'
'&wait_for_selector=test_selector'
- '&browser=True',
+ '&browser=True'
+ '&return_page_source=True',
json={
"html": "test_content",
"cookies": "test_key1=test_value1;test_key2=test_value2",
@@ -40,6 +41,7 @@ def test_integration():
proxy_country='test_country',
wait_for_selector='test_selector',
browser=True,
+ return_page_source=True,
)
expected = {
'content': 'test_content',
@@ -68,7 +70,8 @@ async def test_integration_async(httpx_mock: HTTPXMock):
'&proxy_type=datacenter'
'&proxy_country=test_country'
'&wait_for_selector=test_selector'
- '&browser=true',
+ '&browser=true'
+ '&return_page_source=true',
json={
"html": "test_content",
"cookies": "test_key1=test_value1;test_key2=test_value2",
@@ -87,6 +90,7 @@ async def test_integration_async(httpx_mock: HTTPXMock):
proxy_country='test_country',
wait_for_selector='test_selector',
browser=True,
+ return_page_source=True,
)
expected = {
'content': 'test_content',