Skip to content

Commit 2865690

Browse files
mdmintzmdmintz
committed
Add option to support multiple proxies with auth in parallel
1 parent ec5bb9c commit 2865690

11 files changed

+150
-35
lines changed

‎README.md

+6-5
Original file line numberDiff line numberDiff line change
@@ -499,8 +499,6 @@ pytest my_first_test.py --pdb
499499
--edge # (Shortcut for "--browser=edge".)
500500
--firefox # (Shortcut for "--browser=firefox".)
501501
--safari # (Shortcut for "--browser=safari".)
502-
--cap-file=FILE # (The web browser's desired capabilities to use.)
503-
--cap-string=STRING # (The web browser's desired capabilities to use.)
504502
--settings-file=FILE # (Override default SeleniumBase settings.)
505503
--env=ENV # (Set the test env. Access with "self.env" in tests.)
506504
--account=STR # (Set account. Access with "self.account" in tests.)
@@ -513,11 +511,14 @@ pytest my_first_test.py --pdb
513511
--protocol=PROTOCOL # (The Selenium Grid protocol: http|https.)
514512
--server=SERVER # (The Selenium Grid server/IP used for tests.)
515513
--port=PORT # (The Selenium Grid port used by the test server.)
516-
--proxy=SERVER:PORT # (Connect to a proxy server:port for tests.)
517-
--proxy=USERNAME:PASSWORD@SERVER:PORT # (Use authenticated proxy server.)
518-
--proxy-bypass-list=STRING # (";"-separated hosts to bypass, Eg "*.foo.com")
514+
--cap-file=FILE # (The web browser's desired capabilities to use.)
515+
--cap-string=STRING # (The web browser's desired capabilities to use.)
516+
--proxy=SERVER:PORT # (Connect to a proxy server:port as tests are running)
517+
--proxy=USERNAME:PASSWORD@SERVER:PORT # (Use an authenticated proxy server)
518+
--proxy-bypass-list=STRING # (";"-separated hosts to bypass, Eg "*.foo.com")
519519
--proxy-pac-url=URL # (Connect to a proxy server using a PAC_URL.pac file.)
520520
--proxy-pac-url=USERNAME:PASSWORD@URL # (Authenticated proxy with PAC URL.)
521+
--multi-proxy # (Allow multiple authenticated proxies when multi-threaded.)
521522
--agent=STRING # (Modify the web browser's User-Agent string.)
522523
--mobile # (Use the mobile device emulator while running tests.)
523524
--metrics=STRING # (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".)

‎examples/raw_parameter_script.py

+1
Original file line numberDiff line numberDiff line change
@@ -116,6 +116,7 @@
116116
sb.proxy_string = None
117117
sb.proxy_bypass_list = None
118118
sb.proxy_pac_url = None
119+
sb.multi_proxy = False
119120
sb.swiftshader = False
120121
sb.ad_block_on = False
121122
sb.highlights = None

‎help_docs/customizing_test_runs.md

+6-5
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,6 @@ pytest my_first_test.py --settings-file=custom_settings.py
105105
--edge # (Shortcut for "--browser=edge".)
106106
--firefox # (Shortcut for "--browser=firefox".)
107107
--safari # (Shortcut for "--browser=safari".)
108-
--cap-file=FILE # (The web browser's desired capabilities to use.)
109-
--cap-string=STRING # (The web browser's desired capabilities to use.)
110108
--settings-file=FILE # (Override default SeleniumBase settings.)
111109
--env=ENV # (Set the test env. Access with "self.env" in tests.)
112110
--account=STR # (Set account. Access with "self.account" in tests.)
@@ -119,11 +117,14 @@ pytest my_first_test.py --settings-file=custom_settings.py
119117
--protocol=PROTOCOL # (The Selenium Grid protocol: http|https.)
120118
--server=SERVER # (The Selenium Grid server/IP used for tests.)
121119
--port=PORT # (The Selenium Grid port used by the test server.)
122-
--proxy=SERVER:PORT # (Connect to a proxy server:port for tests.)
123-
--proxy=USERNAME:PASSWORD@SERVER:PORT # (Use authenticated proxy server.)
124-
--proxy-bypass-list=STRING # (";"-separated hosts to bypass, Eg "*.foo.com")
120+
--cap-file=FILE # (The web browser's desired capabilities to use.)
121+
--cap-string=STRING # (The web browser's desired capabilities to use.)
122+
--proxy=SERVER:PORT # (Connect to a proxy server:port as tests are running)
123+
--proxy=USERNAME:PASSWORD@SERVER:PORT # (Use an authenticated proxy server)
124+
--proxy-bypass-list=STRING # (";"-separated hosts to bypass, Eg "*.foo.com")
125125
--proxy-pac-url=URL # (Connect to a proxy server using a PAC_URL.pac file.)
126126
--proxy-pac-url=USERNAME:PASSWORD@URL # (Authenticated proxy with PAC URL.)
127+
--multi-proxy # (Allow multiple authenticated proxies when multi-threaded.)
127128
--agent=STRING # (Modify the web browser's User-Agent string.)
128129
--mobile # (Use the mobile device emulator while running tests.)
129130
--metrics=STRING # (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".)

‎seleniumbase/behave/behave_sb.py

+10-3
Original file line numberDiff line numberDiff line change
@@ -25,11 +25,12 @@
2525
-D port=PORT (The Selenium Grid port used by the test server.)
2626
-D cap-file=FILE (The web browser's desired capabilities to use.)
2727
-D cap-string=STRING (The web browser's desired capabilities to use.)
28-
-D proxy=SERVER:PORT (Connect to a proxy server:port for tests.)
29-
-D proxy=USERNAME:PASSWORD@SERVER:PORT (Use authenticated proxy server.)
28+
-D proxy=SERVER:PORT (Connect to a proxy server:port as tests are running)
29+
-D proxy=USERNAME:PASSWORD@SERVER:PORT (Use an authenticated proxy server)
3030
-D proxy-bypass-list=STRING (";"-separated hosts to bypass, Eg "*.foo.com")
3131
-D proxy-pac-url=URL (Connect to a proxy server using a PAC_URL.pac file.)
3232
-D proxy-pac-url=USERNAME:PASSWORD@URL (Authenticated proxy with PAC URL.)
33+
-D multi-proxy (Allow multiple authenticated proxies when multi-threaded.)
3334
-D agent=STRING (Modify the web browser's User-Agent string.)
3435
-D mobile (Use the mobile device emulator while running tests.)
3536
-D metrics=STRING (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".)
@@ -223,6 +224,7 @@ def get_configured_sb(context):
223224
sb.proxy_string = None
224225
sb.proxy_bypass_list = None
225226
sb.proxy_pac_url = None
227+
sb.multi_proxy = False
226228
sb.enable_3d_apis = False
227229
sb.swiftshader = False
228230
sb.ad_block_on = False
@@ -746,6 +748,10 @@ def get_configured_sb(context):
746748
proxy_pac_url = sb.proxy_pac_url # revert to default
747749
sb.proxy_pac_url = proxy_pac_url
748750
continue
751+
# Handle: -D multi-proxy / multi_proxy
752+
if low_key in ["multi-proxy", "multi_proxy"]:
753+
sb.multi_proxy = True
754+
continue
749755
# Handle: -D enable-3d-apis / enable_3d_apis
750756
if low_key in ["enable-3d-apis", "enable_3d_apis"]:
751757
sb.enable_3d_apis = True
@@ -1121,7 +1127,8 @@ def _perform_behave_unconfigure_():
11211127
from seleniumbase.core import log_helper
11221128
from seleniumbase.core import proxy_helper
11231129

1124-
proxy_helper.remove_proxy_zip_if_present()
1130+
if hasattr(sb_config, "multi_proxy") and not sb_config.multi_proxy:
1131+
proxy_helper.remove_proxy_zip_if_present()
11251132
if hasattr(sb_config, "reuse_session") and sb_config.reuse_session:
11261133
# Close the shared browser session
11271134
if sb_config.shared_driver:

‎seleniumbase/core/browser_launcher.py

+79-14
Original file line numberDiff line numberDiff line change
@@ -295,45 +295,74 @@ def _was_driver_repaired():
295295
return os.path.exists(file_path)
296296

297297

298+
def _set_proxy_filenames():
299+
DOWNLOADS_DIR = constants.Files.DOWNLOADS_FOLDER
300+
for num in range(1000):
301+
PROXY_ZIP_PATH = os.path.join(DOWNLOADS_DIR, "proxy_%s.zip" % num)
302+
PROXY_DIR_PATH = os.path.join(DOWNLOADS_DIR, "proxy_ext_dir_%s" % num)
303+
if os.path.exists(PROXY_ZIP_PATH) or os.path.exists(PROXY_DIR_PATH):
304+
continue
305+
proxy_helper.PROXY_ZIP_PATH = PROXY_ZIP_PATH
306+
proxy_helper.PROXY_DIR_PATH = PROXY_DIR_PATH
307+
return
308+
# Exceeded upper bound. Use Defaults:
309+
PROXY_ZIP_PATH = os.path.join(DOWNLOADS_DIR, "proxy.zip")
310+
PROXY_DIR_PATH = os.path.join(DOWNLOADS_DIR, "proxy_ext_dir")
311+
proxy_helper.PROXY_ZIP_PATH = PROXY_ZIP_PATH
312+
proxy_helper.PROXY_DIR_PATH = PROXY_DIR_PATH
313+
314+
298315
def _add_chrome_proxy_extension(
299-
chrome_options, proxy_string, proxy_user, proxy_pass, zip_it=True
316+
chrome_options,
317+
proxy_string,
318+
proxy_user,
319+
proxy_pass,
320+
zip_it=True,
321+
multi_proxy=False,
300322
):
301323
"""Implementation of https://stackoverflow.com/a/35293284 for
302324
https://stackoverflow.com/questions/12848327/
303325
(Run Selenium on a proxy server that requires authentication.)"""
304326
arg_join = " ".join(sys.argv)
305-
if not ("-n" in sys.argv or " -n=" in arg_join or arg_join == "-c"):
327+
if (
328+
not ("-n" in sys.argv or " -n=" in arg_join or arg_join == "-c")
329+
and not multi_proxy
330+
):
306331
# Single-threaded
307332
if zip_it:
308333
proxy_helper.create_proxy_ext(proxy_string, proxy_user, proxy_pass)
309-
proxy_zip = PROXY_ZIP_PATH
334+
proxy_zip = proxy_helper.PROXY_ZIP_PATH
310335
chrome_options.add_extension(proxy_zip)
311336
else:
312337
proxy_helper.create_proxy_ext(
313338
proxy_string, proxy_user, proxy_pass, zip_it=False
314339
)
315-
chrome_options = add_chrome_ext_dir(chrome_options, PROXY_DIR_PATH)
316-
340+
proxy_dir_path = proxy_helper.PROXY_DIR_PATH
341+
chrome_options = add_chrome_ext_dir(chrome_options, proxy_dir_path)
317342
else:
318-
# Pytest multithreaded test
343+
# Multi-threaded
319344
if zip_it:
320345
proxy_zip_lock = fasteners.InterProcessLock(PROXY_ZIP_LOCK)
321346
with proxy_zip_lock:
322-
if not os.path.exists(PROXY_ZIP_PATH):
347+
if multi_proxy:
348+
_set_proxy_filenames()
349+
if not os.path.exists(proxy_helper.PROXY_ZIP_PATH):
323350
proxy_helper.create_proxy_ext(
324351
proxy_string, proxy_user, proxy_pass
325352
)
326-
proxy_zip = PROXY_ZIP_PATH
353+
proxy_zip = proxy_helper.PROXY_ZIP_PATH
327354
chrome_options.add_extension(proxy_zip)
328355
else:
329356
proxy_dir_lock = fasteners.InterProcessLock(PROXY_DIR_LOCK)
330357
with proxy_dir_lock:
331-
if not os.path.exists(PROXY_DIR_PATH):
358+
if multi_proxy:
359+
_set_proxy_filenames()
360+
if not os.path.exists(proxy_helper.PROXY_DIR_PATH):
332361
proxy_helper.create_proxy_ext(
333362
proxy_string, proxy_user, proxy_pass, False
334363
)
335364
chrome_options = add_chrome_ext_dir(
336-
chrome_options, PROXY_DIR_PATH
365+
chrome_options, proxy_helper.PROXY_DIR_PATH
337366
)
338367
return chrome_options
339368

@@ -406,6 +435,7 @@ def _set_chrome_options(
406435
proxy_pass,
407436
proxy_bypass_list,
408437
proxy_pac_url,
438+
multi_proxy,
409439
user_agent,
410440
recorder_ext,
411441
disable_js,
@@ -656,7 +686,12 @@ def _set_chrome_options(
656686
if is_using_uc(undetectable, browser_name):
657687
zip_it = False # undetected-chromedriver needs a folder ext
658688
chrome_options = _add_chrome_proxy_extension(
659-
chrome_options, proxy_string, proxy_user, proxy_pass, zip_it
689+
chrome_options,
690+
proxy_string,
691+
proxy_user,
692+
proxy_pass,
693+
zip_it,
694+
multi_proxy,
660695
)
661696
chrome_options.add_argument("--proxy-server=%s" % proxy_string)
662697
if proxy_bypass_list:
@@ -669,7 +704,12 @@ def _set_chrome_options(
669704
if is_using_uc(undetectable, browser_name):
670705
zip_it = False
671706
chrome_options = _add_chrome_proxy_extension(
672-
chrome_options, None, proxy_user, proxy_pass, zip_it
707+
chrome_options,
708+
None,
709+
proxy_user,
710+
proxy_pass,
711+
zip_it,
712+
multi_proxy,
673713
)
674714
chrome_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url)
675715
if browser_name != constants.Browser.OPERA:
@@ -1000,6 +1040,7 @@ def get_driver(
10001040
proxy_string=None,
10011041
proxy_bypass_list=None,
10021042
proxy_pac_url=None,
1043+
multi_proxy=None,
10031044
user_agent=None,
10041045
cap_file=None,
10051046
cap_string=None,
@@ -1205,6 +1246,7 @@ def get_driver(
12051246
proxy_pass,
12061247
proxy_bypass_list,
12071248
proxy_pac_url,
1249+
multi_proxy,
12081250
user_agent,
12091251
cap_file,
12101252
cap_string,
@@ -1257,6 +1299,7 @@ def get_driver(
12571299
proxy_pass,
12581300
proxy_bypass_list,
12591301
proxy_pac_url,
1302+
multi_proxy,
12601303
user_agent,
12611304
recorder_ext,
12621305
disable_js,
@@ -1309,6 +1352,7 @@ def get_remote_driver(
13091352
proxy_pass,
13101353
proxy_bypass_list,
13111354
proxy_pac_url,
1355+
multi_proxy,
13121356
user_agent,
13131357
cap_file,
13141358
cap_string,
@@ -1430,6 +1474,7 @@ def get_remote_driver(
14301474
proxy_pass,
14311475
proxy_bypass_list,
14321476
proxy_pac_url,
1477+
multi_proxy,
14331478
user_agent,
14341479
recorder_ext,
14351480
disable_js,
@@ -1618,6 +1663,7 @@ def get_remote_driver(
16181663
proxy_pass,
16191664
proxy_bypass_list,
16201665
proxy_pac_url,
1666+
multi_proxy,
16211667
user_agent,
16221668
recorder_ext,
16231669
disable_js,
@@ -1738,6 +1784,7 @@ def get_remote_driver(
17381784
proxy_pass,
17391785
proxy_bypass_list,
17401786
proxy_pac_url,
1787+
multi_proxy,
17411788
user_agent,
17421789
recorder_ext,
17431790
disable_js,
@@ -1856,6 +1903,7 @@ def get_local_driver(
18561903
proxy_pass,
18571904
proxy_bypass_list,
18581905
proxy_pac_url,
1906+
multi_proxy,
18591907
user_agent,
18601908
recorder_ext,
18611909
disable_js,
@@ -2389,7 +2437,12 @@ def get_local_driver(
23892437
if proxy_string:
23902438
if proxy_auth:
23912439
edge_options = _add_chrome_proxy_extension(
2392-
edge_options, proxy_string, proxy_user, proxy_pass
2440+
edge_options,
2441+
proxy_string,
2442+
proxy_user,
2443+
proxy_pass,
2444+
zip_it=True,
2445+
multi_proxy=multi_proxy,
23932446
)
23942447
edge_options.add_argument("--proxy-server=%s" % proxy_string)
23952448
if proxy_bypass_list:
@@ -2399,7 +2452,12 @@ def get_local_driver(
23992452
elif proxy_pac_url:
24002453
if proxy_auth:
24012454
edge_options = _add_chrome_proxy_extension(
2402-
edge_options, None, proxy_user, proxy_pass
2455+
edge_options,
2456+
None,
2457+
proxy_user,
2458+
proxy_pass,
2459+
zip_it=True,
2460+
multi_proxy=multi_proxy,
24032461
)
24042462
edge_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url)
24052463
edge_options.add_argument("--test-type")
@@ -2614,6 +2672,7 @@ def get_local_driver(
26142672
proxy_pass,
26152673
proxy_bypass_list,
26162674
proxy_pac_url,
2675+
multi_proxy,
26172676
user_agent,
26182677
recorder_ext,
26192678
disable_js,
@@ -2668,6 +2727,7 @@ def get_local_driver(
26682727
proxy_pass,
26692728
proxy_bypass_list,
26702729
proxy_pac_url,
2730+
multi_proxy,
26712731
user_agent,
26722732
recorder_ext,
26732733
disable_js,
@@ -2796,6 +2856,10 @@ def get_local_driver(
27962856
uc_driver_version = None
27972857
if is_using_uc(undetectable, browser_name):
27982858
uc_driver_version = get_uc_driver_version()
2859+
if multi_proxy:
2860+
from seleniumbase import config as sb_config
2861+
2862+
sb_config.multi_proxy = True
27992863
if (
28002864
LOCAL_CHROMEDRIVER
28012865
and os.path.exists(LOCAL_CHROMEDRIVER)
@@ -3133,6 +3197,7 @@ def get_local_driver(
31333197
proxy_pass,
31343198
proxy_bypass_list,
31353199
proxy_pac_url,
3200+
multi_proxy,
31363201
user_agent,
31373202
recorder_ext,
31383203
disable_js,

‎seleniumbase/core/proxy_helper.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True):
106106
zf.writestr("manifest.json", manifest_json)
107107
zf.close()
108108
else:
109-
proxy_ext_dir = os.path.join(downloads_path, "proxy_ext_dir")
109+
proxy_ext_dir = PROXY_DIR_PATH
110110
if not os.path.exists(proxy_ext_dir):
111111
os.mkdir(proxy_ext_dir)
112112
manifest_file = os.path.join(proxy_ext_dir, "manifest.json")

0 commit comments

Comments
 (0)