diff --git a/README.md b/README.md index 72912775870..29da28b9653 100755 --- a/README.md +++ b/README.md @@ -499,8 +499,6 @@ pytest my_first_test.py --pdb --edge # (Shortcut for "--browser=edge".) --firefox # (Shortcut for "--browser=firefox".) --safari # (Shortcut for "--browser=safari".) ---cap-file=FILE # (The web browser's desired capabilities to use.) ---cap-string=STRING # (The web browser's desired capabilities to use.) --settings-file=FILE # (Override default SeleniumBase settings.) --env=ENV # (Set the test env. Access with "self.env" in tests.) --account=STR # (Set account. Access with "self.account" in tests.) @@ -513,11 +511,14 @@ pytest my_first_test.py --pdb --protocol=PROTOCOL # (The Selenium Grid protocol: http|https.) --server=SERVER # (The Selenium Grid server/IP used for tests.) --port=PORT # (The Selenium Grid port used by the test server.) ---proxy=SERVER:PORT # (Connect to a proxy server:port for tests.) ---proxy=USERNAME:PASSWORD@SERVER:PORT # (Use authenticated proxy server.) ---proxy-bypass-list=STRING # (";"-separated hosts to bypass, Eg "*.foo.com") +--cap-file=FILE # (The web browser's desired capabilities to use.) +--cap-string=STRING # (The web browser's desired capabilities to use.) +--proxy=SERVER:PORT # (Connect to a proxy server:port as tests are running) +--proxy=USERNAME:PASSWORD@SERVER:PORT # (Use an authenticated proxy server) +--proxy-bypass-list=STRING # (";"-separated hosts to bypass, Eg "*.foo.com") --proxy-pac-url=URL # (Connect to a proxy server using a PAC_URL.pac file.) --proxy-pac-url=USERNAME:PASSWORD@URL # (Authenticated proxy with PAC URL.) +--multi-proxy # (Allow multiple authenticated proxies when multi-threaded.) --agent=STRING # (Modify the web browser's User-Agent string.) --mobile # (Use the mobile device emulator while running tests.) --metrics=STRING # (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".) diff --git a/examples/proxy_test.py b/examples/proxy_test.py index 8a3a96afe9c..b4ef7770142 100644 --- a/examples/proxy_test.py +++ b/examples/proxy_test.py @@ -18,7 +18,7 @@ def test_proxy(self): print("Skipping test for using Safari.") self.skip("Skipping test for using Safari.") settings.SKIP_JS_WAITS = True - if not self.page_load_strategy == "none": + if not self.page_load_strategy == "none" and not self.undetectable: # This page takes too long to load otherwise self.get_new_driver(page_load_strategy="none") self.open("https://ipinfo.io/") diff --git a/examples/raw_parameter_script.py b/examples/raw_parameter_script.py index 048f2032693..5d8c96aa23d 100644 --- a/examples/raw_parameter_script.py +++ b/examples/raw_parameter_script.py @@ -116,6 +116,7 @@ sb.proxy_string = None sb.proxy_bypass_list = None sb.proxy_pac_url = None + sb.multi_proxy = False sb.swiftshader = False sb.ad_block_on = False sb.highlights = None diff --git a/examples/test_hack_search.py b/examples/test_hack_search.py index 47ec0e6ebca..b3b61faa9bf 100644 --- a/examples/test_hack_search.py +++ b/examples/test_hack_search.py @@ -13,11 +13,11 @@ def test_hack_search(self): print("\n This test is not for Headless Mode.") self.skip('Do not use "--headless" with this test.') self.open("https://google.com/ncr") - self.assert_element('input[title="Search"]') + self.assert_element('[title="Search"]') self.sleep(0.5) self.set_attribute('[action="/search"]', "action", "//bing.com/search") self.set_attributes('[value="Google Search"]', "value", "Bing Search") - self.type('input[title="Search"]', "GitHub SeleniumBase Docs Install") + self.type('[title="Search"]', "GitHub SeleniumBase Docs Install") self.sleep(0.5) self.js_click('[value="Bing Search"]') self.highlight("h1.b_logo", loops=8) diff --git a/help_docs/customizing_test_runs.md b/help_docs/customizing_test_runs.md index 5745182e7eb..a3654fc8f9e 100644 --- a/help_docs/customizing_test_runs.md +++ b/help_docs/customizing_test_runs.md @@ -105,8 +105,6 @@ pytest my_first_test.py --settings-file=custom_settings.py --edge # (Shortcut for "--browser=edge".) --firefox # (Shortcut for "--browser=firefox".) --safari # (Shortcut for "--browser=safari".) ---cap-file=FILE # (The web browser's desired capabilities to use.) ---cap-string=STRING # (The web browser's desired capabilities to use.) --settings-file=FILE # (Override default SeleniumBase settings.) --env=ENV # (Set the test env. Access with "self.env" in tests.) --account=STR # (Set account. Access with "self.account" in tests.) @@ -119,11 +117,14 @@ pytest my_first_test.py --settings-file=custom_settings.py --protocol=PROTOCOL # (The Selenium Grid protocol: http|https.) --server=SERVER # (The Selenium Grid server/IP used for tests.) --port=PORT # (The Selenium Grid port used by the test server.) ---proxy=SERVER:PORT # (Connect to a proxy server:port for tests.) ---proxy=USERNAME:PASSWORD@SERVER:PORT # (Use authenticated proxy server.) ---proxy-bypass-list=STRING # (";"-separated hosts to bypass, Eg "*.foo.com") +--cap-file=FILE # (The web browser's desired capabilities to use.) +--cap-string=STRING # (The web browser's desired capabilities to use.) +--proxy=SERVER:PORT # (Connect to a proxy server:port as tests are running) +--proxy=USERNAME:PASSWORD@SERVER:PORT # (Use an authenticated proxy server) +--proxy-bypass-list=STRING # (";"-separated hosts to bypass, Eg "*.foo.com") --proxy-pac-url=URL # (Connect to a proxy server using a PAC_URL.pac file.) --proxy-pac-url=USERNAME:PASSWORD@URL # (Authenticated proxy with PAC URL.) +--multi-proxy # (Allow multiple authenticated proxies when multi-threaded.) --agent=STRING # (Modify the web browser's User-Agent string.) --mobile # (Use the mobile device emulator while running tests.) --metrics=STRING # (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".) diff --git a/mkdocs_build/requirements.txt b/mkdocs_build/requirements.txt index d229a08ae8e..e73613ef777 100644 --- a/mkdocs_build/requirements.txt +++ b/mkdocs_build/requirements.txt @@ -14,7 +14,7 @@ click==8.1.3 ghp-import==2.1.0 readme-renderer==37.3 pymdown-extensions==9.11 -importlib-metadata==6.2.0 +importlib-metadata==6.3.0 pipdeptree==2.7.0 bleach==6.0.0 lunr==0.6.2 diff --git a/requirements.txt b/requirements.txt index 946ef330061..ce0e4b1ef39 100755 --- a/requirements.txt +++ b/requirements.txt @@ -79,13 +79,15 @@ soupsieve==2.4;python_version>="3.7" beautifulsoup4==4.12.2 cryptography==36.0.2;python_version<"3.7" cryptography==40.0.1;python_version>="3.7" -pygments==2.14.0 +pygments==2.14.0;python_version<"3.7" +pygments==2.15.0;python_version>="3.7" pyreadline3==3.4.1;platform_system=="Windows" tabcompleter==1.1.0 pdbp==1.2.8 colorama==0.4.5;python_version<"3.7" colorama==0.4.6;python_version>="3.7" exceptiongroup==1.1.1;python_version>="3.7" +future-breakpoint==2.0.0;python_version<"3.7" importlib-metadata==4.2.0;python_version<"3.8" pycparser==2.21 pyotp==2.7.0;python_version<"3.7" diff --git a/seleniumbase/__version__.py b/seleniumbase/__version__.py index 4448a51f927..3f644ce6743 100755 --- a/seleniumbase/__version__.py +++ b/seleniumbase/__version__.py @@ -1,2 +1,2 @@ # seleniumbase package -__version__ = "4.13.22" +__version__ = "4.13.23" diff --git a/seleniumbase/behave/behave_sb.py b/seleniumbase/behave/behave_sb.py index 9dabcccbacc..d15701bdbe8 100644 --- a/seleniumbase/behave/behave_sb.py +++ b/seleniumbase/behave/behave_sb.py @@ -25,11 +25,12 @@ -D port=PORT (The Selenium Grid port used by the test server.) -D cap-file=FILE (The web browser's desired capabilities to use.) -D cap-string=STRING (The web browser's desired capabilities to use.) --D proxy=SERVER:PORT (Connect to a proxy server:port for tests.) --D proxy=USERNAME:PASSWORD@SERVER:PORT (Use authenticated proxy server.) +-D proxy=SERVER:PORT (Connect to a proxy server:port as tests are running) +-D proxy=USERNAME:PASSWORD@SERVER:PORT (Use an authenticated proxy server) -D proxy-bypass-list=STRING (";"-separated hosts to bypass, Eg "*.foo.com") -D proxy-pac-url=URL (Connect to a proxy server using a PAC_URL.pac file.) -D proxy-pac-url=USERNAME:PASSWORD@URL (Authenticated proxy with PAC URL.) +-D multi-proxy (Allow multiple authenticated proxies when multi-threaded.) -D agent=STRING (Modify the web browser's User-Agent string.) -D mobile (Use the mobile device emulator while running tests.) -D metrics=STRING (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".) @@ -223,6 +224,7 @@ def get_configured_sb(context): sb.proxy_string = None sb.proxy_bypass_list = None sb.proxy_pac_url = None + sb.multi_proxy = False sb.enable_3d_apis = False sb.swiftshader = False sb.ad_block_on = False @@ -746,6 +748,10 @@ def get_configured_sb(context): proxy_pac_url = sb.proxy_pac_url # revert to default sb.proxy_pac_url = proxy_pac_url continue + # Handle: -D multi-proxy / multi_proxy + if low_key in ["multi-proxy", "multi_proxy"]: + sb.multi_proxy = True + continue # Handle: -D enable-3d-apis / enable_3d_apis if low_key in ["enable-3d-apis", "enable_3d_apis"]: sb.enable_3d_apis = True @@ -1121,7 +1127,8 @@ def _perform_behave_unconfigure_(): from seleniumbase.core import log_helper from seleniumbase.core import proxy_helper - proxy_helper.remove_proxy_zip_if_present() + if hasattr(sb_config, "multi_proxy") and not sb_config.multi_proxy: + proxy_helper.remove_proxy_zip_if_present() if hasattr(sb_config, "reuse_session") and sb_config.reuse_session: # Close the shared browser session if sb_config.shared_driver: diff --git a/seleniumbase/core/browser_launcher.py b/seleniumbase/core/browser_launcher.py index 42d12b7e6ee..dbc4dcf62e7 100644 --- a/seleniumbase/core/browser_launcher.py +++ b/seleniumbase/core/browser_launcher.py @@ -12,6 +12,7 @@ import zipfile from selenium import webdriver from selenium.webdriver.chrome.service import Service as ChromeService +from selenium.webdriver.common.service import utils as service_utils from selenium.webdriver.edge.service import Service as EdgeService from selenium.webdriver.firefox.service import Service as FirefoxService from selenium.webdriver.safari.service import Service as SafariService @@ -295,45 +296,74 @@ def _was_driver_repaired(): return os.path.exists(file_path) +def _set_proxy_filenames(): + DOWNLOADS_DIR = constants.Files.DOWNLOADS_FOLDER + for num in range(1000): + PROXY_ZIP_PATH = os.path.join(DOWNLOADS_DIR, "proxy_%s.zip" % num) + PROXY_DIR_PATH = os.path.join(DOWNLOADS_DIR, "proxy_ext_dir_%s" % num) + if os.path.exists(PROXY_ZIP_PATH) or os.path.exists(PROXY_DIR_PATH): + continue + proxy_helper.PROXY_ZIP_PATH = PROXY_ZIP_PATH + proxy_helper.PROXY_DIR_PATH = PROXY_DIR_PATH + return + # Exceeded upper bound. Use Defaults: + PROXY_ZIP_PATH = os.path.join(DOWNLOADS_DIR, "proxy.zip") + PROXY_DIR_PATH = os.path.join(DOWNLOADS_DIR, "proxy_ext_dir") + proxy_helper.PROXY_ZIP_PATH = PROXY_ZIP_PATH + proxy_helper.PROXY_DIR_PATH = PROXY_DIR_PATH + + def _add_chrome_proxy_extension( - chrome_options, proxy_string, proxy_user, proxy_pass, zip_it=True + chrome_options, + proxy_string, + proxy_user, + proxy_pass, + zip_it=True, + multi_proxy=False, ): """Implementation of https://stackoverflow.com/a/35293284 for https://stackoverflow.com/questions/12848327/ (Run Selenium on a proxy server that requires authentication.)""" arg_join = " ".join(sys.argv) - if not ("-n" in sys.argv or " -n=" in arg_join or arg_join == "-c"): + if ( + not ("-n" in sys.argv or " -n=" in arg_join or arg_join == "-c") + and not multi_proxy + ): # Single-threaded if zip_it: proxy_helper.create_proxy_ext(proxy_string, proxy_user, proxy_pass) - proxy_zip = PROXY_ZIP_PATH + proxy_zip = proxy_helper.PROXY_ZIP_PATH chrome_options.add_extension(proxy_zip) else: proxy_helper.create_proxy_ext( proxy_string, proxy_user, proxy_pass, zip_it=False ) - chrome_options = add_chrome_ext_dir(chrome_options, PROXY_DIR_PATH) - + proxy_dir_path = proxy_helper.PROXY_DIR_PATH + chrome_options = add_chrome_ext_dir(chrome_options, proxy_dir_path) else: - # Pytest multithreaded test + # Multi-threaded if zip_it: proxy_zip_lock = fasteners.InterProcessLock(PROXY_ZIP_LOCK) with proxy_zip_lock: - if not os.path.exists(PROXY_ZIP_PATH): + if multi_proxy: + _set_proxy_filenames() + if not os.path.exists(proxy_helper.PROXY_ZIP_PATH): proxy_helper.create_proxy_ext( proxy_string, proxy_user, proxy_pass ) - proxy_zip = PROXY_ZIP_PATH + proxy_zip = proxy_helper.PROXY_ZIP_PATH chrome_options.add_extension(proxy_zip) else: proxy_dir_lock = fasteners.InterProcessLock(PROXY_DIR_LOCK) with proxy_dir_lock: - if not os.path.exists(PROXY_DIR_PATH): + if multi_proxy: + _set_proxy_filenames() + if not os.path.exists(proxy_helper.PROXY_DIR_PATH): proxy_helper.create_proxy_ext( proxy_string, proxy_user, proxy_pass, False ) chrome_options = add_chrome_ext_dir( - chrome_options, PROXY_DIR_PATH + chrome_options, proxy_helper.PROXY_DIR_PATH ) return chrome_options @@ -406,6 +436,7 @@ def _set_chrome_options( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, recorder_ext, disable_js, @@ -656,7 +687,12 @@ def _set_chrome_options( if is_using_uc(undetectable, browser_name): zip_it = False # undetected-chromedriver needs a folder ext chrome_options = _add_chrome_proxy_extension( - chrome_options, proxy_string, proxy_user, proxy_pass, zip_it + chrome_options, + proxy_string, + proxy_user, + proxy_pass, + zip_it, + multi_proxy, ) chrome_options.add_argument("--proxy-server=%s" % proxy_string) if proxy_bypass_list: @@ -669,7 +705,12 @@ def _set_chrome_options( if is_using_uc(undetectable, browser_name): zip_it = False chrome_options = _add_chrome_proxy_extension( - chrome_options, None, proxy_user, proxy_pass, zip_it + chrome_options, + None, + proxy_user, + proxy_pass, + zip_it, + multi_proxy, ) chrome_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url) if browser_name != constants.Browser.OPERA: @@ -691,7 +732,12 @@ def _set_chrome_options( # To access the Debugger, go to: chrome://inspect/#devices # while a Chromium driver is running. # Info: https://chromedevtools.github.io/devtools-protocol/ - chrome_options.add_argument("--remote-debugging-port=9222") + sys_argv = sys.argv + arg_join = " ".join(sys_argv) + debug_port = 9222 + if ("-n" in sys.argv) or (" -n=" in arg_join) or ("-c" in sys.argv): + debug_port = service_utils.free_port() + chrome_options.add_argument("--remote-debugging-port=%s" % debug_port) if swiftshader: chrome_options.add_argument("--use-gl=swiftshader") elif not is_using_uc(undetectable, browser_name): @@ -1000,6 +1046,7 @@ def get_driver( proxy_string=None, proxy_bypass_list=None, proxy_pac_url=None, + multi_proxy=None, user_agent=None, cap_file=None, cap_string=None, @@ -1205,6 +1252,7 @@ def get_driver( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, cap_file, cap_string, @@ -1257,6 +1305,7 @@ def get_driver( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, recorder_ext, disable_js, @@ -1309,6 +1358,7 @@ def get_remote_driver( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, cap_file, cap_string, @@ -1430,6 +1480,7 @@ def get_remote_driver( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, recorder_ext, disable_js, @@ -1618,6 +1669,7 @@ def get_remote_driver( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, recorder_ext, disable_js, @@ -1738,6 +1790,7 @@ def get_remote_driver( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, recorder_ext, disable_js, @@ -1856,6 +1909,7 @@ def get_local_driver( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, recorder_ext, disable_js, @@ -2389,7 +2443,12 @@ def get_local_driver( if proxy_string: if proxy_auth: edge_options = _add_chrome_proxy_extension( - edge_options, proxy_string, proxy_user, proxy_pass + edge_options, + proxy_string, + proxy_user, + proxy_pass, + zip_it=True, + multi_proxy=multi_proxy, ) edge_options.add_argument("--proxy-server=%s" % proxy_string) if proxy_bypass_list: @@ -2399,7 +2458,12 @@ def get_local_driver( elif proxy_pac_url: if proxy_auth: edge_options = _add_chrome_proxy_extension( - edge_options, None, proxy_user, proxy_pass + edge_options, + None, + proxy_user, + proxy_pass, + zip_it=True, + multi_proxy=multi_proxy, ) edge_options.add_argument("--proxy-pac-url=%s" % proxy_pac_url) edge_options.add_argument("--test-type") @@ -2419,7 +2483,12 @@ def get_local_driver( # To access the Debugger, go to: edge://inspect/#devices # while a Chromium driver is running. # Info: https://chromedevtools.github.io/devtools-protocol/ - edge_options.add_argument("--remote-debugging-port=9222") + sys_argv = sys.argv + arg_join = " ".join(sys_argv) + free_port = 9222 + if ("-n" in sys.argv or " -n=" in args or args == "-c"): + free_port = service_utils.free_port() + edge_options.add_argument("--remote-debugging-port=%s" % free_port) if swiftshader: edge_options.add_argument("--use-gl=swiftshader") else: @@ -2469,7 +2538,14 @@ def get_local_driver( log_path=os.devnull, ) # https://stackoverflow.com/a/56638103/7058266 - edge_options.add_argument("--remote-debugging-port=9222") + sys_argv = sys.argv + arg_join = " ".join(sys_argv) + free_port = 9222 + if ("-n" in sys.argv or " -n=" in args or args == "-c"): + free_port = service_utils.free_port() + edge_options.add_argument( + "--remote-debugging-port=%s" % free_port + ) return Edge(service=service, options=edge_options) if not auto_upgrade_edgedriver: raise # Not an obvious fix. @@ -2535,7 +2611,14 @@ def get_local_driver( log_path=os.devnull, ) # https://stackoverflow.com/a/56638103/7058266 - edge_options.add_argument("--remote-debugging-port=9222") + sys_argv = sys.argv + arg_join = " ".join(sys_argv) + free_port = 9222 + if ("-n" in sys.argv or " -n=" in args or args == "-c"): + free_port = service_utils.free_port() + edge_options.add_argument( + "--remote-debugging-port=%s" % free_port + ) return Edge(service=service, options=edge_options) if not auto_upgrade_edgedriver: raise # Not an obvious fix. @@ -2614,6 +2697,7 @@ def get_local_driver( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, recorder_ext, disable_js, @@ -2668,6 +2752,7 @@ def get_local_driver( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, recorder_ext, disable_js, @@ -2796,6 +2881,10 @@ def get_local_driver( uc_driver_version = None if is_using_uc(undetectable, browser_name): uc_driver_version = get_uc_driver_version() + if multi_proxy: + from seleniumbase import config as sb_config + + sb_config.multi_proxy = True if ( LOCAL_CHROMEDRIVER and os.path.exists(LOCAL_CHROMEDRIVER) @@ -3133,6 +3222,7 @@ def get_local_driver( proxy_pass, proxy_bypass_list, proxy_pac_url, + multi_proxy, user_agent, recorder_ext, disable_js, diff --git a/seleniumbase/core/proxy_helper.py b/seleniumbase/core/proxy_helper.py index 26d2a02a3c8..0b3ce0ddc90 100644 --- a/seleniumbase/core/proxy_helper.py +++ b/seleniumbase/core/proxy_helper.py @@ -106,7 +106,7 @@ def create_proxy_ext(proxy_string, proxy_user, proxy_pass, zip_it=True): zf.writestr("manifest.json", manifest_json) zf.close() else: - proxy_ext_dir = os.path.join(downloads_path, "proxy_ext_dir") + proxy_ext_dir = PROXY_DIR_PATH if not os.path.exists(proxy_ext_dir): os.mkdir(proxy_ext_dir) manifest_file = os.path.join(proxy_ext_dir, "manifest.json") diff --git a/seleniumbase/fixtures/base_case.py b/seleniumbase/fixtures/base_case.py index efec33f9d82..c32021b1f75 100644 --- a/seleniumbase/fixtures/base_case.py +++ b/seleniumbase/fixtures/base_case.py @@ -3517,6 +3517,7 @@ def get_new_driver( proxy=None, proxy_bypass_list=None, proxy_pac_url=None, + multi_proxy=None, agent=None, switch_to=True, cap_file=None, @@ -3571,6 +3572,7 @@ def get_new_driver( proxy - if using a proxy server, specify the "host:port" combo here proxy_bypass_list - ";"-separated hosts to bypass (Eg. "*.foo.com") proxy_pac_url - designates the proxy PAC URL to use (Chromium-only) + multi_proxy - allow multiple proxies with auth while multi-threaded switch_to - the option to switch to the new driver (default = True) cap_file - the file containing desired capabilities for the browser cap_string - the string with desired capabilities for the browser @@ -3663,6 +3665,8 @@ def get_new_driver( proxy_bypass_list = self.proxy_bypass_list if proxy_pac_url is None: proxy_pac_url = self.proxy_pac_url + if multi_proxy is None: + multi_proxy = self.multi_proxy user_agent = agent if user_agent is None: user_agent = self.user_agent @@ -3761,6 +3765,7 @@ def get_new_driver( proxy_string=proxy_string, proxy_bypass_list=proxy_bypass_list, proxy_pac_url=proxy_pac_url, + multi_proxy=multi_proxy, user_agent=user_agent, cap_file=cap_file, cap_string=cap_string, @@ -13416,6 +13421,7 @@ def setUp(self, masterqa_mode=False): self.proxy_string = sb_config.proxy_string self.proxy_bypass_list = sb_config.proxy_bypass_list self.proxy_pac_url = sb_config.proxy_pac_url + self.multi_proxy = sb_config.multi_proxy self.user_agent = sb_config.user_agent self.mobile_emulator = sb_config.mobile_emulator self.device_metrics = sb_config.device_metrics @@ -13762,6 +13768,7 @@ def setUp(self, masterqa_mode=False): proxy=self.proxy_string, proxy_bypass_list=self.proxy_bypass_list, proxy_pac_url=self.proxy_pac_url, + multi_proxy=self.multi_proxy, agent=self.user_agent, switch_to=True, cap_file=self.cap_file, diff --git a/seleniumbase/plugins/driver_manager.py b/seleniumbase/plugins/driver_manager.py index 28a4fce41ed..a141db10be0 100644 --- a/seleniumbase/plugins/driver_manager.py +++ b/seleniumbase/plugins/driver_manager.py @@ -63,6 +63,7 @@ def Driver( proxy=None, # Use proxy. Format: "SERVER:PORT" or "USER:PASS@SERVER:PORT". proxy_bypass_list=None, # Skip proxy when using the listed domains. proxy_pac_url=None, # Use PAC file. (Format: URL or USERNAME:PASSWORD@URL) + multi_proxy=False, # Allow multiple proxies with auth when multi-threaded. agent=None, # Modify the web browser's User-Agent string. cap_file=None, # The desired capabilities to use with a Selenium Grid. cap_string=None, # The desired capabilities to use with a Selenium Grid. @@ -403,6 +404,7 @@ def Driver( proxy_string=proxy_string, proxy_bypass_list=proxy_bypass_list, proxy_pac_url=proxy_pac_url, + multi_proxy=multi_proxy, user_agent=user_agent, cap_file=cap_file, cap_string=cap_string, diff --git a/seleniumbase/plugins/pytest_plugin.py b/seleniumbase/plugins/pytest_plugin.py index 550fe1b0e3b..1b8343c132c 100644 --- a/seleniumbase/plugins/pytest_plugin.py +++ b/seleniumbase/plugins/pytest_plugin.py @@ -40,11 +40,12 @@ def pytest_addoption(parser): --port=PORT (The Selenium Grid port used by the test server.) --cap-file=FILE (The web browser's desired capabilities to use.) --cap-string=STRING (The web browser's desired capabilities to use.) - --proxy=SERVER:PORT (Connect to a proxy server:port for tests.) - --proxy=USERNAME:PASSWORD@SERVER:PORT (Use authenticated proxy server.) + --proxy=SERVER:PORT (Connect to a proxy server:port as tests are running) + --proxy=USERNAME:PASSWORD@SERVER:PORT (Use an authenticated proxy server) --proxy-bypass-list=STRING (";"-separated hosts to bypass, Eg "*.foo.com") --proxy-pac-url=URL (Connect to a proxy server using a PAC_URL.pac file.) --proxy-pac-url=USERNAME:PASSWORD@URL (Authenticated proxy with PAC URL.) + --multi-proxy (Allow multiple authenticated proxies when multi-threaded.) --agent=STRING (Modify the web browser's User-Agent string.) --mobile (Use the mobile device emulator while running tests.) --metrics=STRING (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".) @@ -488,6 +489,16 @@ def pytest_addoption(parser): A username:password@URL string Default: None.""", ) + parser.addoption( + "--multi-proxy", + "--multi_proxy", + action="store_true", + dest="multi_proxy", + default=False, + help="""If you need to run multi-threaded tests with + multiple proxies that require authentication, + set this to allow multiple configurations.""", + ) parser.addoption( "--agent", "--user-agent", @@ -1421,6 +1432,7 @@ def pytest_configure(config): sb_config.proxy_string = config.getoption("proxy_string") sb_config.proxy_bypass_list = config.getoption("proxy_bypass_list") sb_config.proxy_pac_url = config.getoption("proxy_pac_url") + sb_config.multi_proxy = config.getoption("multi_proxy") sb_config.cap_file = config.getoption("cap_file") sb_config.cap_string = config.getoption("cap_string") sb_config.settings_file = config.getoption("settings_file") @@ -1930,7 +1942,8 @@ def pytest_terminal_summary(terminalreporter): def _perform_pytest_unconfigure_(): from seleniumbase.core import proxy_helper - proxy_helper.remove_proxy_zip_if_present() + if not sb_config.multi_proxy: + proxy_helper.remove_proxy_zip_if_present() if hasattr(sb_config, "reuse_session") and sb_config.reuse_session: # Close the shared browser session if sb_config.shared_driver: diff --git a/seleniumbase/plugins/sb_manager.py b/seleniumbase/plugins/sb_manager.py index 8adcd502478..be3109eb42e 100644 --- a/seleniumbase/plugins/sb_manager.py +++ b/seleniumbase/plugins/sb_manager.py @@ -35,6 +35,7 @@ def SB( proxy=None, # Use proxy. Format: "SERVER:PORT" or "USER:PASS@SERVER:PORT". proxy_bypass_list=None, # Skip proxy when using the listed domains. proxy_pac_url=None, # Use PAC file. (Format: URL or USERNAME:PASSWORD@URL) + multi_proxy=False, # Allow multiple proxies with auth when multi-threaded. agent=None, # Modify the web browser's User-Agent string. cap_file=None, # The desired capabilities to use with a Selenium Grid. cap_string=None, # The desired capabilities to use with a Selenium Grid. @@ -663,6 +664,7 @@ def SB( sb_config.proxy_string = proxy_string sb_config.proxy_bypass_list = proxy_bypass_list sb_config.proxy_pac_url = proxy_pac_url + sb_config.multi_proxy = multi_proxy sb_config.enable_3d_apis = enable_3d_apis sb_config.swiftshader = swiftshader sb_config.ad_block_on = ad_block_on @@ -760,6 +762,7 @@ def SB( sb.proxy_string = sb_config.proxy_string sb.proxy_bypass_list = sb_config.proxy_bypass_list sb.proxy_pac_url = sb_config.proxy_pac_url + sb.multi_proxy = sb_config.multi_proxy sb.enable_3d_apis = sb_config.enable_3d_apis sb.swiftshader = sb_config.swiftshader sb.ad_block_on = sb_config.ad_block_on @@ -803,7 +806,8 @@ def SB( log_helper.log_folder_setup(sb_config.log_path) download_helper.reset_downloads_folder() - proxy_helper.remove_proxy_zip_if_present() + if not sb_config.multi_proxy: + proxy_helper.remove_proxy_zip_if_present() start_time = time.time() sb.setUp() test_passed = True # This can change later diff --git a/seleniumbase/plugins/selenium_plugin.py b/seleniumbase/plugins/selenium_plugin.py index 4e90818eab4..5e8af942d1c 100644 --- a/seleniumbase/plugins/selenium_plugin.py +++ b/seleniumbase/plugins/selenium_plugin.py @@ -24,11 +24,12 @@ class SeleniumBrowser(Plugin): --port=PORT (The Selenium Grid port used by the test server.) --cap-file=FILE (The web browser's desired capabilities to use.) --cap-string=STRING (The web browser's desired capabilities to use.) - --proxy=SERVER:PORT (Connect to a proxy server:port for tests.) - --proxy=USERNAME:PASSWORD@SERVER:PORT (Use authenticated proxy server.) + --proxy=SERVER:PORT (Connect to a proxy server:port as tests are running) + --proxy=USERNAME:PASSWORD@SERVER:PORT (Use an authenticated proxy server) --proxy-bypass-list=STRING (";"-separated hosts to bypass, Eg "*.foo.com") --proxy-pac-url=URL (Connect to a proxy server using a PAC_URL.pac file.) --proxy-pac-url=USERNAME:PASSWORD@URL (Authenticated proxy with PAC URL.) + --multi-proxy (Allow multiple authenticated proxies when multi-threaded.) --agent=STRING (Modify the web browser's User-Agent string.) --mobile (Use the mobile device emulator while running tests.) --metrics=STRING (Set mobile metrics: "CSSWidth,CSSHeight,PixelRatio".) @@ -255,6 +256,16 @@ def options(self, parser, env): A username:password@URL string Default: None.""", ) + parser.addoption( + "--multi-proxy", + "--multi_proxy", + action="store_true", + dest="multi_proxy", + default=False, + help="""If you need to run multi-threaded tests with + multiple proxies that require authentication, + set this to allow multiple configurations.""", + ) parser.addoption( "--agent", "--user-agent", @@ -1055,6 +1066,7 @@ def beforeTest(self, test): test.test.proxy_string = self.options.proxy_string test.test.proxy_bypass_list = self.options.proxy_bypass_list test.test.proxy_pac_url = self.options.proxy_pac_url + test.test.multi_proxy = self.options.multi_proxy test.test.user_agent = self.options.user_agent test.test.mobile_emulator = self.options.mobile_emulator test.test.device_metrics = self.options.device_metrics @@ -1195,13 +1207,15 @@ def beforeTest(self, test): sb_config._SMALL_TIMEOUT = settings.SMALL_TIMEOUT sb_config._LARGE_TIMEOUT = settings.LARGE_TIMEOUT sb_config._context_of_runner = False # Context Manager Compatibility + sb_config.multi_proxy = self.options.multi_proxy # The driver will be received later self.driver = None test.test.driver = self.driver def finalize(self, result): """This runs after all tests have completed with nosetests.""" - proxy_helper.remove_proxy_zip_if_present() + if not sb_config.multi_proxy: + proxy_helper.remove_proxy_zip_if_present() def afterTest(self, test): try: diff --git a/seleniumbase/undetected/__init__.py b/seleniumbase/undetected/__init__.py index 4d00e5e74dc..50f913028b2 100644 --- a/seleniumbase/undetected/__init__.py +++ b/seleniumbase/undetected/__init__.py @@ -141,7 +141,28 @@ def __init__( options._session = self debug_host = "127.0.0.1" debug_port = 9222 + import requests + + special_port_free = False # If the port isn't free, don't use 9222 + try: + res = requests.get("http://127.0.0.1:9222") + if res.status_code != 200: + raise Exception("The port is free! It will be used!") + except Exception: + # Use port 9222, which outputs to chrome://inspect/#devices + special_port_free = True + sys_argv = sys.argv + arg_join = " ".join(sys_argv) + from seleniumbase import config as sb_config + + if ( + (("-n" in sys.argv) or (" -n=" in arg_join) or ("-c" in sys.argv)) + or (hasattr(sb_config, "multi_proxy") and sb_config.multi_proxy) + or not special_port_free + ): + debug_port = selenium.webdriver.common.service.utils.free_port() if hasattr(options, "_remote_debugging_port"): + # The user chooses the port. Errors happen if the port is taken. debug_port = options._remote_debugging_port if not options.debugger_address: options.debugger_address = "%s:%d" % (debug_host, debug_port) diff --git a/setup.py b/setup.py index 584baf79202..90c88d33051 100755 --- a/setup.py +++ b/setup.py @@ -203,13 +203,15 @@ "beautifulsoup4==4.12.2", 'cryptography==36.0.2;python_version<"3.7"', 'cryptography==40.0.1;python_version>="3.7"', - "pygments==2.14.0", + 'pygments==2.14.0;python_version<"3.7"', + 'pygments==2.15.0;python_version>="3.7"', 'pyreadline3==3.4.1;platform_system=="Windows"', "tabcompleter==1.1.0", "pdbp==1.2.8", 'colorama==0.4.5;python_version<"3.7"', 'colorama==0.4.6;python_version>="3.7"', 'exceptiongroup==1.1.1;python_version>="3.7"', + 'future-breakpoint==2.0.0;python_version<"3.7"', 'importlib-metadata==4.2.0;python_version<"3.8"', "pycparser==2.21", 'pyotp==2.7.0;python_version<"3.7"',