Merge branch master into master-slp

Author: Anselm Kruis

Lib/imp.py

@@ -334,6 +334,12 @@ def load_dynamic(name, path, file=None):
         """
         import importlib.machinery
         loader = importlib.machinery.ExtensionFileLoader(name, path)
-        return loader.load_module()
+
+        # Issue #24748: Skip the sys.modules check in _load_module_shim;
+        # always load new extension
+        spec = importlib.machinery.ModuleSpec(
+            name=name, loader=loader, origin=path)
+        return _load(spec)
+
 else:
     load_dynamic = None

Lib/test/imp_dummy.py

@@ -0,0 +1,3 @@
+# Fodder for test of issue24748 in test_imp
+
+dummy_name = True

Lib/test/test_imp.py

@@ -3,6 +3,7 @@
 except ImportError:
     _thread = None
 import importlib
+import importlib.util
 import os
 import os.path
 import shutil
@@ -275,6 +276,29 @@ def test_load_module_extension_file_is_None(self):
             self.skipTest("found module doesn't appear to be a C extension")
         imp.load_module(name, None, *found[1:])
 
+    @requires_load_dynamic
+    def test_issue24748_load_module_skips_sys_modules_check(self):
+        name = 'test.imp_dummy'
+        try:
+            del sys.modules[name]
+        except KeyError:
+            pass
+        try:
+            module = importlib.import_module(name)
+            spec = importlib.util.find_spec('_testmultiphase')
+            module = imp.load_dynamic(name, spec.origin)
+            self.assertEqual(module.__name__, name)
+            self.assertEqual(module.__spec__.name, name)
+            self.assertEqual(module.__spec__.origin, spec.origin)
+            self.assertRaises(AttributeError, getattr, module, 'dummy_name')
+            self.assertEqual(module.int_const, 1969)
+            self.assertIs(sys.modules[name], module)
+        finally:
+            try:
+                del sys.modules[name]
+            except KeyError:
+                pass
+
     @unittest.skipIf(sys.dont_write_bytecode,
         "test meaningful only when writing bytecode")
     def test_bug7732(self):

Lib/test/test_time.py

@@ -177,6 +177,19 @@ def _bounds_checking(self, func):
     def test_strftime_bounding_check(self):
         self._bounds_checking(lambda tup: time.strftime('', tup))
 
+    def test_strftime_format_check(self):
+        # Test that strftime does not crash on invalid format strings
+        # that may trigger a buffer overread. When not triggered,
+        # strftime may succeed or raise ValueError depending on
+        # the platform.
+        for x in [ '', 'A', '%A', '%AA' ]:
+            for y in range(0x0, 0x10):
+                for z in [ '%', 'A%', 'AA%', '%A%', 'A%A%', '%#' ]:
+                    try:
+                        time.strftime(x * y + z)
+                    except ValueError:
+                        pass
+
     def test_default_values_for_zero(self):
         # Make sure that using all zeros uses the proper default
         # values.  No test for daylight savings since strftime() does

Lib/test/test_warnings.py renamed to Lib/test/test_warnings/__init__.py

@@ -7,7 +7,7 @@
 from test import support
 from test.support.script_helper import assert_python_ok, assert_python_failure
 
-from test import warning_tests
+from test.test_warnings.data import stacklevel as warning_tests
 
 import warnings as original_warnings
 
@@ -194,11 +194,11 @@ def test_once(self):
             self.module.resetwarnings()
             self.module.filterwarnings("once", category=UserWarning)
             message = UserWarning("FilterTests.test_once")
-            self.module.warn_explicit(message, UserWarning, "test_warnings.py",
+            self.module.warn_explicit(message, UserWarning, "__init__.py",
                                     42)
             self.assertEqual(w[-1].message, message)
             del w[:]
-            self.module.warn_explicit(message, UserWarning, "test_warnings.py",
+            self.module.warn_explicit(message, UserWarning, "__init__.py",
                                     13)
             self.assertEqual(len(w), 0)
             self.module.warn_explicit(message, UserWarning, "test_warnings2.py",
@@ -304,10 +304,10 @@ def test_filename(self):
                     module=self.module) as w:
                 warning_tests.inner("spam1")
                 self.assertEqual(os.path.basename(w[-1].filename),
-                                    "warning_tests.py")
+                                    "stacklevel.py")
                 warning_tests.outer("spam2")
                 self.assertEqual(os.path.basename(w[-1].filename),
-                                    "warning_tests.py")
+                                    "stacklevel.py")
 
     def test_stacklevel(self):
         # Test stacklevel argument
@@ -317,25 +317,36 @@ def test_stacklevel(self):
                     module=self.module) as w:
                 warning_tests.inner("spam3", stacklevel=1)
                 self.assertEqual(os.path.basename(w[-1].filename),
-                                    "warning_tests.py")
+                                    "stacklevel.py")
                 warning_tests.outer("spam4", stacklevel=1)
                 self.assertEqual(os.path.basename(w[-1].filename),
-                                    "warning_tests.py")
+                                    "stacklevel.py")
 
                 warning_tests.inner("spam5", stacklevel=2)
                 self.assertEqual(os.path.basename(w[-1].filename),
-                                    "test_warnings.py")
+                                    "__init__.py")
                 warning_tests.outer("spam6", stacklevel=2)
                 self.assertEqual(os.path.basename(w[-1].filename),
-                                    "warning_tests.py")
+                                    "stacklevel.py")
                 warning_tests.outer("spam6.5", stacklevel=3)
                 self.assertEqual(os.path.basename(w[-1].filename),
-                                    "test_warnings.py")
+                                    "__init__.py")
 
                 warning_tests.inner("spam7", stacklevel=9999)
                 self.assertEqual(os.path.basename(w[-1].filename),
                                     "sys")
 
+    def test_stacklevel_import(self):
+        # Issue #24305: With stacklevel=2, module-level warnings should work.
+        support.unload('test.test_warnings.data.import_warning')
+        with warnings_state(self.module):
+            with original_warnings.catch_warnings(record=True,
+                    module=self.module) as w:
+                self.module.simplefilter('always')
+                import test.test_warnings.data.import_warning
+                self.assertEqual(len(w), 1)
+                self.assertEqual(w[0].filename, __file__)
+
     def test_missing_filename_not_main(self):
         # If __file__ is not specified and __main__ is not the module name,
         # then __file__ should be set to the module name.

Lib/test/test_warnings/__main__.py

@@ -0,0 +1,3 @@
+import unittest
+
+unittest.main('test.test_warnings')

Lib/test/test_warnings/data/import_warning.py

@@ -0,0 +1,3 @@
+import warnings
+
+warnings.warn('module-level warning', DeprecationWarning, stacklevel=2)

Lib/test/warning_tests.py renamed to Lib/test/test_warnings/data/stacklevel.py

@@ -160,6 +160,20 @@ def _getcategory(category):
     return cat
 
 
+def _is_internal_frame(frame):
+    """Signal whether the frame is an internal CPython implementation detail."""
+    filename = frame.f_code.co_filename
+    return 'importlib' in filename and '_bootstrap' in filename
+
+
+def _next_external_frame(frame):
+    """Find the next frame that doesn't involve CPython internals."""
+    frame = frame.f_back
+    while frame is not None and _is_internal_frame(frame):
+        frame = frame.f_back
+    return frame
+
+
 # Code typically replaced by _warnings
 def warn(message, category=None, stacklevel=1):
     """Issue a warning, or maybe ignore it or raise an exception."""
@@ -174,13 +188,23 @@ def warn(message, category=None, stacklevel=1):
                         "not '{:s}'".format(type(category).__name__))
     # Get context information
     try:
-        caller = sys._getframe(stacklevel)
+        if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)):
+            # If frame is too small to care or if the warning originated in
+            # internal code, then do not try to hide any frames.
+            frame = sys._getframe(stacklevel)
+        else:
+            frame = sys._getframe(1)
+            # Look for one frame less since the above line starts us off.
+            for x in range(stacklevel-1):
+                frame = _next_external_frame(frame)
+                if frame is None:
+                    raise ValueError
     except ValueError:
         globals = sys.__dict__
         lineno = 1
     else:
-        globals = caller.f_globals
-        lineno = caller.f_lineno
+        globals = frame.f_globals
+        lineno = frame.f_lineno
     if '__name__' in globals:
         module = globals['__name__']
     else:
@@ -374,7 +398,6 @@ def __exit__(self, *exc_info):
     defaultaction = _defaultaction
     onceregistry = _onceregistry
     _warnings_defaults = True
-
 except ImportError:
     filters = []
     defaultaction = "default"

Lib/warnings.py

@@ -495,132 +495,30 @@ def register_X_browsers():
 #
 
 if sys.platform[:3] == "win":
-
     class WindowsDefault(BaseBrowser):
-        # Windows Default opening arguments.
-
-        cmd = "start"
-        newwindow = ""
-        newtab = ""
-
         def open(self, url, new=0, autoraise=True):
-            # Format the command for optional arguments and add the url.
-            if new == 1:
-                self.cmd += " " + self.newwindow
-            elif new == 2:
-                self.cmd += " " + self.newtab
-            self.cmd += " " + url
             try:
-                subprocess.call(self.cmd, shell=True)
+                os.startfile(url)
             except OSError:
                 # [Error 22] No application is associated with the specified
                 # file for this operation: '<URL>'
                 return False
             else:
                 return True
 
-
-    # Windows Sub-Classes for commonly used browsers.
-
-    class InternetExplorer(WindowsDefault):
-        """Launcher class for Internet Explorer browser"""
-
-        cmd = "start iexplore.exe"
-        newwindow = ""
-        newtab = ""
-
-
-    class WinChrome(WindowsDefault):
-        """Launcher class for windows specific Google Chrome browser"""
-
-        cmd = "start chrome.exe"
-        newwindow = "-new-window"
-        newtab = "-new-tab"
-
-
-    class WinFirefox(WindowsDefault):
-        """Launcher class for windows specific Firefox browser"""
-
-        cmd = "start firefox.exe"
-        newwindow = "-new-window"
-        newtab = "-new-tab"
-
-
-    class WinOpera(WindowsDefault):
-        """Launcher class for windows specific Opera browser"""
-
-        cmd = "start opera"
-        newwindow = ""
-        newtab = ""
-
-
-    class WinSeaMonkey(WindowsDefault):
-        """Launcher class for windows specific SeaMonkey browser"""
-
-        cmd = "start seamonkey"
-        newwinow = ""
-        newtab = ""
-
-
     _tryorder = []
     _browsers = {}
 
-    # First try to use the default Windows browser.
+    # First try to use the default Windows browser
     register("windows-default", WindowsDefault)
 
-    def find_windows_browsers():
-        """ Access the windows registry to determine
-        what browsers are on the system.
-        """
-
-        import winreg
-        HKLM = winreg.HKEY_LOCAL_MACHINE
-        subkey = r'Software\Clients\StartMenuInternet'
-        read32 = winreg.KEY_READ | winreg.KEY_WOW64_32KEY
-        read64 = winreg.KEY_READ | winreg.KEY_WOW64_64KEY
-        key32 = winreg.OpenKey(HKLM, subkey, access=read32)
-        key64 = winreg.OpenKey(HKLM, subkey, access=read64)
-
-        # Return a list of browsers found in the registry
-        # Check if there are any different browsers in the
-        # 32 bit location instead of the 64 bit location.
-        browsers = []
-        i = 0
-        while True:
-            try:
-                browsers.append(winreg.EnumKey(key32, i))
-            except EnvironmentError:
-                break
-            i += 1
-
-        i = 0
-        while True:
-            try:
-                browsers.append(winreg.EnumKey(key64, i))
-            except EnvironmentError:
-                break
-            i += 1
-
-        winreg.CloseKey(key32)
-        winreg.CloseKey(key64)
-
-        return browsers
-
-    # Detect some common windows browsers
-    for browser in find_windows_browsers():
-        browser = browser.lower()
-        if "iexplore" in browser:
-            register("iexplore", None, InternetExplorer("iexplore"))
-        elif "chrome" in browser:
-            register("chrome", None, WinChrome("chrome"))
-        elif "firefox" in browser:
-            register("firefox", None, WinFirefox("firefox"))
-        elif "opera" in browser:
-            register("opera", None, WinOpera("opera"))
-        elif "seamonkey" in browser:
-            register("seamonkey", None, WinSeaMonkey("seamonkey"))
-        else:
-            register(browser, None, WindowsDefault(browser))
+    # Detect some common Windows browsers, fallback to IE
+    iexplore = os.path.join(os.environ.get("PROGRAMFILES", "C:\\Program Files"),
+                            "Internet Explorer\\IEXPLORE.EXE")
+    for browser in ("firefox", "firebird", "seamonkey", "mozilla",
+                    "netscape", "opera", iexplore):
+        if shutil.which(browser):
+            register(browser, None, BackgroundBrowser(browser))
 
 #
 # Platform support for MacOS