bpo-45020: Freeze some of the modules imported during startup.

Lib/test/test_cmd_line_script.py

@@ -474,7 +474,6 @@ def test_dash_m_errors(self):
                 br'ModuleNotFoundError'),
             ('builtins.x.y', br'Error while finding module specification.*'
                 br'ModuleNotFoundError.*No module named.*not a package'),
-            ('os.path', br'loader.*cannot handle'),
             ('importlib', br'No module named.*'
                 br'is a package and cannot be directly executed'),
             ('importlib.nonexistent', br'No module named'),

Lib/test/test_imp.py

@@ -16,6 +16,9 @@
 import _imp
 
 
+OS_PATH_NAME = os.path.__name__
+
+
 def requires_load_dynamic(meth):
     """Decorator to skip a test if not running under CPython or lacking
     imp.load_dynamic()."""
@@ -213,15 +216,17 @@ def test_load_from_source(self):
         # state after reversion. Reinitialising the module contents
         # and just reverting os.environ to its previous state is an OK
         # workaround
-        orig_path = os.path
-        orig_getenv = os.getenv
-        with os_helper.EnvironmentVarGuard():
-            x = imp.find_module("os")
-            self.addCleanup(x[0].close)
-            new_os = imp.load_module("os", *x)
-            self.assertIs(os, new_os)
-            self.assertIs(orig_path, new_os.path)
-            self.assertIsNot(orig_getenv, new_os.getenv)
+        with import_helper.CleanImport('os', 'os.path', OS_PATH_NAME):
+            import os
+            orig_path = os.path
+            orig_getenv = os.getenv
+            with os_helper.EnvironmentVarGuard():
+                x = imp.find_module("os")
+                self.addCleanup(x[0].close)
+                new_os = imp.load_module("os", *x)
+                self.assertIs(os, new_os)
+                self.assertIs(orig_path, new_os.path)
+                self.assertIsNot(orig_getenv, new_os.getenv)
 
     @requires_load_dynamic
     def test_issue15828_load_extensions(self):

Lib/test/test_import/__init__.py

@@ -21,7 +21,7 @@
 from test.support import os_helper
 from test.support import (is_jython, swap_attr, swap_item, cpython_only)
 from test.support.import_helper import (
-    forget, make_legacy_pyc, unlink, unload, DirsOnSysPath)
+    forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport)
 from test.support.os_helper import (
     TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE, temp_dir)
 from test.support import script_helper
@@ -86,8 +86,10 @@ def test_from_import_missing_attr_raises_ImportError(self):
             from importlib import something_that_should_not_exist_anywhere
 
     def test_from_import_missing_attr_has_name_and_path(self):
-        with self.assertRaises(ImportError) as cm:
-            from os import i_dont_exist
+        with CleanImport('os'):
+            import os
+            with self.assertRaises(ImportError) as cm:
+                from os import i_dont_exist
         self.assertEqual(cm.exception.name, 'os')
         self.assertEqual(cm.exception.path, os.__file__)
         self.assertRegex(str(cm.exception), r"cannot import name 'i_dont_exist' from 'os' \(.*os.py\)")

Lib/test/test_pydoc.py

@@ -23,6 +23,7 @@
 import textwrap
 from io import StringIO
 from collections import namedtuple
+from test.support import import_helper
 from test.support import os_helper
 from test.support.script_helper import assert_python_ok, assert_python_failure
 from test.support import threading_helper
@@ -728,6 +729,7 @@ def test_synopsis(self):
     @unittest.skipIf(sys.flags.optimize >= 2,
                      'Docstrings are omitted with -OO and above')
     def test_synopsis_sourceless(self):
+        os = import_helper.import_fresh_module('os')
         expected = os.__doc__.splitlines()[0]
         filename = os.__cached__
         synopsis = pydoc.synopsis(filename)

Lib/test/test_threading.py

@@ -1026,8 +1026,9 @@ def test_4_daemon_threads(self):
 
             def random_io():
                 '''Loop for a while sleeping random tiny amounts and doing some I/O.'''
+                import test.test_threading as mod
                 while True:
-                    with open(os.__file__, 'rb') as in_f:
+                    with open(mod.__file__, 'rb') as in_f:
                         stuff = in_f.read(200)
                         with open(os.devnull, 'wb') as null_f:
                             null_f.write(stuff)

Lib/test/test_trace.py

@@ -376,7 +376,7 @@ def test_coverage(self):
 
     def test_coverage_ignore(self):
         # Ignore all files, nothing should be traced nor printed
-        libpath = os.path.normpath(os.path.dirname(os.__file__))
+        libpath = os.path.normpath(os.path.dirname(os.path.dirname(__file__)))
         # sys.prefix does not work when running from a checkout
         tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,
                              libpath], trace=0, count=1)

Makefile.pre.in

@@ -751,24 +751,40 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES)
 # BEGIN: freezing modules
 
 Python/frozen_modules/importlib__bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py
-	$(srcdir)/Programs/_freeze_module importlib._bootstrap \
-		$(srcdir)/Lib/importlib/_bootstrap.py \
-		$(srcdir)/Python/frozen_modules/importlib__bootstrap.h
+	$(srcdir)/Programs/_freeze_module importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/frozen_modules/importlib__bootstrap.h
 
 Python/frozen_modules/importlib__bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py
-	$(srcdir)/Programs/_freeze_module importlib._bootstrap_external \
-		$(srcdir)/Lib/importlib/_bootstrap_external.py \
-		$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h
+	$(srcdir)/Programs/_freeze_module importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h
 
 Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
-	$(srcdir)/Programs/_freeze_module zipimport \
-		$(srcdir)/Lib/zipimport.py \
-		$(srcdir)/Python/frozen_modules/zipimport.h
+	$(srcdir)/Programs/_freeze_module zipimport $(srcdir)/Lib/zipimport.py $(srcdir)/Python/frozen_modules/zipimport.h
+
+Python/frozen_modules/abc.h: Programs/_freeze_module Lib/abc.py
+	$(srcdir)/Programs/_freeze_module abc $(srcdir)/Lib/abc.py $(srcdir)/Python/frozen_modules/abc.h
+
+Python/frozen_modules/io.h: Programs/_freeze_module Lib/io.py
+	$(srcdir)/Programs/_freeze_module io $(srcdir)/Lib/io.py $(srcdir)/Python/frozen_modules/io.h
+
+Python/frozen_modules/_collections_abc.h: Programs/_freeze_module Lib/_collections_abc.py
+	$(srcdir)/Programs/_freeze_module _collections_abc $(srcdir)/Lib/_collections_abc.py $(srcdir)/Python/frozen_modules/_collections_abc.h
+
+Python/frozen_modules/_sitebuiltins.h: Programs/_freeze_module Lib/_sitebuiltins.py
+	$(srcdir)/Programs/_freeze_module _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py $(srcdir)/Python/frozen_modules/_sitebuiltins.h
+
+Python/frozen_modules/genericpath.h: Programs/_freeze_module Lib/genericpath.py
+	$(srcdir)/Programs/_freeze_module genericpath $(srcdir)/Lib/genericpath.py $(srcdir)/Python/frozen_modules/genericpath.h
+
+Python/frozen_modules/ntpath.h: Programs/_freeze_module Lib/ntpath.py
+	$(srcdir)/Programs/_freeze_module ntpath $(srcdir)/Lib/ntpath.py $(srcdir)/Python/frozen_modules/ntpath.h
+
+Python/frozen_modules/posixpath.h: Programs/_freeze_module Lib/posixpath.py
+	$(srcdir)/Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py $(srcdir)/Python/frozen_modules/posixpath.h
+
+Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py
+	$(srcdir)/Programs/_freeze_module stat $(srcdir)/Lib/stat.py $(srcdir)/Python/frozen_modules/stat.h
 
 Python/frozen_modules/hello.h: Programs/_freeze_module Tools/freeze/flag.py
-	$(srcdir)/Programs/_freeze_module hello \
-		$(srcdir)/Tools/freeze/flag.py \
-		$(srcdir)/Python/frozen_modules/hello.h
+	$(srcdir)/Programs/_freeze_module hello $(srcdir)/Tools/freeze/flag.py $(srcdir)/Python/frozen_modules/hello.h
 
 # END: freezing modules
 
@@ -1006,6 +1022,14 @@ FROZEN_FILES = \
 		$(srcdir)/Python/frozen_modules/importlib__bootstrap.h \
 		$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h \
 		$(srcdir)/Python/frozen_modules/zipimport.h \
+		$(srcdir)/Python/frozen_modules/abc.h \
+		$(srcdir)/Python/frozen_modules/io.h \
+		$(srcdir)/Python/frozen_modules/_collections_abc.h \
+		$(srcdir)/Python/frozen_modules/_sitebuiltins.h \
+		$(srcdir)/Python/frozen_modules/genericpath.h \
+		$(srcdir)/Python/frozen_modules/ntpath.h \
+		$(srcdir)/Python/frozen_modules/posixpath.h \
+		$(srcdir)/Python/frozen_modules/stat.h \
 		$(srcdir)/Python/frozen_modules/hello.h
 # End FROZEN_FILES
 

Misc/NEWS.d/next/Build/2021-09-14-10-07-23.bpo-45020._VGGPv.rst

@@ -0,0 +1,4 @@
+Freeze stdlib modules that are imported during startup.  This provides
+significant performance improvements to startup.  If necessary, use the
+previously added "-X frozen_modules=off" commandline option to force
+importing the source modules.

Python/frozen.c

@@ -41,6 +41,14 @@
 #include "frozen_modules/importlib__bootstrap.h"
 #include "frozen_modules/importlib__bootstrap_external.h"
 #include "frozen_modules/zipimport.h"
+#include "frozen_modules/abc.h"
+#include "frozen_modules/io.h"
+#include "frozen_modules/_collections_abc.h"
+#include "frozen_modules/_sitebuiltins.h"
+#include "frozen_modules/genericpath.h"
+#include "frozen_modules/ntpath.h"
+#include "frozen_modules/posixpath.h"
+#include "frozen_modules/stat.h"
 #include "frozen_modules/hello.h"
 /* End includes */
 
@@ -54,6 +62,17 @@ static const struct _frozen _PyImport_FrozenModules[] = {
         (int)sizeof(_Py_M__importlib__bootstrap_external)},
     {"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
 
+    /* stdlib */
+    {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc)},
+    {"io", _Py_M__io, (int)sizeof(_Py_M__io)},
+    {"_collections_abc", _Py_M___collections_abc,
+        (int)sizeof(_Py_M___collections_abc)},
+    {"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins)},
+    {"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath)},
+    {"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath)},
+    {"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)},
+    {"stat", _Py_M__stat, (int)sizeof(_Py_M__stat)},
+
     /* Test module */
     {"__hello__", _Py_M__hello, (int)sizeof(_Py_M__hello)},
     {"__phello__", _Py_M__hello, -(int)sizeof(_Py_M__hello)},