Skip to content

Commit cbeb819

Browse files
ericsnowcurrentlyericsnowcurrently
authored
bpo-45020: Freeze some of the modules imported during startup. (gh-28335)
Doing this provides significant performance gains for runtime startup (~15% with all the imported modules frozen). We don't yet freeze all the imported modules because there are a few hiccups in the build systems we need to sort out first. (See bpo-45186 and bpo-45188.) Note that in PR GH-28320 we added a command-line flag (-X frozen_modules=[on|off]) that allows users to opt out of (or into) using frozen modules. The default is still "off" but we will change it to "on" as soon as we can do it in a way that does not cause contributors pain. https://bugs.python.org/issue45020
1 parent 1a9ef57 commit cbeb819

19 files changed

+6879
-30
lines changed

‎Lib/test/test_cmd_line_script.py

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -474,7 +474,6 @@ def test_dash_m_errors(self):
474474
br'ModuleNotFoundError'),
475475
('builtins.x.y', br'Error while finding module specification.*'
476476
br'ModuleNotFoundError.*No module named.*not a package'),
477-
('os.path', br'loader.*cannot handle'),
478477
('importlib', br'No module named.*'
479478
br'is a package and cannot be directly executed'),
480479
('importlib.nonexistent', br'No module named'),

‎Lib/test/test_imp.py

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,9 @@
1616
import _imp
1717

1818

19+
OS_PATH_NAME = os.path.__name__
20+
21+
1922
def requires_load_dynamic(meth):
2023
"""Decorator to skip a test if not running under CPython or lacking
2124
imp.load_dynamic()."""
@@ -213,15 +216,17 @@ def test_load_from_source(self):
213216
# state after reversion. Reinitialising the module contents
214217
# and just reverting os.environ to its previous state is an OK
215218
# workaround
216-
orig_path = os.path
217-
orig_getenv = os.getenv
218-
with os_helper.EnvironmentVarGuard():
219-
x = imp.find_module("os")
220-
self.addCleanup(x[0].close)
221-
new_os = imp.load_module("os", *x)
222-
self.assertIs(os, new_os)
223-
self.assertIs(orig_path, new_os.path)
224-
self.assertIsNot(orig_getenv, new_os.getenv)
219+
with import_helper.CleanImport('os', 'os.path', OS_PATH_NAME):
220+
import os
221+
orig_path = os.path
222+
orig_getenv = os.getenv
223+
with os_helper.EnvironmentVarGuard():
224+
x = imp.find_module("os")
225+
self.addCleanup(x[0].close)
226+
new_os = imp.load_module("os", *x)
227+
self.assertIs(os, new_os)
228+
self.assertIs(orig_path, new_os.path)
229+
self.assertIsNot(orig_getenv, new_os.getenv)
225230

226231
@requires_load_dynamic
227232
def test_issue15828_load_extensions(self):

‎Lib/test/test_import/__init__.py

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
from test.support import os_helper
2222
from test.support import (is_jython, swap_attr, swap_item, cpython_only)
2323
from test.support.import_helper import (
24-
forget, make_legacy_pyc, unlink, unload, DirsOnSysPath)
24+
forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport)
2525
from test.support.os_helper import (
2626
TESTFN, rmtree, temp_umask, TESTFN_UNENCODABLE, temp_dir)
2727
from test.support import script_helper
@@ -86,8 +86,10 @@ def test_from_import_missing_attr_raises_ImportError(self):
8686
from importlib import something_that_should_not_exist_anywhere
8787

8888
def test_from_import_missing_attr_has_name_and_path(self):
89-
with self.assertRaises(ImportError) as cm:
90-
from os import i_dont_exist
89+
with CleanImport('os'):
90+
import os
91+
with self.assertRaises(ImportError) as cm:
92+
from os import i_dont_exist
9193
self.assertEqual(cm.exception.name, 'os')
9294
self.assertEqual(cm.exception.path, os.__file__)
9395
self.assertRegex(str(cm.exception), r"cannot import name 'i_dont_exist' from 'os' \(.*os.py\)")

‎Lib/test/test_pydoc.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import textwrap
2424
from io import StringIO
2525
from collections import namedtuple
26+
from test.support import import_helper
2627
from test.support import os_helper
2728
from test.support.script_helper import assert_python_ok, assert_python_failure
2829
from test.support import threading_helper
@@ -728,6 +729,7 @@ def test_synopsis(self):
728729
@unittest.skipIf(sys.flags.optimize >= 2,
729730
'Docstrings are omitted with -OO and above')
730731
def test_synopsis_sourceless(self):
732+
os = import_helper.import_fresh_module('os')
731733
expected = os.__doc__.splitlines()[0]
732734
filename = os.__cached__
733735
synopsis = pydoc.synopsis(filename)

‎Lib/test/test_threading.py

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1026,8 +1026,9 @@ def test_4_daemon_threads(self):
10261026
10271027
def random_io():
10281028
'''Loop for a while sleeping random tiny amounts and doing some I/O.'''
1029+
import test.test_threading as mod
10291030
while True:
1030-
with open(os.__file__, 'rb') as in_f:
1031+
with open(mod.__file__, 'rb') as in_f:
10311032
stuff = in_f.read(200)
10321033
with open(os.devnull, 'wb') as null_f:
10331034
null_f.write(stuff)

‎Lib/test/test_trace.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -376,7 +376,7 @@ def test_coverage(self):
376376

377377
def test_coverage_ignore(self):
378378
# Ignore all files, nothing should be traced nor printed
379-
libpath = os.path.normpath(os.path.dirname(os.__file__))
379+
libpath = os.path.normpath(os.path.dirname(os.path.dirname(__file__)))
380380
# sys.prefix does not work when running from a checkout
381381
tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix,
382382
libpath], trace=0, count=1)

‎Makefile.pre.in

Lines changed: 36 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -751,24 +751,40 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES)
751751
# BEGIN: freezing modules
752752

753753
Python/frozen_modules/importlib__bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py
754-
$(srcdir)/Programs/_freeze_module importlib._bootstrap \
755-
$(srcdir)/Lib/importlib/_bootstrap.py \
756-
$(srcdir)/Python/frozen_modules/importlib__bootstrap.h
754+
$(srcdir)/Programs/_freeze_module importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/frozen_modules/importlib__bootstrap.h
757755

758756
Python/frozen_modules/importlib__bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py
759-
$(srcdir)/Programs/_freeze_module importlib._bootstrap_external \
760-
$(srcdir)/Lib/importlib/_bootstrap_external.py \
761-
$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h
757+
$(srcdir)/Programs/_freeze_module importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h
762758

763759
Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
764-
$(srcdir)/Programs/_freeze_module zipimport \
765-
$(srcdir)/Lib/zipimport.py \
766-
$(srcdir)/Python/frozen_modules/zipimport.h
760+
$(srcdir)/Programs/_freeze_module zipimport $(srcdir)/Lib/zipimport.py $(srcdir)/Python/frozen_modules/zipimport.h
761+
762+
Python/frozen_modules/abc.h: Programs/_freeze_module Lib/abc.py
763+
$(srcdir)/Programs/_freeze_module abc $(srcdir)/Lib/abc.py $(srcdir)/Python/frozen_modules/abc.h
764+
765+
Python/frozen_modules/io.h: Programs/_freeze_module Lib/io.py
766+
$(srcdir)/Programs/_freeze_module io $(srcdir)/Lib/io.py $(srcdir)/Python/frozen_modules/io.h
767+
768+
Python/frozen_modules/_collections_abc.h: Programs/_freeze_module Lib/_collections_abc.py
769+
$(srcdir)/Programs/_freeze_module _collections_abc $(srcdir)/Lib/_collections_abc.py $(srcdir)/Python/frozen_modules/_collections_abc.h
770+
771+
Python/frozen_modules/_sitebuiltins.h: Programs/_freeze_module Lib/_sitebuiltins.py
772+
$(srcdir)/Programs/_freeze_module _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py $(srcdir)/Python/frozen_modules/_sitebuiltins.h
773+
774+
Python/frozen_modules/genericpath.h: Programs/_freeze_module Lib/genericpath.py
775+
$(srcdir)/Programs/_freeze_module genericpath $(srcdir)/Lib/genericpath.py $(srcdir)/Python/frozen_modules/genericpath.h
776+
777+
Python/frozen_modules/ntpath.h: Programs/_freeze_module Lib/ntpath.py
778+
$(srcdir)/Programs/_freeze_module ntpath $(srcdir)/Lib/ntpath.py $(srcdir)/Python/frozen_modules/ntpath.h
779+
780+
Python/frozen_modules/posixpath.h: Programs/_freeze_module Lib/posixpath.py
781+
$(srcdir)/Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py $(srcdir)/Python/frozen_modules/posixpath.h
782+
783+
Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py
784+
$(srcdir)/Programs/_freeze_module stat $(srcdir)/Lib/stat.py $(srcdir)/Python/frozen_modules/stat.h
767785

768786
Python/frozen_modules/hello.h: Programs/_freeze_module Tools/freeze/flag.py
769-
$(srcdir)/Programs/_freeze_module hello \
770-
$(srcdir)/Tools/freeze/flag.py \
771-
$(srcdir)/Python/frozen_modules/hello.h
787+
$(srcdir)/Programs/_freeze_module hello $(srcdir)/Tools/freeze/flag.py $(srcdir)/Python/frozen_modules/hello.h
772788

773789
# END: freezing modules
774790

@@ -1006,6 +1022,14 @@ FROZEN_FILES = \
10061022
$(srcdir)/Python/frozen_modules/importlib__bootstrap.h \
10071023
$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h \
10081024
$(srcdir)/Python/frozen_modules/zipimport.h \
1025+
$(srcdir)/Python/frozen_modules/abc.h \
1026+
$(srcdir)/Python/frozen_modules/io.h \
1027+
$(srcdir)/Python/frozen_modules/_collections_abc.h \
1028+
$(srcdir)/Python/frozen_modules/_sitebuiltins.h \
1029+
$(srcdir)/Python/frozen_modules/genericpath.h \
1030+
$(srcdir)/Python/frozen_modules/ntpath.h \
1031+
$(srcdir)/Python/frozen_modules/posixpath.h \
1032+
$(srcdir)/Python/frozen_modules/stat.h \
10091033
$(srcdir)/Python/frozen_modules/hello.h
10101034
# End FROZEN_FILES
10111035

‎Misc/NEWS.d/next/Build/2021-09-14-10-07-23.bpo-45020._VGGPv.rst

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
Freeze stdlib modules that are imported during startup. This provides
2+
significant performance improvements to startup. If necessary, use the
3+
previously added "-X frozen_modules=off" commandline option to force
4+
importing the source modules.

‎Python/frozen.c

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,14 @@
4141
#include "frozen_modules/importlib__bootstrap.h"
4242
#include "frozen_modules/importlib__bootstrap_external.h"
4343
#include "frozen_modules/zipimport.h"
44+
#include "frozen_modules/abc.h"
45+
#include "frozen_modules/io.h"
46+
#include "frozen_modules/_collections_abc.h"
47+
#include "frozen_modules/_sitebuiltins.h"
48+
#include "frozen_modules/genericpath.h"
49+
#include "frozen_modules/ntpath.h"
50+
#include "frozen_modules/posixpath.h"
51+
#include "frozen_modules/stat.h"
4452
#include "frozen_modules/hello.h"
4553
/* End includes */
4654

@@ -54,6 +62,17 @@ static const struct _frozen _PyImport_FrozenModules[] = {
5462
(int)sizeof(_Py_M__importlib__bootstrap_external)},
5563
{"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
5664

65+
/* stdlib */
66+
{"abc", _Py_M__abc, (int)sizeof(_Py_M__abc)},
67+
{"io", _Py_M__io, (int)sizeof(_Py_M__io)},
68+
{"_collections_abc", _Py_M___collections_abc,
69+
(int)sizeof(_Py_M___collections_abc)},
70+
{"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins)},
71+
{"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath)},
72+
{"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath)},
73+
{"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)},
74+
{"stat", _Py_M__stat, (int)sizeof(_Py_M__stat)},
75+
5776
/* Test module */
5877
{"__hello__", _Py_M__hello, (int)sizeof(_Py_M__hello)},
5978
{"__phello__", _Py_M__hello, -(int)sizeof(_Py_M__hello)},

‎Python/frozen_modules/MANIFEST

Lines changed: 8 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

0 commit comments

Comments
 (0)