Skip to content

Commit 0905916

Browse files
ericsnowcurrentlyericsnowcurrently
authored
bpo-45020: Freeze os, site, and codecs. (gh-28398)
https://bugs.python.org/issue45020
1 parent 74cc245 commit 0905916

File tree

9 files changed

+79
-31
lines changed

9 files changed

+79
-31
lines changed

‎Lib/test/support/__init__.py

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,13 @@
9999
# option.
100100
LONG_TIMEOUT = 5 * 60.0
101101

102+
# TEST_HOME_DIR refers to the top level directory of the "test" package
103+
# that contains Python's regression test suite
104+
TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__))
105+
TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR)
106+
STDLIB_DIR = os.path.dirname(TEST_HOME_DIR)
107+
REPO_ROOT = os.path.dirname(STDLIB_DIR)
108+
102109

103110
class Error(Exception):
104111
"""Base class for regression test exceptions."""
@@ -148,9 +155,7 @@ def load_tests(*args):
148155
"""
149156
if pattern is None:
150157
pattern = "test*"
151-
top_dir = os.path.dirname( # Lib
152-
os.path.dirname( # test
153-
os.path.dirname(__file__))) # support
158+
top_dir = STDLIB_DIR
154159
package_tests = loader.discover(start_dir=pkg_dir,
155160
top_level_dir=top_dir,
156161
pattern=pattern)
@@ -459,11 +464,6 @@ def requires_debug_ranges(reason='requires co_positions / debug_ranges'):
459464
# PGO task. If this is True, PGO is also True.
460465
PGO_EXTENDED = False
461466

462-
# TEST_HOME_DIR refers to the top level directory of the "test" package
463-
# that contains Python's regression test suite
464-
TEST_SUPPORT_DIR = os.path.dirname(os.path.abspath(__file__))
465-
TEST_HOME_DIR = os.path.dirname(TEST_SUPPORT_DIR)
466-
467467
# TEST_DATA_DIR is used as a target download location for remote resources
468468
TEST_DATA_DIR = os.path.join(TEST_HOME_DIR, "data")
469469

@@ -1390,7 +1390,7 @@ def _platform_specific(self):
13901390
self._env = {k.upper(): os.getenv(k) for k in os.environ}
13911391
self._env["PYTHONHOME"] = os.path.dirname(self.real)
13921392
if sysconfig.is_python_build(True):
1393-
self._env["PYTHONPATH"] = os.path.dirname(os.__file__)
1393+
self._env["PYTHONPATH"] = STDLIB_DIR
13941394

13951395
def __enter__(self):
13961396
os.symlink(self.real, self.link)

‎Lib/test/test_embed.py

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
import contextlib
99
import json
1010
import os
11+
import os.path
1112
import re
1213
import shutil
1314
import subprocess
@@ -52,23 +53,21 @@ def remove_python_envvars():
5253

5354
class EmbeddingTestsMixin:
5455
def setUp(self):
55-
here = os.path.abspath(__file__)
56-
basepath = os.path.dirname(os.path.dirname(os.path.dirname(here)))
5756
exename = "_testembed"
5857
if MS_WINDOWS:
5958
ext = ("_d" if debug_build(sys.executable) else "") + ".exe"
6059
exename += ext
6160
exepath = os.path.dirname(sys.executable)
6261
else:
63-
exepath = os.path.join(basepath, "Programs")
62+
exepath = os.path.join(support.REPO_ROOT, "Programs")
6463
self.test_exe = exe = os.path.join(exepath, exename)
6564
if not os.path.exists(exe):
6665
self.skipTest("%r doesn't exist" % exe)
6766
# This is needed otherwise we get a fatal error:
6867
# "Py_Initialize: Unable to get the locale encoding
6968
# LookupError: no codec search functions registered: can't find encoding"
7069
self.oldcwd = os.getcwd()
71-
os.chdir(basepath)
70+
os.chdir(support.REPO_ROOT)
7271

7372
def tearDown(self):
7473
os.chdir(self.oldcwd)
@@ -1304,7 +1303,10 @@ def test_init_pyvenv_cfg(self):
13041303
lib_dynload = os.path.join(pyvenv_home, 'lib')
13051304
os.makedirs(lib_dynload)
13061305
# getpathp.c uses Lib\os.py as the LANDMARK
1307-
shutil.copyfile(os.__file__, os.path.join(lib_dynload, 'os.py'))
1306+
shutil.copyfile(
1307+
os.path.join(support.STDLIB_DIR, 'os.py'),
1308+
os.path.join(lib_dynload, 'os.py'),
1309+
)
13081310

13091311
filename = os.path.join(tmpdir, 'pyvenv.cfg')
13101312
with open(filename, "w", encoding="utf8") as fp:

‎Lib/test/test_import/__init__.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,8 @@
1919
from unittest import mock
2020

2121
from test.support import os_helper
22-
from test.support import (is_jython, swap_attr, swap_item, cpython_only)
22+
from test.support import (
23+
STDLIB_DIR, is_jython, swap_attr, swap_item, cpython_only)
2324
from test.support.import_helper import (
2425
forget, make_legacy_pyc, unlink, unload, DirsOnSysPath, CleanImport)
2526
from test.support.os_helper import (
@@ -495,7 +496,7 @@ def test_dll_dependency_import(self):
495496

496497
env = None
497498
env = {k.upper(): os.environ[k] for k in os.environ}
498-
env["PYTHONPATH"] = tmp2 + ";" + os.path.dirname(os.__file__)
499+
env["PYTHONPATH"] = tmp2 + ";" + STDLIB_DIR
499500

500501
# Test 1: import with added DLL directory
501502
subprocess.check_call([

‎Lib/test/test_site.py

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -591,7 +591,7 @@ def _calc_sys_path_for_underpth_nosite(self, sys_prefix, lines):
591591
return sys_path
592592

593593
def test_underpth_nosite_file(self):
594-
libpath = os.path.dirname(os.path.dirname(encodings.__file__))
594+
libpath = test.support.STDLIB_DIR
595595
exe_prefix = os.path.dirname(sys.executable)
596596
pth_lines = [
597597
'fake-path-name',
@@ -619,7 +619,7 @@ def test_underpth_nosite_file(self):
619619
)
620620

621621
def test_underpth_file(self):
622-
libpath = os.path.dirname(os.path.dirname(encodings.__file__))
622+
libpath = test.support.STDLIB_DIR
623623
exe_prefix = os.path.dirname(sys.executable)
624624
exe_file = self._create_underpth_exe([
625625
'fake-path-name',
@@ -644,7 +644,7 @@ def test_underpth_file(self):
644644

645645

646646
def test_underpth_dll_file(self):
647-
libpath = os.path.dirname(os.path.dirname(encodings.__file__))
647+
libpath = test.support.STDLIB_DIR
648648
exe_prefix = os.path.dirname(sys.executable)
649649
exe_file = self._create_underpth_exe([
650650
'fake-path-name',

‎Makefile.pre.in

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -762,6 +762,9 @@ Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
762762
Python/frozen_modules/abc.h: Programs/_freeze_module Lib/abc.py
763763
Programs/_freeze_module abc $(srcdir)/Lib/abc.py $(srcdir)/Python/frozen_modules/abc.h
764764

765+
Python/frozen_modules/codecs.h: Programs/_freeze_module Lib/codecs.py
766+
Programs/_freeze_module codecs $(srcdir)/Lib/codecs.py $(srcdir)/Python/frozen_modules/codecs.h
767+
765768
Python/frozen_modules/io.h: Programs/_freeze_module Lib/io.py
766769
Programs/_freeze_module io $(srcdir)/Lib/io.py $(srcdir)/Python/frozen_modules/io.h
767770

@@ -780,6 +783,12 @@ Python/frozen_modules/ntpath.h: Programs/_freeze_module Lib/ntpath.py
780783
Python/frozen_modules/posixpath.h: Programs/_freeze_module Lib/posixpath.py
781784
Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py $(srcdir)/Python/frozen_modules/posixpath.h
782785

786+
Python/frozen_modules/os.h: Programs/_freeze_module Lib/os.py
787+
Programs/_freeze_module os $(srcdir)/Lib/os.py $(srcdir)/Python/frozen_modules/os.h
788+
789+
Python/frozen_modules/site.h: Programs/_freeze_module Lib/site.py
790+
Programs/_freeze_module site $(srcdir)/Lib/site.py $(srcdir)/Python/frozen_modules/site.h
791+
783792
Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py
784793
Programs/_freeze_module stat $(srcdir)/Lib/stat.py $(srcdir)/Python/frozen_modules/stat.h
785794

@@ -1023,12 +1032,15 @@ FROZEN_FILES = \
10231032
Python/frozen_modules/importlib__bootstrap_external.h \
10241033
Python/frozen_modules/zipimport.h \
10251034
Python/frozen_modules/abc.h \
1035+
Python/frozen_modules/codecs.h \
10261036
Python/frozen_modules/io.h \
10271037
Python/frozen_modules/_collections_abc.h \
10281038
Python/frozen_modules/_sitebuiltins.h \
10291039
Python/frozen_modules/genericpath.h \
10301040
Python/frozen_modules/ntpath.h \
10311041
Python/frozen_modules/posixpath.h \
1042+
Python/frozen_modules/os.h \
1043+
Python/frozen_modules/site.h \
10321044
Python/frozen_modules/stat.h \
10331045
Python/frozen_modules/__hello__.h
10341046
# End FROZEN_FILES

‎PCbuild/_freeze_module.vcxproj

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -250,6 +250,11 @@
250250
<IntFile>$(IntDir)abc.g.h</IntFile>
251251
<OutFile>$(PySourcePath)Python\frozen_modules\abc.h</OutFile>
252252
</None>
253+
<None Include="..\Lib\codecs.py">
254+
<ModName>codecs</ModName>
255+
<IntFile>$(IntDir)codecs.g.h</IntFile>
256+
<OutFile>$(PySourcePath)Python\frozen_modules\codecs.h</OutFile>
257+
</None>
253258
<None Include="..\Lib\io.py">
254259
<ModName>io</ModName>
255260
<IntFile>$(IntDir)io.g.h</IntFile>
@@ -280,6 +285,16 @@
280285
<IntFile>$(IntDir)posixpath.g.h</IntFile>
281286
<OutFile>$(PySourcePath)Python\frozen_modules\posixpath.h</OutFile>
282287
</None>
288+
<None Include="..\Lib\os.py">
289+
<ModName>os</ModName>
290+
<IntFile>$(IntDir)os.g.h</IntFile>
291+
<OutFile>$(PySourcePath)Python\frozen_modules\os.h</OutFile>
292+
</None>
293+
<None Include="..\Lib\site.py">
294+
<ModName>site</ModName>
295+
<IntFile>$(IntDir)site.g.h</IntFile>
296+
<OutFile>$(PySourcePath)Python\frozen_modules\site.h</OutFile>
297+
</None>
283298
<None Include="..\Lib\stat.py">
284299
<ModName>stat</ModName>
285300
<IntFile>$(IntDir)stat.g.h</IntFile>

‎PCbuild/_freeze_module.vcxproj.filters

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,9 @@
2828
<None Include="..\Lib\abc.py">
2929
<Filter>Python Files</Filter>
3030
</None>
31+
<None Include="..\Lib\codecs.py">
32+
<Filter>Python Files</Filter>
33+
</None>
3134
<None Include="..\Lib\io.py">
3235
<Filter>Python Files</Filter>
3336
</None>
@@ -46,6 +49,12 @@
4649
<None Include="..\Lib\posixpath.py">
4750
<Filter>Python Files</Filter>
4851
</None>
52+
<None Include="..\Lib\os.py">
53+
<Filter>Python Files</Filter>
54+
</None>
55+
<None Include="..\Lib\site.py">
56+
<Filter>Python Files</Filter>
57+
</None>
4958
<None Include="..\Lib\stat.py">
5059
<Filter>Python Files</Filter>
5160
</None>

‎Python/frozen.c

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -42,12 +42,15 @@
4242
#include "frozen_modules/importlib__bootstrap_external.h"
4343
#include "frozen_modules/zipimport.h"
4444
#include "frozen_modules/abc.h"
45+
#include "frozen_modules/codecs.h"
4546
#include "frozen_modules/io.h"
4647
#include "frozen_modules/_collections_abc.h"
4748
#include "frozen_modules/_sitebuiltins.h"
4849
#include "frozen_modules/genericpath.h"
4950
#include "frozen_modules/ntpath.h"
5051
#include "frozen_modules/posixpath.h"
52+
#include "frozen_modules/os.h"
53+
#include "frozen_modules/site.h"
5154
#include "frozen_modules/stat.h"
5255
#include "frozen_modules/__hello__.h"
5356
/* End includes */
@@ -62,15 +65,21 @@ static const struct _frozen _PyImport_FrozenModules[] = {
6265
(int)sizeof(_Py_M__importlib__bootstrap_external)},
6366
{"zipimport", _Py_M__zipimport, (int)sizeof(_Py_M__zipimport)},
6467

65-
/* stdlib */
68+
/* stdlib - startup, without site (python -S) */
6669
{"abc", _Py_M__abc, (int)sizeof(_Py_M__abc)},
70+
{"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs)},
6771
{"io", _Py_M__io, (int)sizeof(_Py_M__io)},
72+
73+
/* stdlib - startup, with site */
6874
{"_collections_abc", _Py_M___collections_abc,
6975
(int)sizeof(_Py_M___collections_abc)},
7076
{"_sitebuiltins", _Py_M___sitebuiltins, (int)sizeof(_Py_M___sitebuiltins)},
7177
{"genericpath", _Py_M__genericpath, (int)sizeof(_Py_M__genericpath)},
7278
{"ntpath", _Py_M__ntpath, (int)sizeof(_Py_M__ntpath)},
7379
{"posixpath", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)},
80+
{"os.path", _Py_M__posixpath, (int)sizeof(_Py_M__posixpath)},
81+
{"os", _Py_M__os, (int)sizeof(_Py_M__os)},
82+
{"site", _Py_M__site, (int)sizeof(_Py_M__site)},
7483
{"stat", _Py_M__stat, (int)sizeof(_Py_M__stat)},
7584

7685
/* Test module */

‎Tools/scripts/freeze_modules.py

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -66,26 +66,26 @@ def find_tool():
6666
# on a builtin zip file instead of a filesystem.
6767
'zipimport',
6868
]),
69-
('stdlib', [
70-
# For the moment we skip codecs, encodings.*, os, and site.
71-
# These modules have different generated files depending on
72-
# if a debug or non-debug build. (See bpo-45186 and bpo-45188.)
73-
# without site (python -S)
69+
('stdlib - startup, without site (python -S)', [
7470
'abc',
75-
#'codecs',
76-
# '<encodings.*>',
71+
'codecs',
72+
# For now we do not freeze the encodings, due # to the noise all
73+
# those extra modules add to the text printed during the build.
74+
# (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.)
75+
#'<encodings.*>',
7776
'io',
78-
# with site
77+
]),
78+
('stdlib - startup, with site', [
7979
'_collections_abc',
8080
'_sitebuiltins',
8181
'genericpath',
8282
'ntpath',
8383
'posixpath',
8484
# We must explicitly mark os.path as a frozen module
8585
# even though it will never be imported.
86-
#f'{OS_PATH} : os.path',
87-
#'os',
88-
#'site',
86+
f'{OS_PATH} : os.path',
87+
'os',
88+
'site',
8989
'stat',
9090
]),
9191
('Test module', [

0 commit comments

Comments
 (0)