Skip to content

Commit 7c801e0

Browse files
ericsnowcurrentlyericsnowcurrently
authored
bpo-45020: Fix some corner cases for frozen module generation. (gh-28538)
This also includes some cleanup in preparation for a PR to make the "make all" output less noisy. https://bugs.python.org/issue45020
1 parent bfe26bb commit 7c801e0

File tree

5 files changed

+130
-69
lines changed

5 files changed

+130
-69
lines changed

‎Makefile.pre.in

Lines changed: 48 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -736,25 +736,54 @@ Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
736736
############################################################################
737737
# frozen modules (including importlib)
738738

739+
# FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py.
740+
FROZEN_FILES_IN = \
741+
Lib/importlib/_bootstrap.py \
742+
Lib/importlib/_bootstrap_external.py \
743+
Lib/zipimport.py \
744+
Lib/abc.py \
745+
Lib/codecs.py \
746+
Lib/io.py \
747+
Lib/_collections_abc.py \
748+
Lib/_sitebuiltins.py \
749+
Lib/genericpath.py \
750+
Lib/ntpath.py \
751+
Lib/posixpath.py \
752+
Lib/os.py \
753+
Lib/site.py \
754+
Lib/stat.py \
755+
Lib/__hello__.py
756+
# End FROZEN_FILES_IN
757+
FROZEN_FILES_OUT = \
758+
Python/frozen_modules/importlib._bootstrap.h \
759+
Python/frozen_modules/importlib._bootstrap_external.h \
760+
Python/frozen_modules/zipimport.h \
761+
Python/frozen_modules/abc.h \
762+
Python/frozen_modules/codecs.h \
763+
Python/frozen_modules/io.h \
764+
Python/frozen_modules/_collections_abc.h \
765+
Python/frozen_modules/_sitebuiltins.h \
766+
Python/frozen_modules/genericpath.h \
767+
Python/frozen_modules/ntpath.h \
768+
Python/frozen_modules/posixpath.h \
769+
Python/frozen_modules/os.h \
770+
Python/frozen_modules/site.h \
771+
Python/frozen_modules/stat.h \
772+
Python/frozen_modules/__hello__.h
773+
# End FROZEN_FILES_OUT
774+
739775
Programs/_freeze_module.o: Programs/_freeze_module.c Makefile
740776

741777
Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN)
742778
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
743779

744-
Tools/scripts/freeze_modules.py: Programs/_freeze_module
745-
746-
.PHONY: regen-frozen
747-
regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES)
748-
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py
749-
@echo "The Makefile was updated, you may need to re-run make."
750-
751780
# BEGIN: freezing modules
752781

753-
Python/frozen_modules/importlib__bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py
754-
Programs/_freeze_module importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/frozen_modules/importlib__bootstrap.h
782+
Python/frozen_modules/importlib._bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py
783+
Programs/_freeze_module importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/frozen_modules/importlib._bootstrap.h
755784

756-
Python/frozen_modules/importlib__bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py
757-
Programs/_freeze_module importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h
785+
Python/frozen_modules/importlib._bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py
786+
Programs/_freeze_module importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h
758787

759788
Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
760789
Programs/_freeze_module zipimport $(srcdir)/Lib/zipimport.py $(srcdir)/Python/frozen_modules/zipimport.h
@@ -797,6 +826,13 @@ Python/frozen_modules/__hello__.h: Programs/_freeze_module Lib/__hello__.py
797826

798827
# END: freezing modules
799828

829+
Tools/scripts/freeze_modules.py: Programs/_freeze_module
830+
831+
.PHONY: regen-frozen
832+
regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN)
833+
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py
834+
@echo "The Makefile was updated, you may need to re-run make."
835+
800836
# We keep this renamed target around for folks with muscle memory.
801837
.PHONY: regen-importlib
802838
regen-importlib: regen-frozen
@@ -1026,26 +1062,7 @@ regen-opcode-targets:
10261062
Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \
10271063
$(srcdir)/Python/condvar.h
10281064

1029-
# FROZEN_FILES is auto-generated by Tools/scripts/freeze_modules.py.
1030-
FROZEN_FILES = \
1031-
Python/frozen_modules/importlib__bootstrap.h \
1032-
Python/frozen_modules/importlib__bootstrap_external.h \
1033-
Python/frozen_modules/zipimport.h \
1034-
Python/frozen_modules/abc.h \
1035-
Python/frozen_modules/codecs.h \
1036-
Python/frozen_modules/io.h \
1037-
Python/frozen_modules/_collections_abc.h \
1038-
Python/frozen_modules/_sitebuiltins.h \
1039-
Python/frozen_modules/genericpath.h \
1040-
Python/frozen_modules/ntpath.h \
1041-
Python/frozen_modules/posixpath.h \
1042-
Python/frozen_modules/os.h \
1043-
Python/frozen_modules/site.h \
1044-
Python/frozen_modules/stat.h \
1045-
Python/frozen_modules/__hello__.h
1046-
# End FROZEN_FILES
1047-
1048-
Python/frozen.o: $(FROZEN_FILES)
1065+
Python/frozen.o: $(FROZEN_FILES_OUT)
10491066

10501067
# Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to
10511068
# follow our naming conventions. dtrace(1) uses the output filename to generate

‎PCbuild/_freeze_module.vcxproj

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -232,13 +232,13 @@
232232
<!-- BEGIN frozen modules -->
233233
<None Include="..\Lib\importlib\_bootstrap.py">
234234
<ModName>importlib._bootstrap</ModName>
235-
<IntFile>$(IntDir)importlib__bootstrap.g.h</IntFile>
236-
<OutFile>$(PySourcePath)Python\frozen_modules\importlib__bootstrap.h</OutFile>
235+
<IntFile>$(IntDir)importlib._bootstrap.g.h</IntFile>
236+
<OutFile>$(PySourcePath)Python\frozen_modules\importlib._bootstrap.h</OutFile>
237237
</None>
238238
<None Include="..\Lib\importlib\_bootstrap_external.py">
239239
<ModName>importlib._bootstrap_external</ModName>
240-
<IntFile>$(IntDir)importlib__bootstrap_external.g.h</IntFile>
241-
<OutFile>$(PySourcePath)Python\frozen_modules\importlib__bootstrap_external.h</OutFile>
240+
<IntFile>$(IntDir)importlib._bootstrap_external.g.h</IntFile>
241+
<OutFile>$(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h</OutFile>
242242
</None>
243243
<None Include="..\Lib\zipimport.py">
244244
<ModName>zipimport</ModName>

‎Python/frozen.c

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -38,8 +38,8 @@
3838
#include "Python.h"
3939

4040
/* Includes for frozen modules: */
41-
#include "frozen_modules/importlib__bootstrap.h"
42-
#include "frozen_modules/importlib__bootstrap_external.h"
41+
#include "frozen_modules/importlib._bootstrap.h"
42+
#include "frozen_modules/importlib._bootstrap_external.h"
4343
#include "frozen_modules/zipimport.h"
4444
#include "frozen_modules/abc.h"
4545
#include "frozen_modules/codecs.h"

‎Tools/scripts/freeze_modules.py

Lines changed: 37 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,9 @@
1111
import subprocess
1212
import sys
1313
import textwrap
14+
import time
1415

15-
from update_file import updating_file_with_tmpfile
16+
from update_file import updating_file_with_tmpfile, update_file_with_tmpfile
1617

1718

1819
ROOT_DIR = os.path.dirname(os.path.dirname(os.path.dirname(__file__)))
@@ -272,7 +273,7 @@ def resolve_frozen_file(frozenid, destdir=MODULES_DIR):
272273
except AttributeError:
273274
raise ValueError(f'unsupported frozenid {frozenid!r}')
274275
# We use a consistent naming convention for all frozen modules.
275-
frozenfile = frozenid.replace('.', '_') + '.h'
276+
frozenfile = f'{frozenid}.h'
276277
if not destdir:
277278
return frozenfile
278279
return os.path.join(destdir, frozenfile)
@@ -542,30 +543,40 @@ def regen_frozen(modules):
542543

543544

544545
def regen_makefile(modules):
546+
pyfiles = []
545547
frozenfiles = []
546548
rules = ['']
547549
for src in _iter_sources(modules):
548550
header = relpath_for_posix_display(src.frozenfile, ROOT_DIR)
549551
frozenfiles.append(f'\t\t{header} \\')
550552

551553
pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR)
552-
# Note that we freeze the module to the target .h file
553-
# instead of going through an intermediate file like we used to.
554-
rules.append(f'{header}: Programs/_freeze_module {pyfile}')
555-
rules.append(
556-
(f'\tPrograms/_freeze_module {src.frozenid} '
557-
f'$(srcdir)/{pyfile} $(srcdir)/{header}'))
558-
rules.append('')
559-
554+
pyfiles.append(f'\t\t{pyfile} \\')
555+
556+
freeze = (f'Programs/_freeze_module {src.frozenid} '
557+
f'$(srcdir)/{pyfile} $(srcdir)/{header}')
558+
rules.extend([
559+
f'{header}: Programs/_freeze_module {pyfile}',
560+
f'\t{freeze}',
561+
'',
562+
])
563+
pyfiles[-1] = pyfiles[-1].rstrip(" \\")
560564
frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")
561565

562566
print(f'# Updating {os.path.relpath(MAKEFILE)}')
563567
with updating_file_with_tmpfile(MAKEFILE) as (infile, outfile):
564568
lines = infile.readlines()
565569
lines = replace_block(
566570
lines,
567-
"FROZEN_FILES =",
568-
"# End FROZEN_FILES",
571+
"FROZEN_FILES_IN =",
572+
"# End FROZEN_FILES_IN",
573+
pyfiles,
574+
MAKEFILE,
575+
)
576+
lines = replace_block(
577+
lines,
578+
"FROZEN_FILES_OUT =",
579+
"# End FROZEN_FILES_OUT",
569580
frozenfiles,
570581
MAKEFILE,
571582
)
@@ -625,13 +636,15 @@ def regen_pcbuild(modules):
625636

626637
def freeze_module(modname, pyfile=None, destdir=MODULES_DIR):
627638
"""Generate the frozen module .h file for the given module."""
639+
tmpsuffix = f'.{int(time.time())}'
628640
for modname, pyfile, ispkg in resolve_modules(modname, pyfile):
629641
frozenfile = resolve_frozen_file(modname, destdir)
630-
_freeze_module(modname, pyfile, frozenfile)
642+
_freeze_module(modname, pyfile, frozenfile, tmpsuffix)
631643

632644

633-
def _freeze_module(frozenid, pyfile, frozenfile):
634-
tmpfile = frozenfile + '.new'
645+
def _freeze_module(frozenid, pyfile, frozenfile, tmpsuffix):
646+
tmpfile = f'{frozenfile}.{int(time.time())}'
647+
print(tmpfile)
635648

636649
argv = [TOOL, frozenid, pyfile, tmpfile]
637650
print('#', ' '.join(os.path.relpath(a) for a in argv), flush=True)
@@ -642,7 +655,7 @@ def _freeze_module(frozenid, pyfile, frozenfile):
642655
sys.exit(f'ERROR: missing {TOOL}; you need to run "make regen-frozen"')
643656
raise # re-raise
644657

645-
os.replace(tmpfile, frozenfile)
658+
update_file_with_tmpfile(frozenfile, tmpfile, create=True)
646659

647660

648661
#######################################
@@ -652,15 +665,18 @@ def main():
652665
# Expand the raw specs, preserving order.
653666
modules = list(parse_frozen_specs(destdir=MODULES_DIR))
654667

668+
# Regen build-related files.
669+
regen_makefile(modules)
670+
regen_pcbuild(modules)
671+
655672
# Freeze the target modules.
673+
tmpsuffix = f'.{int(time.time())}'
656674
for src in _iter_sources(modules):
657-
_freeze_module(src.frozenid, src.pyfile, src.frozenfile)
675+
_freeze_module(src.frozenid, src.pyfile, src.frozenfile, tmpsuffix)
658676

659-
# Regen build-related files.
660-
regen_manifest(modules)
677+
# Regen files dependent of frozen file details.
661678
regen_frozen(modules)
662-
regen_makefile(modules)
663-
regen_pcbuild(modules)
679+
regen_manifest(modules)
664680

665681

666682
if __name__ == '__main__':

‎Tools/scripts/update_file.py

Lines changed: 39 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -46,19 +46,47 @@ def updating_file_with_tmpfile(filename, tmpfile=None):
4646
update_file_with_tmpfile(filename, tmpfile)
4747

4848

49-
def update_file_with_tmpfile(filename, tmpfile):
50-
with open(filename, 'rb') as f:
51-
old_contents = f.read()
52-
with open(tmpfile, 'rb') as f:
53-
new_contents = f.read()
54-
if old_contents != new_contents:
49+
def update_file_with_tmpfile(filename, tmpfile, *, create=False):
50+
try:
51+
targetfile = open(filename, 'rb')
52+
except FileNotFoundError:
53+
if not create:
54+
raise # re-raise
55+
outcome = 'created'
5556
os.replace(tmpfile, filename)
5657
else:
57-
os.unlink(tmpfile)
58+
with targetfile:
59+
old_contents = targetfile.read()
60+
with open(tmpfile, 'rb') as f:
61+
new_contents = f.read()
62+
# Now compare!
63+
if old_contents != new_contents:
64+
outcome = 'updated'
65+
os.replace(tmpfile, filename)
66+
else:
67+
outcome = 'same'
68+
os.unlink(tmpfile)
69+
return outcome
5870

5971

6072
if __name__ == '__main__':
61-
if len(sys.argv) != 3:
62-
print("Usage: %s <path to be updated> <path with new contents>" % (sys.argv[0],))
63-
sys.exit(1)
64-
update_file_with_tmpfile(sys.argv[1], sys.argv[2])
73+
import argparse
74+
parser = argparse.ArgumentParser()
75+
parser.add_argument('--create', action='store_true')
76+
parser.add_argument('--exitcode', action='store_true')
77+
parser.add_argument('filename', help='path to be updated')
78+
parser.add_argument('tmpfile', help='path with new contents')
79+
args = parser.parse_args()
80+
kwargs = vars(args)
81+
setexitcode = kwargs.pop('exitcode')
82+
83+
outcome = update_file_with_tmpfile(**kwargs)
84+
if setexitcode:
85+
if outcome == 'same':
86+
sys.exit(0)
87+
elif outcome == 'updated':
88+
sys.exit(1)
89+
elif outcome == 'created':
90+
sys.exit(2)
91+
else:
92+
raise NotImplementedError

0 commit comments

Comments
 (0)