Skip to content

Commit 044e8d8

Browse files
ericsnowcurrentlyericsnowcurrently
authored
bpo-45019: Add a tool to generate list of modules to include for frozen modules (gh-27980)
Frozen modules must be added to several files in order to work properly. Before this change this had to be done manually. Here we add a tool to generate the relevant lines in those files instead. This helps us avoid mistakes and omissions. https://bugs.python.org/issue45019
1 parent 5246dbc commit 044e8d8

19 files changed

+828
-194
lines changed

‎.gitattributes

+1-2
Original file line numberDiff line numberDiff line change
@@ -46,8 +46,7 @@ Modules/clinic/*.h linguist-generated=true
4646
Objects/clinic/*.h linguist-generated=true
4747
PC/clinic/*.h linguist-generated=true
4848
Python/clinic/*.h linguist-generated=true
49-
Python/importlib.h linguist-generated=true
50-
Python/importlib_external.h linguist-generated=true
49+
Python/frozen_modules/*.h linguist-generated=true
5150
Include/internal/pycore_ast.h linguist-generated=true
5251
Python/Python-ast.c linguist-generated=true
5352
Include/opcode.h linguist-generated=true

‎.github/workflows/build.yml

+1
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,7 @@ jobs:
7171
make regen-stdlib-module-names
7272
- name: Check for changes
7373
run: |
74+
git add -u
7475
changes=$(git status --porcelain)
7576
# Check for changes in regenerated files
7677
if ! test -z "$changes"

‎.gitignore

+1-1
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ Modules/Setup.config
6868
Modules/Setup.local
6969
Modules/config.c
7070
Modules/ld_so_aix
71-
Programs/_freeze_importlib
71+
Programs/_freeze_module
7272
Programs/_testembed
7373
PC/python_nt*.h
7474
PC/pythonnt_rc*.h

‎Doc/c-api/init.rst

+1-1
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ to 1 and ``-bb`` sets :c:data:`Py_BytesWarningFlag` to 2.
110110
Suppress error messages when calculating the module search path in
111111
:c:func:`Py_GetPath`.
112112

113-
Private flag used by ``_freeze_importlib`` and ``frozenmain`` programs.
113+
Private flag used by ``_freeze_module`` and ``frozenmain`` programs.
114114

115115
.. c:var:: int Py_HashRandomizationFlag
116116

‎Makefile.pre.in

+53-40
Original file line numberDiff line numberDiff line change
@@ -574,8 +574,8 @@ coverage-lcov:
574574
@echo "lcov report at $(COVERAGE_REPORT)/index.html"
575575
@echo
576576

577-
# Force regeneration of parser and importlib
578-
coverage-report: regen-token regen-importlib
577+
# Force regeneration of parser and frozen modules
578+
coverage-report: regen-token regen-frozen
579579
@ # build with coverage info
580580
$(MAKE) coverage
581581
@ # run tests, ignore failures
@@ -734,45 +734,60 @@ Programs/_testembed: Programs/_testembed.o $(LIBRARY_DEPS)
734734
$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(BLDLIBRARY) $(LIBS) $(MODLIBS) $(SYSLIBS)
735735

736736
############################################################################
737-
# Importlib
737+
# frozen modules (including importlib)
738738

739-
Programs/_freeze_importlib.o: Programs/_freeze_importlib.c Makefile
739+
Programs/_freeze_module.o: Programs/_freeze_module.c Makefile
740740

741-
Programs/_freeze_importlib: Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN)
742-
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_importlib.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
741+
Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN)
742+
$(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS)
743743

744+
Tools/scripts/freeze_modules.py: Programs/_freeze_module
745+
746+
.PHONY: regen-frozen
747+
regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES)
748+
$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py
749+
@echo "The Makefile was updated, you may need to re-run make."
750+
751+
# BEGIN: freezing modules
752+
753+
Python/frozen_modules/importlib__bootstrap.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap.py
754+
$(srcdir)/Programs/_freeze_module importlib._bootstrap \
755+
$(srcdir)/Lib/importlib/_bootstrap.py \
756+
$(srcdir)/Python/frozen_modules/importlib__bootstrap.h
757+
758+
Python/frozen_modules/importlib__bootstrap_external.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap_external.py
759+
$(srcdir)/Programs/_freeze_module importlib._bootstrap_external \
760+
$(srcdir)/Lib/importlib/_bootstrap_external.py \
761+
$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h
762+
763+
Python/frozen_modules/zipimport.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/zipimport.py
764+
$(srcdir)/Programs/_freeze_module zipimport \
765+
$(srcdir)/Lib/zipimport.py \
766+
$(srcdir)/Python/frozen_modules/zipimport.h
767+
768+
Python/frozen_modules/hello.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Tools/freeze/flag.py
769+
$(srcdir)/Programs/_freeze_module hello \
770+
$(srcdir)/Tools/freeze/flag.py \
771+
$(srcdir)/Python/frozen_modules/hello.h
772+
773+
# END: freezing modules
774+
775+
# We keep this renamed target around for folks with muscle memory.
744776
.PHONY: regen-importlib
745-
regen-importlib: Programs/_freeze_importlib
746-
# Regenerate Python/importlib_external.h
747-
# from Lib/importlib/_bootstrap_external.py using _freeze_importlib
748-
./Programs/_freeze_importlib importlib._bootstrap_external \
749-
$(srcdir)/Lib/importlib/_bootstrap_external.py \
750-
$(srcdir)/Python/importlib_external.h.new
751-
$(UPDATE_FILE) $(srcdir)/Python/importlib_external.h $(srcdir)/Python/importlib_external.h.new
752-
# Regenerate Python/importlib.h from Lib/importlib/_bootstrap.py
753-
# using _freeze_importlib
754-
./Programs/_freeze_importlib importlib._bootstrap \
755-
$(srcdir)/Lib/importlib/_bootstrap.py \
756-
$(srcdir)/Python/importlib.h.new
757-
$(UPDATE_FILE) $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib.h.new
758-
# Regenerate Python/importlib_zipimport.h from Lib/zipimport.py
759-
# using _freeze_importlib
760-
./Programs/_freeze_importlib zipimport \
761-
$(srcdir)/Lib/zipimport.py \
762-
$(srcdir)/Python/importlib_zipimport.h.new
763-
$(UPDATE_FILE) $(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/importlib_zipimport.h.new
777+
regen-importlib: regen-frozen
764778

779+
############################################################################
780+
# ABI
765781

766782
regen-limited-abi: all
767783
$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.txt
768784

769-
770785
############################################################################
771786
# Regenerate all generated files
772787

773788
regen-all: regen-opcode regen-opcode-targets regen-typeslots \
774-
regen-token regen-ast regen-keyword regen-importlib clinic \
775-
regen-pegen-metaparser regen-pegen regen-frozen regen-test-frozenmain
789+
regen-token regen-ast regen-keyword regen-frozen clinic \
790+
regen-pegen-metaparser regen-pegen regen-test-frozenmain
776791
@echo
777792
@echo "Note: make regen-stdlib-module-names and autoconf should be run manually"
778793

@@ -884,15 +899,6 @@ regen-opcode:
884899
$(srcdir)/Include/opcode.h.new
885900
$(UPDATE_FILE) $(srcdir)/Include/opcode.h $(srcdir)/Include/opcode.h.new
886901

887-
.PHONY: regen-frozen
888-
regen-frozen: Programs/_freeze_importlib
889-
# Regenerate code for frozen module "__hello__".
890-
./Programs/_freeze_importlib hello \
891-
$(srcdir)/Tools/freeze/flag.py \
892-
$(srcdir)/Python/frozen_hello.h.new
893-
$(UPDATE_FILE) $(srcdir)/Python/frozen_hello.h \
894-
$(srcdir)/Python/frozen_hello.h.new
895-
896902
.PHONY: regen-token
897903
regen-token:
898904
# Regenerate Doc/library/token-list.inc from Grammar/Tokens
@@ -995,8 +1001,15 @@ regen-opcode-targets:
9951001
Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \
9961002
$(srcdir)/Python/condvar.h
9971003

998-
Python/frozen.o: $(srcdir)/Python/importlib.h $(srcdir)/Python/importlib_external.h \
999-
$(srcdir)/Python/importlib_zipimport.h $(srcdir)/Python/frozen_hello.h
1004+
# FROZEN_FILES is auto-generated by Tools/scripts/freeze_modules.py.
1005+
FROZEN_FILES = \
1006+
$(srcdir)/Python/frozen_modules/importlib__bootstrap.h \
1007+
$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h \
1008+
$(srcdir)/Python/frozen_modules/zipimport.h \
1009+
$(srcdir)/Python/frozen_modules/hello.h
1010+
# End FROZEN_FILES
1011+
1012+
Python/frozen.o: $(FROZEN_FILES)
10001013

10011014
# Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to
10021015
# follow our naming conventions. dtrace(1) uses the output filename to generate
@@ -1918,7 +1931,7 @@ clean-retain-profile: pycremoval
19181931
find build -name '*.py[co]' -exec rm -f {} ';' || true
19191932
-rm -f pybuilddir.txt
19201933
-rm -f Lib/lib2to3/*Grammar*.pickle
1921-
-rm -f Programs/_testembed Programs/_freeze_importlib
1934+
-rm -f Programs/_testembed Programs/_freeze_module
19221935
-find build -type f -a ! -name '*.gc??' -exec rm -f {} ';'
19231936
-rm -f Include/pydtrace_probes.h
19241937
-rm -f profile-gen-stamp

‎Misc/NEWS.d/next/Build/2021-08-26-13-10-46.bpo-45019.e0mo49.rst

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
Generate lines in relevant files for frozen modules. Up until now each of
2+
the files had to be edited manually. This change makes it easier to add to
3+
and modify the frozen modules.

‎PCbuild/_freeze_importlib.vcxproj renamed to ‎PCbuild/_freeze_module.vcxproj

+20-15
Original file line numberDiff line numberDiff line change
@@ -69,7 +69,7 @@
6969
<PropertyGroup Label="Globals">
7070
<ProjectGuid>{19C0C13F-47CA-4432-AFF3-799A296A4DDC}</ProjectGuid>
7171
<Keyword>Win32Proj</Keyword>
72-
<RootNamespace>_freeze_importlib</RootNamespace>
72+
<RootNamespace>_freeze_module</RootNamespace>
7373
<SupportPGO>false</SupportPGO>
7474
</PropertyGroup>
7575
<Import Project="python.props" />
@@ -95,7 +95,7 @@
9595
</Link>
9696
</ItemDefinitionGroup>
9797
<ItemGroup>
98-
<ClCompile Include="..\Programs\_freeze_importlib.c" />
98+
<ClCompile Include="..\Programs\_freeze_module.c" />
9999
</ItemGroup>
100100
<ItemGroup>
101101
<ProjectReference Include="pythoncore.vcxproj">
@@ -108,31 +108,33 @@
108108
</ProjectReference>
109109
</ItemGroup>
110110
<ItemGroup>
111+
<!-- BEGIN frozen modules -->
111112
<None Include="..\Lib\importlib\_bootstrap.py">
112113
<ModName>importlib._bootstrap</ModName>
113-
<IntFile>$(IntDir)importlib.g.h</IntFile>
114-
<OutFile>$(PySourcePath)Python\importlib.h</OutFile>
114+
<IntFile>$(IntDir)importlib__bootstrap.g.h</IntFile>
115+
<OutFile>$(PySourcePath)Python\frozen_modules\importlib__bootstrap.h</OutFile>
115116
</None>
116117
<None Include="..\Lib\importlib\_bootstrap_external.py">
117118
<ModName>importlib._bootstrap_external</ModName>
118-
<IntFile>$(IntDir)importlib_external.g.h</IntFile>
119-
<OutFile>$(PySourcePath)Python\importlib_external.h</OutFile>
119+
<IntFile>$(IntDir)importlib__bootstrap_external.g.h</IntFile>
120+
<OutFile>$(PySourcePath)Python\frozen_modules\importlib__bootstrap_external.h</OutFile>
120121
</None>
121122
<None Include="..\Lib\zipimport.py">
122123
<ModName>zipimport</ModName>
123-
<IntFile>$(IntDir)importlib_zipimport.g.h</IntFile>
124-
<OutFile>$(PySourcePath)Python\importlib_zipimport.h</OutFile>
124+
<IntFile>$(IntDir)zipimport.g.h</IntFile>
125+
<OutFile>$(PySourcePath)Python\frozen_modules\zipimport.h</OutFile>
125126
</None>
126127
<None Include="..\Tools\freeze\flag.py">
127128
<ModName>hello</ModName>
128-
<IntFile>$(IntDir)frozen_hello.g.h</IntFile>
129-
<OutFile>$(PySourcePath)Python\frozen_hello.h</OutFile>
129+
<IntFile>$(IntDir)ello.g.h</IntFile>
130+
<OutFile>$(PySourcePath)Python\frozen_modules\hello.h</OutFile>
130131
</None>
132+
<!-- END frozen modules -->
131133
</ItemGroup>
132134
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
133135
<ImportGroup Label="ExtensionTargets">
134136
</ImportGroup>
135-
<Target Name="_RebuildImportLib">
137+
<Target Name="_RebuildFrozen">
136138
<Exec Command='"$(TargetPath)" "%(None.ModName)" "%(None.FullPath)" "%(None.IntFile)"' />
137139

138140
<Copy SourceFiles="%(None.IntFile)"
@@ -143,15 +145,18 @@
143145

144146
<Message Text="Updated files: @(_Updated->'%(Filename)%(Extension)',', ')"
145147
Condition="'@(_Updated)' != ''" Importance="high" />
146-
<Warning Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
148+
<Warning Text="Frozen modules (e.g. importlib) were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
147149
Condition="'@(_Updated)' != '' and $(Configuration) == 'Debug'" />
148-
<Error Text="Frozen importlib files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
150+
<Error Text="Frozen (e.g. importlib) files were updated. Please rebuild to pick up the changes.%0D%0A%0D%0AIf you are not developing on Windows but you see this error on a continuous integration build, please run 'make regen-all' and commit anything that changes."
149151
Condition="'@(_Updated)' != '' and $(Configuration) == 'Release'" />
150152
</Target>
153+
<Target Name="RebuildFrozen" AfterTargets="AfterBuild" Condition="$(Configuration) == 'Debug' or $(Configuration) == 'Release'"
154+
DependsOnTargets="_RebuildFrozen">
155+
</Target>
151156
<Target Name="RebuildImportLib" AfterTargets="AfterBuild" Condition="$(Configuration) == 'Debug' or $(Configuration) == 'Release'"
152-
DependsOnTargets="_RebuildImportLib">
157+
DependsOnTargets="_RebuildFrozen">
153158
</Target>
154-
<Target Name="_CleanImportLib" BeforeTargets="CoreClean">
159+
<Target Name="_CleanFrozen" BeforeTargets="CoreClean">
155160
<ItemGroup>
156161
<Clean Include="%(None.IntFile)" />
157162
</ItemGroup>

‎PCbuild/_freeze_importlib.vcxproj.filters renamed to ‎PCbuild/_freeze_module.vcxproj.filters

+9-4
Original file line numberDiff line numberDiff line change
@@ -10,19 +10,24 @@
1010
</Filter>
1111
</ItemGroup>
1212
<ItemGroup>
13-
<ClCompile Include="..\Programs\_freeze_importlib.c">
13+
<ClCompile Include="..\Programs\_freeze_module.c">
1414
<Filter>Source Files</Filter>
1515
</ClCompile>
1616
</ItemGroup>
1717
<ItemGroup>
18+
<!-- BEGIN frozen modules -->
1819
<None Include="..\Lib\importlib\_bootstrap.py">
19-
<Filter>Source Files</Filter>
20+
<Filter>Python Files</Filter>
21+
</None>
22+
<None Include="..\Lib\importlib\_bootstrap_external.py">
23+
<Filter>Python Files</Filter>
2024
</None>
2125
<None Include="..\Lib\zipimport.py">
2226
<Filter>Python Files</Filter>
2327
</None>
24-
<None Include="..\Lib\importlib\_bootstrap_external.py">
28+
<None Include="..\Tools\freeze\flag.py">
2529
<Filter>Python Files</Filter>
2630
</None>
31+
<!-- END frozen modules -->
2732
</ItemGroup>
28-
</Project>
33+
</Project>

‎PCbuild/pcbuild.proj

+2-2
Original file line numberDiff line numberDiff line change
@@ -72,8 +72,8 @@
7272
<BuildInParallel>false</BuildInParallel>
7373
</Projects>
7474

75-
<!-- _freeze_importlib -->
76-
<Projects2 Condition="$(Platform) != 'ARM' and $(Platform) != 'ARM64'" Include="_freeze_importlib.vcxproj" />
75+
<!-- _freeze_module -->
76+
<Projects2 Condition="$(Platform) != 'ARM' and $(Platform) != 'ARM64'" Include="_freeze_module.vcxproj" />
7777
<!-- python[w].exe -->
7878
<Projects2 Include="python.vcxproj;pythonw.vcxproj" />
7979
<Projects2 Include="python_uwp.vcxproj;pythonw_uwp.vcxproj" Condition="$(IncludeUwp)" />

‎PCbuild/pcbuild.sln

+1-1
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "pywlauncher", "pywlauncher.
7575
{7B2727B5-5A3F-40EE-A866-43A13CD31446} = {7B2727B5-5A3F-40EE-A866-43A13CD31446}
7676
EndProjectSection
7777
EndProject
78-
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_importlib", "_freeze_importlib.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}"
78+
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_freeze_module", "_freeze_module.vcxproj", "{19C0C13F-47CA-4432-AFF3-799A296A4DDC}"
7979
EndProject
8080
Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "_overlapped", "_overlapped.vcxproj", "{EB6E69DD-04BF-4543-9B92-49FAABCEAC2E}"
8181
EndProject

‎PCbuild/readme.txt

+4-3
Original file line numberDiff line numberDiff line change
@@ -115,9 +115,10 @@ _testembed
115115

116116
These are miscellaneous sub-projects that don't really fit the other
117117
categories:
118-
_freeze_importlib
119-
_freeze_importlib.exe, used to regenerate Python\importlib.h after
120-
changes have been made to Lib\importlib\_bootstrap.py
118+
_freeze_module
119+
_freeze_module.exe, used to regenerate frozen modules in Python
120+
after changes have been made to the corresponding source files
121+
(e.g. Lib\importlib\_bootstrap.py).
121122
pyshellext
122123
pyshellext.dll, the shell extension deployed with the launcher
123124
python3dll

0 commit comments

Comments
 (0)