diff --git a/.gitignore b/.gitignore index c3fc748ea97c33..549523f1e560a5 100644 --- a/.gitignore +++ b/.gitignore @@ -126,6 +126,8 @@ Tools/ssl/win32 Python/frozen_modules/*.h # The manifest can be generated at any time with "make regen-frozen". Python/frozen_modules/MANIFEST +Python/frozen_modules/ready-stamp +Python/frozen_modules/generated-stamp # Two-trick pony for OSX and other case insensitive file systems: # Ignore ./python binary on Unix but still look into ./Python/ directory. diff --git a/Makefile.pre.in b/Makefile.pre.in index 5564c1bae1ce2a..27a5c62dad2229 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -772,58 +772,112 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/__hello__.h # End FROZEN_FILES_OUT -Programs/_freeze_module.o: Programs/_freeze_module.c Makefile - -Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) - $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) - -# BEGIN: freezing modules - -Python/frozen_modules/importlib._bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py - Programs/_freeze_module importlib._bootstrap $(srcdir)/Lib/importlib/_bootstrap.py $(srcdir)/Python/frozen_modules/importlib._bootstrap.h +# the 3.0 release (6046c5e0298c25515ea58abc8ab87f7413e3f743) +OLD_TIMESTAMP=200812031729.10 -Python/frozen_modules/importlib._bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py - Programs/_freeze_module importlib._bootstrap_external $(srcdir)/Lib/importlib/_bootstrap_external.py $(srcdir)/Python/frozen_modules/importlib._bootstrap_external.h +FROZEN_DATA=Python/frozen_modules +FROZEN_READY=$(FROZEN_DATA)/ready-stamp +FROZEN_GENERATED=$(FROZEN_DATA)/generated-stamp -Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py - Programs/_freeze_module zipimport $(srcdir)/Lib/zipimport.py $(srcdir)/Python/frozen_modules/zipimport.h +.PHONY: clean-frozen +clean-frozen: + rm -rf $(FROZEN_DATA)/*-stamp +# rm -f $(FROZEN_DATA)/*.h +# rm -f $(FROZEN_FILES_OUT) -Python/frozen_modules/abc.h: Programs/_freeze_module Lib/abc.py - Programs/_freeze_module abc $(srcdir)/Lib/abc.py $(srcdir)/Python/frozen_modules/abc.h +.FORCE: -Python/frozen_modules/codecs.h: Programs/_freeze_module Lib/codecs.py - Programs/_freeze_module codecs $(srcdir)/Lib/codecs.py $(srcdir)/Python/frozen_modules/codecs.h - -Python/frozen_modules/io.h: Programs/_freeze_module Lib/io.py - Programs/_freeze_module io $(srcdir)/Lib/io.py $(srcdir)/Python/frozen_modules/io.h - -Python/frozen_modules/_collections_abc.h: Programs/_freeze_module Lib/_collections_abc.py - Programs/_freeze_module _collections_abc $(srcdir)/Lib/_collections_abc.py $(srcdir)/Python/frozen_modules/_collections_abc.h - -Python/frozen_modules/_sitebuiltins.h: Programs/_freeze_module Lib/_sitebuiltins.py - Programs/_freeze_module _sitebuiltins $(srcdir)/Lib/_sitebuiltins.py $(srcdir)/Python/frozen_modules/_sitebuiltins.h - -Python/frozen_modules/genericpath.h: Programs/_freeze_module Lib/genericpath.py - Programs/_freeze_module genericpath $(srcdir)/Lib/genericpath.py $(srcdir)/Python/frozen_modules/genericpath.h - -Python/frozen_modules/ntpath.h: Programs/_freeze_module Lib/ntpath.py - Programs/_freeze_module ntpath $(srcdir)/Lib/ntpath.py $(srcdir)/Python/frozen_modules/ntpath.h +# This should always run before any frozen module .h rules. +# It gets all the frozen-related stamp files in the proper state. +# +# This file is created by this rule, with an old timestamp. +# It is deleted as soon as the state of the files changes. +$(FROZEN_READY): .FORCE + @mkdir -p $(FROZEN_DATA) + @touch -t $(OLD_TIMESTAMP) $@ + @rm -rf $(FROZEN_GENERATED) + +# This records that one or more frozen module .h files was built. It also +# prints an informative header when the first frozen module is handled. +# It is run at the beginning of the recipe for each frozen module .h file. +# Note that the rule for each module will print a progress marker on +# the same line as the others. +.PHONY: .generating-frozen-module +.generating-frozen-module: + @if ( mkdir $(FROZEN_GENERATED) ) 2> /dev/null; then \ + chmod 555 $(FROZEN_GENERATED); \ + echo "=== Re-generating the frozen modules. ==="; \ + echo "(For verbose output, run make regen-frozen.)"; \ + else \ + touch $(FROZEN_GENERATED); \ + fi + @rm -f $(FROZEN_READY) -Python/frozen_modules/posixpath.h: Programs/_freeze_module Lib/posixpath.py - Programs/_freeze_module posixpath $(srcdir)/Lib/posixpath.py $(srcdir)/Python/frozen_modules/posixpath.h +# This should always run after all frozen module .h rules finish. +# If at least one frozen module rules actually ran then this rule +# finishes printing the line where the progress markers were printed. +# This rule also removes all the other frozen-related stamp files. +# +# This file is created by the .generating-frozen-module rule if at least +# one of the frozen module rules is run. Otherwise it is created here, +# with an old timestamp. It is always an empty directory. +# timestamp. It is updated to "now" in each frozem module .h rule +# if the module was actually updated. +$(FROZEN_GENERATED): $(FROZEN_FILES_OUT) .FORCE + @if test -e $(FROZEN_GENERATED); then \ + echo ""; \ + else \ + touch $@; \ + chmod 555 $(FROZEN_GENERATED); \ + fi + @touch -t $(OLD_TIMESTAMP) $@ + @rm -f $(FROZEN_READY) -Python/frozen_modules/os.h: Programs/_freeze_module Lib/os.py - Programs/_freeze_module os $(srcdir)/Lib/os.py $(srcdir)/Python/frozen_modules/os.h +Programs/_freeze_module.o: Programs/_freeze_module.c Makefile -Python/frozen_modules/site.h: Programs/_freeze_module Lib/site.py - Programs/_freeze_module site $(srcdir)/Lib/site.py $(srcdir)/Python/frozen_modules/site.h +Programs/_freeze_module: Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(FROZEN_READY) + $(LINKCC) $(PY_CORE_LDFLAGS) -o $@ Programs/_freeze_module.o $(LIBRARY_OBJS_OMIT_FROZEN) $(LIBS) $(MODLIBS) $(SYSLIBS) -Python/frozen_modules/stat.h: Programs/_freeze_module Lib/stat.py - Programs/_freeze_module stat $(srcdir)/Lib/stat.py $(srcdir)/Python/frozen_modules/stat.h +have_frozen_files=$(wildcard $(FROZEN_FILES_OUT)) -Python/frozen_modules/__hello__.h: Programs/_freeze_module Lib/__hello__.py - Programs/_freeze_module __hello__ $(srcdir)/Lib/__hello__.py $(srcdir)/Python/frozen_modules/__hello__.h +$(FROZEN_DATA)/%.h: Programs/_freeze_module + $(eval exists = $(wildcard $@)) + $(eval pyfile = Lib/$(subst .,/,$(*F)).py) + $(eval pyfilefull = $(srcdir)/$(pyfile)) + @if ( mkdir $(FROZEN_GENERATED) ) 2> /dev/null; then \ + chmod 555 $(FROZEN_GENERATED); \ + echo "=== Re-generating the frozen modules. ==="; \ + echo "(For verbose output, run make regen-frozen.)"; \ + else \ + touch $(FROZEN_GENERATED); \ + fi + @rm -f $(FROZEN_READY) + @Programs/_freeze_module $(*F) "$(pyfilefull)" $@.new + @if $(UPDATE_FILE) --create --exitcode $@ $@.new; then \ + if test $@ -ot Python/frozen.o; then touch -r Python/frozen.o $@; fi; \ + if test $@ -ot "$(pyfilefull)"; then touch -r "$(pyfilefull)" $@; fi; \ + fi + @if test -n "$(exists)" && test -n "$(findstring $(pyfile),$?)"; then echo -n " $(@F) "; \ + elif test -z "$(have_frozen_files)"; then echo -n "."; \ + elif test -z "$(exists)"; then echo -n " $(@F) "; \ + else echo -n "."; fi +# BEGIN: freezing modules +Python/frozen_modules/importlib._bootstrap.h: $(srcdir)/Lib/importlib/_bootstrap.py +Python/frozen_modules/importlib._bootstrap_external.h: $(srcdir)/Lib/importlib/_bootstrap_external.py +Python/frozen_modules/zipimport.h: $(srcdir)/Lib/zipimport.py +Python/frozen_modules/abc.h: $(srcdir)/Lib/abc.py +Python/frozen_modules/codecs.h: $(srcdir)/Lib/codecs.py +Python/frozen_modules/io.h: $(srcdir)/Lib/io.py +Python/frozen_modules/_collections_abc.h: $(srcdir)/Lib/_collections_abc.py +Python/frozen_modules/_sitebuiltins.h: $(srcdir)/Lib/_sitebuiltins.py +Python/frozen_modules/genericpath.h: $(srcdir)/Lib/genericpath.py +Python/frozen_modules/ntpath.h: $(srcdir)/Lib/ntpath.py +Python/frozen_modules/posixpath.h: $(srcdir)/Lib/posixpath.py +Python/frozen_modules/os.h: $(srcdir)/Lib/os.py +Python/frozen_modules/site.h: $(srcdir)/Lib/site.py +Python/frozen_modules/stat.h: $(srcdir)/Lib/stat.py +Python/frozen_modules/__hello__.h: $(srcdir)/Lib/__hello__.py # END: freezing modules Tools/scripts/freeze_modules.py: Programs/_freeze_module @@ -1062,7 +1116,7 @@ regen-opcode-targets: Python/ceval.o: $(srcdir)/Python/opcode_targets.h $(srcdir)/Python/ceval_gil.h \ $(srcdir)/Python/condvar.h -Python/frozen.o: $(FROZEN_FILES_OUT) +Python/frozen.o: $(FROZEN_FILES_OUT) $(FROZEN_GENERATED) # Generate DTrace probe macros, then rename them (PYTHON_ -> PyDTrace_) to # follow our naming conventions. dtrace(1) uses the output filename to generate @@ -1988,6 +2042,7 @@ clean-retain-profile: pycremoval -find build -type f -a ! -name '*.gc??' -exec rm -f {} ';' -rm -f Include/pydtrace_probes.h -rm -f profile-gen-stamp + -rm -rf $(FROZEN_DATA)/*-stamp profile-removal: find . -name '*.gc??' -exec rm -f {} ';' diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index aa799d763a55b9..caf37132e84d00 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -545,7 +545,7 @@ def regen_frozen(modules): def regen_makefile(modules): pyfiles = [] frozenfiles = [] - rules = [''] + rules = [] for src in _iter_sources(modules): header = relpath_for_posix_display(src.frozenfile, ROOT_DIR) frozenfiles.append(f'\t\t{header} \\') @@ -553,12 +553,8 @@ def regen_makefile(modules): pyfile = relpath_for_posix_display(src.pyfile, ROOT_DIR) pyfiles.append(f'\t\t{pyfile} \\') - freeze = (f'Programs/_freeze_module {src.frozenid} ' - f'$(srcdir)/{pyfile} $(srcdir)/{header}') rules.extend([ - f'{header}: Programs/_freeze_module {pyfile}', - f'\t{freeze}', - '', + f'{header}: $(srcdir)/{pyfile}', ]) pyfiles[-1] = pyfiles[-1].rstrip(" \\") frozenfiles[-1] = frozenfiles[-1].rstrip(" \\")