gh-97669: Create Tools/build/ directory (#97963)

Create Tools/build/ directory. Move the following scripts from
Tools/scripts/ to Tools/build/:

* check_extension_modules.py
* deepfreeze.py
* freeze_modules.py
* generate_global_objects.py
* generate_levenshtein_examples.py
* generate_opcode_h.py
* generate_re_casefix.py
* generate_sre_constants.py
* generate_stdlib_module_names.py
* generate_token.py
* parse_html5_entities.py
* smelly.py
* stable_abi.py
* umarshal.py
* update_file.py
* verify_ensurepip_wheels.py

Update references to these scripts.

Author: vstinner

.github/CODEOWNERS

@@ -55,7 +55,7 @@ Python/traceback.c            @iritkatriel
 /Lib/html/                    @ezio-melotti
 /Lib/_markupbase.py           @ezio-melotti
 /Lib/test/test_html*.py       @ezio-melotti
-/Tools/scripts/*html5*        @ezio-melotti
+/Tools/build/parse_html5_entities.py   @ezio-melotti
 
 # Import (including importlib).
 # Ignoring importlib.h so as to not get flagged on

.github/workflows/verify-ensurepip-wheels.yml

@@ -6,12 +6,12 @@ on:
     paths:
       - 'Lib/ensurepip/_bundled/**'
       - '.github/workflows/verify-ensurepip-wheels.yml'
-      - 'Tools/scripts/verify_ensurepip_wheels.py'
+      - 'Tools/build/verify_ensurepip_wheels.py'
   pull_request:
     paths:
       - 'Lib/ensurepip/_bundled/**'
       - '.github/workflows/verify-ensurepip-wheels.yml'
-      - 'Tools/scripts/verify_ensurepip_wheels.py'
+      - 'Tools/build/verify_ensurepip_wheels.py'
 
 permissions:
   contents: read
@@ -29,4 +29,4 @@ jobs:
         with:
           python-version: '3'
       - name: Compare checksums of bundled pip and setuptools to ones published on PyPI
-        run: ./Tools/scripts/verify_ensurepip_wheels.py
+        run: ./Tools/build/verify_ensurepip_wheels.py

.gitignore

@@ -143,7 +143,7 @@ Tools/ssl/win32
 Tools/freeze/test/outdir
 
 # The frozen modules are always generated by the build so we don't
-# keep them in the repo.  Also see Tools/scripts/freeze_modules.py.
+# keep them in the repo.  Also see Tools/build/freeze_modules.py.
 Python/frozen_modules/*.h
 # The manifest can be generated at any time with "make regen-frozen".
 Python/frozen_modules/MANIFEST

Doc/library/token-list.inc

@@ -8,7 +8,7 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-// The data structure & init here are inspired by Tools/scripts/deepfreeze.py.
+// The data structure & init here are inspired by Tools/build/deepfreeze.py.
 
 // All field names generated by ASCII_STR() have a common prefix,
 // to help avoid collisions with keywords, etc.
@@ -25,7 +25,7 @@ extern "C" {
 
 // XXX Order by frequency of use?
 
-/* The following is auto-generated by Tools/scripts/generate_global_objects.py. */
+/* The following is auto-generated by Tools/build/generate_global_objects.py. */
 struct _Py_global_strings {
     struct {
         STRUCT_FOR_STR(anon_dictcomp, "<dictcomp>")

Include/internal/pycore_global_strings.h

@@ -1,4 +1,4 @@
-/* Auto-generated by Tools/scripts/generate_token.py */
+/* Auto-generated by Tools/build/generate_token.py */
 
 /* Token types */
 #ifndef Py_INTERNAL_TOKEN_H

Include/internal/pycore_opcode.h

@@ -261,7 +261,7 @@
 
 
 # HTML5 named character references
-# Generated by 'Tools/scripts/parse_html5_entities.py'
+# Generated by Tools/build/parse_html5_entities.py
 # from https://html.spec.whatwg.org/entities.json and
 # https://html.spec.whatwg.org/multipage/named-characters.html.
 # Map HTML5 named character references to the equivalent Unicode character(s).

Include/internal/pycore_runtime_init_generated.h

@@ -285,7 +285,7 @@ BUILDPYTHON=	python$(BUILDEXE)
 HOSTRUNNER= @HOSTRUNNER@
 
 PYTHON_FOR_REGEN?=@PYTHON_FOR_REGEN@
-UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/update_file.py
+UPDATE_FILE=$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/update_file.py
 PYTHON_FOR_BUILD=@PYTHON_FOR_BUILD@
 # Single-platform builds depend on $(BUILDPYTHON). Cross builds use an
 # external "build Python" and have an empty PYTHON_FOR_BUILD_DEPS.
@@ -705,7 +705,7 @@ coverage-report: regen-token regen-frozen
 .PHONY=clinic
 clinic: check-clean-src $(srcdir)/Modules/_blake2/blake2s_impl.c
 	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/clinic/clinic.py --make --srcdir $(srcdir)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_global_objects.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py
 
 # Build the interpreter
 $(BUILDPYTHON):	Programs/python.o $(LINK_PYTHON_DEPS)
@@ -907,7 +907,7 @@ sharedmods: $(SHAREDMODS) pybuilddir.txt
 
 # dependency on BUILDPYTHON ensures that the target is run last
 checksharedmods: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON)
-	@$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/scripts/check_extension_modules.py
+	@$(RUNSHARED) $(PYTHON_FOR_BUILD) $(srcdir)/Tools/build/check_extension_modules.py
 
 rundsymutil: sharedmods $(PYTHON_FOR_BUILD_DEPS) $(BUILDPYTHON)
 	@if [ ! -z $(DSYMUTIL) ] ; then \
@@ -961,13 +961,13 @@ regen-test-frozenmain: $(BUILDPYTHON)
 .PHONY: regen-test-levenshtein
 regen-test-levenshtein:
 	# Regenerate Lib/test/levenshtein_examples.json
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_levenshtein_examples.py Lib/test/levenshtein_examples.json
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_levenshtein_examples.py Lib/test/levenshtein_examples.json
 
 .PHONY: regen-re
 regen-re: $(BUILDPYTHON)
 	# Regenerate Lib/re/_casefix.py
-	# using Tools/scripts/generate_re_casefix.py
-	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/generate_re_casefix.py $(srcdir)/Lib/re/_casefix.py
+	# using Tools/build/generate_re_casefix.py
+	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/generate_re_casefix.py $(srcdir)/Lib/re/_casefix.py
 
 Programs/_testembed: Programs/_testembed.o $(LINK_PYTHON_DEPS)
 	$(LINKCC) $(PY_CORE_LDFLAGS) $(LINKFORSHARED) -o $@ Programs/_testembed.o $(LINK_PYTHON_OBJS) $(LIBS) $(MODLIBS) $(SYSLIBS)
@@ -1013,7 +1013,7 @@ _bootstrap_python: $(LIBRARY_OBJS_OMIT_FROZEN) Programs/_bootstrap_python.o Modu
 # 2) deepfreeze modules with external build Python.
 #
 
-# FROZEN_FILES_* are auto-generated by Tools/scripts/freeze_modules.py.
+# FROZEN_FILES_* are auto-generated by Tools/build/freeze_modules.py.
 FROZEN_FILES_IN = \
 		Lib/importlib/_bootstrap.py \
 		Lib/importlib/_bootstrap_external.py \
@@ -1149,11 +1149,11 @@ Python/frozen_modules/frozen_only.h: Tools/freeze/flag.py $(FREEZE_MODULE_DEPS)
 
 # END: freezing modules
 
-Tools/scripts/freeze_modules.py: $(FREEZE_MODULE)
+Tools/build/freeze_modules.py: $(FREEZE_MODULE)
 
 .PHONY: regen-frozen
-regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN)
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/freeze_modules.py
+regen-frozen: Tools/build/freeze_modules.py $(FROZEN_FILES_IN)
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/freeze_modules.py
 	@echo "The Makefile was updated, you may need to re-run make."
 
 ############################################################################
@@ -1162,11 +1162,11 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES_IN)
 .PHONY: regen-deepfreeze
 regen-deepfreeze: $(DEEPFREEZE_OBJS)
 
-DEEPFREEZE_DEPS=$(srcdir)/Tools/scripts/deepfreeze.py $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT)
+DEEPFREEZE_DEPS=$(srcdir)/Tools/build/deepfreeze.py $(FREEZE_MODULE_DEPS) $(FROZEN_FILES_OUT)
 
 # BEGIN: deepfreeze modules
 Python/deepfreeze/deepfreeze.c: $(DEEPFREEZE_DEPS)
-	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py \
+	$(PYTHON_FOR_FREEZE) $(srcdir)/Tools/build/deepfreeze.py \
 	Python/frozen_modules/importlib._bootstrap.h:importlib._bootstrap \
 	Python/frozen_modules/importlib._bootstrap_external.h:importlib._bootstrap_external \
 	Python/frozen_modules/zipimport.h:zipimport \
@@ -1203,8 +1203,8 @@ regen-importlib: regen-frozen
 # Global objects
 
 .PHONY: regen-global-objects
-regen-global-objects: $(srcdir)/Tools/scripts/generate_global_objects.py
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_global_objects.py
+regen-global-objects: $(srcdir)/Tools/build/generate_global_objects.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_global_objects.py
 	@echo "Note: Global objects can be added or removed by other tools (e.g. deepfreeze), "
 	@echo "      so be sure to re-run regen-global-objects after those tools."
 
@@ -1220,7 +1220,7 @@ check-abidump: all
 	abidiff $(srcdir)/Doc/data/python$(LDVERSION).abi "libpython$(LDVERSION).so" --drop-private-types --no-architecture --no-added-syms
 
 regen-limited-abi: all
-	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml
+	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --generate-all $(srcdir)/Misc/stable_abi.toml
 
 ############################################################################
 # Regenerate all generated files
@@ -1331,8 +1331,8 @@ regen-ast:
 .PHONY: regen-opcode
 regen-opcode:
 	# Regenerate Include/opcode.h from Lib/opcode.py
-	# using Tools/scripts/generate_opcode_h.py
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_opcode_h.py \
+	# using Tools/build/generate_opcode_h.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_opcode_h.py \
 		$(srcdir)/Lib/opcode.py \
 		$(srcdir)/Include/opcode.h.new \
 		$(srcdir)/Include/internal/pycore_opcode.h.new
@@ -1342,23 +1342,23 @@ regen-opcode:
 .PHONY: regen-token
 regen-token:
 	# Regenerate Doc/library/token-list.inc from Grammar/Tokens
-	# using Tools/scripts/generate_token.py
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py rst \
+	# using Tools/build/generate_token.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_token.py rst \
 		$(srcdir)/Grammar/Tokens \
 		$(srcdir)/Doc/library/token-list.inc
 	# Regenerate Include/internal/pycore_token.h from Grammar/Tokens
-	# using Tools/scripts/generate_token.py
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py h \
+	# using Tools/build/generate_token.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_token.py h \
 		$(srcdir)/Grammar/Tokens \
 		$(srcdir)/Include/internal/pycore_token.h
 	# Regenerate Parser/token.c from Grammar/Tokens
-	# using Tools/scripts/generate_token.py
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py c \
+	# using Tools/build/generate_token.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_token.py c \
 		$(srcdir)/Grammar/Tokens \
 		$(srcdir)/Parser/token.c
 	# Regenerate Lib/token.py from Grammar/Tokens
-	# using Tools/scripts/generate_token.py
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_token.py py \
+	# using Tools/build/generate_token.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_token.py py \
 		$(srcdir)/Grammar/Tokens \
 		$(srcdir)/Lib/token.py
 
@@ -1375,16 +1375,16 @@ regen-keyword:
 .PHONY: regen-stdlib-module-names
 regen-stdlib-module-names: all Programs/_testembed
 	# Regenerate Python/stdlib_module_names.h
-	# using Tools/scripts/generate_stdlib_module_names.py
+	# using Tools/build/generate_stdlib_module_names.py
 	$(RUNSHARED) ./$(BUILDPYTHON) \
-		$(srcdir)/Tools/scripts/generate_stdlib_module_names.py \
+		$(srcdir)/Tools/build/generate_stdlib_module_names.py \
 		> $(srcdir)/Python/stdlib_module_names.h.new
 	$(UPDATE_FILE) $(srcdir)/Python/stdlib_module_names.h $(srcdir)/Python/stdlib_module_names.h.new
 
 regen-sre:
 	# Regenerate Modules/_sre/sre_constants.h and Modules/_sre/sre_targets.h
-	# from Lib/re/_constants.py using Tools/scripts/generate_sre_constants.py
-	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/scripts/generate_sre_constants.py \
+	# from Lib/re/_constants.py using Tools/build/generate_sre_constants.py
+	$(PYTHON_FOR_REGEN) $(srcdir)/Tools/build/generate_sre_constants.py \
 		$(srcdir)/Lib/re/_constants.py \
 		$(srcdir)/Modules/_sre/sre_constants.h \
 		$(srcdir)/Modules/_sre/sre_targets.h
@@ -2511,7 +2511,7 @@ distclean: clobber docclean
 
 # Check that all symbols exported by libpython start with "Py" or "_Py"
 smelly: all
-	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/smelly.py
+	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/smelly.py
 
 # Find files with funny names
 funny:
@@ -2549,7 +2549,7 @@ patchcheck: all
 	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/patchcheck/patchcheck.py
 
 check-limited-abi: all
-	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/scripts/stable_abi.py --all $(srcdir)/Misc/stable_abi.toml
+	$(RUNSHARED) ./$(BUILDPYTHON) $(srcdir)/Tools/build/stable_abi.py --all $(srcdir)/Misc/stable_abi.toml
 
 .PHONY: update-config
 update-config:

Include/internal/pycore_token.h

@@ -2,7 +2,7 @@
 # Please append new items at the end.
 
 # The syntax of this file is not fixed.
-# It is designed to be read only by Tools/stable_abi.py, which can change
+# It is designed to be read only by Tools/build/stable_abi.py, which can change
 # without notice.
 
 # For the history of the stable ABI prior to this file,

Include/opcode.h

@@ -3,7 +3,7 @@
  *
  * regular expression matching engine
  *
- * Auto-generated by Tools/scripts/generate_sre_constants.py from
+ * Auto-generated by Tools/build/generate_sre_constants.py from
  * Lib/re/_constants.py.
  *
  * Copyright (c) 1997-2001 by Secret Labs AB.  All rights reserved.

Lib/html/entities.py

@@ -3,7 +3,7 @@
  *
  * regular expression matching engine
  *
- * Auto-generated by Tools/scripts/generate_sre_constants.py from
+ * Auto-generated by Tools/build/generate_sre_constants.py from
  * Lib/re/_constants.py.
  *
  * Copyright (c) 1997-2001 by Secret Labs AB.  All rights reserved.

Lib/token.py

@@ -395,7 +395,7 @@
           DependsOnTargets="FindPythonForBuild"
           Condition="$(Configuration) != 'PGUpdate'">
     <!-- BEGIN deepfreeze rule -->
-	<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\scripts\deepfreeze.py" ^
+	<Exec Command='$(PythonForBuild) "$(PySourcePath)Tools\build\deepfreeze.py" ^
 		 "$(PySourcePath)Python\frozen_modules\importlib._bootstrap.h:importlib._bootstrap" ^
 		 "$(PySourcePath)Python\frozen_modules\importlib._bootstrap_external.h:importlib._bootstrap_external" ^
 		 "$(PySourcePath)Python\frozen_modules\zipimport.h:zipimport" ^

Makefile.pre.in

@@ -13,7 +13,7 @@
     <_ASTOutputs Include="$(PySourcePath)Python\Python-ast.c">
       <Argument>-C</Argument>
     </_ASTOutputs>
-    <_OpcodeSources Include="$(PySourcePath)Tools\scripts\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" />
+    <_OpcodeSources Include="$(PySourcePath)Tools\build\generate_opcode_h.py;$(PySourcePath)Lib\opcode.py" />
     <_OpcodeOutputs Include="$(PySourcePath)Include\opcode.h;$(PySourcePath)Include\internal\pycore_opcode.h;$(PySourcePath)Python\opcode_targets.h" />
     <_TokenSources Include="$(PySourcePath)Grammar\Tokens" />
     <_TokenOutputs Include="$(PySourcePath)Doc\library\token-list.inc">
@@ -59,7 +59,7 @@
           Inputs="@(_OpcodeSources)" Outputs="@(_OpcodeOutputs)"
           DependsOnTargets="FindPythonForBuild">
     <Message Text="Regenerate @(_OpcodeOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
-    <Exec Command="$(PythonForBuild) Tools\scripts\generate_opcode_h.py Lib\opcode.py Include\opcode.h Include\internal\pycore_opcode.h"
+    <Exec Command="$(PythonForBuild) Tools\build\generate_opcode_h.py Lib\opcode.py Include\opcode.h Include\internal\pycore_opcode.h"
           WorkingDirectory="$(PySourcePath)" />
     <Exec Command="$(PythonForBuild) Python\makeopcodetargets.py Python\opcode_targets.h"
           WorkingDirectory="$(PySourcePath)" />
@@ -69,7 +69,7 @@
           Inputs="@(_TokenSources)" Outputs="@(_TokenOutputs)"
           DependsOnTargets="FindPythonForBuild">
     <Message Text="Regenerate @(_TokenOutputs->'%(Filename)%(Extension)',' ')" Importance="high" />
-    <Exec Command="$(PythonForBuild) Tools\scripts\generate_token.py %(_TokenOutputs.Format) Grammar\Tokens &quot;%(_TokenOutputs.Identity)&quot;"
+    <Exec Command="$(PythonForBuild) Tools\build\generate_token.py %(_TokenOutputs.Format) Grammar\Tokens &quot;%(_TokenOutputs.Identity)&quot;"
           WorkingDirectory="$(PySourcePath)" />
     <Touch Files="@(_TokenOutputs)" />
   </Target>
@@ -85,7 +85,7 @@
   <Target Name="_RegenGlobalObjects"
           DependsOnTargets="FindPythonForBuild">
     <Message Text="Regenerate Global Objects" Importance="high" />
-    <Exec Command="$(PythonForBuild) Tools\scripts\generate_global_objects.py"
+    <Exec Command="$(PythonForBuild) Tools\build\generate_global_objects.py"
           WorkingDirectory="$(PySourcePath)" />
   </Target>
 

Misc/stable_abi.toml

@@ -2,7 +2,7 @@
 /* Frozen modules bootstrap
  *
  * Limited and restricted Python interpreter to run
- * "Tools/scripts/deepfreeze.py" on systems with no or older Python
+ * "Tools/build/deepfreeze.py" on systems with no or older Python
  * interpreter.
  */
 

Modules/_sre/sre_constants.h

@@ -2,7 +2,7 @@
    modules into frozen modules (like Lib/importlib/_bootstrap.py
    into Python/importlib.h).
 
-   This is used directly by Tools/scripts/freeze_modules.py, and indirectly by "make regen-frozen".
+   This is used directly by Tools/build/freeze_modules.py, and indirectly by "make regen-frozen".
 
    See Python/frozen.c for more info.