diff --git a/Makefile.pre.in b/Makefile.pre.in index fbd4c3a23fd81b..eda8917977110f 100644 --- a/Makefile.pre.in +++ b/Makefile.pre.in @@ -486,6 +486,10 @@ DEEPFREEZE_OBJS = \ Python/deepfreeze/zipimport.o \ Python/deepfreeze/abc.o \ Python/deepfreeze/codecs.o \ + Python/deepfreeze/encodings.o \ + Python/deepfreeze/encodings.aliases.o \ + Python/deepfreeze/encodings.ascii.o \ + Python/deepfreeze/encodings.utf_8.o \ Python/deepfreeze/io.o \ Python/deepfreeze/_collections_abc.o \ Python/deepfreeze/_sitebuiltins.o \ @@ -1008,6 +1012,18 @@ Python/deepfreeze/abc.c: Python/frozen_modules/abc.h $(DEEPFREEZE_DEPS) Python/deepfreeze/codecs.c: Python/frozen_modules/codecs.h $(DEEPFREEZE_DEPS) $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/codecs.h -m codecs -o Python/deepfreeze/codecs.c +Python/deepfreeze/encodings.c: Python/frozen_modules/encodings.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/encodings.h -m encodings -o Python/deepfreeze/encodings.c + +Python/deepfreeze/encodings.aliases.c: Python/frozen_modules/encodings.aliases.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/encodings.aliases.h -m encodings.aliases -o Python/deepfreeze/encodings.aliases.c + +Python/deepfreeze/encodings.ascii.c: Python/frozen_modules/encodings.ascii.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/encodings.ascii.h -m encodings.ascii -o Python/deepfreeze/encodings.ascii.c + +Python/deepfreeze/encodings.utf_8.c: Python/frozen_modules/encodings.utf_8.h $(DEEPFREEZE_DEPS) + $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/encodings.utf_8.h -m encodings.utf_8 -o Python/deepfreeze/encodings.utf_8.c + Python/deepfreeze/io.c: Python/frozen_modules/io.h $(DEEPFREEZE_DEPS) $(PYTHON_FOR_FREEZE) $(srcdir)/Tools/scripts/deepfreeze.py Python/frozen_modules/io.h -m io -o Python/deepfreeze/io.c @@ -1098,6 +1114,10 @@ FROZEN_FILES_IN = \ Lib/zipimport.py \ Lib/abc.py \ Lib/codecs.py \ + Lib/encodings/__init__.py \ + Lib/encodings/aliases.py \ + Lib/encodings/ascii.py \ + Lib/encodings/utf_8.py \ Lib/io.py \ Lib/_collections_abc.py \ Lib/_sitebuiltins.py \ @@ -1123,6 +1143,10 @@ FROZEN_FILES_OUT = \ Python/frozen_modules/zipimport.h \ Python/frozen_modules/abc.h \ Python/frozen_modules/codecs.h \ + Python/frozen_modules/encodings.h \ + Python/frozen_modules/encodings.aliases.h \ + Python/frozen_modules/encodings.ascii.h \ + Python/frozen_modules/encodings.utf_8.h \ Python/frozen_modules/io.h \ Python/frozen_modules/_collections_abc.h \ Python/frozen_modules/_sitebuiltins.h \ @@ -1171,6 +1195,18 @@ Python/frozen_modules/abc.h: Lib/abc.py $(FREEZE_MODULE_DEPS) Python/frozen_modules/codecs.h: Lib/codecs.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) codecs $(srcdir)/Lib/codecs.py Python/frozen_modules/codecs.h +Python/frozen_modules/encodings.h: Lib/encodings/__init__.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings $(srcdir)/Lib/encodings/__init__.py Python/frozen_modules/encodings.h + +Python/frozen_modules/encodings.aliases.h: Lib/encodings/aliases.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings.aliases $(srcdir)/Lib/encodings/aliases.py Python/frozen_modules/encodings.aliases.h + +Python/frozen_modules/encodings.ascii.h: Lib/encodings/ascii.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings.ascii $(srcdir)/Lib/encodings/ascii.py Python/frozen_modules/encodings.ascii.h + +Python/frozen_modules/encodings.utf_8.h: Lib/encodings/utf_8.py $(FREEZE_MODULE_DEPS) + $(FREEZE_MODULE) encodings.utf_8 $(srcdir)/Lib/encodings/utf_8.py Python/frozen_modules/encodings.utf_8.h + Python/frozen_modules/io.h: Lib/io.py $(FREEZE_MODULE_DEPS) $(FREEZE_MODULE) io $(srcdir)/Lib/io.py Python/frozen_modules/io.h diff --git a/Misc/NEWS.d/next/Build/2021-12-10-16-06-13.bpo-45653.FQo2rA.rst b/Misc/NEWS.d/next/Build/2021-12-10-16-06-13.bpo-45653.FQo2rA.rst new file mode 100644 index 00000000000000..fd9f7bd7561244 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2021-12-10-16-06-13.bpo-45653.FQo2rA.rst @@ -0,0 +1 @@ +Parts of the :mod:`encodings` package is now frozen. diff --git a/PCbuild/_freeze_module.vcxproj b/PCbuild/_freeze_module.vcxproj index 59519cade26705..cbef85283bfaf9 100644 --- a/PCbuild/_freeze_module.vcxproj +++ b/PCbuild/_freeze_module.vcxproj @@ -272,6 +272,34 @@ $(IntDir)codecs.g.c $(PySourcePath)Python\deepfreeze\df.codecs.c + + encodings + $(IntDir)encodings.g.h + $(PySourcePath)Python\frozen_modules\encodings.h + $(IntDir)encodings.g.c + $(PySourcePath)Python\deepfreeze\df.encodings.c + + + encodings.aliases + $(IntDir)encodings.aliases.g.h + $(PySourcePath)Python\frozen_modules\encodings.aliases.h + $(IntDir)encodings.aliases.g.c + $(PySourcePath)Python\deepfreeze\df.encodings.aliases.c + + + encodings.ascii + $(IntDir)encodings.ascii.g.h + $(PySourcePath)Python\frozen_modules\encodings.ascii.h + $(IntDir)encodings.ascii.g.c + $(PySourcePath)Python\deepfreeze\df.encodings.ascii.c + + + encodings.utf_8 + $(IntDir)encodings.utf_8.g.h + $(PySourcePath)Python\frozen_modules\encodings.utf_8.h + $(IntDir)encodings.utf_8.g.c + $(PySourcePath)Python\deepfreeze\df.encodings.utf_8.c + io $(IntDir)io.g.h diff --git a/PCbuild/_freeze_module.vcxproj.filters b/PCbuild/_freeze_module.vcxproj.filters index d107849c856213..2026355ce522b2 100644 --- a/PCbuild/_freeze_module.vcxproj.filters +++ b/PCbuild/_freeze_module.vcxproj.filters @@ -420,6 +420,18 @@ Python Files + + Python Files + + + Python Files + + + Python Files + + + Python Files + Python Files diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 83ae2f08749aa8..767d0806204191 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -532,6 +532,10 @@ + + + + diff --git a/Python/codecs.c b/Python/codecs.c index 343b6e2d03396e..d2f0e6bbd81458 100644 --- a/Python/codecs.c +++ b/Python/codecs.c @@ -13,6 +13,7 @@ Copyright (c) Corporation for National Research Initiatives. #include "pycore_interp.h" // PyInterpreterState.codec_search_path #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_ucnhash.h" // _PyUnicode_Name_CAPI +#include "pycore_fileutils.h" // _Py_join_relfile() #include const char *Py_hexdigits = "0123456789abcdef"; @@ -1405,6 +1406,66 @@ static PyObject *surrogateescape_errors(PyObject *self, PyObject *exc) return PyCodec_SurrogateEscapeErrors(exc); } +/* Set encodings.__path__ for frozen encodings package + * + * The encodings package is frozen but most encodings modules are not. The + * __path__ attribute of the encodings package must be reset so importlib is + * able to find the pure Python modules. + * Returns -1 on error + */ +static int +_set_encodings_path(PyObject *mod) { + int rc = -1; + PyObject *path = NULL; + PyObject *encodings_dir = NULL; + + // borrowed ref + PyObject *stdlib_dir = PySys_GetObject("_stdlib_dir"); + if (stdlib_dir == NULL) { + rc = 0; + goto exit; + } + + Py_ssize_t size; + wchar_t *stdlib_dirw = PyUnicode_AsWideCharString(stdlib_dir, &size); + if (stdlib_dirw == NULL) { + goto exit; + } + + // encodings_dir = os.path.join(sys._stdlib_dir, "encodings") + wchar_t *encodings_dirw = _Py_join_relfile(stdlib_dirw, L"encodings"); + PyMem_Free((void *)stdlib_dirw); + if (encodings_dirw == NULL) { + PyErr_NoMemory(); + goto exit; + } + + encodings_dir = PyUnicode_FromWideChar(encodings_dirw, -1); + PyMem_RawFree((void *)encodings_dirw); + if (encodings_dir == NULL) { + goto exit; + } + + path = PyList_New(0); + if (path == NULL) { + goto exit; + } + if (PyList_Append(path, encodings_dir) < 0) { + goto exit; + } + // encodings.__path__ = [encodings_dir] + if (PyObject_SetAttrString(mod, "__path__", path) < 0) { + goto exit; + } + + rc = 0; + + exit: + Py_XDECREF(path); + Py_XDECREF(encodings_dir); + return rc; +} + static int _PyCodecRegistry_Init(void) { static struct { @@ -1531,6 +1592,11 @@ static int _PyCodecRegistry_Init(void) if (mod == NULL) { return -1; } + + if (_set_encodings_path(mod) < 0) { + Py_DECREF(mod); + return -1; + } Py_DECREF(mod); interp->codecs_initialized = 1; return 0; diff --git a/Python/frozen.c b/Python/frozen.c index 25cf0a8d37c789..4f139361f1d355 100644 --- a/Python/frozen.c +++ b/Python/frozen.c @@ -44,6 +44,10 @@ #include "frozen_modules/zipimport.h" #include "frozen_modules/abc.h" #include "frozen_modules/codecs.h" +#include "frozen_modules/encodings.h" +#include "frozen_modules/encodings.aliases.h" +#include "frozen_modules/encodings.ascii.h" +#include "frozen_modules/encodings.utf_8.h" #include "frozen_modules/io.h" #include "frozen_modules/_collections_abc.h" #include "frozen_modules/_sitebuiltins.h" @@ -72,6 +76,10 @@ extern PyObject *_Py_get_importlib__bootstrap_external_toplevel(void); extern PyObject *_Py_get_zipimport_toplevel(void); extern PyObject *_Py_get_abc_toplevel(void); extern PyObject *_Py_get_codecs_toplevel(void); +extern PyObject *_Py_get_encodings_toplevel(void); +extern PyObject *_Py_get_encodings_aliases_toplevel(void); +extern PyObject *_Py_get_encodings_ascii_toplevel(void); +extern PyObject *_Py_get_encodings_utf_8_toplevel(void); extern PyObject *_Py_get_io_toplevel(void); extern PyObject *_Py_get__collections_abc_toplevel(void); extern PyObject *_Py_get__sitebuiltins_toplevel(void); @@ -110,6 +118,10 @@ static const struct _frozen stdlib_modules[] = { /* stdlib - startup, without site (python -S) */ {"abc", _Py_M__abc, (int)sizeof(_Py_M__abc), GET_CODE(abc)}, {"codecs", _Py_M__codecs, (int)sizeof(_Py_M__codecs), GET_CODE(codecs)}, + {"encodings", _Py_M__encodings, -(int)sizeof(_Py_M__encodings), GET_CODE(encodings)}, + {"encodings.aliases", _Py_M__encodings_aliases, (int)sizeof(_Py_M__encodings_aliases), GET_CODE(encodings_aliases)}, + {"encodings.ascii", _Py_M__encodings_ascii, (int)sizeof(_Py_M__encodings_ascii), GET_CODE(encodings_ascii)}, + {"encodings.utf_8", _Py_M__encodings_utf_8, (int)sizeof(_Py_M__encodings_utf_8), GET_CODE(encodings_utf_8)}, {"io", _Py_M__io, (int)sizeof(_Py_M__io), GET_CODE(io)}, /* stdlib - startup, with site */ diff --git a/Tools/scripts/freeze_modules.py b/Tools/scripts/freeze_modules.py index cbe8bf1ce60cdc..d27398acc109eb 100644 --- a/Tools/scripts/freeze_modules.py +++ b/Tools/scripts/freeze_modules.py @@ -49,10 +49,10 @@ ('stdlib - startup, without site (python -S)', [ 'abc', 'codecs', - # For now we do not freeze the encodings, due # to the noise all - # those extra modules add to the text printed during the build. - # (See https://github.com/python/cpython/pull/28398#pullrequestreview-756856469.) - #'', + '', # encodings.__init__ + 'encodings.aliases', + 'encodings.ascii', + 'encodings.utf_8', 'io', ]), ('stdlib - startup, with site', [