bpo-45019: Do some cleanup related to frozen modules. (gh-28319)

There are a few things I missed in gh-27980. This is a follow-up that will make subsequent PRs cleaner. It includes fixes to tests and tools that reference the frozen modules.

https://bugs.python.org/issue45019

Author: ericsnowcurrently

.gitattributes

@@ -47,6 +47,7 @@ Objects/clinic/*.h          linguist-generated=true
 PC/clinic/*.h               linguist-generated=true
 Python/clinic/*.h           linguist-generated=true
 Python/frozen_modules/*.h   linguist-generated=true
+Python/frozen_modules/MANIFEST  linguist-generated=true
 Include/internal/pycore_ast.h   linguist-generated=true
 Python/Python-ast.c         linguist-generated=true
 Include/opcode.h            linguist-generated=true

.gitignore

@@ -121,6 +121,13 @@ Tools/msi/obj
 Tools/ssl/amd64
 Tools/ssl/win32
 
+# TODO: Once we auto-regen frozem modules for Windows builds
+# we can drop the .h files from the repo and ignore them here.
+# At that point we will rely the frozen manifest file to identify
+# changed generated files.  We'll drop the entry for it then.
+# See: Tools/scripts/freeze_modules.py.
+#Python/frozen_modules/*.h
+
 # Two-trick pony for OSX and other case insensitive file systems:
 # Ignore ./python binary on Unix but still look into ./Python/ directory.
 /python

Lib/ctypes/test/test_values.py

@@ -2,9 +2,12 @@
 A testcase which accesses *values* in a dll.
 """
 
+import imp
+import importlib.util
 import unittest
 import sys
 from ctypes import *
+from test.support import import_helper, captured_stdout
 
 import _ctypes_test
 
@@ -55,41 +58,32 @@ class struct_frozen(Structure):
 
         ft = FrozenTable.in_dll(pythonapi, "PyImport_FrozenModules")
         # ft is a pointer to the struct_frozen entries:
-        items = []
-        # _frozen_importlib changes size whenever importlib._bootstrap
-        # changes, so it gets a special case.  We should make sure it's
-        # found, but don't worry about its size too much.  The same
-        # applies to _frozen_importlib_external.
-        bootstrap_seen = []
-        bootstrap_expected = [
-                b'_frozen_importlib',
-                b'_frozen_importlib_external',
-                b'zipimport',
-                ]
+        modules = []
         for entry in ft:
             # This is dangerous. We *can* iterate over a pointer, but
             # the loop will not terminate (maybe with an access
             # violation;-) because the pointer instance has no size.
             if entry.name is None:
                 break
-
-            if entry.name in bootstrap_expected:
-                bootstrap_seen.append(entry.name)
-                self.assertTrue(entry.size,
-                    "{!r} was reported as having no size".format(entry.name))
-                continue
-            items.append((entry.name.decode("ascii"), entry.size))
-
-        expected = [("__hello__", 164),
-                    ("__phello__", -164),
-                    ("__phello__.spam", 164),
-                    ]
-        self.assertEqual(items, expected, "PyImport_FrozenModules example "
+            modname = entry.name.decode("ascii")
+            modules.append(modname)
+            with self.subTest(modname):
+                # Do a sanity check on entry.size and entry.code.
+                self.assertGreater(abs(entry.size), 10)
+                self.assertTrue([entry.code[i] for i in range(abs(entry.size))])
+                # Check the module's package-ness.
+                spec = importlib.util.find_spec(modname)
+                if entry.size < 0:
+                    # It's a package.
+                    self.assertIsNotNone(spec.submodule_search_locations)
+                else:
+                    self.assertIsNone(spec.submodule_search_locations)
+
+        expected = imp._frozen_module_names()
+        self.maxDiff = None
+        self.assertEqual(modules, expected, "PyImport_FrozenModules example "
             "in Doc/library/ctypes.rst may be out of date")
 
-        self.assertEqual(sorted(bootstrap_seen), bootstrap_expected,
-            "frozen bootstrap modules did not match PyImport_FrozenModules")
-
         from ctypes import _pointer_type_cache
         del _pointer_type_cache[struct_frozen]
 

Lib/imp.py

@@ -9,7 +9,7 @@
 from _imp import (lock_held, acquire_lock, release_lock,
                   get_frozen_object, is_frozen_package,
                   init_frozen, is_builtin, is_frozen,
-                  _fix_co_filename)
+                  _fix_co_filename, _frozen_module_names)
 try:
     from _imp import create_dynamic
 except ImportError:

Makefile.pre.in

@@ -750,22 +750,22 @@ regen-frozen: Tools/scripts/freeze_modules.py $(FROZEN_FILES)
 
 # BEGIN: freezing modules
 
-Python/frozen_modules/importlib__bootstrap.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap.py
+Python/frozen_modules/importlib__bootstrap.h: Programs/_freeze_module Lib/importlib/_bootstrap.py
 	$(srcdir)/Programs/_freeze_module importlib._bootstrap \
 		$(srcdir)/Lib/importlib/_bootstrap.py \
 		$(srcdir)/Python/frozen_modules/importlib__bootstrap.h
 
-Python/frozen_modules/importlib__bootstrap_external.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/importlib/_bootstrap_external.py
+Python/frozen_modules/importlib__bootstrap_external.h: Programs/_freeze_module Lib/importlib/_bootstrap_external.py
 	$(srcdir)/Programs/_freeze_module importlib._bootstrap_external \
 		$(srcdir)/Lib/importlib/_bootstrap_external.py \
 		$(srcdir)/Python/frozen_modules/importlib__bootstrap_external.h
 
-Python/frozen_modules/zipimport.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Lib/zipimport.py
+Python/frozen_modules/zipimport.h: Programs/_freeze_module Lib/zipimport.py
 	$(srcdir)/Programs/_freeze_module zipimport \
 		$(srcdir)/Lib/zipimport.py \
 		$(srcdir)/Python/frozen_modules/zipimport.h
 
-Python/frozen_modules/hello.h: $(srcdir)/Programs/_freeze_module $(srcdir)/Tools/freeze/flag.py
+Python/frozen_modules/hello.h: Programs/_freeze_module Tools/freeze/flag.py
 	$(srcdir)/Programs/_freeze_module hello \
 		$(srcdir)/Tools/freeze/flag.py \
 		$(srcdir)/Python/frozen_modules/hello.h

Python/clinic/import.c.h

@@ -47,7 +47,7 @@
 /* Note that a negative size indicates a package. */
 
 static const struct _frozen _PyImport_FrozenModules[] = {
-    /* importlib */
+    /* import system */
     {"_frozen_importlib", _Py_M__importlib__bootstrap,
         (int)sizeof(_Py_M__importlib__bootstrap)},
     {"_frozen_importlib_external", _Py_M__importlib__bootstrap_external,

Python/frozen.c

@@ -0,0 +1,7 @@
+This directory contains the generated .h files for all the frozen
+modules.  Python/frozen.c depends on these files.
+
+Note that, other than the required frozen modules, none of these files
+are committed into the repo.
+
+See Tools/scripts/freeze_modules.py for more info.

Python/frozen_modules/MANIFEST

@@ -16,6 +16,7 @@
 #include "code.h"
 #include "importdl.h"
 #include "pydtrace.h"
+#include <stdbool.h>
 
 #ifdef HAVE_FCNTL_H
 #include <fcntl.h>
@@ -1049,6 +1050,32 @@ _imp_create_builtin(PyObject *module, PyObject *spec)
 
 /* Frozen modules */
 
+static PyObject *
+list_frozen_module_names(bool force)
+{
+    PyObject *names = PyList_New(0);
+    if (names == NULL) {
+        return NULL;
+    }
+    for (const struct _frozen *p = PyImport_FrozenModules; ; p++) {
+        if (p->name == NULL) {
+            break;
+        }
+        PyObject *name = PyUnicode_FromString(p->name);
+        if (name == NULL) {
+            Py_DECREF(names);
+            return NULL;
+        }
+        int res = PyList_Append(names, name);
+        Py_DECREF(name);
+        if (res != 0) {
+            Py_DECREF(names);
+            return NULL;
+        }
+    }
+    return names;
+}
+
 static const struct _frozen *
 find_frozen(PyObject *name)
 {
@@ -1954,6 +1981,19 @@ _imp_is_frozen_impl(PyObject *module, PyObject *name)
     return PyBool_FromLong((long) (p == NULL ? 0 : p->size));
 }
 
+/*[clinic input]
+_imp._frozen_module_names
+
+Returns the list of available frozen modules.
+[clinic start generated code]*/
+
+static PyObject *
+_imp__frozen_module_names_impl(PyObject *module)
+/*[clinic end generated code: output=80609ef6256310a8 input=76237fbfa94460d2]*/
+{
+    return list_frozen_module_names(true);
+}
+
 /* Common implementation for _imp.exec_dynamic and _imp.exec_builtin */
 static int
 exec_builtin_or_dynamic(PyObject *mod) {
@@ -2114,6 +2154,7 @@ static PyMethodDef imp_methods[] = {
     _IMP_INIT_FROZEN_METHODDEF
     _IMP_IS_BUILTIN_METHODDEF
     _IMP_IS_FROZEN_METHODDEF
+    _IMP__FROZEN_MODULE_NAMES_METHODDEF
     _IMP_CREATE_DYNAMIC_METHODDEF
     _IMP_EXEC_DYNAMIC_METHODDEF
     _IMP_EXEC_BUILTIN_METHODDEF