Add a Test to Verify All Stdlib Modules Can Be Imported in an Isolated Subinterpreter

[ericsnowcurrently]

(This is a follow-up to gh-103092 and gh-104108.)

It may be worth having a test that tries importing every stdlib module in a subinterpreter that has its own GIL. This would ensure that all our builtin/extension modules are isolated (implement multi-phase init and have a Py_MOD_PER_INTERPRETER_GIL_SUPPORTED slot).

(Such a test should fail until gh-103092 is finished.)

Linked PRs

• GH-104289: Test importing all stdlib modules in subinterpreters #133392

[tonybaloney]

This is the script I'm using to do this:

from test.libregrtest.findtests import findtests

import _xxsubinterpreters as interpreters

# Get a list of tests
test_names = findtests()
skip_tests = [
    "test_httpservers", # hangs?
    "test_imaplib" # hangs?
]


def run_test():
    import unittest

    test_cases = unittest.defaultTestLoader.loadTestsFromName(f"test.{test_name}")
    reasons = ""
    pass_count = 0
    fail_count = 0
    for case in test_cases:
        r = unittest.result.TestResult()
        case.run(r)
        if r.wasSuccessful():
            pass_count += r.testsRun
        else:
            for failedcase, reason in r.failures:
                reasons += "---------------------------------------------------------------\n"
                reasons += f"Test case {failedcase} failed:\n"
                reasons += reason
                reasons += "\n---------------------------------------------------------------\n"
                fail_count += 1

            for failedcase, reason in r.errors:
                reasons += (
                    "---------------------------------------------------------------\n"
                )
                reasons += f"Test case {failedcase} failed with errors:\n"
                reasons += reason
                reasons += "\n---------------------------------------------------------------\n"
                fail_count += 1
    print(f"Test {test_name} passed {pass_count} tests and failed {fail_count} tests")
    if fail_count > 0:
        print(reasons)

interp = interpreters.create()

for test in test_names:
    # Run the test suite
    if test in skip_tests:
        print(f"Skipping test {test}")
        continue
    print(f"Running test {test}")
    try:
        result = interpreters.run_func(interp, run_test, shared={"test_name": test})
    except Exception as e:
        print(f"Test {test} failed with exception {e}")
        continue