Skip to content

bpo-46417: Finalize structseq types at exit #30645

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Jan 21, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions Include/internal/pycore_floatobject.h
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ extern "C" {
extern void _PyFloat_InitState(PyInterpreterState *);
extern PyStatus _PyFloat_InitTypes(PyInterpreterState *);
extern void _PyFloat_Fini(PyInterpreterState *);
extern void _PyFloat_FiniType(PyInterpreterState *);


/* other API */
Expand Down
1 change: 1 addition & 0 deletions Include/internal/pycore_long.h
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ extern "C" {
/* runtime lifecycle */

extern PyStatus _PyLong_InitTypes(PyInterpreterState *);
extern void _PyLong_FiniTypes(PyInterpreterState *interp);


/* other API */
Expand Down
1 change: 1 addition & 0 deletions Include/internal/pycore_pyerrors.h
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ extern "C" {
/* runtime lifecycle */

extern PyStatus _PyErr_InitTypes(PyInterpreterState *);
extern void _PyErr_FiniTypes(PyInterpreterState *);


/* other API */
Expand Down
2 changes: 2 additions & 0 deletions Include/internal/pycore_pylifecycle.h
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ extern PyStatus _PySys_Create(
extern PyStatus _PySys_ReadPreinitWarnOptions(PyWideStringList *options);
extern PyStatus _PySys_ReadPreinitXOptions(PyConfig *config);
extern int _PySys_UpdateConfig(PyThreadState *tstate);
extern void _PySys_Fini(PyInterpreterState *interp);
extern PyStatus _PyBuiltins_AddExceptions(PyObject * bltinmod);
extern PyStatus _Py_HashRandomization_Init(const PyConfig *);

Expand All @@ -81,6 +82,7 @@ extern void _PyTraceMalloc_Fini(void);
extern void _PyWarnings_Fini(PyInterpreterState *interp);
extern void _PyAST_Fini(PyInterpreterState *interp);
extern void _PyAtExit_Fini(PyInterpreterState *interp);
extern void _PyThread_FiniType(PyInterpreterState *interp);

extern PyStatus _PyGILState_Init(_PyRuntimeState *runtime);
extern PyStatus _PyGILState_SetTstate(PyThreadState *tstate);
Expand Down
2 changes: 2 additions & 0 deletions Include/internal/pycore_typeobject.h
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,8 @@ struct type_cache {

extern PyStatus _PyTypes_InitSlotDefs(void);

extern void _PyStaticType_Dealloc(PyTypeObject *type);


#ifdef __cplusplus
}
Expand Down
3 changes: 3 additions & 0 deletions Include/structseq.h
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ PyAPI_FUNC(void) PyStructSequence_InitType(PyTypeObject *type,
PyAPI_FUNC(int) PyStructSequence_InitType2(PyTypeObject *type,
PyStructSequence_Desc *desc);
#endif
#ifdef Py_BUILD_CORE
PyAPI_FUNC(void) _PyStructSequence_FiniType(PyTypeObject *type);
#endif
PyAPI_FUNC(PyTypeObject*) PyStructSequence_NewType(PyStructSequence_Desc *desc);

PyAPI_FUNC(PyObject *) PyStructSequence_New(PyTypeObject* type);
Expand Down
55 changes: 55 additions & 0 deletions Lib/test/_test_embed_structseq.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
import sys
import types
import unittest


# bpo-46417: Test that structseq types used by the sys module are still
# valid when Py_Finalize()/Py_Initialize() are called multiple times.
class TestStructSeq(unittest.TestCase):
# test PyTypeObject members
def check_structseq(self, obj_type):
# ob_refcnt
self.assertGreaterEqual(sys.getrefcount(obj_type), 1)
# tp_base
self.assertTrue(issubclass(obj_type, tuple))
# tp_bases
self.assertEqual(obj_type.__bases__, (tuple,))
# tp_dict
self.assertIsInstance(obj_type.__dict__, types.MappingProxyType)
# tp_mro
self.assertEqual(obj_type.__mro__, (obj_type, tuple, object))
# tp_name
self.assertIsInstance(type.__name__, str)
# tp_subclasses
self.assertEqual(obj_type.__subclasses__(), [])

def test_sys_attrs(self):
for attr_name in (
'flags', # FlagsType
'float_info', # FloatInfoType
'hash_info', # Hash_InfoType
'int_info', # Int_InfoType
'thread_info', # ThreadInfoType
'version_info', # VersionInfoType
):
with self.subTest(attr=attr_name):
attr = getattr(sys, attr_name)
self.check_structseq(type(attr))

def test_sys_funcs(self):
func_names = ['get_asyncgen_hooks'] # AsyncGenHooksType
if hasattr(sys, 'getwindowsversion'):
func_names.append('getwindowsversion') # WindowsVersionType
for func_name in func_names:
with self.subTest(func=func_name):
func = getattr(sys, func_name)
obj = func()
self.check_structseq(type(obj))


try:
unittest.main()
except SystemExit as exc:
if exc.args[0] != 0:
raise
print("Tests passed")
12 changes: 12 additions & 0 deletions Lib/test/test_embed.py
Original file line number Diff line number Diff line change
Expand Up @@ -329,6 +329,18 @@ def test_run_main_loop(self):
self.assertEqual(out, "Py_RunMain(): sys.argv=['-c', 'arg2']\n" * nloop)
self.assertEqual(err, '')

def test_finalize_structseq(self):
# bpo-46417: Py_Finalize() clears structseq static types. Check that
# sys attributes using struct types still work when
# Py_Finalize()/Py_Initialize() is called multiple times.
# print() calls type->tp_repr(instance) and so checks that the types
# are still working properly.
script = support.findfile('_test_embed_structseq.py')
with open(script, encoding="utf-8") as fp:
code = fp.read()
out, err = self.run_embedded_interpreter("test_repeated_init_exec", code)
self.assertEqual(out, 'Tests passed\n' * INIT_LOOPS)


class InitConfigTests(EmbeddingTestsMixin, unittest.TestCase):
maxDiff = 4096
Expand Down
8 changes: 8 additions & 0 deletions Objects/floatobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -2082,6 +2082,14 @@ _PyFloat_Fini(PyInterpreterState *interp)
#endif
}

void
_PyFloat_FiniType(PyInterpreterState *interp)
{
if (_Py_IsMainInterpreter(interp)) {
_PyStructSequence_FiniType(&FloatInfoType);
}
}

/* Print summary info about the state of the optimized allocator */
void
_PyFloat_DebugMallocStats(FILE *out)
Expand Down
11 changes: 11 additions & 0 deletions Objects/longobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -5949,3 +5949,14 @@ _PyLong_InitTypes(PyInterpreterState *interp)

return _PyStatus_OK();
}


void
_PyLong_FiniTypes(PyInterpreterState *interp)
{
if (!_Py_IsMainInterpreter(interp)) {
return;
}

_PyStructSequence_FiniType(&Int_InfoType);
}
30 changes: 30 additions & 0 deletions Objects/structseq.c
Original file line number Diff line number Diff line change
Expand Up @@ -532,6 +532,36 @@ PyStructSequence_InitType(PyTypeObject *type, PyStructSequence_Desc *desc)
(void)PyStructSequence_InitType2(type, desc);
}


void
_PyStructSequence_FiniType(PyTypeObject *type)
{
// Ensure that the type is initialized
assert(type->tp_name != NULL);
assert(type->tp_base == &PyTuple_Type);

// Cannot delete a type if it still has subclasses
if (type->tp_subclasses != NULL) {
return;
}

// Undo PyStructSequence_NewType()
type->tp_name = NULL;
PyMem_Free(type->tp_members);

_PyStaticType_Dealloc(type);
assert(Py_REFCNT(type) == 1);
// Undo Py_INCREF(type) of _PyStructSequence_InitType().
// Don't use Py_DECREF(): static type must not be deallocated
Py_SET_REFCNT(type, 0);

// Make sure that _PyStructSequence_InitType() will initialize
// the type again
assert(Py_REFCNT(type) == 0);
assert(type->tp_name == NULL);
}


PyTypeObject *
PyStructSequence_NewType(PyStructSequence_Desc *desc)
{
Expand Down
23 changes: 21 additions & 2 deletions Objects/typeobject.c
Original file line number Diff line number Diff line change
Expand Up @@ -4070,10 +4070,27 @@ type_setattro(PyTypeObject *type, PyObject *name, PyObject *value)
extern void
_PyDictKeys_DecRef(PyDictKeysObject *keys);


void
_PyStaticType_Dealloc(PyTypeObject *type)
{
// _PyStaticType_Dealloc() must not be called if a type has subtypes.
// A subtype can inherit attributes and methods of its parent type,
// and a type must no longer be used once it's deallocated.
assert(type->tp_subclasses == NULL);

Py_CLEAR(type->tp_dict);
Py_CLEAR(type->tp_bases);
Py_CLEAR(type->tp_mro);
Py_CLEAR(type->tp_cache);
Py_CLEAR(type->tp_subclasses);
type->tp_flags &= ~Py_TPFLAGS_READY;
}


static void
type_dealloc(PyTypeObject *type)
{
PyHeapTypeObject *et;
PyObject *tp, *val, *tb;

/* Assert this is a heap-allocated type object */
Expand All @@ -4082,8 +4099,8 @@ type_dealloc(PyTypeObject *type)
PyErr_Fetch(&tp, &val, &tb);
remove_all_subclasses(type, type->tp_bases);
PyErr_Restore(tp, val, tb);

PyObject_ClearWeakRefs((PyObject *)type);
et = (PyHeapTypeObject *)type;
Py_XDECREF(type->tp_base);
Py_XDECREF(type->tp_dict);
Py_XDECREF(type->tp_bases);
Expand All @@ -4094,6 +4111,8 @@ type_dealloc(PyTypeObject *type)
* of most other objects. It's okay to cast it to char *.
*/
PyObject_Free((char *)type->tp_doc);

PyHeapTypeObject *et = (PyHeapTypeObject *)type;
Py_XDECREF(et->ht_name);
Py_XDECREF(et->ht_qualname);
Py_XDECREF(et->ht_slots);
Expand Down
40 changes: 40 additions & 0 deletions Programs/_testembed.c
Original file line number Diff line number Diff line change
Expand Up @@ -15,12 +15,18 @@
#include <stdlib.h> // putenv()
#include <wchar.h>

int main_argc;
char **main_argv;

/*********************************************************
* Embedded interpreter tests that need a custom exe
*
* Executed via 'EmbeddingTests' in Lib/test/test_capi.py
*********************************************************/

// Use to display the usage
#define PROGRAM "test_embed"

/* Use path starting with "./" avoids a search along the PATH */
#define PROGRAM_NAME L"./_testembed"

Expand Down Expand Up @@ -113,6 +119,36 @@ PyInit_embedded_ext(void)
return PyModule_Create(&embedded_ext);
}

/****************************************************************************
* Call Py_Initialize()/Py_Finalize() multiple times and execute Python code
***************************************************************************/

// Used by bpo-46417 to test that structseq types used by the sys module are
// cleared properly and initialized again properly when Python is finalized
// multiple times.
static int test_repeated_init_exec(void)
{
if (main_argc < 3) {
fprintf(stderr, "usage: %s test_repeated_init_exec CODE\n", PROGRAM);
exit(1);
}
const char *code = main_argv[2];

for (int i=1; i <= INIT_LOOPS; i++) {
fprintf(stderr, "--- Loop #%d ---\n", i);
fflush(stderr);

_testembed_Py_Initialize();
int err = PyRun_SimpleString(code);
Py_Finalize();
if (err) {
return 1;
}
}
return 0;
}


/*****************************************************
* Test forcing a particular IO encoding
*****************************************************/
Expand Down Expand Up @@ -1880,6 +1916,7 @@ struct TestCase

static struct TestCase TestCases[] = {
// Python initialization
{"test_repeated_init_exec", test_repeated_init_exec},
{"test_forced_io_encoding", test_forced_io_encoding},
{"test_repeated_init_and_subinterpreters", test_repeated_init_and_subinterpreters},
{"test_repeated_init_and_inittab", test_repeated_init_and_inittab},
Expand Down Expand Up @@ -1946,6 +1983,9 @@ static struct TestCase TestCases[] = {

int main(int argc, char *argv[])
{
main_argc = argc;
main_argv = argv;

if (argc > 1) {
for (struct TestCase *tc = TestCases; tc && tc->name; tc++) {
if (strcmp(argv[1], tc->name) == 0)
Expand Down
11 changes: 11 additions & 0 deletions Python/errors.c
Original file line number Diff line number Diff line change
Expand Up @@ -1241,6 +1241,17 @@ _PyErr_InitTypes(PyInterpreterState *interp)
}


void
_PyErr_FiniTypes(PyInterpreterState *interp)
{
if (!_Py_IsMainInterpreter(interp)) {
return;
}

_PyStructSequence_FiniType(&UnraisableHookArgsType);
}


static PyObject *
make_unraisable_hook_args(PyThreadState *tstate, PyObject *exc_type,
PyObject *exc_value, PyObject *exc_tb,
Expand Down
6 changes: 6 additions & 0 deletions Python/pylifecycle.c
Original file line number Diff line number Diff line change
Expand Up @@ -1666,11 +1666,17 @@ flush_std_files(void)
static void
finalize_interp_types(PyInterpreterState *interp)
{
_PySys_Fini(interp);
_PyExc_Fini(interp);
_PyFrame_Fini(interp);
_PyAsyncGen_Fini(interp);
_PyContext_Fini(interp);
_PyFloat_FiniType(interp);
_PyLong_FiniTypes(interp);
_PyThread_FiniType(interp);
_PyErr_FiniTypes(interp);
_PyTypes_Fini(interp);

// Call _PyUnicode_ClearInterned() before _PyDict_Fini() since it uses
// a dict internally.
_PyUnicode_ClearInterned(interp);
Expand Down
15 changes: 15 additions & 0 deletions Python/sysmodule.c
Original file line number Diff line number Diff line change
Expand Up @@ -3102,6 +3102,21 @@ _PySys_Create(PyThreadState *tstate, PyObject **sysmod_p)
}


void
_PySys_Fini(PyInterpreterState *interp)
{
if (_Py_IsMainInterpreter(interp)) {
_PyStructSequence_FiniType(&VersionInfoType);
_PyStructSequence_FiniType(&FlagsType);
#if defined(MS_WINDOWS)
_PyStructSequence_FiniType(&WindowsVersionType);
#endif
_PyStructSequence_FiniType(&Hash_InfoType);
_PyStructSequence_FiniType(&AsyncGenHooksType);
}
}


static PyObject *
makepathobject(const wchar_t *path, wchar_t delim)
{
Expand Down
11 changes: 11 additions & 0 deletions Python/thread.c
Original file line number Diff line number Diff line change
Expand Up @@ -243,3 +243,14 @@ PyThread_GetInfo(void)
PyStructSequence_SET_ITEM(threadinfo, pos++, value);
return threadinfo;
}


void
_PyThread_FiniType(PyInterpreterState *interp)
{
if (!_Py_IsMainInterpreter(interp)) {
return;
}

_PyStructSequence_FiniType(&ThreadInfoType);
}