From 0d25bcd6093bb32e9ae9db102258d8ca7b8f2785 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 May 2019 12:18:16 -0400 Subject: [PATCH 1/5] Add a NEWS entry. --- .../Core and Builtins/2019-05-07-12-18-11.bpo-36737.XAo6LY.rst | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2019-05-07-12-18-11.bpo-36737.XAo6LY.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2019-05-07-12-18-11.bpo-36737.XAo6LY.rst b/Misc/NEWS.d/next/Core and Builtins/2019-05-07-12-18-11.bpo-36737.XAo6LY.rst new file mode 100644 index 00000000000000..7a2c647dbff31e --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2019-05-07-12-18-11.bpo-36737.XAo6LY.rst @@ -0,0 +1,2 @@ +Move PyRuntimeState.warnings into per-interpreter state (via "module +state"). From 8be07f4e363a847e00caeed810aec09e521bb890 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Tue, 7 May 2019 11:19:21 -0400 Subject: [PATCH 2/5] Move the warnings state to PyInterpreterState. --- Include/internal/pycore_pystate.h | 3 +- Python/_warnings.c | 282 +++++++++++++++++++++--------- 2 files changed, 202 insertions(+), 83 deletions(-) diff --git a/Include/internal/pycore_pystate.h b/Include/internal/pycore_pystate.h index 67bcd147e2829f..69ceecba40dac5 100644 --- a/Include/internal/pycore_pystate.h +++ b/Include/internal/pycore_pystate.h @@ -90,6 +90,8 @@ struct _is { PyObject *pyexitmodule; uint64_t tstate_next_unique_id; + + struct _warnings_runtime_state warnings; }; PyAPI_FUNC(struct _is*) _PyInterpreterState_LookUpID(PY_INT64_T); @@ -179,7 +181,6 @@ typedef struct pyruntimestate { int nexitfuncs; struct _gc_runtime_state gc; - struct _warnings_runtime_state warnings; struct _ceval_runtime_state ceval; struct _gilstate_runtime_state gilstate; diff --git a/Python/_warnings.c b/Python/_warnings.c index 388b29954081eb..c7284bd2903b40 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -15,6 +15,108 @@ _Py_IDENTIFIER(default); _Py_IDENTIFIER(ignore); #endif + +/*************************************************************************/ + +typedef struct _warnings_runtime_state WarningsState; + +/* Forward declaration of the _warnings module definition. */ +static struct PyModuleDef warningsmodule; + +/* Given a module object, get its per-module state. */ +static WarningsState * +_Warnings_GetState() +{ + PyThreadState *tstate = PyThreadState_GET(); + if (tstate == NULL) { + PyErr_SetString(PyExc_RuntimeError, + "_Warnings_GetState: could not identify current interpreter"); + return NULL; + } + return &tstate->interp->warnings; +} + +/* Clear the given warnings module state. */ +static void +_Warnings_ClearState(WarningsState *st) +{ + Py_CLEAR(st->filters); + Py_CLEAR(st->once_registry); + Py_CLEAR(st->default_action); +} + +static PyObject * +init_filters(void) +{ +#ifdef Py_DEBUG + /* Py_DEBUG builds show all warnings by default */ + return PyList_New(0); +#else + /* Other builds ignore a number of warning categories by default */ + PyObject *filters = PyList_New(5); + if (filters == NULL) { + return NULL; + } + + size_t pos = 0; /* Post-incremented in each use. */ + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_DeprecationWarning, &PyId_default, "__main__")); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_DeprecationWarning, &PyId_ignore, NULL)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore, NULL)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ImportWarning, &PyId_ignore, NULL)); + PyList_SET_ITEM(filters, pos++, + create_filter(PyExc_ResourceWarning, &PyId_ignore, NULL)); + + for (size_t x = 0; x < pos; x++) { + if (PyList_GET_ITEM(filters, x) == NULL) { + Py_DECREF(filters); + return NULL; + } + } + return filters; +#endif +} + +/* Initialize the given warnings module state. */ +static int +_Warnings_InitState(WarningsState *st) +{ + if (st->filters == NULL) { + st->filters = init_filters(); + if (st->filters == NULL) { + goto error; + } + } + + if (st->once_registry == NULL) { + st->once_registry = PyDict_New(); + if (st->once_registry == NULL) { + goto error; + } + } + + if (st->default_action == NULL) { + st->default_action = PyUnicode_FromString("default"); + if (st->default_action == NULL) { + goto error; + } + } + + st->filters_version = 0; + + return 0; + +error: + _Warnings_ClearState(st); + return -1; +} + + +/*************************************************************************/ + static int check_matched(PyObject *obj, PyObject *arg) { @@ -93,7 +195,7 @@ get_warnings_attr(_Py_Identifier *attr_id, int try_import) static PyObject * -get_once_registry(void) +get_once_registry(WarningsState *st) { PyObject *registry; _Py_IDENTIFIER(onceregistry); @@ -102,8 +204,8 @@ get_once_registry(void) if (registry == NULL) { if (PyErr_Occurred()) return NULL; - assert(_PyRuntime.warnings.once_registry); - return _PyRuntime.warnings.once_registry; + assert(st->once_registry); + return st->once_registry; } if (!PyDict_Check(registry)) { PyErr_Format(PyExc_TypeError, @@ -113,13 +215,13 @@ get_once_registry(void) Py_DECREF(registry); return NULL; } - Py_SETREF(_PyRuntime.warnings.once_registry, registry); + Py_SETREF(st->once_registry, registry); return registry; } static PyObject * -get_default_action(void) +get_default_action(WarningsState *st) { PyObject *default_action; _Py_IDENTIFIER(defaultaction); @@ -129,8 +231,8 @@ get_default_action(void) if (PyErr_Occurred()) { return NULL; } - assert(_PyRuntime.warnings.default_action); - return _PyRuntime.warnings.default_action; + assert(st->default_action); + return st->default_action; } if (!PyUnicode_Check(default_action)) { PyErr_Format(PyExc_TypeError, @@ -140,7 +242,7 @@ get_default_action(void) Py_DECREF(default_action); return NULL; } - Py_SETREF(_PyRuntime.warnings.default_action, default_action); + Py_SETREF(st->default_action, default_action); return default_action; } @@ -154,6 +256,10 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, Py_ssize_t i; PyObject *warnings_filters; _Py_IDENTIFIER(filters); + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + return NULL; + } warnings_filters = get_warnings_attr(&PyId_filters, 0); if (warnings_filters == NULL) { @@ -161,17 +267,17 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, return NULL; } else { - Py_SETREF(_PyRuntime.warnings.filters, warnings_filters); + Py_SETREF(st->filters, warnings_filters); } - PyObject *filters = _PyRuntime.warnings.filters; + PyObject *filters = st->filters; if (filters == NULL || !PyList_Check(filters)) { PyErr_SetString(PyExc_ValueError, MODULE_NAME ".filters must be a list"); return NULL; } - /* _PyRuntime.warnings.filters could change while we are iterating over it. */ + /* WarningsState.filters could change while we are iterating over it. */ for (i = 0; i < PyList_GET_SIZE(filters); i++) { PyObject *tmp_item, *action, *msg, *cat, *mod, *ln_obj; Py_ssize_t ln; @@ -232,7 +338,7 @@ get_filter(PyObject *category, PyObject *text, Py_ssize_t lineno, Py_DECREF(tmp_item); } - action = get_default_action(); + action = get_default_action(st); if (action != NULL) { Py_INCREF(Py_None); *item = Py_None; @@ -252,16 +358,20 @@ already_warned(PyObject *registry, PyObject *key, int should_set) if (key == NULL) return -1; + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + return -1; + } version_obj = _PyDict_GetItemIdWithError(registry, &PyId_version); if (version_obj == NULL || !PyLong_CheckExact(version_obj) - || PyLong_AsLong(version_obj) != _PyRuntime.warnings.filters_version) + || PyLong_AsLong(version_obj) != st->filters_version) { if (PyErr_Occurred()) { return -1; } PyDict_Clear(registry); - version_obj = PyLong_FromLong(_PyRuntime.warnings.filters_version); + version_obj = PyLong_FromLong(st->filters_version); if (version_obj == NULL) return -1; if (_PyDict_SetItemId(registry, &PyId_version, version_obj) < 0) { @@ -567,11 +677,15 @@ warn_explicit(PyObject *category, PyObject *message, if (_PyUnicode_EqualToASCIIString(action, "once")) { if (registry == NULL || registry == Py_None) { - registry = get_once_registry(); + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + goto cleanup; + } + registry = get_once_registry(st); if (registry == NULL) goto cleanup; } - /* _PyRuntime.warnings.once_registry[(text, category)] = 1 */ + /* WarningsState.once_registry[(text, category)] = 1 */ rc = update_registry(registry, text, category, 0); } else if (_PyUnicode_EqualToASCIIString(action, "module")) { @@ -925,7 +1039,11 @@ warnings_warn_explicit(PyObject *self, PyObject *args, PyObject *kwds) static PyObject * warnings_filters_mutated(PyObject *self, PyObject *args) { - _PyRuntime.warnings.filters_version++; + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + return NULL; + } + st->filters_version++; Py_RETURN_NONE; } @@ -1202,51 +1320,59 @@ create_filter(PyObject *category, _Py_Identifier *id, const char *modname) #endif -static PyObject * -init_filters(void) +static int +warnings_traverse(PyObject *m, visitproc visit, void *arg) { -#ifdef Py_DEBUG - /* Py_DEBUG builds show all warnings by default */ - return PyList_New(0); -#else - /* Other builds ignore a number of warning categories by default */ - PyObject *filters = PyList_New(5); - if (filters == NULL) { - return NULL; + // During runtime finalization we need to keep the state around a little longer. + if (_PyRuntime.finalizing) { + return 0; + } + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + return -1; } + Py_VISIT(st->filters); + Py_VISIT(st->once_registry); + return 0; +} - size_t pos = 0; /* Post-incremented in each use. */ - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, &PyId_default, "__main__")); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_DeprecationWarning, &PyId_ignore, NULL)); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_PendingDeprecationWarning, &PyId_ignore, NULL)); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ImportWarning, &PyId_ignore, NULL)); - PyList_SET_ITEM(filters, pos++, - create_filter(PyExc_ResourceWarning, &PyId_ignore, NULL)); +static int +warnings_clear(PyObject *m) +{ + // During runtime finalization we need to keep the state around a little longer. + if (_PyRuntime.finalizing) { + return 0; + } + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + return -1; + } + _Warnings_ClearState(st); + return 0; +} - for (size_t x = 0; x < pos; x++) { - if (PyList_GET_ITEM(filters, x) == NULL) { - Py_DECREF(filters); - return NULL; - } +static void +warnings_free(PyObject *m) +{ + // During runtime finalization we need to keep the state around a little longer. + if (_PyRuntime.finalizing) { + return; } - return filters; -#endif + WarningsState *st = _Warnings_GetState(); + assert(st != NULL); + _Warnings_ClearState(st); } static struct PyModuleDef warningsmodule = { PyModuleDef_HEAD_INIT, - MODULE_NAME, - warnings__doc__, - 0, - warnings_functions, - NULL, - NULL, - NULL, - NULL + MODULE_NAME, /* m_name */ + warnings__doc__, /* m_doc */ + 0, /* m_size */ + warnings_functions, /* m_methods */ + NULL, /* m_reload */ + warnings_traverse, /* m_traverse */ + warnings_clear, /* m_clear */ + (freefunc)warnings_free /* m_free */ }; @@ -1256,49 +1382,41 @@ _PyWarnings_Init(void) PyObject *m; m = PyModule_Create(&warningsmodule); - if (m == NULL) + if (m == NULL) { return NULL; + } - struct _warnings_runtime_state *state = &_PyRuntime.warnings; - if (state->filters == NULL) { - state->filters = init_filters(); - if (state->filters == NULL) - return NULL; + WarningsState *st = _Warnings_GetState(); + if (st == NULL) { + // XXX decref m? + return NULL; } - Py_INCREF(state->filters); - if (PyModule_AddObject(m, "filters", state->filters) < 0) + if (_Warnings_InitState(st) < 0) { return NULL; - - if (state->once_registry == NULL) { - state->once_registry = PyDict_New(); - if (state->once_registry == NULL) - return NULL; } - Py_INCREF(state->once_registry); - if (PyModule_AddObject(m, "_onceregistry", - state->once_registry) < 0) + + Py_INCREF(st->filters); + if (PyModule_AddObject(m, "filters", st->filters) < 0) { return NULL; + } - if (state->default_action == NULL) { - state->default_action = PyUnicode_FromString("default"); - if (state->default_action == NULL) - return NULL; + Py_INCREF(st->once_registry); + if (PyModule_AddObject(m, "_onceregistry", st->once_registry) < 0) { + return NULL; } - Py_INCREF(state->default_action); - if (PyModule_AddObject(m, "_defaultaction", - state->default_action) < 0) + + Py_INCREF(st->default_action); + if (PyModule_AddObject(m, "_defaultaction", st->default_action) < 0) { return NULL; + } - state->filters_version = 0; return m; } - +// We need this to ensure that warnings still work until late in finalization. void _PyWarnings_Fini(_PyRuntimeState *runtime) { - struct _warnings_runtime_state *state = &runtime->warnings; - Py_CLEAR(state->filters); - Py_CLEAR(state->once_registry); - Py_CLEAR(state->default_action); + PyInterpreterState *interp = runtime->finalizing->interp; + _Warnings_ClearState(&interp->warnings); } From 1857804f66146ad6c674a55665d18e8a4eb6badf Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 May 2019 12:05:50 -0400 Subject: [PATCH 3/5] Use _PyWarnings_Fini() explicitly rather than normal module fini. --- Include/internal/pycore_pylifecycle.h | 2 +- Python/_warnings.c | 52 +++------------------------ Python/pylifecycle.c | 2 +- Python/pystate.c | 4 +++ 4 files changed, 10 insertions(+), 50 deletions(-) diff --git a/Include/internal/pycore_pylifecycle.h b/Include/internal/pycore_pylifecycle.h index adb1f5d90a5960..7144bbcda7cb72 100644 --- a/Include/internal/pycore_pylifecycle.h +++ b/Include/internal/pycore_pylifecycle.h @@ -81,7 +81,7 @@ extern void PyLong_Fini(void); extern void _PyFaulthandler_Fini(void); extern void _PyHash_Fini(void); extern int _PyTraceMalloc_Fini(void); -extern void _PyWarnings_Fini(_PyRuntimeState *runtime); +extern void _PyWarnings_Fini(PyInterpreterState *interp); extern void _PyGILState_Init( _PyRuntimeState *runtime, diff --git a/Python/_warnings.c b/Python/_warnings.c index c7284bd2903b40..36f06ffa34e8e7 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1320,49 +1320,6 @@ create_filter(PyObject *category, _Py_Identifier *id, const char *modname) #endif -static int -warnings_traverse(PyObject *m, visitproc visit, void *arg) -{ - // During runtime finalization we need to keep the state around a little longer. - if (_PyRuntime.finalizing) { - return 0; - } - WarningsState *st = _Warnings_GetState(); - if (st == NULL) { - return -1; - } - Py_VISIT(st->filters); - Py_VISIT(st->once_registry); - return 0; -} - -static int -warnings_clear(PyObject *m) -{ - // During runtime finalization we need to keep the state around a little longer. - if (_PyRuntime.finalizing) { - return 0; - } - WarningsState *st = _Warnings_GetState(); - if (st == NULL) { - return -1; - } - _Warnings_ClearState(st); - return 0; -} - -static void -warnings_free(PyObject *m) -{ - // During runtime finalization we need to keep the state around a little longer. - if (_PyRuntime.finalizing) { - return; - } - WarningsState *st = _Warnings_GetState(); - assert(st != NULL); - _Warnings_ClearState(st); -} - static struct PyModuleDef warningsmodule = { PyModuleDef_HEAD_INIT, MODULE_NAME, /* m_name */ @@ -1370,9 +1327,9 @@ static struct PyModuleDef warningsmodule = { 0, /* m_size */ warnings_functions, /* m_methods */ NULL, /* m_reload */ - warnings_traverse, /* m_traverse */ - warnings_clear, /* m_clear */ - (freefunc)warnings_free /* m_free */ + NULL, /* m_traverse */ + NULL, /* m_clear */ + NULL /* m_free */ }; @@ -1415,8 +1372,7 @@ _PyWarnings_Init(void) // We need this to ensure that warnings still work until late in finalization. void -_PyWarnings_Fini(_PyRuntimeState *runtime) +_PyWarnings_Fini(PyInterpreterState *interp) { - PyInterpreterState *interp = runtime->finalizing->interp; _Warnings_ClearState(&interp->warnings); } diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index bd4d1d92662a69..32902aa0d597d0 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1288,7 +1288,7 @@ Py_FinalizeEx(void) PyDict_Fini(); PySlice_Fini(); _PyGC_Fini(runtime); - _PyWarnings_Fini(runtime); + _PyWarnings_Fini(interp); _Py_HashRandomization_Fini(); _PyArg_Fini(); PyAsyncGen_Fini(); diff --git a/Python/pystate.c b/Python/pystate.c index e9c4c7d8376bf7..44acfed6b98355 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -5,6 +5,7 @@ #include "pycore_coreconfig.h" #include "pycore_pymem.h" #include "pycore_pystate.h" +#include "pycore_pylifecycle.h" /* -------------------------------------------------------------------------- CAUTION @@ -257,6 +258,9 @@ _PyInterpreterState_Clear(_PyRuntimeState *runtime, PyInterpreterState *interp) Py_CLEAR(interp->after_forkers_parent); Py_CLEAR(interp->after_forkers_child); #endif + if (runtime->finalizing == NULL) { + _PyWarnings_Fini(interp); + } // XXX Once we have one allocator per interpreter (i.e. // per-interpreter GC) we must ensure that all of the interpreter's // objects have been cleaned up at the point. From eb95dfaa9268179bdbf8e39f91a796a45c2dd838 Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Wed, 8 May 2019 12:11:38 -0400 Subject: [PATCH 4/5] Decref the module if init fails. --- Python/_warnings.c | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index 36f06ffa34e8e7..98d07f3788579e 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -1345,29 +1345,35 @@ _PyWarnings_Init(void) WarningsState *st = _Warnings_GetState(); if (st == NULL) { - // XXX decref m? - return NULL; + goto error; } if (_Warnings_InitState(st) < 0) { - return NULL; + goto error; } Py_INCREF(st->filters); if (PyModule_AddObject(m, "filters", st->filters) < 0) { - return NULL; + goto error; } Py_INCREF(st->once_registry); if (PyModule_AddObject(m, "_onceregistry", st->once_registry) < 0) { - return NULL; + goto error; } Py_INCREF(st->default_action); if (PyModule_AddObject(m, "_defaultaction", st->default_action) < 0) { - return NULL; + goto error; } return m; + +error: + if (st != NULL) { + _Warnings_ClearState(st); + } + Py_DECREF(m); + return NULL; } // We need this to ensure that warnings still work until late in finalization. From 6094432a61892cd9f3e1df6217fe21a7d294d8ad Mon Sep 17 00:00:00 2001 From: Eric Snow Date: Thu, 9 May 2019 15:46:32 -0400 Subject: [PATCH 5/5] Fix some compiler errors. --- Python/_warnings.c | 53 +++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 27 deletions(-) diff --git a/Python/_warnings.c b/Python/_warnings.c index 98d07f3788579e..0b192580e1073e 100644 --- a/Python/_warnings.c +++ b/Python/_warnings.c @@ -45,6 +45,32 @@ _Warnings_ClearState(WarningsState *st) Py_CLEAR(st->default_action); } +#ifndef Py_DEBUG +static PyObject * +create_filter(PyObject *category, _Py_Identifier *id, const char *modname) +{ + PyObject *modname_obj = NULL; + PyObject *action_str = _PyUnicode_FromId(id); + if (action_str == NULL) { + return NULL; + } + + /* Default to "no module name" for initial filter set */ + if (modname != NULL) { + modname_obj = PyUnicode_InternFromString(modname); + if (modname_obj == NULL) { + return NULL; + } + } else { + modname_obj = Py_None; + } + + /* This assumes the line number is zero for now. */ + return PyTuple_Pack(5, action_str, Py_None, + category, modname_obj, _PyLong_Zero); +} +#endif + static PyObject * init_filters(void) { @@ -1293,33 +1319,6 @@ static PyMethodDef warnings_functions[] = { }; -#ifndef Py_DEBUG -static PyObject * -create_filter(PyObject *category, _Py_Identifier *id, const char *modname) -{ - PyObject *modname_obj = NULL; - PyObject *action_str = _PyUnicode_FromId(id); - if (action_str == NULL) { - return NULL; - } - - /* Default to "no module name" for initial filter set */ - if (modname != NULL) { - modname_obj = PyUnicode_InternFromString(modname); - if (modname_obj == NULL) { - return NULL; - } - } else { - modname_obj = Py_None; - } - - /* This assumes the line number is zero for now. */ - return PyTuple_Pack(5, action_str, Py_None, - category, modname_obj, _PyLong_Zero); -} -#endif - - static struct PyModuleDef warningsmodule = { PyModuleDef_HEAD_INIT, MODULE_NAME, /* m_name */