Merge remote-tracking branch 'cpython/main' into pythongh-86682

Author: zooba

Doc/c-api/init.rst

@@ -1239,12 +1239,25 @@ All of the following functions must be called after :c:func:`Py_Initialize`.
    The global interpreter lock need not be held, but may be held if it is
    necessary to serialize calls to this function.
 
+   .. audit-event:: cpython.PyThreadState_New id c.PyThreadState_New
+
+      Raise an auditing event ``cpython.PyThreadState_New`` with Python's thread
+      id as the argument. The event will be raised from the thread creating the new
+      ``PyThreadState``, which may not be the new thread.
+
 
 .. c:function:: void PyThreadState_Clear(PyThreadState *tstate)
 
    Reset all information in a thread state object.  The global interpreter lock
    must be held.
 
+   .. audit-event:: cpython.PyThreadState_Clear id c.PyThreadState_Clear
+
+      Raise an auditing event ``cpython.PyThreadState_Clear`` with Python's
+      thread id as the argument. The event may be raised from a different thread
+      than the one being cleared. Exceptions raised from a hook will be treated
+      as unraisable and will not abort the operation.
+
    .. versionchanged:: 3.9
       This function now calls the :c:member:`PyThreadState.on_delete` callback.
       Previously, that happened in :c:func:`PyThreadState_Delete`.

Doc/library/_thread.rst

@@ -57,6 +57,8 @@ This module defines the following constants and functions:
    When the function raises a :exc:`SystemExit` exception, it is silently
    ignored.
 
+   .. audit-event:: _thread.start_new_thread function,args,kwargs start_new_thread
+
    .. versionchanged:: 3.8
       :func:`sys.unraisablehook` is now used to handle unhandled exceptions.
 

Include/cpython/modsupport.h

@@ -106,5 +106,3 @@ PyAPI_FUNC(PyObject * const *) _PyArg_UnpackKeywordsWithVararg(
                            (minpos), (maxpos), (minkw), (buf)))
 
 PyAPI_FUNC(PyObject *) _PyModule_CreateInitialized(PyModuleDef*, int apiver);
-
-PyAPI_DATA(const char *) _Py_PackageContext;

Include/internal/pycore_context.h

@@ -18,6 +18,10 @@ void _PyContext_Fini(PyInterpreterState *);
 
 /* other API */
 
+typedef struct {
+    PyObject_HEAD
+} _PyContextTokenMissing;
+
 #ifndef WITH_FREELISTS
 // without freelists
 #  define PyContext_MAXFREELIST 0

Include/internal/pycore_dict.h

@@ -9,6 +9,9 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+#include "pycore_dict_state.h"
+#include "pycore_runtime.h"         // _PyRuntime
+
 
 /* runtime lifecycle */
 
@@ -17,25 +20,6 @@ extern void _PyDict_Fini(PyInterpreterState *interp);
 
 /* other API */
 
-#ifndef WITH_FREELISTS
-// without freelists
-#  define PyDict_MAXFREELIST 0
-#endif
-
-#ifndef PyDict_MAXFREELIST
-#  define PyDict_MAXFREELIST 80
-#endif
-
-struct _Py_dict_state {
-#if PyDict_MAXFREELIST > 0
-    /* Dictionary reuse scheme to save calls to malloc and free */
-    PyDictObject *free_list[PyDict_MAXFREELIST];
-    int numfree;
-    PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST];
-    int keys_numfree;
-#endif
-};
-
 typedef struct {
     /* Cached hash code of me_key. */
     Py_hash_t me_hash;
@@ -152,13 +136,11 @@ struct _dictvalues {
      (PyDictUnicodeEntry*)(&((int8_t*)((dk)->dk_indices))[(size_t)1 << (dk)->dk_log2_index_bytes]))
 #define DK_IS_UNICODE(dk) ((dk)->dk_kind != DICT_KEYS_GENERAL)
 
-extern uint64_t _pydict_global_version;
-
-#define DICT_MAX_WATCHERS 8
 #define DICT_VERSION_INCREMENT (1 << DICT_MAX_WATCHERS)
 #define DICT_VERSION_MASK (DICT_VERSION_INCREMENT - 1)
 
-#define DICT_NEXT_VERSION() (_pydict_global_version += DICT_VERSION_INCREMENT)
+#define DICT_NEXT_VERSION() \
+    (_PyRuntime.dict_state.global_version += DICT_VERSION_INCREMENT)
 
 void
 _PyDict_SendEvent(int watcher_bits,

Include/internal/pycore_dict_state.h

@@ -0,0 +1,47 @@
+#ifndef Py_INTERNAL_DICT_STATE_H
+#define Py_INTERNAL_DICT_STATE_H
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef Py_BUILD_CORE
+#  error "this header requires Py_BUILD_CORE define"
+#endif
+
+
+struct _Py_dict_runtime_state {
+    /*Global counter used to set ma_version_tag field of dictionary.
+     * It is incremented each time that a dictionary is created and each
+     * time that a dictionary is modified. */
+    uint64_t global_version;
+    uint32_t next_keys_version;
+};
+
+
+#ifndef WITH_FREELISTS
+// without freelists
+#  define PyDict_MAXFREELIST 0
+#endif
+
+#ifndef PyDict_MAXFREELIST
+#  define PyDict_MAXFREELIST 80
+#endif
+
+#define DICT_MAX_WATCHERS 8
+
+struct _Py_dict_state {
+#if PyDict_MAXFREELIST > 0
+    /* Dictionary reuse scheme to save calls to malloc and free */
+    PyDictObject *free_list[PyDict_MAXFREELIST];
+    PyDictKeysObject *keys_free_list[PyDict_MAXFREELIST];
+    int numfree;
+    int keys_numfree;
+#endif
+    PyDict_WatchCallback watchers[DICT_MAX_WATCHERS];
+};
+
+
+#ifdef __cplusplus
+}
+#endif
+#endif   /* !Py_INTERNAL_DICT_STATE_H */

Include/internal/pycore_fileutils.h

@@ -10,6 +10,11 @@ extern "C" {
 
 #include <locale.h>   /* struct lconv */
 
+
+struct _fileutils_state {
+    int force_ascii;
+};
+
 typedef enum {
     _Py_ERROR_UNKNOWN=0,
     _Py_ERROR_STRICT,

Include/internal/pycore_floatobject.h

@@ -19,6 +19,18 @@ extern void _PyFloat_FiniType(PyInterpreterState *);
 
 /* other API */
 
+enum _py_float_format_type {
+    _py_float_format_unknown,
+    _py_float_format_ieee_big_endian,
+    _py_float_format_ieee_little_endian,
+};
+
+struct _Py_float_runtime_state {
+    enum _py_float_format_type float_format;
+    enum _py_float_format_type double_format;
+};
+
+
 #ifndef WITH_FREELISTS
 // without freelists
 #  define PyFloat_MAXFREELIST 0

Include/internal/pycore_function.h

@@ -8,6 +8,10 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+struct _py_func_runtime_state {
+    uint32_t next_version;
+};
+
 extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr);
 
 extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func);

Include/internal/pycore_global_objects.h

@@ -10,6 +10,8 @@ extern "C" {
 
 #include "pycore_gc.h"              // PyGC_Head
 #include "pycore_global_strings.h"  // struct _Py_global_strings
+#include "pycore_hamt.h"            // PyHamtNode_Bitmap
+#include "pycore_context.h"         // _PyContextTokenMissing
 #include "pycore_typeobject.h"      // pytype_slotdef
 
 
@@ -52,6 +54,10 @@ struct _Py_global_objects {
 
         _PyGC_Head_UNUSED _tuple_empty_gc_not_used;
         PyTupleObject tuple_empty;
+
+        _PyGC_Head_UNUSED _hamt_bitmap_node_empty_gc_not_used;
+        PyHamtNode_Bitmap hamt_bitmap_node_empty;
+        _PyContextTokenMissing context_token_missing;
     } singletons;
 
     PyObject *interned;
@@ -76,6 +82,9 @@ struct _Py_interp_cached_objects {
 struct _Py_interp_static_objects {
     struct {
         int _not_used;
+        // hamt_empty is here instead of global because of its weakreflist.
+        _PyGC_Head_UNUSED _hamt_empty_gc_not_used;
+        PyHamtObject hamt_empty;
     } singletons;
 };
 

Include/internal/pycore_global_objects_fini_generated.h

@@ -28,10 +28,6 @@ extern PyTypeObject _PyHamtKeys_Type;
 extern PyTypeObject _PyHamtValues_Type;
 extern PyTypeObject _PyHamtItems_Type;
 
-/* runtime lifecycle */
-
-void _PyHamt_Fini(PyInterpreterState *);
-
 
 /* other API */
 
@@ -53,6 +49,13 @@ typedef struct {
 } PyHamtObject;
 
 
+typedef struct {
+    PyObject_VAR_HEAD
+    uint32_t b_bitmap;
+    PyObject *b_array[1];
+} PyHamtNode_Bitmap;
+
+
 /* A struct to hold the state of depth-first traverse of the tree.
 
    HAMT is an immutable collection.  Iterators will hold a strong reference

Include/internal/pycore_hamt.h

@@ -32,6 +32,8 @@ struct _import_runtime_state {
         _PyTime_t accumulated;
         int header;
     } find_and_load;
+    /* Package context -- the full module name for package imports */
+    const char * pkgcontext;
 };
 
 

Include/internal/pycore_import.h

@@ -14,7 +14,7 @@ extern "C" {
 #include "pycore_ast_state.h"     // struct ast_state
 #include "pycore_code.h"          // struct callable_cache
 #include "pycore_context.h"       // struct _Py_context_state
-#include "pycore_dict.h"          // struct _Py_dict_state
+#include "pycore_dict_state.h"    // struct _Py_dict_state
 #include "pycore_exceptions.h"    // struct _Py_exc_state
 #include "pycore_floatobject.h"   // struct _Py_float_state
 #include "pycore_genobject.h"     // struct _Py_async_gen_state
@@ -28,6 +28,7 @@ extern "C" {
 
 
 struct _pending_calls {
+    int busy;
     PyThread_type_lock lock;
     /* Request for running pending calls. */
     _Py_atomic_int calls_to_do;
@@ -170,8 +171,6 @@ struct _is {
     // Initialized to _PyEval_EvalFrameDefault().
     _PyFrameEvalFunction eval_frame;
 
-    PyDict_WatchCallback dict_watchers[DICT_MAX_WATCHERS];
-
     Py_ssize_t co_extra_user_count;
     freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];
 

Include/internal/pycore_interp.h

@@ -5,6 +5,36 @@
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
-uint64_t _Py_KeyedHash(uint64_t, const char *, Py_ssize_t);
 
+struct pyhash_runtime_state {
+    struct {
+#ifndef MS_WINDOWS
+        int fd;
+        dev_t st_dev;
+        ino_t st_ino;
+#else
+    // This is a placeholder so the struct isn't empty on Windows.
+    int _not_used;
+#endif
+    } urandom_cache;
+};
+
+#ifndef MS_WINDOWS
+# define _py_urandom_cache_INIT \
+    { \
+        .fd = -1, \
+    }
+#else
+# define _py_urandom_cache_INIT {0}
 #endif
+
+#define pyhash_state_INIT \
+    { \
+        .urandom_cache = _py_urandom_cache_INIT, \
+    }
+
+
+uint64_t _Py_KeyedHash(uint64_t, const char *, Py_ssize_t);
+
+
+#endif  // Py_INTERNAL_HASH_H

Include/internal/pycore_pyhash.h

@@ -14,10 +14,6 @@ extern "C" {
 struct _PyArgv;
 struct pyruntimestate;
 
-/* True if the main interpreter thread exited due to an unhandled
- * KeyboardInterrupt exception, suggesting the user pressed ^C. */
-PyAPI_DATA(int) _Py_UnhandledKeyboardInterrupt;
-
 extern int _Py_SetFileSystemEncoding(
     const char *encoding,
     const char *errors);

Include/internal/pycore_pylifecycle.h

@@ -9,12 +9,16 @@ extern "C" {
 #endif
 
 #include "pycore_atomic.h"          /* _Py_atomic_address */
+#include "pycore_dict_state.h"      // struct _Py_dict_runtime_state
 #include "pycore_dtoa.h"            // struct _dtoa_runtime_state
+#include "pycore_floatobject.h"     // struct _Py_float_runtime_state
+#include "pycore_function.h"        // struct _func_runtime_state
 #include "pycore_gil.h"             // struct _gil_runtime_state
 #include "pycore_global_objects.h"  // struct _Py_global_objects
 #include "pycore_import.h"          // struct _import_runtime_state
 #include "pycore_interp.h"          // PyInterpreterState
 #include "pycore_pymem.h"           // struct _pymem_allocators
+#include "pycore_pyhash.h"          // struct pyhash_runtime_state
 #include "pycore_obmalloc.h"        // struct obmalloc_state
 #include "pycore_unicodeobject.h"   // struct _Py_unicode_runtime_ids
 
@@ -92,6 +96,12 @@ typedef struct pyruntimestate {
 
     struct _pymem_allocators allocators;
     struct _obmalloc_state obmalloc;
+    struct pyhash_runtime_state pyhash_state;
+    struct {
+        /* True if the main interpreter thread exited due to an unhandled
+         * KeyboardInterrupt exception, suggesting the user pressed ^C. */
+        int unhandled_keyboard_interrupt;
+    } signals;
 
     struct pyinterpreters {
         PyThread_type_lock mutex;
@@ -131,6 +141,7 @@ typedef struct pyruntimestate {
         struct _PyTraceMalloc_Config config;
     } tracemalloc;
     struct _dtoa_runtime_state dtoa;
+    struct _fileutils_state fileutils;
 
     PyPreConfig preconfig;
 
@@ -140,7 +151,10 @@ typedef struct pyruntimestate {
     void *open_code_userdata;
     _Py_AuditHookEntry *audit_hook_head;
 
-    struct _Py_unicode_runtime_ids unicode_ids;
+    struct _Py_float_runtime_state float_state;
+    struct _Py_unicode_runtime_state unicode_state;
+    struct _Py_dict_runtime_state dict_state;
+    struct _py_func_runtime_state func_state;
 
     struct {
         /* Used to set PyTypeObject.tp_version_tag */