diff --git a/Include/coreconfig.h b/Include/coreconfig.h index 0ed3222a15e142..ce44e3dd90e0cc 100644 --- a/Include/coreconfig.h +++ b/Include/coreconfig.h @@ -1,11 +1,12 @@ #ifndef Py_PYCORECONFIG_H #define Py_PYCORECONFIG_H +#ifndef Py_LIMITED_API + #ifdef __cplusplus extern "C" { #endif -#ifndef Py_LIMITED_API typedef struct { const char *prefix; const char *msg; @@ -32,10 +33,31 @@ typedef struct { #define _Py_INIT_FAILED(err) \ (err.msg != NULL) -#endif /* !defined(Py_LIMITED_API) */ + +typedef struct { + /* Enable UTF-8 mode? + Set by -X utf8 command line option and PYTHONUTF8 environment variable. + If set to -1 (default), inherit Py_UTF8Mode value. */ + int utf8_mode; + + PyMemAllocatorEx raw_alloc; +} _PyConfigCtx; + +PyAPI_FUNC(void) _PyConfigCtx_Init(_PyConfigCtx *ctx); + + +/* Functions implemented in obmalloc.c */ +PyAPI_FUNC(void *) _PyMem_RawMallocCtx(const _PyConfigCtx *ctx, size_t size); +PyAPI_FUNC(void *) _PyMem_RawCallocCtx(const _PyConfigCtx *ctx, size_t nelem, size_t elsize); +PyAPI_FUNC(void *) _PyMem_RawReallocCtx(const _PyConfigCtx *ctx, void *ptr, size_t new_size); +PyAPI_FUNC(void) _PyMem_RawFreeCtx(const _PyConfigCtx *ctx, void *ptr); +PyAPI_FUNC(char *) _PyMem_RawStrdupCtx(const _PyConfigCtx *ctx, const char *str); +PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const _PyConfigCtx *ctx, const wchar_t *str); typedef struct { + _PyConfigCtx ctx; + /* Install signal handlers? Yes by default. */ int install_signal_handlers; @@ -102,11 +124,6 @@ typedef struct { char *filesystem_encoding; char *filesystem_errors; - /* Enable UTF-8 mode? - Set by -X utf8 command line option and PYTHONUTF8 environment variable. - If set to -1 (default), inherit Py_UTF8Mode value. */ - int utf8_mode; - wchar_t *pycache_prefix; /* PYTHONPYCACHEPREFIX, -X pycache_prefix=PATH */ wchar_t *program_name; /* Program name, see also Py_GetProgramName() */ @@ -309,7 +326,8 @@ typedef struct { # define _PyCoreConfig_WINDOWS_INIT #endif -#define _PyCoreConfig_INIT \ +/* _PyCoreConfig_STATIC_INIT must not be used: use _PyCoreConfig_Init() */ +#define _PyCoreConfig_STATIC_INIT \ (_PyCoreConfig){ \ .install_signal_handlers = 1, \ .use_environment = -1, \ @@ -317,7 +335,6 @@ typedef struct { .faulthandler = -1, \ .tracemalloc = -1, \ .coerce_c_locale = -1, \ - .utf8_mode = -1, \ .argc = -1, \ .nmodule_search_path = -1, \ .isolated = -1, \ @@ -336,10 +353,11 @@ typedef struct { ._install_importlib = 1, \ ._check_hash_pycs_mode = "default", \ ._frozen = -1} -/* Note: _PyCoreConfig_INIT sets other fields to 0/NULL */ +/* Note: _PyCoreConfig_STATIC_INIT sets other fields to 0/NULL */ + +PyAPI_FUNC(void) _PyCoreConfig_Init(_PyCoreConfig *config); -#ifndef Py_LIMITED_API PyAPI_FUNC(_PyInitError) _PyCoreConfig_Read(_PyCoreConfig *config); PyAPI_FUNC(void) _PyCoreConfig_Clear(_PyCoreConfig *); PyAPI_FUNC(int) _PyCoreConfig_Copy( @@ -362,9 +380,9 @@ PyAPI_FUNC(int) _PyCoreConfig_GetEnvDup( /* Used by _testcapi.get_global_config() and _testcapi.get_core_config() */ PyAPI_FUNC(PyObject *) _Py_GetGlobalVariablesAsDict(void); PyAPI_FUNC(PyObject *) _PyCoreConfig_AsDict(const _PyCoreConfig *config); -#endif #ifdef __cplusplus } #endif +#endif /* !Py_LIMITED_API */ #endif /* !Py_PYCORECONFIG_H */ diff --git a/Include/fileutils.h b/Include/fileutils.h index fdd60fffcd5514..4a65ad37497b23 100644 --- a/Include/fileutils.h +++ b/Include/fileutils.h @@ -34,7 +34,9 @@ typedef enum { PyAPI_FUNC(_Py_error_handler) _Py_GetErrorHandler(const char *errors); + PyAPI_FUNC(int) _Py_DecodeLocaleEx( + const _PyConfigCtx *ctx, const char *arg, wchar_t **wstr, size_t *wlen, @@ -43,6 +45,7 @@ PyAPI_FUNC(int) _Py_DecodeLocaleEx( _Py_error_handler errors); PyAPI_FUNC(int) _Py_EncodeLocaleEx( + const _PyConfigCtx *ctx, const wchar_t *text, char **str, size_t *error_pos, @@ -51,6 +54,7 @@ PyAPI_FUNC(int) _Py_EncodeLocaleEx( _Py_error_handler errors); #endif + #ifndef Py_LIMITED_API PyAPI_FUNC(PyObject *) _Py_device_encoding(int); diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index d577e099d1f551..ec67d7dc148e1f 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -8,7 +8,13 @@ extern "C" { # error "Py_BUILD_CORE must be defined to include this header" #endif +PyAPI_FUNC(wchar_t*) _Py_DecodeLocaleCtx( + const _PyConfigCtx *ctx, + const char* arg, + size_t *wlen); + PyAPI_FUNC(int) _Py_DecodeUTF8Ex( + const _PyConfigCtx *ctx, const char *arg, Py_ssize_t arglen, wchar_t **wstr, @@ -17,6 +23,7 @@ PyAPI_FUNC(int) _Py_DecodeUTF8Ex( _Py_error_handler errors); PyAPI_FUNC(int) _Py_EncodeUTF8Ex( + const _PyConfigCtx *ctx, const wchar_t *text, char **str, size_t *error_pos, diff --git a/Include/internal/pycore_pathconfig.h b/Include/internal/pycore_pathconfig.h index 267e690976da87..b690597ffbd165 100644 --- a/Include/internal/pycore_pathconfig.h +++ b/Include/internal/pycore_pathconfig.h @@ -9,12 +9,15 @@ extern "C" { #endif PyAPI_FUNC(void) _Py_wstrlist_clear( + const _PyConfigCtx *ctx, int len, wchar_t **list); PyAPI_FUNC(wchar_t**) _Py_wstrlist_copy( + const _PyConfigCtx *ctx, int len, wchar_t **list); PyAPI_FUNC(_PyInitError) _Py_wstrlist_append( + const _PyConfigCtx *ctx, int *len, wchar_t ***list, const wchar_t *str); @@ -23,6 +26,8 @@ PyAPI_FUNC(PyObject*) _Py_wstrlist_as_pylist( wchar_t **list); typedef struct _PyPathConfig { + _PyConfigCtx ctx; + /* Full path to the Python program */ wchar_t *program_full_path; wchar_t *prefix; @@ -44,17 +49,21 @@ typedef struct _PyPathConfig { int site_import; } _PyPathConfig; -#define _PyPathConfig_INIT \ - {.module_search_path = NULL, \ - .isolated = -1, \ - .site_import = -1} -/* Note: _PyPathConfig_INIT sets other fields to 0/NULL */ +#define _PyPathConfig_STATIC_INIT \ + (_PyPathConfig){ \ + .module_search_path = NULL, \ + .isolated = -1, \ + .site_import = -1} +/* Note: _PyPathConfig_STATIC_INIT sets other fields to 0/NULL */ -PyAPI_DATA(_PyPathConfig) _Py_path_config; +PyAPI_FUNC(void) _PyPathConfig_StaticInit( + _PyPathConfig *config); PyAPI_FUNC(void) _PyPathConfig_ClearGlobal(void); PyAPI_FUNC(_PyInitError) _PyPathConfig_SetGlobal( - const struct _PyPathConfig *config); + const _PyPathConfig *config); + +PyAPI_FUNC(_PyPathConfig*) _PyPathConfig_GetGlobal(void); PyAPI_FUNC(_PyInitError) _PyPathConfig_Calculate_impl( _PyPathConfig *config, diff --git a/Include/pymem.h b/Include/pymem.h index 23457adb5a45e9..09bfbf7280dbe3 100644 --- a/Include/pymem.h +++ b/Include/pymem.h @@ -79,9 +79,6 @@ PyAPI_FUNC(char *) _PyMem_RawStrdup(const char *str); /* strdup() using PyMem_Malloc() */ PyAPI_FUNC(char *) _PyMem_Strdup(const char *str); - -/* wcsdup() using PyMem_RawMalloc() */ -PyAPI_FUNC(wchar_t*) _PyMem_RawWcsdup(const wchar_t *str); #endif /* Macros. */ @@ -198,6 +195,7 @@ PyAPI_FUNC(void) PyMem_SetAllocator(PyMemAllocatorDomain domain, PyAPI_FUNC(void) PyMem_SetupDebugHooks(void); #endif /* Py_LIMITED_API */ + /* bpo-35053: expose _Py_tracemalloc_config for performance: _Py_NewReference() needs an efficient check to test if tracemalloc is tracing. diff --git a/Modules/_testcapimodule.c b/Modules/_testcapimodule.c index 56a08a418b361f..75ce05c2138edf 100644 --- a/Modules/_testcapimodule.c +++ b/Modules/_testcapimodule.c @@ -4622,7 +4622,8 @@ encode_locale_ex(PyObject *self, PyObject *args) char *str = NULL; size_t error_pos; const char *reason = NULL; - int ret = _Py_EncodeLocaleEx(wstr, + PyInterpreterState *interp = _PyInterpreterState_Get(); + int ret = _Py_EncodeLocaleEx(&interp->core_config.ctx, wstr, &str, &error_pos, &reason, current_locale, error_handler); PyMem_Free(wstr); @@ -4666,7 +4667,8 @@ decode_locale_ex(PyObject *self, PyObject *args) wchar_t *wstr = NULL; size_t wlen = 0; const char *reason = NULL; - int ret = _Py_DecodeLocaleEx(str, + PyInterpreterState *interp = _PyInterpreterState_Get(); + int ret = _Py_DecodeLocaleEx(&interp->core_config.ctx, str, &wstr, &wlen, &reason, current_locale, error_handler); diff --git a/Modules/getpath.c b/Modules/getpath.c index 18df795c93fc96..05d28f5ec17cc5 100644 --- a/Modules/getpath.c +++ b/Modules/getpath.c @@ -118,6 +118,8 @@ extern "C" { : _Py_INIT_NO_MEMORY() typedef struct { + _PyConfigCtx ctx; + wchar_t *path_env; /* PATH environment variable */ wchar_t *pythonpath; /* PYTHONPATH define */ @@ -337,7 +339,7 @@ add_exe_suffix(wchar_t *progpath) bytes long. */ static int -search_for_prefix(const _PyCoreConfig *core_config, +search_for_prefix(const _PyCoreConfig *core_config, _PyPathConfig *config, PyCalculatePath *calculate, wchar_t *prefix) { size_t n; @@ -404,10 +406,10 @@ search_for_prefix(const _PyCoreConfig *core_config, static void -calculate_prefix(const _PyCoreConfig *core_config, +calculate_prefix(const _PyCoreConfig *core_config, _PyPathConfig *config, PyCalculatePath *calculate, wchar_t *prefix) { - calculate->prefix_found = search_for_prefix(core_config, calculate, prefix); + calculate->prefix_found = search_for_prefix(core_config, config, calculate, prefix); if (!calculate->prefix_found) { if (!core_config->_frozen) { fprintf(stderr, @@ -599,7 +601,7 @@ calculate_program_full_path(const _PyCoreConfig *core_config, execpath[0] == SEP) { size_t len; - wchar_t *path = Py_DecodeLocale(execpath, &len); + wchar_t *path = Py_DecodeLocaleCtx(execpath, &len); if (path == NULL) { return DECODE_LOCALE_ERR("executable path", len); } @@ -653,7 +655,7 @@ calculate_program_full_path(const _PyCoreConfig *core_config, } #endif - config->program_full_path = _PyMem_RawWcsdup(program_full_path); + config->program_full_path = _PyMem_RawWcsdup(&config->ctx, program_full_path); if (config->program_full_path == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -662,7 +664,10 @@ calculate_program_full_path(const _PyCoreConfig *core_config, static _PyInitError -calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_path) +calculate_argv0_path(PyCalculatePath *calculate, + const _PyCoreConfig *core_config, + _PyPathConfig *config, + const wchar_t *program_full_path) { wcsncpy(calculate->argv0_path, program_full_path, MAXPATHLEN); calculate->argv0_path[MAXPATHLEN] = '\0'; @@ -688,7 +693,7 @@ calculate_argv0_path(PyCalculatePath *calculate, const wchar_t *program_full_pat ** build-directory-specific logic to find Lib and such. */ size_t len; - wchar_t* wbuf = Py_DecodeLocale(modPath, &len); + wchar_t* wbuf = Py_DecodeLocaleCtx(modPath, &len); if (wbuf == NULL) { return DECODE_LOCALE_ERR("framework location", len); } @@ -834,7 +839,7 @@ calculate_module_search_path(const _PyCoreConfig *core_config, bufsz += wcslen(exec_prefix) + 1; /* Allocate the buffer */ - wchar_t *buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); + wchar_t *buf = _PyMem_RawMallocCtx(&config->ctx, bufsz * sizeof(wchar_t)); if (buf == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -891,30 +896,32 @@ calculate_module_search_path(const _PyCoreConfig *core_config, static _PyInitError calculate_init(PyCalculatePath *calculate, + _PyPathConfig *config, const _PyCoreConfig *core_config) { + _PyConfigCtx *ctx = &calculate->ctx; size_t len; const char *path = getenv("PATH"); if (path) { - calculate->path_env = Py_DecodeLocale(path, &len); + calculate->path_env = _Py_DecodeLocaleCtx(ctx, path, &len); if (!calculate->path_env) { return DECODE_LOCALE_ERR("PATH environment variable", len); } } - calculate->pythonpath = Py_DecodeLocale(PYTHONPATH, &len); + calculate->pythonpath = _Py_DecodeLocaleCtx(ctx, PYTHONPATH, &len); if (!calculate->pythonpath) { return DECODE_LOCALE_ERR("PYTHONPATH define", len); } - calculate->prefix = Py_DecodeLocale(PREFIX, &len); + calculate->prefix = _Py_DecodeLocaleCtx(ctx, PREFIX, &len); if (!calculate->prefix) { return DECODE_LOCALE_ERR("PREFIX define", len); } - calculate->exec_prefix = Py_DecodeLocale(EXEC_PREFIX, &len); + calculate->exec_prefix = _Py_DecodeLocaleCtx(ctx, EXEC_PREFIX, &len); if (!calculate->prefix) { return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); } - calculate->lib_python = Py_DecodeLocale("lib/python" VERSION, &len); + calculate->lib_python = _Py_DecodeLocaleCtx(ctx, "lib/python" VERSION, &len); if (!calculate->lib_python) { return DECODE_LOCALE_ERR("EXEC_PREFIX define", len); } @@ -925,11 +932,11 @@ calculate_init(PyCalculatePath *calculate, static void calculate_free(PyCalculatePath *calculate) { - PyMem_RawFree(calculate->pythonpath); - PyMem_RawFree(calculate->prefix); - PyMem_RawFree(calculate->exec_prefix); - PyMem_RawFree(calculate->lib_python); - PyMem_RawFree(calculate->path_env); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->pythonpath); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->prefix); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->exec_prefix); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->lib_python); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->path_env); } @@ -944,7 +951,7 @@ calculate_path_impl(const _PyCoreConfig *core_config, return err; } - err = calculate_argv0_path(calculate, config->program_full_path); + err = calculate_argv0_path(calculate, core_config, config, config->program_full_path); if (_Py_INIT_FAILED(err)) { return err; } @@ -953,7 +960,7 @@ calculate_path_impl(const _PyCoreConfig *core_config, wchar_t prefix[MAXPATHLEN+1]; memset(prefix, 0, sizeof(prefix)); - calculate_prefix(core_config, calculate, prefix); + calculate_prefix(core_config, config, calculate, prefix); calculate_zip_path(calculate, prefix); @@ -976,14 +983,14 @@ calculate_path_impl(const _PyCoreConfig *core_config, calculate_reduce_prefix(calculate, prefix); - config->prefix = _PyMem_RawWcsdup(prefix); + config->prefix = _PyMem_RawWcsdup(&config->ctx, prefix); if (config->prefix == NULL) { return _Py_INIT_NO_MEMORY(); } calculate_reduce_exec_prefix(calculate, exec_prefix); - config->exec_prefix = _PyMem_RawWcsdup(exec_prefix); + config->exec_prefix = _PyMem_RawWcsdup(&config->ctx, exec_prefix); if (config->exec_prefix == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -997,8 +1004,9 @@ _PyPathConfig_Calculate_impl(_PyPathConfig *config, const _PyCoreConfig *core_co { PyCalculatePath calculate; memset(&calculate, 0, sizeof(calculate)); + _PyConfigCtx_Init(&calculate.ctx); - _PyInitError err = calculate_init(&calculate, core_config); + _PyInitError err = calculate_init(&calculate, config, core_config); if (_Py_INIT_FAILED(err)) { goto done; } diff --git a/Modules/main.c b/Modules/main.c index 8e66ddded419a3..fb2833032b33fa 100644 --- a/Modules/main.c +++ b/Modules/main.c @@ -2,6 +2,7 @@ #include "Python.h" #include "osdefs.h" +#include "pycore_fileutils.h" #include "pycore_getopt.h" #include "pycore_pathconfig.h" #include "pycore_pylifecycle.h" @@ -304,6 +305,7 @@ pymain_run_command(wchar_t *command, PyCompilerFlags *cf) /* Main program */ typedef struct { + _PyConfigCtx ctx; wchar_t **argv; int nwarnoption; /* Number of -W command line options */ wchar_t **warnoptions; /* Command line -W options */ @@ -315,6 +317,8 @@ typedef struct { /* Structure used by Py_Main() to pass data to subfunctions */ typedef struct { + _PyConfigCtx ctx; + /* Input arguments */ int argc; int use_bytes_argv; @@ -346,9 +350,9 @@ typedef struct { static wchar_t* -pymain_wstrdup(_PyMain *pymain, const wchar_t *str) +pymain_wstrdup(_PyMain *pymain, _PyConfigCtx *ctx, const wchar_t *str) { - wchar_t *str2 = _PyMem_RawWcsdup(str); + wchar_t *str2 = _PyMem_RawWcsdup(ctx, str); if (str2 == NULL) { pymain->err = _Py_INIT_NO_MEMORY(); return NULL; @@ -366,7 +370,7 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, if (pymain->use_bytes_argv) { /* +1 for a the NULL terminator */ size_t size = sizeof(wchar_t*) * (pymain->argc + 1); - wchar_t** argv = (wchar_t **)PyMem_RawMalloc(size); + wchar_t** argv = (wchar_t **)_PyMem_RawMallocCtx(&cmdline->ctx, size); if (argv == NULL) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; @@ -374,9 +378,9 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, for (int i = 0; i < pymain->argc; i++) { size_t len; - wchar_t *arg = Py_DecodeLocale(pymain->bytes_argv[i], &len); + wchar_t *arg = _Py_DecodeLocaleCtx(&cmdline->ctx, pymain->bytes_argv[i], &len); if (arg == NULL) { - _Py_wstrlist_clear(i, argv); + _Py_wstrlist_clear(&cmdline->ctx, i, argv); pymain->err = DECODE_LOCALE_ERR("command line arguments", (Py_ssize_t)len); return -1; @@ -398,7 +402,7 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, else { program = L""; } - config->program = pymain_wstrdup(pymain, program); + config->program = pymain_wstrdup(pymain, &config->ctx, program); if (config->program == NULL) { return -1; } @@ -410,23 +414,21 @@ pymain_init_cmdline_argv(_PyMain *pymain, _PyCoreConfig *config, static void pymain_clear_cmdline(_PyMain *pymain, _PyCmdline *cmdline) { - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _Py_wstrlist_clear(cmdline->nwarnoption, cmdline->warnoptions); + _Py_wstrlist_clear(&cmdline->ctx, cmdline->nwarnoption, cmdline->warnoptions); cmdline->nwarnoption = 0; cmdline->warnoptions = NULL; - _Py_wstrlist_clear(cmdline->nenv_warnoption, cmdline->env_warnoptions); + _Py_wstrlist_clear(&cmdline->ctx, cmdline->nenv_warnoption, cmdline->env_warnoptions); cmdline->nenv_warnoption = 0; cmdline->env_warnoptions = NULL; if (pymain->use_bytes_argv && cmdline->argv != NULL) { - _Py_wstrlist_clear(pymain->argc, cmdline->argv); + _Py_wstrlist_clear(&cmdline->ctx, pymain->argc, cmdline->argv); } cmdline->argv = NULL; - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + memset(cmdline, 0, sizeof(*cmdline)); + _PyConfigCtx_Init(&cmdline->ctx); } @@ -435,7 +437,7 @@ pymain_clear_pymain(_PyMain *pymain) { #define CLEAR(ATTR) \ do { \ - PyMem_RawFree(ATTR); \ + _PyMem_RawFreeCtx(&pymain->ctx, ATTR); \ ATTR = NULL; \ } while (0) @@ -445,20 +447,6 @@ pymain_clear_pymain(_PyMain *pymain) #undef CLEAR } -static void -pymain_clear_config(_PyCoreConfig *config) -{ - /* Clear core config with the memory allocator - used by pymain_read_conf() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _PyCoreConfig_Clear(config); - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -} - - static void pymain_free(_PyMain *pymain) { @@ -471,17 +459,12 @@ pymain_free(_PyMain *pymain) _PyPathConfig_ClearGlobal(); _Py_ClearStandardStreamEncoding(); - /* Force the allocator used by pymain_read_conf() */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - pymain_clear_pymain(pymain); - _Py_wstrlist_clear(orig_argc, orig_argv); + _Py_wstrlist_clear(&pymain->ctx, orig_argc, orig_argv); orig_argc = 0; orig_argv = NULL; - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); #ifdef __INSURE__ /* Insure++ is a memory analysis tool that aids in discovering @@ -526,21 +509,21 @@ pymain_sys_path_add_path0(PyInterpreterState *interp, PyObject *path0) _PyInitError -_Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str) +_Py_wstrlist_append(const _PyConfigCtx *ctx, int *len, wchar_t ***list, const wchar_t *str) { if (*len == INT_MAX) { /* len+1 would overflow */ return _Py_INIT_NO_MEMORY(); } - wchar_t *str2 = _PyMem_RawWcsdup(str); + wchar_t *str2 = _PyMem_RawWcsdup(ctx, str); if (str2 == NULL) { return _Py_INIT_NO_MEMORY(); } size_t size = (*len + 1) * sizeof(list[0]); - wchar_t **list2 = (wchar_t **)PyMem_RawRealloc(*list, size); + wchar_t **list2 = (wchar_t **)_PyMem_RawReallocCtx(ctx, *list, size); if (list2 == NULL) { - PyMem_RawFree(str2); + _PyMem_RawFreeCtx(ctx, str2); return _Py_INIT_NO_MEMORY(); } list2[*len] = str2; @@ -551,9 +534,10 @@ _Py_wstrlist_append(int *len, wchar_t ***list, const wchar_t *str) static int -pymain_wstrlist_append(_PyMain *pymain, int *len, wchar_t ***list, const wchar_t *str) +pymain_wstrlist_append(_PyMain *pymain, _PyCoreConfig *config, + int *len, wchar_t ***list, const wchar_t *str) { - _PyInitError err = _Py_wstrlist_append(len, list, str); + _PyInitError err = _Py_wstrlist_append(&config->ctx, len, list, str); if (_Py_INIT_FAILED(err)) { pymain->err = err; return -1; @@ -584,7 +568,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, that look like options are left for the command to interpret. */ size_t len = wcslen(_PyOS_optarg) + 1 + 1; - wchar_t *command = PyMem_RawMalloc(sizeof(wchar_t) * len); + wchar_t *command = _PyMem_RawMallocCtx(&pymain->ctx, sizeof(wchar_t) * len); if (command == NULL) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; @@ -600,7 +584,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, /* -m is the last option; following arguments that look like options are left for the module to interpret. */ - pymain->module = pymain_wstrdup(pymain, _PyOS_optarg); + pymain->module = pymain_wstrdup(pymain, &pymain->ctx, _PyOS_optarg); if (pymain->module == NULL) { return -1; } @@ -689,7 +673,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, break; case 'W': - if (pymain_wstrlist_append(pymain, + if (pymain_wstrlist_append(pymain, config, &cmdline->nwarnoption, &cmdline->warnoptions, _PyOS_optarg) < 0) { @@ -698,7 +682,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, break; case 'X': - if (pymain_wstrlist_append(pymain, + if (pymain_wstrlist_append(pymain, config, &config->nxoption, &config->xoptions, _PyOS_optarg) < 0) { @@ -726,7 +710,7 @@ pymain_parse_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, && _PyOS_optind < pymain->argc && wcscmp(cmdline->argv[_PyOS_optind], L"-") != 0) { - pymain->filename = pymain_wstrdup(pymain, cmdline->argv[_PyOS_optind]); + pymain->filename = pymain_wstrdup(pymain, &pymain->ctx, cmdline->argv[_PyOS_optind]); if (pymain->filename == NULL) { return -1; } @@ -797,7 +781,8 @@ static _PyInitError config_add_warnings_optlist(_PyCoreConfig *config, int len, wchar_t **options) { for (int i = 0; i < len; i++) { - _PyInitError err = _Py_wstrlist_append(&config->nwarnoption, + _PyInitError err = _Py_wstrlist_append(&config->ctx, + &config->nwarnoption, &config->warnoptions, options[i]); if (_Py_INIT_FAILED(err)) { @@ -831,7 +816,8 @@ config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline) */ if (config->dev_mode) { - err = _Py_wstrlist_append(&config->nwarnoption, + err = _Py_wstrlist_append(&config->ctx, + &config->nwarnoption, &config->warnoptions, L"default"); if (_Py_INIT_FAILED(err)) { @@ -865,7 +851,8 @@ config_init_warnoptions(_PyCoreConfig *config, _PyCmdline *cmdline) else { filter = L"default::BytesWarning"; } - err = _Py_wstrlist_append(&config->nwarnoption, + err = _Py_wstrlist_append(&config->ctx, + &config->nwarnoption, &config->warnoptions, filter); if (_Py_INIT_FAILED(err)) { @@ -900,7 +887,8 @@ cmdline_init_env_warnoptions(_PyMain *pymain, const _PyCoreConfig *config, warning != NULL; warning = WCSTOK(NULL, L",", &context)) { - _PyInitError err = _Py_wstrlist_append(&cmdline->nenv_warnoption, + _PyInitError err = _Py_wstrlist_append(&config->ctx, + &cmdline->nenv_warnoption, &cmdline->env_warnoptions, warning); if (_Py_INIT_FAILED(err)) { @@ -982,10 +970,10 @@ pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdlin /* Ensure at least one (empty) argument is seen */ static wchar_t *empty_argv[1] = {L""}; argc = 1; - argv = _Py_wstrlist_copy(1, empty_argv); + argv = _Py_wstrlist_copy(&config->ctx, 1, empty_argv); } else { - argv = _Py_wstrlist_copy(argc, &cmdline->argv[_PyOS_optind]); + argv = _Py_wstrlist_copy(&config->ctx, argc, &cmdline->argv[_PyOS_optind]); } if (argv == NULL) { @@ -1003,9 +991,9 @@ pymain_init_core_argv(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdlin arg0 = L"-m"; } if (arg0 != NULL) { - arg0 = _PyMem_RawWcsdup(arg0); + arg0 = _PyMem_RawWcsdup(&config->ctx, arg0); if (arg0 == NULL) { - _Py_wstrlist_clear(argc, argv); + _Py_wstrlist_clear(&config->ctx, argc, argv); pymain->err = _Py_INIT_NO_MEMORY(); return -1; } @@ -1293,11 +1281,13 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, #ifdef MS_WINDOWS int init_legacy_encoding = Py_LegacyWindowsFSEncodingFlag; #endif - _PyCoreConfig save_config = _PyCoreConfig_INIT; + _PyCoreConfig save_config; int res = -1; int locale_coerced = 0; int loops = 0; + _PyCoreConfig_Init(&save_config); + if (_PyCoreConfig_Copy(&save_config, config) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); goto done; @@ -1307,7 +1297,7 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, _Py_SetLocaleFromEnv(LC_CTYPE); while (1) { - int utf8_mode = config->utf8_mode; + int utf8_mode = config->ctx.utf8_mode; int encoding_changed = 0; /* Watchdog to prevent an infinite loop */ @@ -1320,7 +1310,9 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, /* bpo-34207: Py_DecodeLocale() and Py_EncodeLocale() depend on Py_UTF8Mode and Py_LegacyWindowsFSEncodingFlag. */ - Py_UTF8Mode = config->utf8_mode; + cmdline->ctx.utf8_mode = config->ctx.utf8_mode; + Py_UTF8Mode = config->ctx.utf8_mode; + /* FIXME: put legacy_windows_fs_encoding into ctx */ #ifdef MS_WINDOWS Py_LegacyWindowsFSEncodingFlag = config->legacy_windows_fs_encoding; #endif @@ -1352,13 +1344,13 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, } if (utf8_mode == -1) { - if (config->utf8_mode == 1) { + if (config->ctx.utf8_mode == 1) { /* UTF-8 Mode enabled */ encoding_changed = 1; } } else { - if (config->utf8_mode != utf8_mode) { + if (config->ctx.utf8_mode != utf8_mode) { encoding_changed = 1; } } @@ -1369,15 +1361,14 @@ pymain_read_conf(_PyMain *pymain, _PyCoreConfig *config, /* Reset the configuration before reading again the configuration, just keep UTF-8 Mode value. */ - int new_utf8_mode = config->utf8_mode; + int new_utf8_mode = config->ctx.utf8_mode; int new_coerce_c_locale = config->coerce_c_locale; if (_PyCoreConfig_Copy(config, &save_config) < 0) { pymain->err = _Py_INIT_NO_MEMORY(); goto done; } pymain_clear_cmdline(pymain, cmdline); - memset(cmdline, 0, sizeof(*cmdline)); - config->utf8_mode = new_utf8_mode; + config->ctx.utf8_mode = new_utf8_mode; config->coerce_c_locale = new_coerce_c_locale; /* The encoding changed: read again the configuration @@ -1686,11 +1677,6 @@ static int pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, _PyCmdline *cmdline) { - pymain->err = _PyRuntime_Initialize(); - if (_Py_INIT_FAILED(pymain->err)) { - return -1; - } - int res = pymain_read_conf(pymain, config, cmdline); if (res < 0) { return -1; @@ -1712,7 +1698,7 @@ pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, } /* For Py_GetArgcArgv(). Cleared by pymain_free(). */ - orig_argv = _Py_wstrlist_copy(pymain->argc, cmdline->argv); + orig_argv = _Py_wstrlist_copy(&pymain->ctx, pymain->argc, cmdline->argv); if (orig_argv == NULL) { pymain->err = _Py_INIT_NO_MEMORY(); return -1; @@ -1736,29 +1722,13 @@ pymain_cmdline_impl(_PyMain *pymain, _PyCoreConfig *config, static int pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) { - /* Force default allocator, since pymain_free() and pymain_clear_config() - must use the same allocator than this function. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); -#ifdef Py_DEBUG - PyMemAllocatorEx default_alloc; - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &default_alloc); -#endif - _PyCmdline cmdline; memset(&cmdline, 0, sizeof(cmdline)); + _PyConfigCtx_Init(&cmdline.ctx); int res = pymain_cmdline_impl(pymain, config, &cmdline); pymain_clear_cmdline(pymain, &cmdline); - -#ifdef Py_DEBUG - /* Make sure that PYMEM_DOMAIN_RAW has not been modified */ - PyMemAllocatorEx cur_alloc; - PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &cur_alloc); - assert(memcmp(&cur_alloc, &default_alloc, sizeof(cur_alloc)) == 0); -#endif - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return res; } @@ -1766,6 +1736,12 @@ pymain_cmdline(_PyMain *pymain, _PyCoreConfig *config) static int pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) { + pymain->err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(pymain->err)) { + return -1; + } + _PyConfigCtx_Init(&pymain->ctx); + /* 754 requires that FP exceptions run in "no stop" mode by default, * and until C vendors implement C99's ways to control FP exceptions, * Python requires non-stop mode. Alas, some platforms enable FP @@ -1775,17 +1751,19 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) fedisableexcept(FE_OVERFLOW); #endif - _PyCoreConfig local_config = _PyCoreConfig_INIT; + _PyCoreConfig local_config; _PyCoreConfig *config = &local_config; + _PyCoreConfig_Init(&local_config); + _PyCoreConfig_GetGlobalConfig(config); int cmd_res = pymain_cmdline(pymain, config); if (cmd_res < 0) { - _Py_FatalInitError(pymain->err); + return -1; } if (cmd_res == 1) { - pymain_clear_config(config); + _PyCoreConfig_Clear(config); return 1; } @@ -1796,15 +1774,15 @@ pymain_init(_PyMain *pymain, PyInterpreterState **interp_p) PyInterpreterState *interp; pymain->err = _Py_InitializeCore(&interp, config); if (_Py_INIT_FAILED(pymain->err)) { - _Py_FatalInitError(pymain->err); + return -1; } *interp_p = interp; - pymain_clear_config(config); + _PyCoreConfig_Clear(config); config = &interp->core_config; if (pymain_init_python_main(pymain, config, interp) < 0) { - _Py_FatalInitError(pymain->err); + return -1; } return 0; } @@ -1815,6 +1793,9 @@ pymain_main(_PyMain *pymain) { PyInterpreterState *interp; int res = pymain_init(pymain, &interp); + if (res < 0) { + _Py_FatalInitError(pymain->err); + } if (res != 1) { if (pymain_run_python(pymain, interp) < 0) { _Py_FatalInitError(pymain->err); diff --git a/Objects/obmalloc.c b/Objects/obmalloc.c index 1c2a32050f9381..871df7b655c34e 100644 --- a/Objects/obmalloc.c +++ b/Objects/obmalloc.c @@ -204,11 +204,11 @@ static struct { }; #define PYDBGRAW_ALLOC \ - {&_PyMem_Debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree} + (PyMemAllocatorEx){&_PyMem_Debug.raw, _PyMem_DebugRawMalloc, _PyMem_DebugRawCalloc, _PyMem_DebugRawRealloc, _PyMem_DebugRawFree} #define PYDBGMEM_ALLOC \ - {&_PyMem_Debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} + (PyMemAllocatorEx){&_PyMem_Debug.mem, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} #define PYDBGOBJ_ALLOC \ - {&_PyMem_Debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} + (PyMemAllocatorEx){&_PyMem_Debug.obj, _PyMem_DebugMalloc, _PyMem_DebugCalloc, _PyMem_DebugRealloc, _PyMem_DebugFree} #ifdef Py_DEBUG static PyMemAllocatorEx _PyMem_Raw = PYDBGRAW_ALLOC; @@ -376,16 +376,6 @@ _PyMem_GetAllocatorsName(void) } -#undef MALLOC_ALLOC -#undef PYMALLOC_ALLOC -#undef PYRAW_ALLOC -#undef PYMEM_ALLOC -#undef PYOBJ_ALLOC -#undef PYDBGRAW_ALLOC -#undef PYDBGMEM_ALLOC -#undef PYDBGOBJ_ALLOC - - static PyObjectArenaAllocator _PyObject_Arena = {NULL, #ifdef MS_WINDOWS _PyObject_ArenaVirtualAlloc, _PyObject_ArenaVirtualFree @@ -427,11 +417,7 @@ _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain) } PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &_PyMem_Debug.raw.alloc); - alloc.ctx = &_PyMem_Debug.raw; - alloc.malloc = _PyMem_DebugRawMalloc; - alloc.calloc = _PyMem_DebugRawCalloc; - alloc.realloc = _PyMem_DebugRawRealloc; - alloc.free = _PyMem_DebugRawFree; + alloc = PYDBGRAW_ALLOC ; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &alloc); } else if (domain == PYMEM_DOMAIN_MEM) { @@ -440,11 +426,7 @@ _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain) } PyMem_GetAllocator(PYMEM_DOMAIN_MEM, &_PyMem_Debug.mem.alloc); - alloc.ctx = &_PyMem_Debug.mem; - alloc.malloc = _PyMem_DebugMalloc; - alloc.calloc = _PyMem_DebugCalloc; - alloc.realloc = _PyMem_DebugRealloc; - alloc.free = _PyMem_DebugFree; + alloc = PYDBGMEM_ALLOC; PyMem_SetAllocator(PYMEM_DOMAIN_MEM, &alloc); } else if (domain == PYMEM_DOMAIN_OBJ) { @@ -453,11 +435,7 @@ _PyMem_SetupDebugHooksDomain(PyMemAllocatorDomain domain) } PyMem_GetAllocator(PYMEM_DOMAIN_OBJ, &_PyMem_Debug.obj.alloc); - alloc.ctx = &_PyMem_Debug.obj; - alloc.malloc = _PyMem_DebugMalloc; - alloc.calloc = _PyMem_DebugCalloc; - alloc.realloc = _PyMem_DebugRealloc; - alloc.free = _PyMem_DebugFree; + alloc = PYDBGOBJ_ALLOC; PyMem_SetAllocator(PYMEM_DOMAIN_OBJ, &alloc); } } @@ -513,6 +491,7 @@ PyObject_SetArenaAllocator(PyObjectArenaAllocator *allocator) _PyObject_Arena = *allocator; } + void * PyMem_RawMalloc(size_t size) { @@ -551,6 +530,44 @@ void PyMem_RawFree(void *ptr) } +void* +_PyMem_RawMallocCtx(const _PyConfigCtx *ctx, size_t size) +{ + const PyMemAllocatorEx *alloc = &ctx->raw_alloc; + /* see PyMem_RawMalloc() */ + if (size > (size_t)PY_SSIZE_T_MAX) + return NULL; + return alloc->malloc(alloc->ctx, size); +} + +void* +_PyMem_RawCallocCtx(const _PyConfigCtx *ctx, size_t nelem, size_t elsize) +{ + const PyMemAllocatorEx *alloc = &ctx->raw_alloc; + /* see PyMem_RawMalloc() */ + if (elsize != 0 && nelem > (size_t)PY_SSIZE_T_MAX / elsize) + return NULL; + return alloc->calloc(alloc->ctx, nelem, elsize); +} + +void* +_PyMem_RawReallocCtx(const _PyConfigCtx *ctx, void *ptr, size_t new_size) +{ + const PyMemAllocatorEx *alloc = &ctx->raw_alloc; + /* see PyMem_RawMalloc() */ + if (new_size > (size_t)PY_SSIZE_T_MAX) + return NULL; + return alloc->realloc(alloc->ctx, ptr, new_size); +} + +void +_PyMem_RawFreeCtx(const _PyConfigCtx *ctx, void *ptr) +{ + const PyMemAllocatorEx *alloc = &ctx->raw_alloc; + alloc->free(alloc->ctx, ptr); +} + + void * PyMem_Malloc(size_t size) { @@ -586,7 +603,7 @@ PyMem_Free(void *ptr) wchar_t* -_PyMem_RawWcsdup(const wchar_t *str) +_PyMem_RawWcsdup(const _PyConfigCtx *ctx, const wchar_t *str) { assert(str != NULL); @@ -596,7 +613,13 @@ _PyMem_RawWcsdup(const wchar_t *str) } size_t size = (len + 1) * sizeof(wchar_t); - wchar_t *str2 = PyMem_RawMalloc(size); + wchar_t *str2; + if (ctx != NULL) { + str2 = _PyMem_RawMallocCtx(ctx, size); + } + else { + str2 = PyMem_RawMalloc(size); + } if (str2 == NULL) { return NULL; } @@ -605,6 +628,19 @@ _PyMem_RawWcsdup(const wchar_t *str) return str2; } +char * +_PyMem_RawStrdupCtx(const _PyConfigCtx *ctx, const char *str) +{ + assert(str != NULL); + size_t size = strlen(str) + 1; + char *copy = _PyMem_RawMallocCtx(ctx, size); + if (copy == NULL) { + return NULL; + } + memcpy(copy, str, size); + return copy; +} + char * _PyMem_RawStrdup(const char *str) { diff --git a/Objects/unicodeobject.c b/Objects/unicodeobject.c index 04ca5f3344470e..78962bc77de13d 100644 --- a/Objects/unicodeobject.c +++ b/Objects/unicodeobject.c @@ -3424,7 +3424,8 @@ unicode_encode_locale(PyObject *unicode, const char *errors, char *str; size_t error_pos; const char *reason; - int res = _Py_EncodeLocaleEx(wstr, &str, &error_pos, &reason, + PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); + int res = _Py_EncodeLocaleEx(&interp->core_config.ctx, wstr, &str, &error_pos, &reason, current_locale, error_handler); if (res != 0) { if (res == -2) { @@ -3452,7 +3453,7 @@ unicode_encode_locale(PyObject *unicode, const char *errors, PyMem_Free(wstr); PyObject *bytes = PyBytes_FromString(str); - PyMem_RawFree(str); + _PyMem_RawFreeCtx(&interp->core_config.ctx, str); return bytes; } @@ -3636,7 +3637,10 @@ unicode_decode_locale(const char *str, Py_ssize_t len, const char *errors, wchar_t *wstr; size_t wlen; const char *reason; - int res = _Py_DecodeLocaleEx(str, &wstr, &wlen, &reason, + + PyInterpreterState *interp = _PyInterpreterState_GET_UNSAFE(); + int res = _Py_DecodeLocaleEx(&interp->core_config.ctx, + str, &wstr, &wlen, &reason, current_locale, error_handler); if (res != 0) { if (res == -2) { @@ -3661,7 +3665,7 @@ unicode_decode_locale(const char *str, Py_ssize_t len, const char *errors, } PyObject *unicode = PyUnicode_FromWideChar(wstr, wlen); - PyMem_RawFree(wstr); + _PyMem_RawFreeCtx(&interp->core_config.ctx, wstr); return unicode; } @@ -4904,7 +4908,8 @@ PyUnicode_DecodeUTF8Stateful(const char *s, non-NULL, write the start of the illegal byte sequence into *wlen. If reason is not NULL, write the decoding error message into *reason. */ int -_Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, +_Py_DecodeUTF8Ex(const _PyConfigCtx *ctx, const char *s, Py_ssize_t size, + wchar_t **wstr, size_t *wlen, const char **reason, _Py_error_handler errors) { const char *orig_s = s; @@ -4934,7 +4939,7 @@ _Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, return -1; } - unicode = PyMem_RawMalloc((size + 1) * sizeof(wchar_t)); + unicode = _PyMem_RawMallocCtx(ctx, (size + 1) * sizeof(wchar_t)); if (!unicode) { return -1; } @@ -4980,7 +4985,7 @@ _Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, unicode[outpos++] = ch; } else { - PyMem_RawFree(unicode ); + _PyMem_RawFreeCtx(ctx, unicode); if (reason != NULL) { switch (ch) { case 0: @@ -5011,11 +5016,15 @@ _Py_DecodeUTF8Ex(const char *s, Py_ssize_t size, wchar_t **wstr, size_t *wlen, return 0; } +/* FIXME: add ctx parameter */ wchar_t* _Py_DecodeUTF8_surrogateescape(const char *arg, Py_ssize_t arglen) { wchar_t *wstr; - int res = _Py_DecodeUTF8Ex(arg, arglen, &wstr, NULL, NULL, 1); + _PyConfigCtx ctx; + + _PyConfigCtx_Init(&ctx); + int res = _Py_DecodeUTF8Ex(&ctx, arg, arglen, &wstr, NULL, NULL, 1); if (res != 0) { return NULL; } @@ -5034,7 +5043,8 @@ _Py_DecodeUTF8_surrogateescape(const char *arg, Py_ssize_t arglen) On memory allocation failure, return -1. */ int -_Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, +_Py_EncodeUTF8Ex(const _PyConfigCtx *ctx, const wchar_t *text, + char **str, size_t *error_pos, const char **reason, int raw_malloc, _Py_error_handler errors) { const Py_ssize_t max_char_size = 4; @@ -5063,7 +5073,7 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, } char *bytes; if (raw_malloc) { - bytes = PyMem_RawMalloc((len + 1) * max_char_size); + bytes = _PyMem_RawMallocCtx(ctx, (len + 1) * max_char_size); } else { bytes = PyMem_Malloc((len + 1) * max_char_size); @@ -5108,7 +5118,7 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, *reason = "encoding error"; } if (raw_malloc) { - PyMem_RawFree(bytes); + _PyMem_RawFreeCtx(ctx, bytes); } else { PyMem_Free(bytes); @@ -5136,7 +5146,7 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, size_t final_size = (p - bytes); char *bytes2; if (raw_malloc) { - bytes2 = PyMem_RawRealloc(bytes, final_size); + bytes2 = _PyMem_RawReallocCtx(ctx, bytes, final_size); } else { bytes2 = PyMem_Realloc(bytes, final_size); @@ -5146,7 +5156,7 @@ _Py_EncodeUTF8Ex(const wchar_t *text, char **str, size_t *error_pos, *error_pos = (size_t)-1; } if (raw_malloc) { - PyMem_RawFree(bytes); + _PyMem_RawFreeCtx(ctx, bytes); } else { PyMem_Free(bytes); diff --git a/PC/getpathp.c b/PC/getpathp.c index ee9d3d258f6857..6d5234c18876bc 100644 --- a/PC/getpathp.c +++ b/PC/getpathp.c @@ -118,6 +118,8 @@ #endif typedef struct { + _PyConfigCtx ctx; + const wchar_t *path_env; /* PATH environment variable */ const wchar_t *home; /* PYTHONHOME environment variable */ @@ -353,7 +355,7 @@ extern const char *PyWin_DLLVersionString; in advance. It could be simplied now Win16/Win32s is dead! */ static wchar_t * -getpythonregpath(HKEY keyBase, int skipcore) +getpythonregpath(_PyConfigCtx *ctx, HKEY keyBase, int skipcore) { HKEY newKey = 0; DWORD dataSize = 0; @@ -375,7 +377,7 @@ getpythonregpath(HKEY keyBase, int skipcore) keyBufLen = sizeof(keyPrefix) + sizeof(WCHAR)*(versionLen-1) + sizeof(keySuffix); - keyBuf = keyBufPtr = PyMem_RawMalloc(keyBufLen); + keyBuf = keyBufPtr = _PyMem_RawMallocCtx(ctx, keyBufLen); if (keyBuf==NULL) { goto done; } @@ -407,7 +409,7 @@ getpythonregpath(HKEY keyBase, int skipcore) /* Allocate a temp array of char buffers, so we only need to loop reading the registry once */ - ppPaths = PyMem_RawMalloc( sizeof(WCHAR *) * numKeys ); + ppPaths = _PyMem_RawMallocCtx(ctx, sizeof(WCHAR *) * numKeys); if (ppPaths==NULL) { goto done; } @@ -435,7 +437,7 @@ getpythonregpath(HKEY keyBase, int skipcore) /* Find the value of the buffer size, malloc, then read it */ RegQueryValueExW(subKey, NULL, 0, NULL, NULL, &reqdSize); if (reqdSize) { - ppPaths[index] = PyMem_RawMalloc(reqdSize); + ppPaths[index] = _PyMem_RawMallocCtx(ctx, reqdSize); if (ppPaths[index]) { RegQueryValueExW(subKey, NULL, 0, NULL, (LPBYTE)ppPaths[index], @@ -452,7 +454,7 @@ getpythonregpath(HKEY keyBase, int skipcore) } /* original datasize from RegQueryInfo doesn't include the \0 */ - dataBuf = PyMem_RawMalloc((dataSize+1) * sizeof(WCHAR)); + dataBuf = _PyMem_RawMallocCtx(ctx, (dataSize+1) * sizeof(WCHAR)); if (dataBuf) { WCHAR *szCur = dataBuf; /* Copy our collected strings */ @@ -484,7 +486,7 @@ getpythonregpath(HKEY keyBase, int skipcore) rc = RegQueryValueExW(newKey, NULL, 0, NULL, (LPBYTE)szCur, &dataSize); if (rc != ERROR_SUCCESS) { - PyMem_RawFree(dataBuf); + _PyMem_RawFreeCtx(ctx, dataBuf); goto done; } } @@ -494,14 +496,15 @@ getpythonregpath(HKEY keyBase, int skipcore) done: /* Loop freeing my temp buffers */ if (ppPaths) { - for(index=0; indexdll_path = _PyMem_RawWcsdup(dll_path); + config->dll_path = _PyMem_RawWcsdup(&config->ctx, dll_path); if (config->dll_path == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -544,7 +547,7 @@ get_program_full_path(const _PyCoreConfig *core_config, return _Py_INIT_ERR("Cannot determine program path"); } - config->program_full_path = PyMem_RawMalloc( + config->program_full_path = _PyMem_RawMallocCtx(&config->ctx, sizeof(wchar_t) * (MAXPATHLEN + 1)); return canonicalize(config->program_full_path, @@ -568,7 +571,7 @@ read_pth_file(_PyPathConfig *config, wchar_t *prefix, const wchar_t *path) size_t bufsiz = MAXPATHLEN; size_t prefixlen = wcslen(prefix); - wchar_t *buf = (wchar_t*)PyMem_RawMalloc(bufsiz * sizeof(wchar_t)); + wchar_t *buf = _PyMem_RawMallocCtx(&config->ctx, bufsiz * sizeof(wchar_t)); buf[0] = '\0'; while (!feof(sp_file)) { @@ -596,16 +599,16 @@ read_pth_file(_PyPathConfig *config, wchar_t *prefix, const wchar_t *path) } DWORD wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, NULL, 0); - wchar_t *wline = (wchar_t*)PyMem_RawMalloc((wn + 1) * sizeof(wchar_t)); + wchar_t *wline = _PyMem_RawMallocCtx(&config->ctx, (wn + 1) * sizeof(wchar_t)); wn = MultiByteToWideChar(CP_UTF8, 0, line, -1, wline, wn + 1); wline[wn] = '\0'; size_t usedsiz = wcslen(buf); while (usedsiz + wn + prefixlen + 4 > bufsiz) { bufsiz += MAXPATHLEN; - buf = (wchar_t*)PyMem_RawRealloc(buf, (bufsiz + 1) * sizeof(wchar_t)); + buf = _PyMem_RawReallocCtx(&config->ctx, buf, (bufsiz + 1) * sizeof(wchar_t)); if (!buf) { - PyMem_RawFree(wline); + _PyMem_RawFreeCtx(&config->ctx, wline); goto error; } } @@ -627,7 +630,7 @@ read_pth_file(_PyPathConfig *config, wchar_t *prefix, const wchar_t *path) wchar_t *b = &buf[usedsiz]; join(b, wline); - PyMem_RawFree(wline); + _PyMem_RawFreeCtx(&config->ctx, wline); } fclose(sp_file); @@ -635,7 +638,7 @@ read_pth_file(_PyPathConfig *config, wchar_t *prefix, const wchar_t *path) return 1; error: - PyMem_RawFree(buf); + _PyMem_RawFreeCtx(&config->ctx, buf); fclose(sp_file); return 0; } @@ -756,8 +759,8 @@ calculate_module_search_path(const _PyCoreConfig *core_config, { int skiphome = calculate->home==NULL ? 0 : 1; #ifdef Py_ENABLE_SHARED - calculate->machine_path = getpythonregpath(HKEY_LOCAL_MACHINE, skiphome); - calculate->user_path = getpythonregpath(HKEY_CURRENT_USER, skiphome); + calculate->machine_path = getpythonregpath(&calculate->ctx, HKEY_LOCAL_MACHINE, skiphome); + calculate->user_path = getpythonregpath(&calculate->ctx, HKEY_CURRENT_USER, skiphome); #endif /* We only use the default relative PYTHONPATH if we haven't anything better to use! */ @@ -805,7 +808,7 @@ calculate_module_search_path(const _PyCoreConfig *core_config, } wchar_t *buf, *start_buf; - buf = PyMem_RawMalloc(bufsz * sizeof(wchar_t)); + buf = _PyMem_RawMallocCtx(&config->ctx, bufsz * sizeof(wchar_t)); if (buf == NULL) { /* We can't exit, so print a warning and limp along */ fprintf(stderr, "Can't malloc dynamic PYTHONPATH.\n"); @@ -978,7 +981,7 @@ calculate_path_impl(const _PyCoreConfig *core_config, } done: - config->prefix = _PyMem_RawWcsdup(prefix); + config->prefix = _PyMem_RawWcsdup(&config->ctx, prefix); if (config->prefix == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -990,8 +993,14 @@ calculate_path_impl(const _PyCoreConfig *core_config, static void calculate_free(PyCalculatePath *calculate) { - PyMem_RawFree(calculate->machine_path); - PyMem_RawFree(calculate->user_path); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->machine_path); + _PyMem_RawFreeCtx(&calculate->ctx, calculate->user_path); +} + + +static void +_PyCalculatePath_Init(PyCalculatePath *calculate) +{ } @@ -1000,6 +1009,7 @@ _PyPathConfig_Calculate_impl(_PyPathConfig *config, const _PyCoreConfig *core_co { PyCalculatePath calculate; memset(&calculate, 0, sizeof(calculate)); + _PyConfigCtx_Init(&calculate.ctx); calculate_init(&calculate, core_config); @@ -1034,9 +1044,11 @@ _Py_CheckPython3(void) } python3_checked = 1; + /* If there is a python3.dll next to the python3y.dll, assume this is a build tree; use that DLL */ - wcscpy(py3path, _Py_path_config.dll_path); + _PyPathConfig* path_config = _PyPathConfig_GetGlobal(); + wcscpy(py3path, path_config->dll_path); s = wcsrchr(py3path, L'\\'); if (!s) { s = py3path; diff --git a/Programs/_freeze_importlib.c b/Programs/_freeze_importlib.c index 3f43757ecb1ff5..c9c8eaa2c3dee2 100644 --- a/Programs/_freeze_importlib.c +++ b/Programs/_freeze_importlib.c @@ -76,7 +76,8 @@ main(int argc, char *argv[]) } text[text_size] = '\0'; - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); config.user_site_directory = 0; config.site_import = 0; config.use_environment = 0; diff --git a/Programs/_testembed.c b/Programs/_testembed.c index 22212bff52d4a1..b52be36dc000d6 100644 --- a/Programs/_testembed.c +++ b/Programs/_testembed.c @@ -139,6 +139,8 @@ static int test_forced_io_encoding(void) static int test_pre_initialization_api(void) { + unsetenv("PYTHONMALLOC"); + /* Leading "./" ensures getpath.c can still find the standard library */ _Py_EMBED_PREINIT_CHECK("Checking Py_DecodeLocale\n"); wchar_t *program = Py_DecodeLocale("./spam", NULL); @@ -235,6 +237,8 @@ static void bpo20891_thread(void *lockp) static int test_bpo20891(void) { + unsetenv("PYTHONMALLOC"); + /* bpo-20891: Calling PyGILState_Ensure in a non-Python thread before calling PyEval_InitThreads() must not crash. PyGILState_Ensure() must call PyEval_InitThreads() for us in this case. */ @@ -278,6 +282,8 @@ static int test_initialize_twice(void) static int test_initialize_pymain(void) { + unsetenv("PYTHONMALLOC"); + wchar_t *argv[] = {L"PYTHON", L"-c", L"import sys; print(f'Py_Main() after Py_Initialize: sys.argv={sys.argv}')", L"arg2"}; @@ -427,7 +433,8 @@ static int test_init_global_config(void) static int test_init_from_config(void) { /* Test _Py_InitializeFromConfig() */ - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); config.install_signal_handlers = 0; /* FIXME: test use_environment */ @@ -461,7 +468,7 @@ static int test_init_from_config(void) putenv("PYTHONUTF8=0"); Py_UTF8Mode = 0; - config.utf8_mode = 1; + config.ctx.utf8_mode = 1; putenv("PYTHONPYCACHEPREFIX=env_pycache_prefix"); config.pycache_prefix = L"conf_pycache_prefix"; @@ -604,11 +611,12 @@ static int test_init_env(void) static int test_init_isolated(void) { /* Test _PyCoreConfig.isolated=1 */ - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); /* Set coerce_c_locale and utf8_mode to not depend on the locale */ config.coerce_c_locale = 0; - config.utf8_mode = 0; + config.ctx.utf8_mode = 0; /* Use path starting with "./" avoids a search along the PATH */ config.program_name = L"./_testembed"; @@ -628,7 +636,8 @@ static int test_init_isolated(void) static int test_init_dev_mode(void) { - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); putenv("PYTHONFAULTHANDLER="); putenv("PYTHONMALLOC="); config.dev_mode = 1; diff --git a/Python/coreconfig.c b/Python/coreconfig.c index a040a865ae1ddb..b96bd0b823e7cc 100644 --- a/Python/coreconfig.c +++ b/Python/coreconfig.c @@ -55,6 +55,26 @@ int Py_LegacyWindowsStdioFlag = 0; /* Uses FileIO instead of WindowsConsoleIO */ #endif +void _PyConfigCtx_Init(_PyConfigCtx *ctx) +{ + _PyInitError err = _PyRuntime_Initialize(); + if (_Py_INIT_FAILED(err)) { + _Py_FatalInitError(err); + } + + ctx->utf8_mode = -1; + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &ctx->raw_alloc); +} + +static void +_PyConfigCtx_Copy(_PyConfigCtx *ctx, const _PyConfigCtx *ctx2) +{ + /* Don't copy raw_alloc, each context keeps is own memory allocators. + Memory allocators must be copied or modified manually. */ + ctx->utf8_mode = ctx2->utf8_mode; +} + + PyObject * _Py_GetGlobalVariablesAsDict(void) { @@ -128,28 +148,28 @@ _Py_GetGlobalVariablesAsDict(void) void -_Py_wstrlist_clear(int len, wchar_t **list) +_Py_wstrlist_clear(const _PyConfigCtx *ctx, int len, wchar_t **list) { for (int i=0; i < len; i++) { - PyMem_RawFree(list[i]); + _PyMem_RawFreeCtx(ctx, list[i]); } - PyMem_RawFree(list); + _PyMem_RawFreeCtx(ctx, list); } wchar_t** -_Py_wstrlist_copy(int len, wchar_t **list) +_Py_wstrlist_copy(const _PyConfigCtx *ctx, int len, wchar_t **list) { assert((len > 0 && list != NULL) || len == 0); size_t size = len * sizeof(list[0]); - wchar_t **list_copy = PyMem_RawMalloc(size); + wchar_t **list_copy = _PyMem_RawMallocCtx(ctx, size); if (list_copy == NULL) { return NULL; } for (int i=0; i < len; i++) { - wchar_t* arg = _PyMem_RawWcsdup(list[i]); + wchar_t* arg = _PyMem_RawWcsdup(ctx, list[i]); if (arg == NULL) { - _Py_wstrlist_clear(i, list_copy); + _Py_wstrlist_clear(ctx, i, list_copy); return NULL; } list_copy[i] = arg; @@ -281,18 +301,26 @@ _Py_ClearStandardStreamEncoding(void) } +void +_PyCoreConfig_Init(_PyCoreConfig *config) +{ + *config = _PyCoreConfig_STATIC_INIT; + _PyConfigCtx_Init(&config->ctx); +} + + /* Free memory allocated in config, but don't clear all attributes */ void _PyCoreConfig_Clear(_PyCoreConfig *config) { #define CLEAR(ATTR) \ do { \ - PyMem_RawFree(ATTR); \ + _PyMem_RawFreeCtx(&config->ctx, ATTR); \ ATTR = NULL; \ } while (0) #define CLEAR_WSTRLIST(LEN, LIST) \ do { \ - _Py_wstrlist_clear(LEN, LIST); \ + _Py_wstrlist_clear(&config->ctx, LEN, LIST); \ LEN = 0; \ LIST = NULL; \ } while (0) @@ -334,11 +362,13 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) { _PyCoreConfig_Clear(config); + _PyConfigCtx_Copy(&config->ctx, &config2->ctx); + #define COPY_ATTR(ATTR) config->ATTR = config2->ATTR #define COPY_STR_ATTR(ATTR) \ do { \ if (config2->ATTR != NULL) { \ - config->ATTR = _PyMem_RawStrdup(config2->ATTR); \ + config->ATTR = _PyMem_RawStrdupCtx(&config->ctx, config2->ATTR); \ if (config->ATTR == NULL) { \ return -1; \ } \ @@ -347,16 +377,17 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) #define COPY_WSTR_ATTR(ATTR) \ do { \ if (config2->ATTR != NULL) { \ - config->ATTR = _PyMem_RawWcsdup(config2->ATTR); \ + config->ATTR = _PyMem_RawWcsdup(&config->ctx, config2->ATTR); \ if (config->ATTR == NULL) { \ return -1; \ } \ } \ } while (0) + /* Pass the context of the current config, not of the copied config */ #define COPY_WSTRLIST(LEN, LIST) \ do { \ if (config2->LIST != NULL) { \ - config->LIST = _Py_wstrlist_copy(config2->LEN, config2->LIST); \ + config->LIST = _Py_wstrlist_copy(&config->ctx, config2->LEN, config2->LIST); \ if (config->LIST == NULL) { \ return -1; \ } \ @@ -381,7 +412,6 @@ _PyCoreConfig_Copy(_PyCoreConfig *config, const _PyCoreConfig *config2) COPY_ATTR(coerce_c_locale); COPY_ATTR(coerce_c_locale_warn); - COPY_ATTR(utf8_mode); COPY_WSTR_ATTR(pycache_prefix); COPY_WSTR_ATTR(module_search_path_env); @@ -472,7 +502,7 @@ _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, return 0; } - wchar_t *copy = _PyMem_RawWcsdup(var); + wchar_t *copy = _PyMem_RawWcsdup(&config->ctx, var); if (copy == NULL) { return -1; } @@ -486,7 +516,7 @@ _PyCoreConfig_GetEnvDup(const _PyCoreConfig *config, } size_t len; - wchar_t *wvar = Py_DecodeLocale(var, &len); + wchar_t *wvar = _Py_DecodeLocaleCtx(&config->ctx, var, &len); if (!wvar) { if (len == (size_t)-2) { return -2; @@ -513,7 +543,7 @@ _PyCoreConfig_GetGlobalConfig(_PyCoreConfig *config) config->ATTR = !(VALUE); \ } - COPY_FLAG(utf8_mode, Py_UTF8Mode); + COPY_FLAG(ctx.utf8_mode, Py_UTF8Mode); COPY_FLAG(isolated, Py_IsolatedFlag); COPY_FLAG(bytes_warning, Py_BytesWarningFlag); COPY_FLAG(inspect, Py_InspectFlag); @@ -552,7 +582,7 @@ _PyCoreConfig_SetGlobalConfig(const _PyCoreConfig *config) VAR = !config->ATTR; \ } - COPY_FLAG(utf8_mode, Py_UTF8Mode); + COPY_FLAG(ctx.utf8_mode, Py_UTF8Mode); COPY_FLAG(isolated, Py_IsolatedFlag); COPY_FLAG(bytes_warning, Py_BytesWarningFlag); COPY_FLAG(inspect, Py_InspectFlag); @@ -589,10 +619,12 @@ config_init_program_name(_PyCoreConfig *config) { assert(config->program_name == NULL); + _PyPathConfig* path_config = _PyPathConfig_GetGlobal(); + /* If Py_SetProgramName() was called, use its value */ - const wchar_t *program_name = _Py_path_config.program_name; + const wchar_t *program_name = path_config->program_name; if (program_name != NULL) { - config->program_name = _PyMem_RawWcsdup(program_name); + config->program_name = _PyMem_RawWcsdup(&config->ctx, program_name); if (config->program_name == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -612,7 +644,7 @@ config_init_program_name(_PyCoreConfig *config) const char *p = _PyCoreConfig_GetEnv(config, "PYTHONEXECUTABLE"); if (p != NULL) { size_t len; - wchar_t* program_name = Py_DecodeLocale(p, &len); + wchar_t* program_name = _Py_DecodeLocaleCtx(&config->ctx, p, &len); if (program_name == NULL) { return DECODE_LOCALE_ERR("PYTHONEXECUTABLE environment " "variable", (Py_ssize_t)len); @@ -628,7 +660,7 @@ config_init_program_name(_PyCoreConfig *config) * the argv0 of the stub executable */ size_t len; - wchar_t* program_name = Py_DecodeLocale(pyvenv_launcher, &len); + wchar_t* program_name = _Py_DecodeLocaleCtx(&config->ctx, pyvenv_launcher, &len); if (program_name == NULL) { return DECODE_LOCALE_ERR("__PYVENV_LAUNCHER__ environment " "variable", (Py_ssize_t)len); @@ -642,7 +674,7 @@ config_init_program_name(_PyCoreConfig *config) /* Use argv[0] by default, if available */ if (config->program != NULL) { - config->program_name = _PyMem_RawWcsdup(config->program); + config->program_name = _PyMem_RawWcsdup(&config->ctx, config->program); if (config->program_name == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -655,7 +687,7 @@ config_init_program_name(_PyCoreConfig *config) #else const wchar_t *default_program_name = L"python3"; #endif - config->program_name = _PyMem_RawWcsdup(default_program_name); + config->program_name = _PyMem_RawWcsdup(&config->ctx, default_program_name); if (config->program_name == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -690,11 +722,12 @@ static _PyInitError config_init_home(_PyCoreConfig *config) { wchar_t *home; + _PyPathConfig* path_config = _PyPathConfig_GetGlobal(); /* If Py_SetPythonHome() was called, use its value */ - home = _Py_path_config.home; + home = path_config->home; if (home) { - config->home = _PyMem_RawWcsdup(home); + config->home = _PyMem_RawWcsdup(&config->ctx, home); if (config->home == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -752,17 +785,17 @@ config_init_utf8_mode(_PyCoreConfig *config) if (sep) { xopt = sep + 1; if (wcscmp(xopt, L"1") == 0) { - config->utf8_mode = 1; + config->ctx.utf8_mode = 1; } else if (wcscmp(xopt, L"0") == 0) { - config->utf8_mode = 0; + config->ctx.utf8_mode = 0; } else { return _Py_INIT_USER_ERR("invalid -X utf8 option value"); } } else { - config->utf8_mode = 1; + config->ctx.utf8_mode = 1; } return _Py_INIT_OK(); } @@ -770,10 +803,10 @@ config_init_utf8_mode(_PyCoreConfig *config) const char *opt = _PyCoreConfig_GetEnv(config, "PYTHONUTF8"); if (opt) { if (strcmp(opt, "1") == 0) { - config->utf8_mode = 1; + config->ctx.utf8_mode = 1; } else if (strcmp(opt, "0") == 0) { - config->utf8_mode = 0; + config->ctx.utf8_mode = 0; } else { return _Py_INIT_USER_ERR("invalid PYTHONUTF8 environment " @@ -976,7 +1009,7 @@ config_init_pycache_prefix(_PyCoreConfig *config) if (xoption) { const wchar_t *sep = wcschr(xoption, L'='); if (sep && wcslen(sep) > 1) { - config->pycache_prefix = _PyMem_RawWcsdup(sep + 1); + config->pycache_prefix = _PyMem_RawWcsdup(&config->ctx, sep + 1); if (config->pycache_prefix == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1052,14 +1085,14 @@ config_init_locale(_PyCoreConfig *config) } #ifndef MS_WINDOWS - if (config->utf8_mode < 0) { + if (config->ctx.utf8_mode < 0) { /* The C locale and the POSIX locale enable the UTF-8 Mode (PEP 540) */ const char *ctype_loc = setlocale(LC_CTYPE, NULL); if (ctype_loc != NULL && (strcmp(ctype_loc, "C") == 0 || strcmp(ctype_loc, "POSIX") == 0)) { - config->utf8_mode = 1; + config->ctx.utf8_mode = 1; } } #endif @@ -1094,7 +1127,7 @@ get_stdio_errors(const _PyCoreConfig *config) static _PyInitError -get_locale_encoding(char **locale_encoding) +get_locale_encoding(_PyConfigCtx *ctx, char **locale_encoding) { #ifdef MS_WINDOWS char encoding[20]; @@ -1108,7 +1141,7 @@ get_locale_encoding(char **locale_encoding) "nl_langinfo(CODESET) failed"); } #endif - *locale_encoding = _PyMem_RawStrdup(encoding); + *locale_encoding = _PyMem_RawStrdupCtx(ctx, encoding); if (*locale_encoding == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1122,14 +1155,14 @@ config_init_stdio_encoding(_PyCoreConfig *config) /* If Py_SetStandardStreamEncoding() have been called, use these parameters. */ if (config->stdio_encoding == NULL && _Py_StandardStreamEncoding != NULL) { - config->stdio_encoding = _PyMem_RawStrdup(_Py_StandardStreamEncoding); + config->stdio_encoding = _PyMem_RawStrdupCtx(&config->ctx, _Py_StandardStreamEncoding); if (config->stdio_encoding == NULL) { return _Py_INIT_NO_MEMORY(); } } if (config->stdio_errors == NULL && _Py_StandardStreamErrors != NULL) { - config->stdio_errors = _PyMem_RawStrdup(_Py_StandardStreamErrors); + config->stdio_errors = _PyMem_RawStrdupCtx(&config->ctx, _Py_StandardStreamErrors); if (config->stdio_errors == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1159,7 +1192,7 @@ config_init_stdio_encoding(_PyCoreConfig *config) /* Does PYTHONIOENCODING contain an encoding? */ if (pythonioencoding[0]) { if (config->stdio_encoding == NULL) { - config->stdio_encoding = _PyMem_RawStrdup(pythonioencoding); + config->stdio_encoding = _PyMem_RawStrdupCtx(&config->ctx, pythonioencoding); if (config->stdio_encoding == NULL) { PyMem_RawFree(pythonioencoding); return _Py_INIT_NO_MEMORY(); @@ -1176,7 +1209,7 @@ config_init_stdio_encoding(_PyCoreConfig *config) } if (config->stdio_errors == NULL && err != NULL) { - config->stdio_errors = _PyMem_RawStrdup(err); + config->stdio_errors = _PyMem_RawStrdupCtx(&config->ctx, err); if (config->stdio_errors == NULL) { PyMem_RawFree(pythonioencoding); return _Py_INIT_NO_MEMORY(); @@ -1187,15 +1220,15 @@ config_init_stdio_encoding(_PyCoreConfig *config) } /* UTF-8 Mode uses UTF-8/surrogateescape */ - if (config->utf8_mode) { + if (config->ctx.utf8_mode) { if (config->stdio_encoding == NULL) { - config->stdio_encoding = _PyMem_RawStrdup("utf-8"); + config->stdio_encoding = _PyMem_RawStrdupCtx(&config->ctx, "utf-8"); if (config->stdio_encoding == NULL) { return _Py_INIT_NO_MEMORY(); } } if (config->stdio_errors == NULL) { - config->stdio_errors = _PyMem_RawStrdup("surrogateescape"); + config->stdio_errors = _PyMem_RawStrdupCtx(&config->ctx, "surrogateescape"); if (config->stdio_errors == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1204,14 +1237,14 @@ config_init_stdio_encoding(_PyCoreConfig *config) /* Choose the default error handler based on the current locale. */ if (config->stdio_encoding == NULL) { - _PyInitError err = get_locale_encoding(&config->stdio_encoding); + _PyInitError err = get_locale_encoding(&config->ctx, &config->stdio_encoding); if (_Py_INIT_FAILED(err)) { return err; } } if (config->stdio_errors == NULL) { const char *errors = get_stdio_errors(config); - config->stdio_errors = _PyMem_RawStrdup(errors); + config->stdio_errors = _PyMem_RawStrdupCtx(&config->ctx, errors); if (config->stdio_errors == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1228,13 +1261,13 @@ config_init_fs_encoding(_PyCoreConfig *config) if (config->legacy_windows_fs_encoding) { /* Legacy Windows filesystem encoding: mbcs/replace */ if (config->filesystem_encoding == NULL) { - config->filesystem_encoding = _PyMem_RawStrdup("mbcs"); + config->filesystem_encoding = _PyMem_RawStrdupCtx(&config->ctx, "mbcs"); if (config->filesystem_encoding == NULL) { return _Py_INIT_NO_MEMORY(); } } if (config->filesystem_errors == NULL) { - config->filesystem_errors = _PyMem_RawStrdup("replace"); + config->filesystem_errors = _PyMem_RawStrdupCtx(&config->ctx, "replace"); if (config->filesystem_errors == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1246,35 +1279,35 @@ config_init_fs_encoding(_PyCoreConfig *config) Note: UTF-8 Mode takes the same code path and the Legacy Windows FS encoding has the priortiy over UTF-8 Mode. */ if (config->filesystem_encoding == NULL) { - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); + config->filesystem_encoding = _PyMem_RawStrdupCtx(&config->ctx, "utf-8"); if (config->filesystem_encoding == NULL) { return _Py_INIT_NO_MEMORY(); } } if (config->filesystem_errors == NULL) { - config->filesystem_errors = _PyMem_RawStrdup("surrogatepass"); + config->filesystem_errors = _PyMem_RawStrdupCtx(&config->ctx, "surrogatepass"); if (config->filesystem_errors == NULL) { return _Py_INIT_NO_MEMORY(); } } #else if (config->filesystem_encoding == NULL) { - if (config->utf8_mode) { + if (config->ctx.utf8_mode) { /* UTF-8 Mode use: utf-8/surrogateescape */ - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); + config->filesystem_encoding = _PyMem_RawStrdupCtx(&config->ctx, "utf-8"); /* errors defaults to surrogateescape above */ } else if (_Py_GetForceASCII()) { - config->filesystem_encoding = _PyMem_RawStrdup("ascii"); + config->filesystem_encoding = _PyMem_RawStrdupCtx(&config->ctx, "ascii"); } else { /* macOS and Android use UTF-8, other platforms use the locale encoding. */ #if defined(__APPLE__) || defined(__ANDROID__) - config->filesystem_encoding = _PyMem_RawStrdup("utf-8"); + config->filesystem_encoding = _PyMem_RawStrdupCtx(&config->ctx, "utf-8"); #else - _PyInitError err = get_locale_encoding(&config->filesystem_encoding); + _PyInitError err = get_locale_encoding(&config->ctx, &config->filesystem_encoding); if (_Py_INIT_FAILED(err)) { return err; } @@ -1288,7 +1321,7 @@ config_init_fs_encoding(_PyCoreConfig *config) if (config->filesystem_errors == NULL) { /* by default, use the "surrogateescape" error handler */ - config->filesystem_errors = _PyMem_RawStrdup("surrogateescape"); + config->filesystem_errors = _PyMem_RawStrdupCtx(&config->ctx, "surrogateescape"); if (config->filesystem_errors == NULL) { return _Py_INIT_NO_MEMORY(); } @@ -1325,7 +1358,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config) #ifdef MS_WINDOWS if (config->legacy_windows_fs_encoding) { - config->utf8_mode = 0; + config->ctx.utf8_mode = 0; } #endif @@ -1349,7 +1382,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config) return err; } - if (config->utf8_mode < 0) { + if (config->ctx.utf8_mode < 0) { err = config_init_utf8_mode(config); if (_Py_INIT_FAILED(err)) { return err; @@ -1370,7 +1403,7 @@ _PyCoreConfig_Read(_PyCoreConfig *config) } } - if (config->utf8_mode < 0 || config->coerce_c_locale < 0) { + if (config->ctx.utf8_mode < 0 || config->coerce_c_locale < 0) { config_init_locale(config); } @@ -1403,8 +1436,8 @@ _PyCoreConfig_Read(_PyCoreConfig *config) if (config->coerce_c_locale < 0) { config->coerce_c_locale = 0; } - if (config->utf8_mode < 0) { - config->utf8_mode = 0; + if (config->ctx.utf8_mode < 0) { + config->ctx.utf8_mode = 0; } if (config->argc < 0) { config->argc = 0; @@ -1492,7 +1525,7 @@ _PyCoreConfig_AsDict(const _PyCoreConfig *config) SET_ITEM_INT(coerce_c_locale_warn); SET_ITEM_STR(filesystem_encoding); SET_ITEM_STR(filesystem_errors); - SET_ITEM_INT(utf8_mode); + SET_ITEM("utf8_mode", PyLong_FromLong(config->ctx.utf8_mode)); SET_ITEM_WSTR(pycache_prefix); SET_ITEM_WSTR(program_name); SET_ITEM_WSTRLIST(argc, argv); diff --git a/Python/fileutils.c b/Python/fileutils.c index c9a8e58dd1221d..594bbf027f13da 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -232,7 +232,7 @@ _Py_GetForceASCII(void) static int -encode_ascii(const wchar_t *text, char **str, +encode_ascii(const _PyConfigCtx *ctx, const wchar_t *text, char **str, size_t *error_pos, const char **reason, int raw_malloc, _Py_error_handler errors) { @@ -249,7 +249,7 @@ encode_ascii(const wchar_t *text, char **str, /* +1 for NULL byte */ if (raw_malloc) { - result = PyMem_RawMalloc(len + 1); + result = _PyMem_RawMallocCtx(ctx, len + 1); } else { result = PyMem_Malloc(len + 1); @@ -272,7 +272,7 @@ encode_ascii(const wchar_t *text, char **str, } else { if (raw_malloc) { - PyMem_RawFree(result); + _PyMem_RawFreeCtx(ctx, result); } else { PyMem_Free(result); @@ -301,7 +301,7 @@ _Py_GetForceASCII(void) #if !defined(HAVE_MBRTOWC) || defined(USE_FORCE_ASCII) static int -decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, +decode_ascii(const _PyConfigCtx *ctx, const char *arg, wchar_t **wstr, size_t *wlen, const char **reason, _Py_error_handler errors) { wchar_t *res; @@ -317,7 +317,7 @@ decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) { return -1; } - res = PyMem_RawMalloc(argsize * sizeof(wchar_t)); + res = _PyMem_RawMallocCtx(ctx, argsize * sizeof(wchar_t)); if (!res) { return -1; } @@ -353,8 +353,9 @@ decode_ascii(const char *arg, wchar_t **wstr, size_t *wlen, #endif /* !HAVE_MBRTOWC */ static int -decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, - const char **reason, _Py_error_handler errors) +decode_current_locale(const _PyConfigCtx *ctx, const char* arg, wchar_t **wstr, + size_t *wlen, const char **reason, + _Py_error_handler errors) { wchar_t *res; size_t argsize; @@ -383,7 +384,7 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t) - 1) { return -1; } - res = (wchar_t *)PyMem_RawMalloc((argsize + 1) * sizeof(wchar_t)); + res = (wchar_t *)_PyMem_RawMallocCtx(ctx, (argsize + 1) * sizeof(wchar_t)); if (!res) { return -1; } @@ -417,7 +418,7 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, if (argsize > PY_SSIZE_T_MAX / sizeof(wchar_t)) { return -1; } - res = (wchar_t*)PyMem_RawMalloc(argsize * sizeof(wchar_t)); + res = (wchar_t*)_PyMem_RawMallocCtx(ctx, argsize * sizeof(wchar_t)); if (!res) { return -1; } @@ -490,7 +491,7 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, /* Cannot use C locale for escaping; manually escape as if charset is ASCII (i.e. escape all bytes > 128. This will still roundtrip correctly in the locale's charset, which must be an ASCII superset. */ - return decode_ascii(arg, wstr, wlen, reason, errors); + return decode_ascii(ctx, arg, wstr, wlen, reason, errors); #endif /* HAVE_MBRTOWC */ } @@ -518,29 +519,29 @@ decode_current_locale(const char* arg, wchar_t **wstr, size_t *wlen, Use the Py_EncodeLocaleEx() function to encode the character string back to a byte string. */ int -_Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen, +_Py_DecodeLocaleEx(const _PyConfigCtx *ctx, const char* arg, wchar_t **wstr, size_t *wlen, const char **reason, int current_locale, _Py_error_handler errors) { if (current_locale) { #ifdef __ANDROID__ - return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason, + return _Py_DecodeUTF8Ex(ctx, arg, strlen(arg), wstr, wlen, reason, errors); #else - return decode_current_locale(arg, wstr, wlen, reason, errors); + return decode_current_locale(ctx, arg, wstr, wlen, reason, errors); #endif } #if defined(__APPLE__) || defined(__ANDROID__) - return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason, + return _Py_DecodeUTF8Ex(ctx, arg, strlen(arg), wstr, wlen, reason, errors); #else - int use_utf8 = (Py_UTF8Mode == 1); + int use_utf8 = (ctx->utf8_mode == 1); #ifdef MS_WINDOWS use_utf8 |= !Py_LegacyWindowsFSEncodingFlag; #endif if (use_utf8) { - return _Py_DecodeUTF8Ex(arg, strlen(arg), wstr, wlen, reason, + return _Py_DecodeUTF8Ex(ctx, arg, strlen(arg), wstr, wlen, reason, errors); } @@ -551,11 +552,11 @@ _Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen, if (force_ascii) { /* force ASCII encoding to workaround mbstowcs() issue */ - return decode_ascii(arg, wstr, wlen, reason, errors); + return decode_ascii(ctx, arg, wstr, wlen, reason, errors); } #endif - return decode_current_locale(arg, wstr, wlen, reason, errors); + return decode_current_locale(ctx, arg, wstr, wlen, reason, errors); #endif /* __APPLE__ or __ANDROID__ */ } @@ -580,10 +581,10 @@ _Py_DecodeLocaleEx(const char* arg, wchar_t **wstr, size_t *wlen, Use the Py_EncodeLocale() function to encode the character string back to a byte string. */ wchar_t* -Py_DecodeLocale(const char* arg, size_t *wlen) +_Py_DecodeLocaleCtx(const _PyConfigCtx *ctx, const char* arg, size_t *wlen) { wchar_t *wstr; - int res = _Py_DecodeLocaleEx(arg, &wstr, wlen, + int res = _Py_DecodeLocaleEx(ctx, arg, &wstr, wlen, NULL, 0, _Py_ERROR_SURROGATEESCAPE); if (res != 0) { @@ -596,9 +597,17 @@ Py_DecodeLocale(const char* arg, size_t *wlen) return wstr; } +wchar_t* +Py_DecodeLocale(const char* arg, size_t *wlen) +{ + _PyConfigCtx ctx; + _PyConfigCtx_Init(&ctx); + return _Py_DecodeLocaleCtx(&ctx, arg, wlen); +} + static int -encode_current_locale(const wchar_t *text, char **str, +encode_current_locale(const _PyConfigCtx *ctx, const wchar_t *text, char **str, size_t *error_pos, const char **reason, int raw_malloc, _Py_error_handler errors) { @@ -661,7 +670,7 @@ encode_current_locale(const wchar_t *text, char **str, size += 1; /* nul byte at the end */ if (raw_malloc) { - result = PyMem_RawMalloc(size); + result = _PyMem_RawMallocCtx(ctx, size); } else { result = PyMem_Malloc(size); @@ -676,7 +685,7 @@ encode_current_locale(const wchar_t *text, char **str, encode_error: if (raw_malloc) { - PyMem_RawFree(result); + _PyMem_RawFreeCtx(ctx, result); } else { PyMem_Free(result); @@ -709,30 +718,31 @@ encode_current_locale(const wchar_t *text, char **str, -3: the error handler 'errors' is not supported. */ static int -encode_locale_ex(const wchar_t *text, char **str, size_t *error_pos, +encode_locale_ex(const _PyConfigCtx *ctx, const wchar_t *text, + char **str, size_t *error_pos, const char **reason, int raw_malloc, int current_locale, _Py_error_handler errors) { if (current_locale) { #ifdef __ANDROID__ - return _Py_EncodeUTF8Ex(text, str, error_pos, reason, + return _Py_EncodeUTF8Ex(ctx, text, str, error_pos, reason, raw_malloc, errors); #else - return encode_current_locale(text, str, error_pos, reason, + return encode_current_locale(ctx, text, str, error_pos, reason, raw_malloc, errors); #endif } #if defined(__APPLE__) || defined(__ANDROID__) - return _Py_EncodeUTF8Ex(text, str, error_pos, reason, + return _Py_EncodeUTF8Ex(ctx, text, str, error_pos, reason, raw_malloc, errors); #else - int use_utf8 = (Py_UTF8Mode == 1); + int use_utf8 = (ctx->utf8_mode == 1); #ifdef MS_WINDOWS use_utf8 |= !Py_LegacyWindowsFSEncodingFlag; #endif if (use_utf8) { - return _Py_EncodeUTF8Ex(text, str, error_pos, reason, + return _Py_EncodeUTF8Ex(ctx, text, str, error_pos, reason, raw_malloc, errors); } @@ -742,12 +752,12 @@ encode_locale_ex(const wchar_t *text, char **str, size_t *error_pos, } if (force_ascii) { - return encode_ascii(text, str, error_pos, reason, + return encode_ascii(ctx, text, str, error_pos, reason, raw_malloc, errors); } #endif - return encode_current_locale(text, str, error_pos, reason, + return encode_current_locale(ctx, text, str, error_pos, reason, raw_malloc, errors); #endif /* __APPLE__ or __ANDROID__ */ } @@ -757,7 +767,10 @@ encode_locale(const wchar_t *text, size_t *error_pos, int raw_malloc, int current_locale) { char *str; - int res = encode_locale_ex(text, &str, error_pos, NULL, + _PyConfigCtx ctx; + + _PyConfigCtx_Init(&ctx); + int res = encode_locale_ex(&ctx, text, &str, error_pos, NULL, raw_malloc, current_locale, _Py_ERROR_SURROGATEESCAPE); if (res != -2 && error_pos) { @@ -798,11 +811,11 @@ _Py_EncodeLocaleRaw(const wchar_t *text, size_t *error_pos) int -_Py_EncodeLocaleEx(const wchar_t *text, char **str, +_Py_EncodeLocaleEx(const _PyConfigCtx *ctx, const wchar_t *text, char **str, size_t *error_pos, const char **reason, int current_locale, _Py_error_handler errors) { - return encode_locale_ex(text, str, error_pos, reason, 1, + return encode_locale_ex(ctx, text, str, error_pos, reason, 1, current_locale, errors); } diff --git a/Python/frozenmain.c b/Python/frozenmain.c index 616090965b1351..e9ed076bff634b 100644 --- a/Python/frozenmain.c +++ b/Python/frozenmain.c @@ -41,7 +41,8 @@ Py_FrozenMain(int argc, char **argv) } } - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); config._frozen = 1; /* Suppress errors from getpath.c */ if ((p = Py_GETENV("PYTHONINSPECT")) && *p != '\0') diff --git a/Python/pathconfig.c b/Python/pathconfig.c index 6a8688059825bf..d7583ba5e8fa94 100644 --- a/Python/pathconfig.c +++ b/Python/pathconfig.c @@ -13,14 +13,33 @@ extern "C" { #endif -_PyPathConfig _Py_path_config = _PyPathConfig_INIT; +/* static initializes all fields to 0/NULL */ +static _PyPathConfig global_path_config; +void +_PyPathConfig_StaticInit(_PyPathConfig *config) +{ + *config = _PyPathConfig_STATIC_INIT; + _PyConfigCtx_Init(&config->ctx); +} + + +static void +pathconfig_global_static_init(void) +{ + if (global_path_config.ctx.raw_alloc.malloc != NULL) { + /* Already initialized */ + return; + } + _PyPathConfig_StaticInit(&global_path_config); +} + static int -copy_wstr(wchar_t **dst, const wchar_t *src) +copy_wstr(const _PyConfigCtx *ctx, wchar_t **dst, const wchar_t *src) { if (src != NULL) { - *dst = _PyMem_RawWcsdup(src); + *dst = _PyMem_RawWcsdup(ctx, src); if (*dst == NULL) { return -1; } @@ -35,15 +54,9 @@ copy_wstr(wchar_t **dst, const wchar_t *src) static void _PyPathConfig_Clear(_PyPathConfig *config) { - /* _PyMem_SetDefaultAllocator() is needed to get a known memory allocator, - since Py_SetPath(), Py_SetPythonHome() and Py_SetProgramName() can be - called before Py_Initialize() which can changes the memory allocator. */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - #define CLEAR(ATTR) \ do { \ - PyMem_RawFree(ATTR); \ + _PyMem_RawFreeCtx(&config->ctx, ATTR); \ ATTR = NULL; \ } while (0) @@ -58,8 +71,6 @@ _PyPathConfig_Clear(_PyPathConfig *config) CLEAR(config->home); CLEAR(config->program_name); #undef CLEAR - - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); } @@ -69,10 +80,8 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, const _PyCoreConfig *core_config) { _PyInitError err; - _PyPathConfig new_config = _PyPathConfig_INIT; - - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _PyPathConfig new_config; + _PyPathConfig_StaticInit(&new_config); /* Calculate program_full_path, prefix, exec_prefix (Unix) or dll_path (Windows), and module_search_path */ @@ -82,11 +91,11 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, } /* Copy home and program_name from core_config */ - if (copy_wstr(&new_config.home, core_config->home) < 0) { + if (copy_wstr(&new_config.ctx, &new_config.home, core_config->home) < 0) { err = _Py_INIT_NO_MEMORY(); goto err; } - if (copy_wstr(&new_config.program_name, core_config->program_name) < 0) { + if (copy_wstr(&new_config.ctx, &new_config.program_name, core_config->program_name) < 0) { err = _Py_INIT_NO_MEMORY(); goto err; } @@ -94,14 +103,10 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, _PyPathConfig_Clear(path_config); *path_config = new_config; - err = _Py_INIT_OK(); - goto done; + return _Py_INIT_OK(); err: _PyPathConfig_Clear(&new_config); - -done: - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return err; } @@ -109,15 +114,18 @@ _PyPathConfig_Calculate(_PyPathConfig *path_config, _PyInitError _PyPathConfig_SetGlobal(const _PyPathConfig *config) { + pathconfig_global_static_init(); + _PyInitError err; - _PyPathConfig new_config = _PyPathConfig_INIT; + _PyPathConfig new_config; + _PyPathConfig_StaticInit(&new_config); PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); #define COPY_ATTR(ATTR) \ do { \ - if (copy_wstr(&new_config.ATTR, config->ATTR) < 0) { \ + if (copy_wstr(&new_config.ctx, &new_config.ATTR, config->ATTR) < 0) { \ _PyPathConfig_Clear(&new_config); \ err = _Py_INIT_NO_MEMORY(); \ goto done; \ @@ -135,9 +143,9 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config) COPY_ATTR(program_name); COPY_ATTR(home); - _PyPathConfig_Clear(&_Py_path_config); + _PyPathConfig_Clear(&global_path_config); /* Steal new_config strings; don't clear new_config */ - _Py_path_config = new_config; + global_path_config = new_config; err = _Py_INIT_OK(); @@ -146,11 +154,19 @@ _PyPathConfig_SetGlobal(const _PyPathConfig *config) return err; } +_PyPathConfig* +_PyPathConfig_GetGlobal(void) +{ + pathconfig_global_static_init(); + return &global_path_config; +} + void _PyPathConfig_ClearGlobal(void) { - _PyPathConfig_Clear(&_Py_path_config); + pathconfig_global_static_init(); + _PyPathConfig_Clear(&global_path_config); } @@ -193,7 +209,8 @@ _PyCoreConfig_SetPathConfig(const _PyCoreConfig *core_config) _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); _PyInitError err; - _PyPathConfig path_config = _PyPathConfig_INIT; + _PyPathConfig path_config; + _PyPathConfig_StaticInit(&path_config); path_config.module_search_path = wstrlist_join(DELIM, core_config->nmodule_search_path, @@ -202,25 +219,25 @@ _PyCoreConfig_SetPathConfig(const _PyCoreConfig *core_config) goto no_memory; } - if (copy_wstr(&path_config.program_full_path, core_config->executable) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.program_full_path, core_config->executable) < 0) { goto no_memory; } - if (copy_wstr(&path_config.prefix, core_config->prefix) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.prefix, core_config->prefix) < 0) { goto no_memory; } #ifdef MS_WINDOWS - if (copy_wstr(&path_config.dll_path, core_config->dll_path) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.dll_path, core_config->dll_path) < 0) { goto no_memory; } #else - if (copy_wstr(&path_config.exec_prefix, core_config->exec_prefix) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.exec_prefix, core_config->exec_prefix) < 0) { goto no_memory; } #endif - if (copy_wstr(&path_config.program_name, core_config->program_name) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.program_name, core_config->program_name) < 0) { goto no_memory; } - if (copy_wstr(&path_config.home, core_config->home) < 0) { + if (copy_wstr(&path_config.ctx, &path_config.home, core_config->home) < 0) { goto no_memory; } @@ -268,7 +285,8 @@ core_config_init_module_search_paths(_PyCoreConfig *config, memcpy(path, sys_path, path_len * sizeof(wchar_t)); path[path_len] = L'\0'; - _PyInitError err = _Py_wstrlist_append(&config->nmodule_search_path, + _PyInitError err = _Py_wstrlist_append(&config->ctx, + &config->nmodule_search_path, &config->module_search_paths, path); PyMem_RawFree(path); @@ -288,9 +306,10 @@ core_config_init_module_search_paths(_PyCoreConfig *config, static _PyInitError _PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config) { - _PyPathConfig path_config = _PyPathConfig_INIT; + _PyPathConfig path_config; _PyInitError err; + _PyPathConfig_StaticInit(&path_config); err = _PyPathConfig_Calculate(&path_config, config); if (_Py_INIT_FAILED(err)) { goto error; @@ -304,14 +323,15 @@ _PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config) } if (config->executable == NULL) { - if (copy_wstr(&config->executable, + if (copy_wstr(&config->ctx, + &config->executable, path_config.program_full_path) < 0) { goto no_memory; } } if (config->prefix == NULL) { - if (copy_wstr(&config->prefix, path_config.prefix) < 0) { + if (copy_wstr(&config->ctx, &config->prefix, path_config.prefix) < 0) { goto no_memory; } } @@ -322,14 +342,14 @@ _PyCoreConfig_CalculatePathConfig(_PyCoreConfig *config) #else wchar_t *exec_prefix = path_config.exec_prefix; #endif - if (copy_wstr(&config->exec_prefix, exec_prefix) < 0) { + if (copy_wstr(&config->ctx, &config->exec_prefix, exec_prefix) < 0) { goto no_memory; } } #ifdef MS_WINDOWS if (config->dll_path == NULL) { - if (copy_wstr(&config->dll_path, path_config.dll_path) < 0) { + if (copy_wstr(&config->ctx, &config->dll_path, path_config.dll_path) < 0) { goto no_memory; } } @@ -373,13 +393,13 @@ _PyCoreConfig_InitPathConfig(_PyCoreConfig *config) } if (config->base_prefix == NULL) { - if (copy_wstr(&config->base_prefix, config->prefix) < 0) { + if (copy_wstr(&config->ctx, &config->base_prefix, config->prefix) < 0) { return _Py_INIT_NO_MEMORY(); } } if (config->base_exec_prefix == NULL) { - if (copy_wstr(&config->base_exec_prefix, config->exec_prefix) < 0) { + if (copy_wstr(&config->ctx, &config->base_exec_prefix, config->exec_prefix) < 0) { return _Py_INIT_NO_MEMORY(); } } @@ -390,13 +410,16 @@ _PyCoreConfig_InitPathConfig(_PyCoreConfig *config) static void pathconfig_global_init(void) { - if (_Py_path_config.module_search_path != NULL) { + pathconfig_global_static_init(); + + if (global_path_config.module_search_path != NULL) { /* Already initialized */ return; } _PyInitError err; - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); err = _PyCoreConfig_Read(&config); if (_Py_INIT_FAILED(err)) { @@ -422,8 +445,10 @@ pathconfig_global_init(void) void Py_SetPath(const wchar_t *path) { + pathconfig_global_static_init(); + if (path == NULL) { - _PyPathConfig_Clear(&_Py_path_config); + _PyPathConfig_Clear(&global_path_config); return; } @@ -431,28 +456,29 @@ Py_SetPath(const wchar_t *path) _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); _PyPathConfig new_config; - new_config.program_full_path = _PyMem_RawWcsdup(Py_GetProgramName()); + _PyPathConfig_StaticInit(&new_config); + new_config.program_full_path = _PyMem_RawWcsdup(&new_config.ctx, Py_GetProgramName()); int alloc_error = (new_config.program_full_path == NULL); - new_config.prefix = _PyMem_RawWcsdup(L""); + new_config.prefix = _PyMem_RawWcsdup(&new_config.ctx, L""); alloc_error |= (new_config.prefix == NULL); #ifdef MS_WINDOWS - new_config.dll_path = _PyMem_RawWcsdup(L""); + new_config.dll_path = _PyMem_RawWcsdup(&new_config.ctx, L""); alloc_error |= (new_config.dll_path == NULL); #else - new_config.exec_prefix = _PyMem_RawWcsdup(L""); + new_config.exec_prefix = _PyMem_RawWcsdup(&new_config.ctx, L""); alloc_error |= (new_config.exec_prefix == NULL); #endif - new_config.module_search_path = _PyMem_RawWcsdup(path); + new_config.module_search_path = _PyMem_RawWcsdup(&new_config.ctx, path); alloc_error |= (new_config.module_search_path == NULL); /* steal the home and program_name values (to leave them unchanged) */ - new_config.home = _Py_path_config.home; - _Py_path_config.home = NULL; - new_config.program_name = _Py_path_config.program_name; - _Py_path_config.program_name = NULL; + new_config.home = global_path_config.home; + global_path_config.home = NULL; + new_config.program_name = global_path_config.program_name; + global_path_config.program_name = NULL; - _PyPathConfig_Clear(&_Py_path_config); - _Py_path_config = new_config; + _PyPathConfig_Clear(&global_path_config); + global_path_config = new_config; PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); @@ -465,6 +491,8 @@ Py_SetPath(const wchar_t *path) void Py_SetPythonHome(const wchar_t *home) { + pathconfig_global_static_init(); + if (home == NULL) { return; } @@ -472,12 +500,12 @@ Py_SetPythonHome(const wchar_t *home) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - PyMem_RawFree(_Py_path_config.home); - _Py_path_config.home = _PyMem_RawWcsdup(home); + _PyMem_RawFreeCtx(&global_path_config.ctx, global_path_config.home); + global_path_config.home = _PyMem_RawWcsdup(&global_path_config.ctx, home); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.home == NULL) { + if (global_path_config.home == NULL) { Py_FatalError("Py_SetPythonHome() failed: out of memory"); } } @@ -486,6 +514,8 @@ Py_SetPythonHome(const wchar_t *home) void Py_SetProgramName(const wchar_t *program_name) { + pathconfig_global_static_init(); + if (program_name == NULL || program_name[0] == L'\0') { return; } @@ -493,12 +523,12 @@ Py_SetProgramName(const wchar_t *program_name) PyMemAllocatorEx old_alloc; _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - PyMem_RawFree(_Py_path_config.program_name); - _Py_path_config.program_name = _PyMem_RawWcsdup(program_name); + _PyMem_RawFreeCtx(&global_path_config.ctx, global_path_config.program_name); + global_path_config.program_name = _PyMem_RawWcsdup(&global_path_config.ctx, program_name); PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - if (_Py_path_config.program_name == NULL) { + if (global_path_config.program_name == NULL) { Py_FatalError("Py_SetProgramName() failed: out of memory"); } } @@ -508,7 +538,7 @@ wchar_t * Py_GetPath(void) { pathconfig_global_init(); - return _Py_path_config.module_search_path; + return global_path_config.module_search_path; } @@ -516,7 +546,7 @@ wchar_t * Py_GetPrefix(void) { pathconfig_global_init(); - return _Py_path_config.prefix; + return global_path_config.prefix; } @@ -527,7 +557,7 @@ Py_GetExecPrefix(void) return Py_GetPrefix(); #else pathconfig_global_init(); - return _Py_path_config.exec_prefix; + return global_path_config.exec_prefix; #endif } @@ -536,7 +566,7 @@ wchar_t * Py_GetProgramFullPath(void) { pathconfig_global_init(); - return _Py_path_config.program_full_path; + return global_path_config.program_full_path; } @@ -544,7 +574,7 @@ wchar_t* Py_GetPythonHome(void) { pathconfig_global_init(); - return _Py_path_config.home; + return global_path_config.home; } @@ -552,7 +582,7 @@ wchar_t * Py_GetProgramName(void) { pathconfig_global_init(); - return _Py_path_config.program_name; + return global_path_config.program_name; } /* Compute argv[0] which will be prepended to sys.argv */ diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 58e16473100e3a..35a06a8ba4291a 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -575,6 +575,9 @@ _Py_InitializeCore_impl(PyInterpreterState **interp_p, } *interp_p = interp; + /* Update the memory allocator of the config and copy the new config */ + _PyCoreConfig_Clear(&interp->core_config); + PyMem_GetAllocator(PYMEM_DOMAIN_RAW, &interp->core_config.ctx.raw_alloc); if (_PyCoreConfig_Copy(&interp->core_config, core_config) < 0) { return _Py_INIT_ERR("failed to copy core config"); } @@ -706,7 +709,8 @@ _Py_InitializeCore(PyInterpreterState **interp_p, /* Copy the configuration, since _PyCoreConfig_Read() modifies it (and the input configuration is read only). */ - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + _PyCoreConfig_Init(&config); /* Set LC_CTYPE to the user preferred locale */ _Py_SetLocaleFromEnv(LC_CTYPE); @@ -898,7 +902,9 @@ Py_InitializeEx(int install_sigs) } _PyInitError err; - _PyCoreConfig config = _PyCoreConfig_INIT; + _PyCoreConfig config; + + _PyCoreConfig_Init(&config); config.install_signal_handlers = install_sigs; err = _Py_InitializeFromConfig(&config); diff --git a/Python/pystate.c b/Python/pystate.c index f86f5a96f07450..d44a4ebab34838 100644 --- a/Python/pystate.c +++ b/Python/pystate.c @@ -148,7 +148,7 @@ PyInterpreterState_New(void) interp->codec_error_registry = NULL; interp->codecs_initialized = 0; interp->fscodec_initialized = 0; - interp->core_config = _PyCoreConfig_INIT; + _PyCoreConfig_Init(&interp->core_config); interp->config = _PyMainInterpreterConfig_INIT; interp->importlib = NULL; interp->import_func = NULL; diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 2284e88d4c1180..c0526b097ed5fd 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1661,47 +1661,57 @@ struct _preinit_entry { typedef struct _preinit_entry *_Py_PreInitEntry; -static _Py_PreInitEntry _preinit_warnoptions = NULL; -static _Py_PreInitEntry _preinit_xoptions = NULL; +typedef struct { + _PyConfigCtx ctx; + _Py_PreInitEntry list; +} _Py_PreInit; + +/* "static" initializes all fields to 0/NULL */ +static _Py_PreInit _preinit_warnoptions; +static _Py_PreInit _preinit_xoptions; + +static void +preinit_static_init(_Py_PreInit *preinit) +{ + if (preinit->ctx.raw_alloc.malloc != NULL) { + /* Already initialized */ + return; + } + _PyConfigCtx_Init(&preinit->ctx); +} + static _Py_PreInitEntry -_alloc_preinit_entry(const wchar_t *value) +preinit_alloc(_Py_PreInit *preinit, const wchar_t *value) { /* To get this to work, we have to initialize the runtime implicitly */ _PyRuntime_Initialize(); - /* Force default allocator, so we can ensure that it also gets used to - * destroy the linked list in _clear_preinit_entries. - */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); - - _Py_PreInitEntry node = PyMem_RawCalloc(1, sizeof(*node)); + _Py_PreInitEntry node = _PyMem_RawCallocCtx(&preinit->ctx, 1, sizeof(*node)); if (node != NULL) { - node->value = _PyMem_RawWcsdup(value); + node->value = _PyMem_RawWcsdup(&preinit->ctx, value); if (node->value == NULL) { - PyMem_RawFree(node); + _PyMem_RawFreeCtx(&preinit->ctx, node); node = NULL; }; }; - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); return node; }; static int -_append_preinit_entry(_Py_PreInitEntry *optionlist, const wchar_t *value) +preinit_append(_Py_PreInit *preinit, const wchar_t *value) { - _Py_PreInitEntry new_entry = _alloc_preinit_entry(value); + _Py_PreInitEntry new_entry = preinit_alloc(preinit, value); if (new_entry == NULL) { return -1; } /* We maintain the linked list in this order so it's easy to play back * the add commands in the same order later on in _Py_InitializeCore */ - _Py_PreInitEntry last_entry = *optionlist; + _Py_PreInitEntry last_entry = preinit->list; if (last_entry == NULL) { - *optionlist = new_entry; + preinit->list = new_entry; } else { while (last_entry->next != NULL) { last_entry = last_entry->next; @@ -1712,50 +1722,44 @@ _append_preinit_entry(_Py_PreInitEntry *optionlist, const wchar_t *value) }; static void -_clear_preinit_entries(_Py_PreInitEntry *optionlist) +preinit_clear(_Py_PreInit *preinit) { - _Py_PreInitEntry current = *optionlist; - *optionlist = NULL; - /* Deallocate the nodes and their contents using the default allocator */ - PyMemAllocatorEx old_alloc; - _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc); + _Py_PreInitEntry current = preinit->list; + preinit->list = NULL; + while (current != NULL) { _Py_PreInitEntry next = current->next; - PyMem_RawFree(current->value); - PyMem_RawFree(current); + _PyMem_RawFreeCtx(&preinit->ctx, current->value); + _PyMem_RawFreeCtx(&preinit->ctx, current); current = next; } - PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc); }; -static void -_clear_all_preinit_options(void) -{ - _clear_preinit_entries(&_preinit_warnoptions); - _clear_preinit_entries(&_preinit_xoptions); -} - static int -_PySys_ReadPreInitOptions(void) +sys_read_preinit_options(void) { + preinit_static_init(&_preinit_warnoptions); + preinit_static_init(&_preinit_xoptions); + /* Rerun the add commands with the actual sys module available */ PyThreadState *tstate = _PyThreadState_GET(); if (tstate == NULL) { /* Still don't have a thread state, so something is wrong! */ return -1; } - _Py_PreInitEntry entry = _preinit_warnoptions; + _Py_PreInitEntry entry = _preinit_warnoptions.list; while (entry != NULL) { PySys_AddWarnOption(entry->value); entry = entry->next; } - entry = _preinit_xoptions; + entry = _preinit_xoptions.list; while (entry != NULL) { PySys_AddXOption(entry->value); entry = entry->next; } - _clear_all_preinit_options(); + preinit_clear(&_preinit_warnoptions); + preinit_clear(&_preinit_xoptions); return 0; }; @@ -1790,9 +1794,11 @@ get_warnoptions(void) void PySys_ResetWarnOptions(void) { + preinit_static_init(&_preinit_warnoptions); + PyThreadState *tstate = _PyThreadState_GET(); if (tstate == NULL) { - _clear_preinit_entries(&_preinit_warnoptions); + preinit_clear(&_preinit_warnoptions); return; } @@ -1829,9 +1835,11 @@ PySys_AddWarnOptionUnicode(PyObject *option) void PySys_AddWarnOption(const wchar_t *s) { + preinit_static_init(&_preinit_warnoptions); + PyThreadState *tstate = _PyThreadState_GET(); if (tstate == NULL) { - _append_preinit_entry(&_preinit_warnoptions, s); + preinit_append(&_preinit_warnoptions, s); return; } PyObject *unicode; @@ -1916,9 +1924,11 @@ _PySys_AddXOptionWithError(const wchar_t *s) void PySys_AddXOption(const wchar_t *s) { + preinit_static_init(&_preinit_xoptions); + PyThreadState *tstate = _PyThreadState_GET(); if (tstate == NULL) { - _append_preinit_entry(&_preinit_xoptions, s); + preinit_append(&_preinit_xoptions, s); return; } if (_PySys_AddXOptionWithError(s) < 0) { @@ -2101,7 +2111,7 @@ make_flags(void) SetFlag(config->use_hash_seed == 0 || config->hash_seed != 0); SetFlag(config->isolated); PyStructSequence_SET_ITEM(seq, pos++, PyBool_FromLong(config->dev_mode)); - SetFlag(config->utf8_mode); + SetFlag(config->ctx.utf8_mode); #undef SetFlag if (PyErr_Occurred()) { @@ -2555,8 +2565,9 @@ _PySys_EndInit(PyObject *sysdict, PyInterpreterState *interp) /* Transfer any sys.warnoptions and sys._xoptions set directly * by an embedding application from the linked list to the module. */ - if (_PySys_ReadPreInitOptions() != 0) + if (sys_read_preinit_options() < 0) { return -1; + } if (PyErr_Occurred()) return -1;