From 2475af9ee2022f14245f6caead3ffce15d11c0b2 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Fri, 12 Apr 2024 16:15:01 +0200 Subject: [PATCH 01/72] Speedup `posixpath.abspath()` for relative paths --- Include/internal/pycore_fileutils.h | 2 +- Lib/posixpath.py | 42 +++++++++++----- Modules/clinic/posixmodule.c.h | 75 ++++++++++++++++++++++++++++- Modules/posixmodule.c | 34 ++++++++++++- Python/fileutils.c | 9 +++- 5 files changed, 146 insertions(+), 16 deletions(-) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 5c55282fa39e6f..715e3349949fe0 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -279,7 +279,7 @@ extern size_t _Py_find_basename(const wchar_t *filename); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(wchar_t*) _Py_normpath(wchar_t *path, Py_ssize_t size); -extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *length); +extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *length); // The Windows Games API family does not provide these functions // so provide our own implementations. Remove them in case they get added diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 8fd49cdc358908..1307e0ed2d053c 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -385,17 +385,37 @@ def normpath(path): return _path_normpath(path) or "." -def abspath(path): - """Return an absolute path.""" - path = os.fspath(path) - if isinstance(path, bytes): - if not path.startswith(b'/'): - path = join(os.getcwdb(), path) - else: - if not path.startswith('/'): - path = join(os.getcwd(), path) - return normpath(path) - +try: + from posix import _path_abspath +except ImportError: + def abspath(path): + """Return an absolute path.""" + path = os.fspath(path) + if isinstance(path, bytes): + if not path.startswith(b'/'): + path = join(os.getcwdb(), path) + else: + if not path.startswith('/'): + path = join(os.getcwd(), path) + return normpath(path) +else: + def abspath(path): + """Return an absolute path.""" + path = os.fspath(path) + if isinstance(path, bytes): + if path.startswith(b'/'): + return os.fsencode(_path_abspath(os.fsdecode(path))) + else: + cwd = os.getcwdb() + path = join(cwd, path) + return os.fsencode(_path_abspath(os.fsdecode(path), len(cwd))) + else: + if path.startswith('/'): + return _path_abspath(path) + else: + cwd = os.getcwd() + path = join(cwd, path) + return _path_abspath(path, len(cwd)) # Return a canonical path (i.e. the absolute location of a file on the # filesystem). diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 0398629e3c10ce..9dd60f4b34f04b 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2303,6 +2303,79 @@ os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } +PyDoc_STRVAR(os__path_abspath__doc__, +"_path_abspath($module, /, path, start=0)\n" +"--\n" +"\n" +"Make path absolute."); + +#define OS__PATH_ABSPATH_METHODDEF \ + {"_path_abspath", _PyCFunction_CAST(os__path_abspath), METH_FASTCALL|METH_KEYWORDS, os__path_abspath__doc__}, + +static PyObject * +os__path_abspath_impl(PyObject *module, PyObject *path, Py_ssize_t start); + +static PyObject * +os__path_abspath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +{ + PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 2 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), &_Py_ID(start), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", "start", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_abspath", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[2]; + Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; + PyObject *path; + Py_ssize_t start = 0; + + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); + if (!args) { + goto exit; + } + path = args[0]; + if (!noptargs) { + goto skip_optional_pos; + } + { + Py_ssize_t ival = -1; + PyObject *iobj = _PyNumber_Index(args[1]); + if (iobj != NULL) { + ival = PyLong_AsSsize_t(iobj); + Py_DECREF(iobj); + } + if (ival == -1 && PyErr_Occurred()) { + goto exit; + } + start = ival; + } +skip_optional_pos: + return_value = os__path_abspath_impl(module, path, start); + +exit: + return return_value; +} + PyDoc_STRVAR(os_mkdir__doc__, "mkdir($module, /, path, mode=511, *, dir_fd=None)\n" "--\n" @@ -12602,4 +12675,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=511f0788a6b90db0 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9d08024d1d89c702 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 5e54cf64cd563e..1023e15cde7edd 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5490,12 +5490,43 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return NULL; } Py_ssize_t norm_len; - wchar_t *norm_path = _Py_normpath_and_size(buffer, len, &norm_len); + wchar_t *norm_path = _Py_normpath_and_size(buffer, len, 0, &norm_len); PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); PyMem_Free(buffer); return result; } +/*[clinic input] +os._path_abspath + + path: object + + start: Py_ssize_t=0 + +Make path absolute. +[clinic start generated code]*/ + +static PyObject * +os__path_abspath_impl(PyObject *module, PyObject *path, Py_ssize_t start) +/*[clinic end generated code: output=69e536dbe18ecf3a input=29df0995bc21a9cf]*/ +{ + if (!PyUnicode_Check(path)) { + PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", + Py_TYPE(path)->tp_name); + return NULL; + } + Py_ssize_t len; + wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); + if (!buffer) { + return NULL; + } + Py_ssize_t abs_len; + wchar_t *abs_path = _Py_normpath_and_size(buffer, len, start, &abs_len); + PyObject *result = PyUnicode_FromWideChar(abs_path, abs_len); + PyMem_Free(buffer); + return result; +} + /*[clinic input] os.mkdir @@ -16800,6 +16831,7 @@ static PyMethodDef posix_methods[] = { OS__GETVOLUMEPATHNAME_METHODDEF OS__PATH_SPLITROOT_METHODDEF OS__PATH_NORMPATH_METHODDEF + OS__PATH_ABSPATH_METHODDEF OS_GETLOADAVG_METHODDEF OS_URANDOM_METHODDEF OS_SETRESUID_METHODDEF diff --git a/Python/fileutils.c b/Python/fileutils.c index 882d3299575cf3..db464e67014c48 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2390,7 +2390,7 @@ _Py_find_basename(const wchar_t *filename) to be the end of the path. 'normsize' will be set to contain the length of the resulting normalized path. */ wchar_t * -_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) +_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *normsize) { assert(path != NULL); if ((size < 0 && !path[0]) || size == 0) { @@ -2455,6 +2455,11 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t *normsize) } #endif /* MS_WINDOWS */ + if (path + start > p1) { + p1 = p2 = path + start; + lastC = *(p1-1); + } + /* if pEnd is specified, check that. Else, check for null terminator */ for (; !IS_END(p1); ++p1) { wchar_t c = *p1; @@ -2522,7 +2527,7 @@ wchar_t * _Py_normpath(wchar_t *path, Py_ssize_t size) { Py_ssize_t norm_length; - return _Py_normpath_and_size(path, size, &norm_length); + return _Py_normpath_and_size(path, size, 0, &norm_length); } From 807b494220ac0372267892c5818c8d8a91a9e966 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Fri, 12 Apr 2024 17:29:51 +0200 Subject: [PATCH 02/72] Fix cwd length for byte paths & bus error --- Lib/posixpath.py | 6 +++--- Python/fileutils.c | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 1307e0ed2d053c..d9c2ba26aae93f 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -406,9 +406,9 @@ def abspath(path): if path.startswith(b'/'): return os.fsencode(_path_abspath(os.fsdecode(path))) else: - cwd = os.getcwdb() - path = join(cwd, path) - return os.fsencode(_path_abspath(os.fsdecode(path), len(cwd))) + cwd = os.fsdecode(os.getcwdb()) + path = join(cwd, os.fsdecode(path)) + return os.fsencode(_path_abspath(path, len(cwd))) else: if path.startswith('/'): return _path_abspath(path) diff --git a/Python/fileutils.c b/Python/fileutils.c index db464e67014c48..b9fade873119c6 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2397,6 +2397,11 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize *normsize = 0; return path; } + // Start beyond end of path + if (start >= size) { + *normsize = size; + return path; + } wchar_t *pEnd = size >= 0 ? &path[size] : NULL; wchar_t *p1 = path; // sequentially scanned address in the path wchar_t *p2 = path; // destination of a scanned character to be ljusted @@ -2455,11 +2460,11 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize } #endif /* MS_WINDOWS */ + // Skip past cwd if (path + start > p1) { p1 = p2 = path + start; lastC = *(p1-1); } - /* if pEnd is specified, check that. Else, check for null terminator */ for (; !IS_END(p1); ++p1) { wchar_t c = *p1; From fa50a4c2ec14fe7ff1233201379d6da48a2e362c Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Sat, 13 Apr 2024 18:00:19 +0000 Subject: [PATCH 03/72] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../2024-04-13-18-00-18.gh-issue-117587.52AB27.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-00-18.gh-issue-117587.52AB27.rst diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-00-18.gh-issue-117587.52AB27.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-00-18.gh-issue-117587.52AB27.rst new file mode 100644 index 00000000000000..d925bff7f13e6b --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-00-18.gh-issue-117587.52AB27.rst @@ -0,0 +1 @@ +Speedup :func:`os.path.abspath` for relative paths on Unix. From 716f5948d03822a21938fa06b40e115aa2a31d26 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Sat, 13 Apr 2024 21:31:17 +0200 Subject: [PATCH 04/72] Fix `_Py_normpath_and_size` when `size=-1` --- Python/fileutils.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/fileutils.c b/Python/fileutils.c index b9fade873119c6..89fea95dac18a1 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2398,7 +2398,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize return path; } // Start beyond end of path - if (start >= size) { + if (size >= 0 && start >= size) { *normsize = size; return path; } @@ -2460,8 +2460,8 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize } #endif /* MS_WINDOWS */ - // Skip past cwd - if (path + start > p1) { + // Skip past cwd, not allowed if size is unknown + if (size >= 0 && path + start > p1) { p1 = p2 = path + start; lastC = *(p1-1); } From cc3a59cd1b589a1b0948cc8de2c64fd6c9296ce8 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Sat, 13 Apr 2024 22:42:24 +0200 Subject: [PATCH 05/72] Update Modules/posixmodule.c Co-authored-by: Pieter Eendebak --- Modules/posixmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 1023e15cde7edd..c674fe026d861a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5503,7 +5503,7 @@ os._path_abspath start: Py_ssize_t=0 -Make path absolute. +Make path absolute from character start. [clinic start generated code]*/ static PyObject * From f5f2c7b752aa0e10b472e5742b79d94a87c3331f Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Sat, 13 Apr 2024 23:43:49 +0200 Subject: [PATCH 06/72] Update Python/fileutils.c Co-authored-by: Pieter Eendebak --- Python/fileutils.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Python/fileutils.c b/Python/fileutils.c index 89fea95dac18a1..fe3d3eef889b02 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2393,6 +2393,7 @@ wchar_t * _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *normsize) { assert(path != NULL); + assert(start>=0); if ((size < 0 && !path[0]) || size == 0) { *normsize = 0; return path; From a1bc125dc00936e4d4c401d098327826352b98dc Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sat, 13 Apr 2024 23:48:40 +0200 Subject: [PATCH 07/72] fastcall & unicode type Co-authored-by: Pieter Eendebak Co-authored-by: Erlend E. Aasland --- Lib/posixpath.py | 4 +- Modules/clinic/posixmodule.c.h | 88 +++++++--------------------------- Modules/posixmodule.c | 23 +++------ Python/fileutils.c | 5 +- 4 files changed, 29 insertions(+), 91 deletions(-) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index d9c2ba26aae93f..1cc8e985338e82 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -404,14 +404,14 @@ def abspath(path): path = os.fspath(path) if isinstance(path, bytes): if path.startswith(b'/'): - return os.fsencode(_path_abspath(os.fsdecode(path))) + return os.fsencode(_path_abspath(os.fsdecode(path), 0)) else: cwd = os.fsdecode(os.getcwdb()) path = join(cwd, os.fsdecode(path)) return os.fsencode(_path_abspath(path, len(cwd))) else: if path.startswith('/'): - return _path_abspath(path) + return _path_abspath(path, 0) else: cwd = os.getcwd() path = join(cwd, path) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 9dd60f4b34f04b..6d1b6548a32900 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2249,54 +2249,28 @@ os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #endif /* defined(MS_WINDOWS) */ PyDoc_STRVAR(os__path_normpath__doc__, -"_path_normpath($module, /, path)\n" +"_path_normpath($module, path, /)\n" "--\n" "\n" "Basic path normalization."); #define OS__PATH_NORMPATH_METHODDEF \ - {"_path_normpath", _PyCFunction_CAST(os__path_normpath), METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__}, + {"_path_normpath", (PyCFunction)os__path_normpath, METH_O, os__path_normpath__doc__}, static PyObject * os__path_normpath_impl(PyObject *module, PyObject *path); static PyObject * -os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_normpath(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 1 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"path", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "_path_normpath", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[1]; PyObject *path; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); - if (!args) { + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_path_normpath", "argument", "str", arg); goto exit; } - path = args[0]; + path = arg; return_value = os__path_normpath_impl(module, path); exit: @@ -2304,59 +2278,32 @@ os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO } PyDoc_STRVAR(os__path_abspath__doc__, -"_path_abspath($module, /, path, start=0)\n" +"_path_abspath($module, path, start, /)\n" "--\n" "\n" -"Make path absolute."); +"Make path absolute from character start."); #define OS__PATH_ABSPATH_METHODDEF \ - {"_path_abspath", _PyCFunction_CAST(os__path_abspath), METH_FASTCALL|METH_KEYWORDS, os__path_abspath__doc__}, + {"_path_abspath", _PyCFunction_CAST(os__path_abspath), METH_FASTCALL, os__path_abspath__doc__}, static PyObject * os__path_abspath_impl(PyObject *module, PyObject *path, Py_ssize_t start); static PyObject * -os__path_abspath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) +os__path_abspath(PyObject *module, PyObject *const *args, Py_ssize_t nargs) { PyObject *return_value = NULL; - #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) - - #define NUM_KEYWORDS 2 - static struct { - PyGC_Head _this_is_not_used; - PyObject_VAR_HEAD - PyObject *ob_item[NUM_KEYWORDS]; - } _kwtuple = { - .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) - .ob_item = { &_Py_ID(path), &_Py_ID(start), }, - }; - #undef NUM_KEYWORDS - #define KWTUPLE (&_kwtuple.ob_base.ob_base) - - #else // !Py_BUILD_CORE - # define KWTUPLE NULL - #endif // !Py_BUILD_CORE - - static const char * const _keywords[] = {"path", "start", NULL}; - static _PyArg_Parser _parser = { - .keywords = _keywords, - .fname = "_path_abspath", - .kwtuple = KWTUPLE, - }; - #undef KWTUPLE - PyObject *argsbuf[2]; - Py_ssize_t noptargs = nargs + (kwnames ? PyTuple_GET_SIZE(kwnames) : 0) - 1; PyObject *path; - Py_ssize_t start = 0; + Py_ssize_t start; - args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 2, 0, argsbuf); - if (!args) { + if (!_PyArg_CheckPositional("_path_abspath", nargs, 2, 2)) { goto exit; } - path = args[0]; - if (!noptargs) { - goto skip_optional_pos; + if (!PyUnicode_Check(args[0])) { + _PyArg_BadArgument("_path_abspath", "argument 1", "str", args[0]); + goto exit; } + path = args[0]; { Py_ssize_t ival = -1; PyObject *iobj = _PyNumber_Index(args[1]); @@ -2369,7 +2316,6 @@ os__path_abspath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyOb } start = ival; } -skip_optional_pos: return_value = os__path_abspath_impl(module, path, start); exit: @@ -12675,4 +12621,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=9d08024d1d89c702 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9fab569d9946cccb input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c674fe026d861a..724ad827bd5674 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5470,20 +5470,16 @@ os__path_islink_impl(PyObject *module, PyObject *path) /*[clinic input] os._path_normpath - path: object + path: unicode + / Basic path normalization. [clinic start generated code]*/ static PyObject * os__path_normpath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/ +/*[clinic end generated code: output=b94d696d828019da input=41e8ebf762cdbef5]*/ { - if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", - Py_TYPE(path)->tp_name); - return NULL; - } Py_ssize_t len; wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); if (!buffer) { @@ -5499,22 +5495,17 @@ os__path_normpath_impl(PyObject *module, PyObject *path) /*[clinic input] os._path_abspath - path: object - - start: Py_ssize_t=0 + path: unicode + start: Py_ssize_t + / Make path absolute from character start. [clinic start generated code]*/ static PyObject * os__path_abspath_impl(PyObject *module, PyObject *path, Py_ssize_t start) -/*[clinic end generated code: output=69e536dbe18ecf3a input=29df0995bc21a9cf]*/ +/*[clinic end generated code: output=69e536dbe18ecf3a input=ab404a44d616f24a]*/ { - if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", - Py_TYPE(path)->tp_name); - return NULL; - } Py_ssize_t len; wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); if (!buffer) { diff --git a/Python/fileutils.c b/Python/fileutils.c index fe3d3eef889b02..1502e0e223ce18 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2387,8 +2387,9 @@ _Py_find_basename(const wchar_t *filename) path, which will be within the original buffer. Guaranteed to not make the path longer, and will not fail. 'size' is the length of the path, if known. If -1, the first null character will be assumed - to be the end of the path. 'normsize' will be set to contain the - length of the resulting normalized path. */ + to be the end of the path. 'start' is the position where to start + normalizing. 'normsize' will be set to contain the length of the + resulting normalized path. */ wchar_t * _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *normsize) { From ae2c040716c162797a40f7f18ae53e31dc8c717a Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Sun, 14 Apr 2024 08:10:36 +0200 Subject: [PATCH 08/72] Revert changes to `os__path_normpath_impl` --- Modules/clinic/posixmodule.c.h | 40 ++++++++++++++++++++++++++++------ Modules/posixmodule.c | 10 ++++++--- 2 files changed, 40 insertions(+), 10 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 6d1b6548a32900..17a37317b25629 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2249,28 +2249,54 @@ os__path_islink(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObj #endif /* defined(MS_WINDOWS) */ PyDoc_STRVAR(os__path_normpath__doc__, -"_path_normpath($module, path, /)\n" +"_path_normpath($module, /, path)\n" "--\n" "\n" "Basic path normalization."); #define OS__PATH_NORMPATH_METHODDEF \ - {"_path_normpath", (PyCFunction)os__path_normpath, METH_O, os__path_normpath__doc__}, + {"_path_normpath", _PyCFunction_CAST(os__path_normpath), METH_FASTCALL|METH_KEYWORDS, os__path_normpath__doc__}, static PyObject * os__path_normpath_impl(PyObject *module, PyObject *path); static PyObject * -os__path_normpath(PyObject *module, PyObject *arg) +os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) { PyObject *return_value = NULL; + #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE) + + #define NUM_KEYWORDS 1 + static struct { + PyGC_Head _this_is_not_used; + PyObject_VAR_HEAD + PyObject *ob_item[NUM_KEYWORDS]; + } _kwtuple = { + .ob_base = PyVarObject_HEAD_INIT(&PyTuple_Type, NUM_KEYWORDS) + .ob_item = { &_Py_ID(path), }, + }; + #undef NUM_KEYWORDS + #define KWTUPLE (&_kwtuple.ob_base.ob_base) + + #else // !Py_BUILD_CORE + # define KWTUPLE NULL + #endif // !Py_BUILD_CORE + + static const char * const _keywords[] = {"path", NULL}; + static _PyArg_Parser _parser = { + .keywords = _keywords, + .fname = "_path_normpath", + .kwtuple = KWTUPLE, + }; + #undef KWTUPLE + PyObject *argsbuf[1]; PyObject *path; - if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("_path_normpath", "argument", "str", arg); + args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); + if (!args) { goto exit; } - path = arg; + path = args[0]; return_value = os__path_normpath_impl(module, path); exit: @@ -12621,4 +12647,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=9fab569d9946cccb input=a9049054013a1b77]*/ +/*[clinic end generated code: output=7d41b1bbfe849fe8 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 724ad827bd5674..91f5502221841d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5470,16 +5470,20 @@ os__path_islink_impl(PyObject *module, PyObject *path) /*[clinic input] os._path_normpath - path: unicode - / + path: object Basic path normalization. [clinic start generated code]*/ static PyObject * os__path_normpath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b94d696d828019da input=41e8ebf762cdbef5]*/ +/*[clinic end generated code: output=b94d696d828019da input=5e90c39e12549dc0]*/ { + if (!PyUnicode_Check(path)) { + PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", + Py_TYPE(path)->tp_name); + return NULL; + } Py_ssize_t len; wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); if (!buffer) { From 95773d957def6b93243359b73b93406c35893a5c Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Sun, 14 Apr 2024 18:42:08 +0200 Subject: [PATCH 09/72] simply `os.fsdecode(os.getcwdb())` --- Lib/posixpath.py | 2 +- Lib/test/test_posixpath.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 1cc8e985338e82..89f3f9aa5cca07 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -406,7 +406,7 @@ def abspath(path): if path.startswith(b'/'): return os.fsencode(_path_abspath(os.fsdecode(path), 0)) else: - cwd = os.fsdecode(os.getcwdb()) + cwd = os.getcwd() path = join(cwd, os.fsdecode(path)) return os.fsencode(_path_abspath(path, len(cwd))) else: diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 248fe2cc5d5ca8..873a374da147b8 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -648,6 +648,7 @@ def test_realpath_resolve_first(self): def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") + (real_getcwdb, os.getcwdb) = (os.getcwdb, lambda: br"/home/user/bar") try: curdir = os.path.split(os.getcwd())[-1] self.assertRaises(TypeError, posixpath.relpath, None) @@ -671,9 +672,10 @@ def test_relpath(self): self.assertEqual(posixpath.relpath("/a", "/a"), '.') self.assertEqual(posixpath.relpath("/a/b", "/a/b"), '.') finally: - os.getcwd = real_getcwd + (os.getcwd, os.getcwdb) = (real_getcwd, real_getcwdb) def test_relpath_bytes(self): + (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") (real_getcwdb, os.getcwdb) = (os.getcwdb, lambda: br"/home/user/bar") try: curdir = os.path.split(os.getcwdb())[-1] @@ -701,7 +703,7 @@ def test_relpath_bytes(self): self.assertRaises(TypeError, posixpath.relpath, b"bytes", "str") self.assertRaises(TypeError, posixpath.relpath, "str", b"bytes") finally: - os.getcwdb = real_getcwdb + (os.getcwd, os.getcwdb) = (real_getcwd, real_getcwdb) def test_commonpath(self): def check(paths, expected): From b77a15df0e21a1b676fb20822fc77a79f0f011be Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sun, 14 Apr 2024 21:25:25 +0200 Subject: [PATCH 10/72] 1st attempt at full C implementation --- Include/internal/pycore_fileutils.h | 2 +- Lib/posixpath.py | 15 +----- Modules/clinic/posixmodule.c.h | 48 ++++--------------- Modules/posixmodule.c | 73 ++++++++++++++++++++++++----- Python/fileutils.c | 2 +- 5 files changed, 75 insertions(+), 65 deletions(-) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 715e3349949fe0..f1aa0169e3f2b8 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -279,7 +279,7 @@ extern size_t _Py_find_basename(const wchar_t *filename); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(wchar_t*) _Py_normpath(wchar_t *path, Py_ssize_t size); -extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *length); +extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, size_t start, Py_ssize_t *length); // The Windows Games API family does not provide these functions // so provide our own implementations. Remove them in case they get added diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 1cc8e985338e82..b2ffae2e67eeb9 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -403,19 +403,8 @@ def abspath(path): """Return an absolute path.""" path = os.fspath(path) if isinstance(path, bytes): - if path.startswith(b'/'): - return os.fsencode(_path_abspath(os.fsdecode(path), 0)) - else: - cwd = os.fsdecode(os.getcwdb()) - path = join(cwd, os.fsdecode(path)) - return os.fsencode(_path_abspath(path, len(cwd))) - else: - if path.startswith('/'): - return _path_abspath(path, 0) - else: - cwd = os.getcwd() - path = join(cwd, path) - return _path_abspath(path, len(cwd)) + return os.fsencode(_path_abspath(os.fsdecode(path))) + return _path_abspath(path) # Return a canonical path (i.e. the absolute location of a file on the # filesystem). diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 17a37317b25629..ad581f5b0d3014 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2303,50 +2303,18 @@ os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } +#if !defined(MS_WINDOWS) + PyDoc_STRVAR(os__path_abspath__doc__, -"_path_abspath($module, path, start, /)\n" +"_path_abspath($module, path, /)\n" "--\n" "\n" "Make path absolute from character start."); #define OS__PATH_ABSPATH_METHODDEF \ - {"_path_abspath", _PyCFunction_CAST(os__path_abspath), METH_FASTCALL, os__path_abspath__doc__}, - -static PyObject * -os__path_abspath_impl(PyObject *module, PyObject *path, Py_ssize_t start); - -static PyObject * -os__path_abspath(PyObject *module, PyObject *const *args, Py_ssize_t nargs) -{ - PyObject *return_value = NULL; - PyObject *path; - Py_ssize_t start; - - if (!_PyArg_CheckPositional("_path_abspath", nargs, 2, 2)) { - goto exit; - } - if (!PyUnicode_Check(args[0])) { - _PyArg_BadArgument("_path_abspath", "argument 1", "str", args[0]); - goto exit; - } - path = args[0]; - { - Py_ssize_t ival = -1; - PyObject *iobj = _PyNumber_Index(args[1]); - if (iobj != NULL) { - ival = PyLong_AsSsize_t(iobj); - Py_DECREF(iobj); - } - if (ival == -1 && PyErr_Occurred()) { - goto exit; - } - start = ival; - } - return_value = os__path_abspath_impl(module, path, start); + {"_path_abspath", (PyCFunction)os__path_abspath, METH_O, os__path_abspath__doc__}, -exit: - return return_value; -} +#endif /* !defined(MS_WINDOWS) */ PyDoc_STRVAR(os_mkdir__doc__, "mkdir($module, /, path, mode=511, *, dir_fd=None)\n" @@ -12100,6 +12068,10 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS__PATH_ISLINK_METHODDEF #endif /* !defined(OS__PATH_ISLINK_METHODDEF) */ +#ifndef OS__PATH_ABSPATH_METHODDEF + #define OS__PATH_ABSPATH_METHODDEF +#endif /* !defined(OS__PATH_ABSPATH_METHODDEF) */ + #ifndef OS_NICE_METHODDEF #define OS_NICE_METHODDEF #endif /* !defined(OS_NICE_METHODDEF) */ @@ -12647,4 +12619,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=7d41b1bbfe849fe8 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=d11f54d12df1df85 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 91f5502221841d..05c005d4f1203c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5496,32 +5496,81 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return result; } +#ifndef MS_WINDOWS + /*[clinic input] os._path_abspath - path: unicode - start: Py_ssize_t + path: object / -Make path absolute from character start. +Make path absolute. [clinic start generated code]*/ static PyObject * -os__path_abspath_impl(PyObject *module, PyObject *path, Py_ssize_t start) -/*[clinic end generated code: output=69e536dbe18ecf3a input=ab404a44d616f24a]*/ +os__path_abspath(PyObject *module, PyObject *path) +/*[clinic end generated code: output=6a6bb40f1ebe86f2 input=21e4ac322da1a31e]*/ { - Py_ssize_t len; - wchar_t *buffer = PyUnicode_AsWideCharString(path, &len); - if (!buffer) { + if (!PyUnicode_Check(path)) { + PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", + Py_TYPE(path)->tp_name); return NULL; } - Py_ssize_t abs_len; - wchar_t *abs_path = _Py_normpath_and_size(buffer, len, start, &abs_len); - PyObject *result = PyUnicode_FromWideChar(abs_path, abs_len); - PyMem_Free(buffer); + + Py_ssize_t rel_path_len; + wchar_t *rel_path = PyUnicode_AsWideCharString(path, &rel_path_len); + if (!rel_path) { + return NULL; + } + + size_t cwd_len; + wchar_t *abs_path; + Py_ssize_t abs_path_len; + if (_Py_isabs(rel_path)) { + cwd_len = 0; + abs_path = rel_path; + abs_path_len = rel_path_len; + } else { + wchar_t cwd[MAXPATHLEN + 1]; + cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; + if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) { + /* unable to get the current directory */ + return posix_error(); + } + + cwd_len = wcslen(cwd); + abs_path_len = cwd_len + 1 + rel_path_len + 1; + if (abs_path_len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { + return posix_error(); + } + + abs_path = *PyMem_RawMalloc(abs_path_len * sizeof(wchar_t)); + if (!abs_path) { + return NULL; + } + + // Join cwd & path + wchar_t *abs_path_dup = abs_path; + memcpy(abs_path_dup, cwd, cwd_len * sizeof(wchar_t)); + abs_path_dup += cwd_len; + + *abs_path_dup = SEP; + abs_path_dup++; + + memcpy(abs_path_dup, rel_path, rel_path_len * sizeof(wchar_t)); + abs_path_dup += rel_path_len; + + *abs_path_dup = '\0'; + } + + abs_path = _Py_normpath_and_size(abs_path, abs_path_len, cwd_len, &abs_path_len); + PyObject *result = PyUnicode_FromWideChar(abs_path, abs_path_len); + PyMem_Free(rel_path); return result; } +#endif + /*[clinic input] os.mkdir diff --git a/Python/fileutils.c b/Python/fileutils.c index 1502e0e223ce18..9d629260b6023e 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2391,7 +2391,7 @@ _Py_find_basename(const wchar_t *filename) normalizing. 'normsize' will be set to contain the length of the resulting normalized path. */ wchar_t * -_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *normsize) +_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, size_t start, Py_ssize_t *normsize) { assert(path != NULL); assert(start>=0); From c63518bd4867cf829b70c2c777fa3e7007ff011f Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Sun, 14 Apr 2024 21:30:59 +0200 Subject: [PATCH 11/72] Update Modules/posixmodule.c Co-authored-by: Erlend E. Aasland --- Modules/posixmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 05c005d4f1203c..cbb9f5c4ec93c2 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5512,8 +5512,8 @@ os__path_abspath(PyObject *module, PyObject *path) /*[clinic end generated code: output=6a6bb40f1ebe86f2 input=21e4ac322da1a31e]*/ { if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, "expected 'str', not '%.200s'", - Py_TYPE(path)->tp_name); + PyErr_Format(PyExc_TypeError, "expected 'str', not %T", + Py_TYPE(path)); return NULL; } From 94b12c208a1c37df74ae79a21358af55559a5ddb Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sun, 14 Apr 2024 21:45:27 +0200 Subject: [PATCH 12/72] 2nd attempt at full C implementation --- Include/internal/pycore_fileutils.h | 2 +- Modules/posixmodule.c | 6 +++--- Python/fileutils.c | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index f1aa0169e3f2b8..715e3349949fe0 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -279,7 +279,7 @@ extern size_t _Py_find_basename(const wchar_t *filename); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(wchar_t*) _Py_normpath(wchar_t *path, Py_ssize_t size); -extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, size_t start, Py_ssize_t *length); +extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *length); // The Windows Games API family does not provide these functions // so provide our own implementations. Remove them in case they get added diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 05c005d4f1203c..4db25091d5f7a3 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -24,6 +24,7 @@ #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_signal.h" // Py_NSIG #include "pycore_time.h" // _PyLong_FromTime_t() +#include "osdefs.h" // SEP #ifdef HAVE_UNISTD_H # include // symlink() @@ -36,7 +37,6 @@ # endif # include # include // UNLEN -# include "osdefs.h" // SEP # if defined(MS_WINDOWS_DESKTOP) || defined(MS_WINDOWS_SYSTEM) # define HAVE_SYMLINK # endif /* MS_WINDOWS_DESKTOP | MS_WINDOWS_SYSTEM */ @@ -5523,7 +5523,7 @@ os__path_abspath(PyObject *module, PyObject *path) return NULL; } - size_t cwd_len; + Py_ssize_t cwd_len; wchar_t *abs_path; Py_ssize_t abs_path_len; if (_Py_isabs(rel_path)) { @@ -5544,7 +5544,7 @@ os__path_abspath(PyObject *module, PyObject *path) return posix_error(); } - abs_path = *PyMem_RawMalloc(abs_path_len * sizeof(wchar_t)); + abs_path = PyMem_RawMalloc(abs_path_len * sizeof(wchar_t)); if (!abs_path) { return NULL; } diff --git a/Python/fileutils.c b/Python/fileutils.c index 9d629260b6023e..1502e0e223ce18 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2391,7 +2391,7 @@ _Py_find_basename(const wchar_t *filename) normalizing. 'normsize' will be set to contain the length of the resulting normalized path. */ wchar_t * -_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, size_t start, Py_ssize_t *normsize) +_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *normsize) { assert(path != NULL); assert(start>=0); From 51c9541ef08926f29ce3623a6f2fa1a26261ac46 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sun, 14 Apr 2024 22:11:36 +0200 Subject: [PATCH 13/72] 3rd attempt at full C implementation --- Modules/clinic/posixmodule.c.h | 4 ++-- Modules/posixmodule.c | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index ad581f5b0d3014..677e0c2f51ba1e 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2309,7 +2309,7 @@ PyDoc_STRVAR(os__path_abspath__doc__, "_path_abspath($module, path, /)\n" "--\n" "\n" -"Make path absolute from character start."); +"Make path absolute."); #define OS__PATH_ABSPATH_METHODDEF \ {"_path_abspath", (PyCFunction)os__path_abspath, METH_O, os__path_abspath__doc__}, @@ -12619,4 +12619,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=d11f54d12df1df85 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=9391700f4021fd83 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 29ccfbbf5591bd..99ca7229f18d7a 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5509,7 +5509,7 @@ Make path absolute. static PyObject * os__path_abspath(PyObject *module, PyObject *path) -/*[clinic end generated code: output=6a6bb40f1ebe86f2 input=21e4ac322da1a31e]*/ +/*[clinic end generated code: output=6a6bb40f1ebe86f2 input=6f97dcc93c4bd71f]*/ { if (!PyUnicode_Check(path)) { PyErr_Format(PyExc_TypeError, "expected 'str', not %T", @@ -5539,12 +5539,12 @@ os__path_abspath(PyObject *module, PyObject *path) } cwd_len = wcslen(cwd); - abs_path_len = cwd_len + 1 + rel_path_len + 1; - if (abs_path_len > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { + abs_path_len = cwd_len + 1 + rel_path_len; + if (abs_path_len + 1 > PY_SSIZE_T_MAX / sizeof(wchar_t)) { return posix_error(); } - abs_path = PyMem_RawMalloc(abs_path_len * sizeof(wchar_t)); + abs_path = PyMem_RawMalloc((abs_path_len + 1) * sizeof(wchar_t)); if (!abs_path) { return NULL; } From ea0b4d1179832ff57212eb188c9450a14f1c910c Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sun, 14 Apr 2024 22:54:49 +0200 Subject: [PATCH 14/72] Fix tests --- Lib/test/test_posixpath.py | 94 ++++++++++++++++++-------------------- Modules/posixmodule.c | 2 +- 2 files changed, 45 insertions(+), 51 deletions(-) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 873a374da147b8..7f4aba2ae3cc88 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -646,64 +646,58 @@ def test_realpath_resolve_first(self): safe_rmdir(ABSTFN + "/k") safe_rmdir(ABSTFN) + @skip_if_ABSTFN_contains_backslash def test_relpath(self): - (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") - (real_getcwdb, os.getcwdb) = (os.getcwdb, lambda: br"/home/user/bar") - try: - curdir = os.path.split(os.getcwd())[-1] - self.assertRaises(TypeError, posixpath.relpath, None) - self.assertRaises(ValueError, posixpath.relpath, "") - self.assertEqual(posixpath.relpath("a"), "a") - self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") - self.assertEqual(posixpath.relpath("a/b"), "a/b") - self.assertEqual(posixpath.relpath("../a/b"), "../a/b") + self.assertRaises(TypeError, posixpath.relpath, None) + self.assertRaises(ValueError, posixpath.relpath, "") + self.assertEqual(posixpath.relpath("a"), "a") + self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") + self.assertEqual(posixpath.relpath("a/b"), "a/b") + self.assertEqual(posixpath.relpath("../a/b"), "../a/b") + with os_helper.change_cwd(ABSTFN): + curdir = basename(ABSTFN) self.assertEqual(posixpath.relpath("a", "../b"), "../"+curdir+"/a") self.assertEqual(posixpath.relpath("a/b", "../c"), "../"+curdir+"/a/b") - self.assertEqual(posixpath.relpath("a", "b/c"), "../../a") - self.assertEqual(posixpath.relpath("a", "a"), ".") - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/x/y/z"), '../../../foo/bar/bat') - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/foo/bar"), 'bat') - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/"), 'foo/bar/bat') - self.assertEqual(posixpath.relpath("/", "/foo/bar/bat"), '../../..') - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/x"), '../foo/bar/bat') - self.assertEqual(posixpath.relpath("/x", "/foo/bar/bat"), '../../../x') - self.assertEqual(posixpath.relpath("/", "/"), '.') - self.assertEqual(posixpath.relpath("/a", "/a"), '.') - self.assertEqual(posixpath.relpath("/a/b", "/a/b"), '.') - finally: - (os.getcwd, os.getcwdb) = (real_getcwd, real_getcwdb) + self.assertEqual(posixpath.relpath("a", "b/c"), "../../a") + self.assertEqual(posixpath.relpath("a", "a"), ".") + self.assertEqual(posixpath.relpath("/foo/bar/bat", "/x/y/z"), '../../../foo/bar/bat') + self.assertEqual(posixpath.relpath("/foo/bar/bat", "/foo/bar"), 'bat') + self.assertEqual(posixpath.relpath("/foo/bar/bat", "/"), 'foo/bar/bat') + self.assertEqual(posixpath.relpath("/", "/foo/bar/bat"), '../../..') + self.assertEqual(posixpath.relpath("/foo/bar/bat", "/x"), '../foo/bar/bat') + self.assertEqual(posixpath.relpath("/x", "/foo/bar/bat"), '../../../x') + self.assertEqual(posixpath.relpath("/", "/"), '.') + self.assertEqual(posixpath.relpath("/a", "/a"), '.') + self.assertEqual(posixpath.relpath("/a/b", "/a/b"), '.') + @skip_if_ABSTFN_contains_backslash def test_relpath_bytes(self): - (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") - (real_getcwdb, os.getcwdb) = (os.getcwdb, lambda: br"/home/user/bar") - try: - curdir = os.path.split(os.getcwdb())[-1] - self.assertRaises(ValueError, posixpath.relpath, b"") - self.assertEqual(posixpath.relpath(b"a"), b"a") - self.assertEqual(posixpath.relpath(posixpath.abspath(b"a")), b"a") - self.assertEqual(posixpath.relpath(b"a/b"), b"a/b") - self.assertEqual(posixpath.relpath(b"../a/b"), b"../a/b") + self.assertRaises(ValueError, posixpath.relpath, b"") + self.assertEqual(posixpath.relpath(b"a"), b"a") + self.assertEqual(posixpath.relpath(posixpath.abspath(b"a")), b"a") + self.assertEqual(posixpath.relpath(b"a/b"), b"a/b") + self.assertEqual(posixpath.relpath(b"../a/b"), b"../a/b") + with os_helper.change_cwd(ABSTFN): + curdir = os.fsencode(basename(ABSTFN)) self.assertEqual(posixpath.relpath(b"a", b"../b"), - b"../"+curdir+b"/a") + b"../"+curdir+b"/a") self.assertEqual(posixpath.relpath(b"a/b", b"../c"), - b"../"+curdir+b"/a/b") - self.assertEqual(posixpath.relpath(b"a", b"b/c"), b"../../a") - self.assertEqual(posixpath.relpath(b"a", b"a"), b".") - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/x/y/z"), b'../../../foo/bar/bat') - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/foo/bar"), b'bat') - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/"), b'foo/bar/bat') - self.assertEqual(posixpath.relpath(b"/", b"/foo/bar/bat"), b'../../..') - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/x"), b'../foo/bar/bat') - self.assertEqual(posixpath.relpath(b"/x", b"/foo/bar/bat"), b'../../../x') - self.assertEqual(posixpath.relpath(b"/", b"/"), b'.') - self.assertEqual(posixpath.relpath(b"/a", b"/a"), b'.') - self.assertEqual(posixpath.relpath(b"/a/b", b"/a/b"), b'.') - - self.assertRaises(TypeError, posixpath.relpath, b"bytes", "str") - self.assertRaises(TypeError, posixpath.relpath, "str", b"bytes") - finally: - (os.getcwd, os.getcwdb) = (real_getcwd, real_getcwdb) + b"../"+curdir+b"/a/b") + self.assertEqual(posixpath.relpath(b"a", b"b/c"), b"../../a") + self.assertEqual(posixpath.relpath(b"a", b"a"), b".") + self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/x/y/z"), b'../../../foo/bar/bat') + self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/foo/bar"), b'bat') + self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/"), b'foo/bar/bat') + self.assertEqual(posixpath.relpath(b"/", b"/foo/bar/bat"), b'../../..') + self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/x"), b'../foo/bar/bat') + self.assertEqual(posixpath.relpath(b"/x", b"/foo/bar/bat"), b'../../../x') + self.assertEqual(posixpath.relpath(b"/", b"/"), b'.') + self.assertEqual(posixpath.relpath(b"/a", b"/a"), b'.') + self.assertEqual(posixpath.relpath(b"/a/b", b"/a/b"), b'.') + + self.assertRaises(TypeError, posixpath.relpath, b"bytes", "str") + self.assertRaises(TypeError, posixpath.relpath, "str", b"bytes") def test_commonpath(self): def check(paths, expected): diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 99ca7229f18d7a..f3845e569020da 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5540,7 +5540,7 @@ os__path_abspath(PyObject *module, PyObject *path) cwd_len = wcslen(cwd); abs_path_len = cwd_len + 1 + rel_path_len; - if (abs_path_len + 1 > PY_SSIZE_T_MAX / sizeof(wchar_t)) { + if (abs_path_len + 1 > (Py_ssize_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { return posix_error(); } From 8d435d69da055caa2c5c9da47a626deda02d13a7 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sun, 14 Apr 2024 23:18:00 +0200 Subject: [PATCH 15/72] Fix tests attempt 2 --- Lib/test/test_posixpath.py | 102 ++++++++++++++++++++----------------- Modules/posixmodule.c | 4 +- 2 files changed, 57 insertions(+), 49 deletions(-) diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 7f4aba2ae3cc88..e3fb227953658b 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -648,56 +648,64 @@ def test_realpath_resolve_first(self): @skip_if_ABSTFN_contains_backslash def test_relpath(self): - self.assertRaises(TypeError, posixpath.relpath, None) - self.assertRaises(ValueError, posixpath.relpath, "") - self.assertEqual(posixpath.relpath("a"), "a") - self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") - self.assertEqual(posixpath.relpath("a/b"), "a/b") - self.assertEqual(posixpath.relpath("../a/b"), "../a/b") - with os_helper.change_cwd(ABSTFN): - curdir = basename(ABSTFN) - self.assertEqual(posixpath.relpath("a", "../b"), "../"+curdir+"/a") - self.assertEqual(posixpath.relpath("a/b", "../c"), - "../"+curdir+"/a/b") - self.assertEqual(posixpath.relpath("a", "b/c"), "../../a") - self.assertEqual(posixpath.relpath("a", "a"), ".") - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/x/y/z"), '../../../foo/bar/bat') - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/foo/bar"), 'bat') - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/"), 'foo/bar/bat') - self.assertEqual(posixpath.relpath("/", "/foo/bar/bat"), '../../..') - self.assertEqual(posixpath.relpath("/foo/bar/bat", "/x"), '../foo/bar/bat') - self.assertEqual(posixpath.relpath("/x", "/foo/bar/bat"), '../../../x') - self.assertEqual(posixpath.relpath("/", "/"), '.') - self.assertEqual(posixpath.relpath("/a", "/a"), '.') - self.assertEqual(posixpath.relpath("/a/b", "/a/b"), '.') + try: + os.mkdir(ABSTFN) + self.assertRaises(TypeError, posixpath.relpath, None) + self.assertRaises(ValueError, posixpath.relpath, "") + self.assertEqual(posixpath.relpath("a"), "a") + self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") + self.assertEqual(posixpath.relpath("a/b"), "a/b") + self.assertEqual(posixpath.relpath("../a/b"), "../a/b") + with os_helper.change_cwd(ABSTFN): + curdir = basename(ABSTFN) + self.assertEqual(posixpath.relpath("a", "../b"), "../"+curdir+"/a") + self.assertEqual(posixpath.relpath("a/b", "../c"), + "../"+curdir+"/a/b") + self.assertEqual(posixpath.relpath("a", "b/c"), "../../a") + self.assertEqual(posixpath.relpath("a", "a"), ".") + self.assertEqual(posixpath.relpath("/foo/bar/bat", "/x/y/z"), '../../../foo/bar/bat') + self.assertEqual(posixpath.relpath("/foo/bar/bat", "/foo/bar"), 'bat') + self.assertEqual(posixpath.relpath("/foo/bar/bat", "/"), 'foo/bar/bat') + self.assertEqual(posixpath.relpath("/", "/foo/bar/bat"), '../../..') + self.assertEqual(posixpath.relpath("/foo/bar/bat", "/x"), '../foo/bar/bat') + self.assertEqual(posixpath.relpath("/x", "/foo/bar/bat"), '../../../x') + self.assertEqual(posixpath.relpath("/", "/"), '.') + self.assertEqual(posixpath.relpath("/a", "/a"), '.') + self.assertEqual(posixpath.relpath("/a/b", "/a/b"), '.') + finally: + safe_rmdir(ABSTFN) @skip_if_ABSTFN_contains_backslash def test_relpath_bytes(self): - self.assertRaises(ValueError, posixpath.relpath, b"") - self.assertEqual(posixpath.relpath(b"a"), b"a") - self.assertEqual(posixpath.relpath(posixpath.abspath(b"a")), b"a") - self.assertEqual(posixpath.relpath(b"a/b"), b"a/b") - self.assertEqual(posixpath.relpath(b"../a/b"), b"../a/b") - with os_helper.change_cwd(ABSTFN): - curdir = os.fsencode(basename(ABSTFN)) - self.assertEqual(posixpath.relpath(b"a", b"../b"), - b"../"+curdir+b"/a") - self.assertEqual(posixpath.relpath(b"a/b", b"../c"), - b"../"+curdir+b"/a/b") - self.assertEqual(posixpath.relpath(b"a", b"b/c"), b"../../a") - self.assertEqual(posixpath.relpath(b"a", b"a"), b".") - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/x/y/z"), b'../../../foo/bar/bat') - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/foo/bar"), b'bat') - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/"), b'foo/bar/bat') - self.assertEqual(posixpath.relpath(b"/", b"/foo/bar/bat"), b'../../..') - self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/x"), b'../foo/bar/bat') - self.assertEqual(posixpath.relpath(b"/x", b"/foo/bar/bat"), b'../../../x') - self.assertEqual(posixpath.relpath(b"/", b"/"), b'.') - self.assertEqual(posixpath.relpath(b"/a", b"/a"), b'.') - self.assertEqual(posixpath.relpath(b"/a/b", b"/a/b"), b'.') - - self.assertRaises(TypeError, posixpath.relpath, b"bytes", "str") - self.assertRaises(TypeError, posixpath.relpath, "str", b"bytes") + try: + os.mkdir(ABSTFN) + self.assertRaises(ValueError, posixpath.relpath, b"") + self.assertEqual(posixpath.relpath(b"a"), b"a") + self.assertEqual(posixpath.relpath(posixpath.abspath(b"a")), b"a") + self.assertEqual(posixpath.relpath(b"a/b"), b"a/b") + self.assertEqual(posixpath.relpath(b"../a/b"), b"../a/b") + with os_helper.change_cwd(ABSTFN): + curdir = os.fsencode(basename(ABSTFN)) + self.assertEqual(posixpath.relpath(b"a", b"../b"), + b"../"+curdir+b"/a") + self.assertEqual(posixpath.relpath(b"a/b", b"../c"), + b"../"+curdir+b"/a/b") + self.assertEqual(posixpath.relpath(b"a", b"b/c"), b"../../a") + self.assertEqual(posixpath.relpath(b"a", b"a"), b".") + self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/x/y/z"), b'../../../foo/bar/bat') + self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/foo/bar"), b'bat') + self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/"), b'foo/bar/bat') + self.assertEqual(posixpath.relpath(b"/", b"/foo/bar/bat"), b'../../..') + self.assertEqual(posixpath.relpath(b"/foo/bar/bat", b"/x"), b'../foo/bar/bat') + self.assertEqual(posixpath.relpath(b"/x", b"/foo/bar/bat"), b'../../../x') + self.assertEqual(posixpath.relpath(b"/", b"/"), b'.') + self.assertEqual(posixpath.relpath(b"/a", b"/a"), b'.') + self.assertEqual(posixpath.relpath(b"/a/b", b"/a/b"), b'.') + + self.assertRaises(TypeError, posixpath.relpath, b"bytes", "str") + self.assertRaises(TypeError, posixpath.relpath, "str", b"bytes") + finally: + safe_rmdir(ABSTFN) def test_commonpath(self): def check(paths, expected): diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f3845e569020da..32846a1da2feb0 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5540,11 +5540,11 @@ os__path_abspath(PyObject *module, PyObject *path) cwd_len = wcslen(cwd); abs_path_len = cwd_len + 1 + rel_path_len; - if (abs_path_len + 1 > (Py_ssize_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { + if ((size_t)abs_path_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { return posix_error(); } - abs_path = PyMem_RawMalloc((abs_path_len + 1) * sizeof(wchar_t)); + abs_path = PyMem_RawMalloc(((size_t)abs_path_len + 1) * sizeof(wchar_t)); if (!abs_path) { return NULL; } From 7057f3b0ec80718336d692a5ad6efac661939826 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Mon, 15 Apr 2024 08:55:52 +0200 Subject: [PATCH 16/72] special case for '' & '.' --- Modules/posixmodule.c | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 32846a1da2feb0..bcf231d1322ddf 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5523,6 +5523,10 @@ os__path_abspath(PyObject *module, PyObject *path) return NULL; } + if (rel_path[0] == '\0' || !wcscmp(rel_path, L".")) { + return posix_getcwd(0); + } + Py_ssize_t cwd_len; wchar_t *abs_path; Py_ssize_t abs_path_len; @@ -5539,7 +5543,8 @@ os__path_abspath(PyObject *module, PyObject *path) } cwd_len = wcslen(cwd); - abs_path_len = cwd_len + 1 + rel_path_len; + int add_sep = cwd[cwd_len - 1] != SEP; + abs_path_len = cwd_len + add_sep + rel_path_len; if ((size_t)abs_path_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { return posix_error(); } @@ -5554,8 +5559,10 @@ os__path_abspath(PyObject *module, PyObject *path) memcpy(abs_path_dup, cwd, cwd_len * sizeof(wchar_t)); abs_path_dup += cwd_len; - *abs_path_dup = SEP; - abs_path_dup++; + if (add_sep) { + *abs_path_dup = SEP; + abs_path_dup++; + } memcpy(abs_path_dup, rel_path, rel_path_len * sizeof(wchar_t)); abs_path_dup += rel_path_len; From fff561e70f14ce57aa3c2b8d2c3df7325e2ea0bd Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Mon, 15 Apr 2024 09:27:35 +0200 Subject: [PATCH 17/72] follow pep 7 --- Modules/posixmodule.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index bcf231d1322ddf..81fcd1a1c4c9da 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5534,7 +5534,8 @@ os__path_abspath(PyObject *module, PyObject *path) cwd_len = 0; abs_path = rel_path; abs_path_len = rel_path_len; - } else { + } + else { wchar_t cwd[MAXPATHLEN + 1]; cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) { From 15c8f0de8919a42d6f07db1e99b0c69c85dec77e Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Mon, 15 Apr 2024 16:34:11 +0200 Subject: [PATCH 18/72] check type with argument clinic --- Modules/clinic/posixmodule.c.h | 22 +++++++++++++++++++++- Modules/posixmodule.c | 12 +++--------- 2 files changed, 24 insertions(+), 10 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 677e0c2f51ba1e..85e6d2c0b23d2c 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2314,6 +2314,26 @@ PyDoc_STRVAR(os__path_abspath__doc__, #define OS__PATH_ABSPATH_METHODDEF \ {"_path_abspath", (PyCFunction)os__path_abspath, METH_O, os__path_abspath__doc__}, +static PyObject * +os__path_abspath_impl(PyObject *module, PyObject *path); + +static PyObject * +os__path_abspath(PyObject *module, PyObject *arg) +{ + PyObject *return_value = NULL; + PyObject *path; + + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_path_abspath", "argument", "str", arg); + goto exit; + } + path = arg; + return_value = os__path_abspath_impl(module, path); + +exit: + return return_value; +} + #endif /* !defined(MS_WINDOWS) */ PyDoc_STRVAR(os_mkdir__doc__, @@ -12619,4 +12639,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=9391700f4021fd83 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e984e2fa69e44555 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 81fcd1a1c4c9da..0c7f3a31d55e95 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5501,22 +5501,16 @@ os__path_normpath_impl(PyObject *module, PyObject *path) /*[clinic input] os._path_abspath - path: object + path: unicode / Make path absolute. [clinic start generated code]*/ static PyObject * -os__path_abspath(PyObject *module, PyObject *path) -/*[clinic end generated code: output=6a6bb40f1ebe86f2 input=6f97dcc93c4bd71f]*/ +os__path_abspath_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ { - if (!PyUnicode_Check(path)) { - PyErr_Format(PyExc_TypeError, "expected 'str', not %T", - Py_TYPE(path)); - return NULL; - } - Py_ssize_t rel_path_len; wchar_t *rel_path = PyUnicode_AsWideCharString(path, &rel_path_len); if (!rel_path) { From 34858a82ff1caeef001cd7358ff7ab767a25e1f6 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Tue, 16 Apr 2024 08:49:46 +0200 Subject: [PATCH 19/72] Fix memory leak & variable names Co-authored-by: Eryk Sun --- Modules/posixmodule.c | 66 +++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 30 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 0c7f3a31d55e95..a8ad0b959a031d 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5511,63 +5511,69 @@ static PyObject * os__path_abspath_impl(PyObject *module, PyObject *path) /*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ { - Py_ssize_t rel_path_len; - wchar_t *rel_path = PyUnicode_AsWideCharString(path, &rel_path_len); - if (!rel_path) { - return NULL; + Py_ssize_t path_buf_len; + wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_buf_len); + wchar_t *path_buf2 = NULL; + PyObject *result = NULL; + if (!path_buf) { + goto exit; } - if (rel_path[0] == '\0' || !wcscmp(rel_path, L".")) { - return posix_getcwd(0); + if (path_buf[0] == '\0' || !wcscmp(path_buf, L".")) { + result = posix_getcwd(0); + goto exit; } - Py_ssize_t cwd_len; wchar_t *abs_path; Py_ssize_t abs_path_len; - if (_Py_isabs(rel_path)) { - cwd_len = 0; - abs_path = rel_path; - abs_path_len = rel_path_len; + if (_Py_isabs(path_buf)) { + abs_path = _Py_normpath_and_size(path_buf, path_buf_len, 0, &abs_path_len); } else { wchar_t cwd[MAXPATHLEN + 1]; cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) { /* unable to get the current directory */ - return posix_error(); + result = posix_error(); + goto exit; } - cwd_len = wcslen(cwd); + Py_ssize_t cwd_len = wcslen(cwd); int add_sep = cwd[cwd_len - 1] != SEP; - abs_path_len = cwd_len + add_sep + rel_path_len; - if ((size_t)abs_path_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { - return posix_error(); + Py_ssize_t path_buf2_len = cwd_len + add_sep + path_buf_len; + if ((size_t)path_buf2_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { + result = posix_error(); + goto exit; } - abs_path = PyMem_RawMalloc(((size_t)abs_path_len + 1) * sizeof(wchar_t)); - if (!abs_path) { - return NULL; + path_buf2 = PyMem_RawMalloc(((size_t)path_buf2_len + 1) * sizeof(wchar_t)); + if (!path_buf2) { + goto exit; } // Join cwd & path - wchar_t *abs_path_dup = abs_path; - memcpy(abs_path_dup, cwd, cwd_len * sizeof(wchar_t)); - abs_path_dup += cwd_len; + wchar_t *p = path_buf2; + memcpy(p, cwd, cwd_len * sizeof(wchar_t)); + p += cwd_len; if (add_sep) { - *abs_path_dup = SEP; - abs_path_dup++; + *p = SEP; + p++; } - memcpy(abs_path_dup, rel_path, rel_path_len * sizeof(wchar_t)); - abs_path_dup += rel_path_len; + memcpy(p, path_buf, path_buf_len * sizeof(wchar_t)); + p += path_buf_len; - *abs_path_dup = '\0'; + *p = '\0'; + abs_path = _Py_normpath_and_size(path_buf2, path_buf2_len, cwd_len + add_sep, &abs_path_len); } - abs_path = _Py_normpath_and_size(abs_path, abs_path_len, cwd_len, &abs_path_len); - PyObject *result = PyUnicode_FromWideChar(abs_path, abs_path_len); - PyMem_Free(rel_path); + result = PyUnicode_FromWideChar(abs_path, abs_path_len); +exit: + if (path_buf) + PyMem_Free(path_buf); + if (path_buf2) + PyMem_Free(path_buf2); return result; } From 21cd24a2b3d5137fb22877a8f0c5ab1c4318815b Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Tue, 16 Apr 2024 09:12:20 +0200 Subject: [PATCH 20/72] Use `PyMem_RawFree` --- Modules/posixmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index a8ad0b959a031d..8d5a802c3592c8 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5573,7 +5573,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) if (path_buf) PyMem_Free(path_buf); if (path_buf2) - PyMem_Free(path_buf2); + PyMem_RawFree(path_buf2); return result; } From 1113ec347ee422511e7c49ce492943c4dc55cd00 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 16 Apr 2024 19:26:29 +0200 Subject: [PATCH 21/72] Update Modules/posixmodule.c Co-authored-by: Eryk Sun --- Modules/posixmodule.c | 75 +++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 34 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 8d5a802c3592c8..64a18191cc2c8f 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5511,69 +5511,76 @@ static PyObject * os__path_abspath_impl(PyObject *module, PyObject *path) /*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ { - Py_ssize_t path_buf_len; - wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_buf_len); - wchar_t *path_buf2 = NULL; + Py_ssize_t path_len, start, abs_len, cwd_len; + wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; PyObject *result = NULL; + + wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); if (!path_buf) { goto exit; } - if (path_buf[0] == '\0' || !wcscmp(path_buf, L".")) { + if (path_len == 0 || !wcscmp(path_buf, L".")) { result = posix_getcwd(0); goto exit; } - wchar_t *abs_path; - Py_ssize_t abs_path_len; if (_Py_isabs(path_buf)) { - abs_path = _Py_normpath_and_size(path_buf, path_buf_len, 0, &abs_path_len); + start = 0; + abs_len = path_len; + abs = path_buf; } else { - wchar_t cwd[MAXPATHLEN + 1]; - cwd[Py_ARRAY_LENGTH(cwd) - 1] = 0; - if (!_Py_wgetcwd(cwd, Py_ARRAY_LENGTH(cwd) - 1)) { - /* unable to get the current directory */ - result = posix_error(); + PyObject *cwd_obj = posix_getcwd(0); + if (!cwd_obj) { + goto exit; + } + cwd_buf = PyUnicode_AsWideCharString(cwd_obj, &cwd_len); + Py_DECREF(cwd_obj); + if (!cwd_buf) { goto exit; } - Py_ssize_t cwd_len = wcslen(cwd); - int add_sep = cwd[cwd_len - 1] != SEP; - Py_ssize_t path_buf2_len = cwd_len + add_sep + path_buf_len; - if ((size_t)path_buf2_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { - result = posix_error(); + int add_sep = cwd_buf[cwd_len - 1] != SEP; + start = cwd_len + add_sep; + abs_len = cwd_len + add_sep + path_len; + + if ((size_t)abs_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { + PyErr_SetString(PyExc_OverflowError, "path is too long"); goto exit; } - path_buf2 = PyMem_RawMalloc(((size_t)path_buf2_len + 1) * sizeof(wchar_t)); - if (!path_buf2) { + abs_buf = PyMem_RawMalloc(((size_t)abs_len + 1) * sizeof(wchar_t)); + if (!abs_buf) { + PyErr_NoMemory(); goto exit; } + abs = abs_buf; // Join cwd & path - wchar_t *p = path_buf2; - memcpy(p, cwd, cwd_len * sizeof(wchar_t)); + wchar_t *p = abs_buf; + memcpy(p, cwd_buf, cwd_len * sizeof(wchar_t)); p += cwd_len; - if (add_sep) { - *p = SEP; - p++; + *(p++) = SEP; } - - memcpy(p, path_buf, path_buf_len * sizeof(wchar_t)); - p += path_buf_len; - - *p = '\0'; - abs_path = _Py_normpath_and_size(path_buf2, path_buf2_len, cwd_len + add_sep, &abs_path_len); + memcpy(p, path_buf, path_len * sizeof(wchar_t)); + p[path_len] = '\0'; } - result = PyUnicode_FromWideChar(abs_path, abs_path_len); + abs = _Py_normpath_and_size(abs, abs_len, start, &abs_len); + result = PyUnicode_FromWideChar(abs, abs_len); + exit: - if (path_buf) + if (path_buf) { PyMem_Free(path_buf); - if (path_buf2) - PyMem_RawFree(path_buf2); + } + if (cwd_buf) { + PyMem_Free(cwd_buf); + } + if (abs_buf) { + PyMem_RawFree(abs_buf); + } return result; } From 7cca80efc2bbda2db86d19f9035eb43e6a7d2883 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 16 Apr 2024 19:31:19 +0200 Subject: [PATCH 22/72] Update Modules/posixmodule.c Co-authored-by: Eryk Sun --- Modules/posixmodule.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 64a18191cc2c8f..ae933f20dd7dbb 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5520,7 +5520,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) goto exit; } - if (path_len == 0 || !wcscmp(path_buf, L".")) { + if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { result = posix_getcwd(0); goto exit; } From 7b6727c66593e74d3344f31ffafe46b172d5cf2c Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Wed, 17 Apr 2024 11:26:01 +0200 Subject: [PATCH 23/72] Update Modules/posixmodule.c Co-authored-by: Erlend E. Aasland --- Modules/posixmodule.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index ae933f20dd7dbb..4f5672a9269230 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5558,8 +5558,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) abs = abs_buf; // Join cwd & path - wchar_t *p = abs_buf; - memcpy(p, cwd_buf, cwd_len * sizeof(wchar_t)); + wchar_t *p = memcpy(abs_buf, cwd_buf, cwd_len * sizeof(wchar_t)); p += cwd_len; if (add_sep) { *(p++) = SEP; From 97516f68bb1feb788897b15ccc5ea7fc41457244 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Wed, 17 Apr 2024 11:35:33 +0200 Subject: [PATCH 24/72] narrow scoping --- Modules/posixmodule.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4f5672a9269230..4711bb32e13957 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5511,8 +5511,8 @@ static PyObject * os__path_abspath_impl(PyObject *module, PyObject *path) /*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ { - Py_ssize_t path_len, start, abs_len, cwd_len; - wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; + Py_ssize_t path_len, start, abs_len; + wchar_t *abs, *abs_buf = NULL; PyObject *result = NULL; wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); @@ -5535,7 +5535,8 @@ os__path_abspath_impl(PyObject *module, PyObject *path) if (!cwd_obj) { goto exit; } - cwd_buf = PyUnicode_AsWideCharString(cwd_obj, &cwd_len); + Py_ssize_t cwd_len; + wchar_t *cwd_buf = PyUnicode_AsWideCharString(cwd_obj, &cwd_len); Py_DECREF(cwd_obj); if (!cwd_buf) { goto exit; @@ -5543,7 +5544,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) int add_sep = cwd_buf[cwd_len - 1] != SEP; start = cwd_len + add_sep; - abs_len = cwd_len + add_sep + path_len; + abs_len = start + path_len; if ((size_t)abs_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { PyErr_SetString(PyExc_OverflowError, "path is too long"); @@ -5565,6 +5566,8 @@ os__path_abspath_impl(PyObject *module, PyObject *path) } memcpy(p, path_buf, path_len * sizeof(wchar_t)); p[path_len] = '\0'; + + PyMem_Free(cwd_buf); } abs = _Py_normpath_and_size(abs, abs_len, start, &abs_len); @@ -5574,9 +5577,6 @@ os__path_abspath_impl(PyObject *module, PyObject *path) if (path_buf) { PyMem_Free(path_buf); } - if (cwd_buf) { - PyMem_Free(cwd_buf); - } if (abs_buf) { PyMem_RawFree(abs_buf); } From 7e87ccd338dbf9f7634e1feb4c48c8d510fdce2c Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Wed, 17 Apr 2024 16:29:39 +0200 Subject: [PATCH 25/72] Free `cwd_buf` for all exits --- Modules/posixmodule.c | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 4711bb32e13957..084099980dedb6 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5512,7 +5512,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) /*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ { Py_ssize_t path_len, start, abs_len; - wchar_t *abs, *abs_buf = NULL; + wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; PyObject *result = NULL; wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); @@ -5536,7 +5536,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) goto exit; } Py_ssize_t cwd_len; - wchar_t *cwd_buf = PyUnicode_AsWideCharString(cwd_obj, &cwd_len); + cwd_buf = PyUnicode_AsWideCharString(cwd_obj, &cwd_len); Py_DECREF(cwd_obj); if (!cwd_buf) { goto exit; @@ -5566,8 +5566,6 @@ os__path_abspath_impl(PyObject *module, PyObject *path) } memcpy(p, path_buf, path_len * sizeof(wchar_t)); p[path_len] = '\0'; - - PyMem_Free(cwd_buf); } abs = _Py_normpath_and_size(abs, abs_len, start, &abs_len); @@ -5577,6 +5575,9 @@ os__path_abspath_impl(PyObject *module, PyObject *path) if (path_buf) { PyMem_Free(path_buf); } + if (cwd_buf) { + PyMem_Free(cwd_buf); + } if (abs_buf) { PyMem_RawFree(abs_buf); } From c06d34fe4a588f778f39e1ef413f6d33a510d82d Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Wed, 17 Apr 2024 18:13:34 +0200 Subject: [PATCH 26/72] rename `start` to `prefix_len` --- Modules/posixmodule.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 084099980dedb6..73f501865f4326 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5511,7 +5511,7 @@ static PyObject * os__path_abspath_impl(PyObject *module, PyObject *path) /*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ { - Py_ssize_t path_len, start, abs_len; + Py_ssize_t path_len, prefix_len, abs_len; wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; PyObject *result = NULL; @@ -5526,7 +5526,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) } if (_Py_isabs(path_buf)) { - start = 0; + prefix_len = 0; abs_len = path_len; abs = path_buf; } @@ -5543,8 +5543,8 @@ os__path_abspath_impl(PyObject *module, PyObject *path) } int add_sep = cwd_buf[cwd_len - 1] != SEP; - start = cwd_len + add_sep; - abs_len = start + path_len; + prefix_len = cwd_len + add_sep; + abs_len = prefix_len + path_len; if ((size_t)abs_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { PyErr_SetString(PyExc_OverflowError, "path is too long"); @@ -5568,7 +5568,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) p[path_len] = '\0'; } - abs = _Py_normpath_and_size(abs, abs_len, start, &abs_len); + abs = _Py_normpath_and_size(abs, abs_len, prefix_len, &abs_len); result = PyUnicode_FromWideChar(abs, abs_len); exit: From 3d2836e664c3e6ef1259ffc2e2843cd231456e0c Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sun, 21 Apr 2024 21:33:03 +0200 Subject: [PATCH 27/72] test different cache key --- .github/workflows/reusable-wasi.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/reusable-wasi.yml b/.github/workflows/reusable-wasi.yml index 60eef7bc478bbf..7a7bc94451a9d5 100644 --- a/.github/workflows/reusable-wasi.yml +++ b/.github/workflows/reusable-wasi.yml @@ -50,7 +50,7 @@ jobs: uses: actions/cache@v4 with: path: ${{ env.CROSS_BUILD_PYTHON }}/config.cache - key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }} + key: ${{ github.job }}-${{ runner.os }}-${{ env.IMAGE_VERSION }}-${{ inputs.config_hash }}-${{ env.pythonLocation }} - name: "Configure build Python" run: python3 Tools/wasm/wasi.py configure-build-python -- --config-cache --with-pydebug - name: "Make build Python" From 111df432559174915192a3d4848940be38eef347 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Tue, 23 Apr 2024 17:04:25 +0200 Subject: [PATCH 28/72] Add C implementation of `ntpath.abspath()` --- Lib/ntpath.py | 4 +- Lib/posixpath.py | 31 +++--- ...-04-13-18-00-18.gh-issue-117587.52AB27.rst | 2 +- Modules/clinic/posixmodule.c.h | 60 ++---------- Modules/posixmodule.c | 97 ++++++++----------- 5 files changed, 70 insertions(+), 124 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index aba18bfe407abf..95b1ba9824b4d6 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -599,7 +599,7 @@ def _abspath_fallback(path): # Return an absolute path. try: - from nt import _getfullpathname + from nt import _path_abspath except ImportError: # not running on Windows - mock up something sensible abspath = _abspath_fallback @@ -608,7 +608,7 @@ def _abspath_fallback(path): def abspath(path): """Return the absolute version of a path.""" try: - return _getfullpathname(normpath(path)) + return _path_abspath(path) except (OSError, ValueError): return _abspath_fallback(path) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 3db3db8c324368..b746d910a83222 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -386,26 +386,29 @@ def normpath(path): return _path_normpath(path) or "." +def _abspath_fallback(path): + """Return an absolute path.""" + path = os.fspath(path) + if isinstance(path, bytes): + if not path.startswith(b'/'): + path = join(os.getcwdb(), path) + else: + if not path.startswith('/'): + path = join(os.getcwd(), path) + return normpath(path) + + try: from posix import _path_abspath except ImportError: - def abspath(path): - """Return an absolute path.""" - path = os.fspath(path) - if isinstance(path, bytes): - if not path.startswith(b'/'): - path = join(os.getcwdb(), path) - else: - if not path.startswith('/'): - path = join(os.getcwd(), path) - return normpath(path) + abspath = _abspath_fallback else: def abspath(path): """Return an absolute path.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_abspath(os.fsdecode(path))) - return _path_abspath(path) + try: + return _path_abspath(path) + except (UnicodeEncodeError, ValueError): + return _abspath_fallback(path) # Return a canonical path (i.e. the absolute location of a file on the # filesystem). diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-00-18.gh-issue-117587.52AB27.rst b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-00-18.gh-issue-117587.52AB27.rst index d925bff7f13e6b..14063a72c0e5d8 100644 --- a/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-00-18.gh-issue-117587.52AB27.rst +++ b/Misc/NEWS.d/next/Core and Builtins/2024-04-13-18-00-18.gh-issue-117587.52AB27.rst @@ -1 +1 @@ -Speedup :func:`os.path.abspath` for relative paths on Unix. +Speedup :func:`os.path.abspath` with a native implementation. diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 85e6d2c0b23d2c..98a91d37d5d1ae 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1785,39 +1785,6 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P #if defined(MS_WINDOWS) -PyDoc_STRVAR(os__getfullpathname__doc__, -"_getfullpathname($module, path, /)\n" -"--\n" -"\n"); - -#define OS__GETFULLPATHNAME_METHODDEF \ - {"_getfullpathname", (PyCFunction)os__getfullpathname, METH_O, os__getfullpathname__doc__}, - -static PyObject * -os__getfullpathname_impl(PyObject *module, path_t *path); - -static PyObject * -os__getfullpathname(PyObject *module, PyObject *arg) -{ - PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_getfullpathname", "path", 0, 0); - - if (!path_converter(arg, &path)) { - goto exit; - } - return_value = os__getfullpathname_impl(module, &path); - -exit: - /* Cleanup for path */ - path_cleanup(&path); - - return return_value; -} - -#endif /* defined(MS_WINDOWS) */ - -#if defined(MS_WINDOWS) - PyDoc_STRVAR(os__getfinalpathname__doc__, "_getfinalpathname($module, path, /)\n" "--\n" @@ -2303,8 +2270,6 @@ os__path_normpath(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyO return return_value; } -#if !defined(MS_WINDOWS) - PyDoc_STRVAR(os__path_abspath__doc__, "_path_abspath($module, path, /)\n" "--\n" @@ -2315,27 +2280,26 @@ PyDoc_STRVAR(os__path_abspath__doc__, {"_path_abspath", (PyCFunction)os__path_abspath, METH_O, os__path_abspath__doc__}, static PyObject * -os__path_abspath_impl(PyObject *module, PyObject *path); +os__path_abspath_impl(PyObject *module, path_t *path); static PyObject * os__path_abspath(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_abspath", "path", 0, 0); - if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("_path_abspath", "argument", "str", arg); + if (!path_converter(arg, &path)) { goto exit; } - path = arg; - return_value = os__path_abspath_impl(module, path); + return_value = os__path_abspath_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } -#endif /* !defined(MS_WINDOWS) */ - PyDoc_STRVAR(os_mkdir__doc__, "mkdir($module, /, path, mode=511, *, dir_fd=None)\n" "--\n" @@ -12052,10 +12016,6 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS__PATH_ISDEVDRIVE_METHODDEF #endif /* !defined(OS__PATH_ISDEVDRIVE_METHODDEF) */ -#ifndef OS__GETFULLPATHNAME_METHODDEF - #define OS__GETFULLPATHNAME_METHODDEF -#endif /* !defined(OS__GETFULLPATHNAME_METHODDEF) */ - #ifndef OS__GETFINALPATHNAME_METHODDEF #define OS__GETFINALPATHNAME_METHODDEF #endif /* !defined(OS__GETFINALPATHNAME_METHODDEF) */ @@ -12088,10 +12048,6 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #define OS__PATH_ISLINK_METHODDEF #endif /* !defined(OS__PATH_ISLINK_METHODDEF) */ -#ifndef OS__PATH_ABSPATH_METHODDEF - #define OS__PATH_ABSPATH_METHODDEF -#endif /* !defined(OS__PATH_ABSPATH_METHODDEF) */ - #ifndef OS_NICE_METHODDEF #define OS_NICE_METHODDEF #endif /* !defined(OS_NICE_METHODDEF) */ @@ -12639,4 +12595,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=e984e2fa69e44555 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=f8bf7c940193ae4f input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 73f501865f4326..85e72ddfd24daa 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4848,40 +4848,6 @@ _PyOS_getfullpathname(const wchar_t *path, wchar_t **abspath_p) } -/* A helper function for abspath on win32 */ -/*[clinic input] -os._getfullpathname - - path: path_t - / - -[clinic start generated code]*/ - -static PyObject * -os__getfullpathname_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=bb8679d56845bc9b input=332ed537c29d0a3e]*/ -{ - wchar_t *abspath; - - if (_PyOS_getfullpathname(path->wide, &abspath) < 0) { - return win32_error_object("GetFullPathNameW", path->object); - } - if (abspath == NULL) { - return PyErr_NoMemory(); - } - - PyObject *str = PyUnicode_FromWideChar(abspath, wcslen(abspath)); - PyMem_RawFree(abspath); - if (str == NULL) { - return NULL; - } - if (path->narrow) { - Py_SETREF(str, PyUnicode_EncodeFSDefault(str)); - } - return str; -} - - /*[clinic input] os._getfinalpathname @@ -5496,39 +5462,55 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return result; } -#ifndef MS_WINDOWS - /*[clinic input] os._path_abspath - path: unicode + path: path_t / Make path absolute. [clinic start generated code]*/ static PyObject * -os__path_abspath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ +os__path_abspath_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=bb40fbf3be7251a4 input=54be6529c3267bd0]*/ { - Py_ssize_t path_len, prefix_len, abs_len; - wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; - PyObject *result = NULL; + Py_ssize_t path_len, abs_len; + wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL, *path_buf = NULL; + PyObject *wide = NULL, *result = NULL; - wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); - if (!path_buf) { +#ifdef MS_WINDOWS + path_len = path->length; + path_buf = (wchar_t *)path->wide; +#else + if (!(wide = PyUnicode_DecodeFSDefaultAndSize(path->narrow, path->length)) || + !(path_buf = PyUnicode_AsWideCharString(wide, &path_len))) + { goto exit; } +#endif + int is_bytes = PyBytes_Check(path->object); +#ifdef MS_WINDOWS + abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len); + if (abs_len == 0) { + result = posix_getcwd(is_bytes); + goto exit; + } + if (_PyOS_getfullpathname(abs, &abs_buf) < 0) { + result = win32_error_object("GetFullPathNameW", path->object); + goto exit; + } + abs = abs_buf; + abs_len = wcslen(abs_buf); +#else if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { - result = posix_getcwd(0); + result = posix_getcwd(is_bytes); goto exit; } if (_Py_isabs(path_buf)) { - prefix_len = 0; - abs_len = path_len; - abs = path_buf; + abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len); } else { PyObject *cwd_obj = posix_getcwd(0); @@ -5543,7 +5525,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) } int add_sep = cwd_buf[cwd_len - 1] != SEP; - prefix_len = cwd_len + add_sep; + Py_ssize_t prefix_len = cwd_len + add_sep; abs_len = prefix_len + path_len; if ((size_t)abs_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { @@ -5553,10 +5535,9 @@ os__path_abspath_impl(PyObject *module, PyObject *path) abs_buf = PyMem_RawMalloc(((size_t)abs_len + 1) * sizeof(wchar_t)); if (!abs_buf) { - PyErr_NoMemory(); + result = PyErr_NoMemory(); goto exit; } - abs = abs_buf; // Join cwd & path wchar_t *p = memcpy(abs_buf, cwd_buf, cwd_len * sizeof(wchar_t)); @@ -5566,15 +5547,24 @@ os__path_abspath_impl(PyObject *module, PyObject *path) } memcpy(p, path_buf, path_len * sizeof(wchar_t)); p[path_len] = '\0'; + abs = _Py_normpath_and_size(abs_buf, abs_len, prefix_len, &abs_len); } +#endif - abs = _Py_normpath_and_size(abs, abs_len, prefix_len, &abs_len); result = PyUnicode_FromWideChar(abs, abs_len); + if (is_bytes) { + Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); + } exit: +#ifndef MS_WINDOWS + if (wide) { + Py_DECREF(wide); + } if (path_buf) { PyMem_Free(path_buf); } +#endif if (cwd_buf) { PyMem_Free(cwd_buf); } @@ -5584,8 +5574,6 @@ os__path_abspath_impl(PyObject *module, PyObject *path) return result; } -#endif - /*[clinic input] os.mkdir @@ -16883,7 +16871,6 @@ static PyMethodDef posix_methods[] = { OS_FPATHCONF_METHODDEF OS_PATHCONF_METHODDEF OS_ABORT_METHODDEF - OS__GETFULLPATHNAME_METHODDEF OS__GETDISKUSAGE_METHODDEF OS__GETFINALPATHNAME_METHODDEF OS__FINDFIRSTFILE_METHODDEF From a538218727480bbaf8e41be2b478f8cf63af6e0f Mon Sep 17 00:00:00 2001 From: nineteendo Date: Tue, 23 Apr 2024 17:25:29 +0200 Subject: [PATCH 29/72] Replace with `nt._path_abspath` --- Lib/ctypes/__init__.py | 2 +- Lib/ntpath.py | 2 +- Lib/test/test_ctypes/test_loading.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index b7ee46d664ab08..bda759509d7dbe 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -380,7 +380,7 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, import nt mode = nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS if '/' in name or '\\' in name: - self._name = nt._getfullpathname(self._name) + self._name = nt._path_abspath(self._name) mode |= nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR class _FuncPtr(_CFuncPtr): diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 95b1ba9824b4d6..287dcb6523a72d 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -583,7 +583,7 @@ def normpath(path): def _abspath_fallback(path): """Return the absolute version of a path as a fallback function in case - `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for + `nt._path_abspath` is not available or raises OSError. See bpo-31047 for more. """ diff --git a/Lib/test/test_ctypes/test_loading.py b/Lib/test/test_ctypes/test_loading.py index b25e81b65cf103..a670daec2f2ab6 100644 --- a/Lib/test/test_ctypes/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -188,11 +188,11 @@ def should_fail(command): should_pass("windll.kernel32.SetDllDirectoryW(None); WinDLL('_sqlite3.dll', winmode=0)") # Full path load without DLL_LOAD_DIR shouldn't find dependency - should_fail("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + + should_fail("WinDLL(nt._path_abspath('_sqlite3.dll'), " + "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32)") # Full path load with DLL_LOAD_DIR should succeed - should_pass("WinDLL(nt._getfullpathname('_sqlite3.dll'), " + + should_pass("WinDLL(nt._path_abspath('_sqlite3.dll'), " + "winmode=nt._LOAD_LIBRARY_SEARCH_SYSTEM32|" + "nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR)") From d36379d027e1e4c3de99291a7eb515c0144647bd Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 23 Apr 2024 17:34:01 +0200 Subject: [PATCH 30/72] Update Lib/posixpath.py Co-authored-by: Eryk Sun --- Lib/posixpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index b746d910a83222..4438986105075b 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -407,7 +407,7 @@ def abspath(path): """Return an absolute path.""" try: return _path_abspath(path) - except (UnicodeEncodeError, ValueError): + except ValueError: return _abspath_fallback(path) # Return a canonical path (i.e. the absolute location of a file on the From 6e645f1c7cfe21a4f076eee41fbe41f4004a9c9d Mon Sep 17 00:00:00 2001 From: nineteendo Date: Wed, 24 Apr 2024 14:28:41 +0200 Subject: [PATCH 31/72] Remove fallback --- Lib/ntpath.py | 3 +++ Lib/posixpath.py | 31 ++++++++++++++---------------- Modules/clinic/posixmodule.c.h | 15 +++++++-------- Modules/posixmodule.c | 35 ++++++++++------------------------ 4 files changed, 34 insertions(+), 50 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 287dcb6523a72d..7ac232423b365d 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -607,7 +607,10 @@ def _abspath_fallback(path): else: # use native Windows method on Windows def abspath(path): """Return the absolute version of a path.""" + path = os.fspath(path) try: + if isinstance(path, bytes): + return os.fsencode(_path_abspath(os.fsdecode(path))) return _path_abspath(path) except (OSError, ValueError): return _abspath_fallback(path) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 4438986105075b..3db3db8c324368 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -386,29 +386,26 @@ def normpath(path): return _path_normpath(path) or "." -def _abspath_fallback(path): - """Return an absolute path.""" - path = os.fspath(path) - if isinstance(path, bytes): - if not path.startswith(b'/'): - path = join(os.getcwdb(), path) - else: - if not path.startswith('/'): - path = join(os.getcwd(), path) - return normpath(path) - - try: from posix import _path_abspath except ImportError: - abspath = _abspath_fallback + def abspath(path): + """Return an absolute path.""" + path = os.fspath(path) + if isinstance(path, bytes): + if not path.startswith(b'/'): + path = join(os.getcwdb(), path) + else: + if not path.startswith('/'): + path = join(os.getcwd(), path) + return normpath(path) else: def abspath(path): """Return an absolute path.""" - try: - return _path_abspath(path) - except ValueError: - return _abspath_fallback(path) + path = os.fspath(path) + if isinstance(path, bytes): + return os.fsencode(_path_abspath(os.fsdecode(path))) + return _path_abspath(path) # Return a canonical path (i.e. the absolute location of a file on the # filesystem). diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 98a91d37d5d1ae..8f0cf42cde9016 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2280,23 +2280,22 @@ PyDoc_STRVAR(os__path_abspath__doc__, {"_path_abspath", (PyCFunction)os__path_abspath, METH_O, os__path_abspath__doc__}, static PyObject * -os__path_abspath_impl(PyObject *module, path_t *path); +os__path_abspath_impl(PyObject *module, PyObject *path); static PyObject * os__path_abspath(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_path_abspath", "path", 0, 0); + PyObject *path; - if (!path_converter(arg, &path)) { + if (!PyUnicode_Check(arg)) { + _PyArg_BadArgument("_path_abspath", "argument", "str", arg); goto exit; } - return_value = os__path_abspath_impl(module, &path); + path = arg; + return_value = os__path_abspath_impl(module, path); exit: - /* Cleanup for path */ - path_cleanup(&path); - return return_value; } @@ -12595,4 +12594,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=f8bf7c940193ae4f input=a9049054013a1b77]*/ +/*[clinic end generated code: output=480c7ac86682ae3b input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 85e72ddfd24daa..2b569dbd2a8187 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5465,47 +5465,40 @@ os__path_normpath_impl(PyObject *module, PyObject *path) /*[clinic input] os._path_abspath - path: path_t + path: unicode / Make path absolute. [clinic start generated code]*/ static PyObject * -os__path_abspath_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=bb40fbf3be7251a4 input=54be6529c3267bd0]*/ +os__path_abspath_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ { Py_ssize_t path_len, abs_len; - wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL, *path_buf = NULL; - PyObject *wide = NULL, *result = NULL; + wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; + PyObject *result = NULL; -#ifdef MS_WINDOWS - path_len = path->length; - path_buf = (wchar_t *)path->wide; -#else - if (!(wide = PyUnicode_DecodeFSDefaultAndSize(path->narrow, path->length)) || - !(path_buf = PyUnicode_AsWideCharString(wide, &path_len))) - { + wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); + if (!path_buf) { goto exit; } -#endif - int is_bytes = PyBytes_Check(path->object); #ifdef MS_WINDOWS abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len); if (abs_len == 0) { - result = posix_getcwd(is_bytes); + result = posix_getcwd(0); goto exit; } if (_PyOS_getfullpathname(abs, &abs_buf) < 0) { - result = win32_error_object("GetFullPathNameW", path->object); + result = win32_error_object("GetFullPathNameW", path); goto exit; } abs = abs_buf; abs_len = wcslen(abs_buf); #else if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { - result = posix_getcwd(is_bytes); + result = posix_getcwd(0); goto exit; } @@ -5552,19 +5545,11 @@ os__path_abspath_impl(PyObject *module, path_t *path) #endif result = PyUnicode_FromWideChar(abs, abs_len); - if (is_bytes) { - Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); - } exit: -#ifndef MS_WINDOWS - if (wide) { - Py_DECREF(wide); - } if (path_buf) { PyMem_Free(path_buf); } -#endif if (cwd_buf) { PyMem_Free(cwd_buf); } From 8fd29847cf841facfd726229cc94b0d98fdc651e Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Wed, 24 Apr 2024 22:43:13 +0200 Subject: [PATCH 32/72] Update Modules/posixmodule.c --- Modules/posixmodule.c | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2b569dbd2a8187..b5e90dbf022bf5 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5547,15 +5547,9 @@ os__path_abspath_impl(PyObject *module, PyObject *path) result = PyUnicode_FromWideChar(abs, abs_len); exit: - if (path_buf) { - PyMem_Free(path_buf); - } - if (cwd_buf) { - PyMem_Free(cwd_buf); - } - if (abs_buf) { - PyMem_RawFree(abs_buf); - } + PyMem_Free(path_buf); + PyMem_Free(cwd_buf); + PyMem_RawFree(abs_buf); return result; } From 4d8eb1ddcc897bd78bd5ca306b0b111f62ca55f2 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Wed, 24 Apr 2024 23:22:04 +0200 Subject: [PATCH 33/72] Follow pep 7 --- Python/fileutils.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Python/fileutils.c b/Python/fileutils.c index 1502e0e223ce18..0c3d32ead1b269 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2391,7 +2391,8 @@ _Py_find_basename(const wchar_t *filename) normalizing. 'normsize' will be set to contain the length of the resulting normalized path. */ wchar_t * -_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *normsize) +_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, + Py_ssize_t *normsize) { assert(path != NULL); assert(start>=0); From c4701234792555176685ef8f8dc2b33480ea81c7 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Thu, 25 Apr 2024 08:20:46 +0200 Subject: [PATCH 34/72] Check for embedded null characters Co-authored-by: Eryk Sun --- Modules/posixmodule.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index b5e90dbf022bf5..c4bbeb83ddc98b 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5490,6 +5490,11 @@ os__path_abspath_impl(PyObject *module, PyObject *path) result = posix_getcwd(0); goto exit; } + if (wcslen(abs) != abs_len) { + PyErr_Format(PyExc_ValueError, + "_path_abspath: embedded null character in path"); + goto exit; + } if (_PyOS_getfullpathname(abs, &abs_buf) < 0) { result = win32_error_object("GetFullPathNameW", path); goto exit; From 3a9af3f8aab7993252b5a046ca9064b35c3a4ad1 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Thu, 25 Apr 2024 10:11:37 +0200 Subject: [PATCH 35/72] Stricter check for embedded null characters Co-authored-by: Eryk Sun --- Modules/posixmodule.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c4bbeb83ddc98b..5c258e2723f185 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5485,16 +5485,16 @@ os__path_abspath_impl(PyObject *module, PyObject *path) } #ifdef MS_WINDOWS + if (wcslen(path_buf) != path_len) { + PyErr_Format(PyExc_ValueError, + "_path_abspath: embedded null character in path"); + goto exit; + } abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len); if (abs_len == 0) { result = posix_getcwd(0); goto exit; } - if (wcslen(abs) != abs_len) { - PyErr_Format(PyExc_ValueError, - "_path_abspath: embedded null character in path"); - goto exit; - } if (_PyOS_getfullpathname(abs, &abs_buf) < 0) { result = win32_error_object("GetFullPathNameW", path); goto exit; From 9fbc62e3d51116d3d8167f71cc79f95d0e3881b4 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Thu, 25 Apr 2024 12:32:03 +0200 Subject: [PATCH 36/72] Support qualified referencing --- Include/internal/pycore_fileutils.h | 3 ++- Lib/test/test_ntpath.py | 1 + Modules/posixmodule.c | 9 +++++---- Python/fileutils.c | 19 ++++++++++++------- 4 files changed, 20 insertions(+), 12 deletions(-) diff --git a/Include/internal/pycore_fileutils.h b/Include/internal/pycore_fileutils.h index 0b14f72eca52a4..858e758f8f458d 100644 --- a/Include/internal/pycore_fileutils.h +++ b/Include/internal/pycore_fileutils.h @@ -279,7 +279,8 @@ extern size_t _Py_find_basename(const wchar_t *filename); // Export for '_testinternalcapi' shared extension PyAPI_FUNC(wchar_t*) _Py_normpath(wchar_t *path, Py_ssize_t size); -extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *length); +extern wchar_t *_Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, + Py_ssize_t *length, int explicit_curdir); // The Windows Games API family does not provide these functions // so provide our own implementations. Remove them in case they get added diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 7f91bf1c2b837a..5573d0e21a06ac 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -836,6 +836,7 @@ def test_abspath(self): tester('ntpath.abspath("")', cwd_dir) tester('ntpath.abspath(" ")', cwd_dir + "\\ ") tester('ntpath.abspath("?")', cwd_dir + "\\?") + tester('ntpath.abspath("./con")', cwd_dir + "\\con") drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index c984d637a74a81..88afa3edce014e 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5499,7 +5499,7 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return NULL; } Py_ssize_t norm_len; - wchar_t *norm_path = _Py_normpath_and_size(buffer, len, 0, &norm_len); + wchar_t *norm_path = _Py_normpath_and_size(buffer, len, 0, &norm_len, 0); PyObject *result = PyUnicode_FromWideChar(norm_path, norm_len); PyMem_Free(buffer); return result; @@ -5533,7 +5533,8 @@ os__path_abspath_impl(PyObject *module, PyObject *path) "_path_abspath: embedded null character in path"); goto exit; } - abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len); + // Preserve `.\` for qualified referencing + abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len, 1); if (abs_len == 0) { result = posix_getcwd(0); goto exit; @@ -5551,7 +5552,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) } if (_Py_isabs(path_buf)) { - abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len); + abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len, 0); } else { PyObject *cwd_obj = posix_getcwd(0); @@ -5588,7 +5589,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) } memcpy(p, path_buf, path_len * sizeof(wchar_t)); p[path_len] = '\0'; - abs = _Py_normpath_and_size(abs_buf, abs_len, prefix_len, &abs_len); + abs = _Py_normpath_and_size(abs_buf, abs_len, prefix_len, &abs_len, 0); } #endif diff --git a/Python/fileutils.c b/Python/fileutils.c index 21f74bb87dfaa6..39a577cca3d5ee 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2482,10 +2482,11 @@ _Py_find_basename(const wchar_t *filename) the path, if known. If -1, the first null character will be assumed to be the end of the path. 'start' is the position where to start normalizing. 'normsize' will be set to contain the length of the - resulting normalized path. */ + resulting normalized path. If 'explicit_curdir' is set, a leading + '.\' is preserved */ wchar_t * _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, - Py_ssize_t *normsize) + Py_ssize_t *normsize, int explicit_curdir) { assert(path != NULL); assert(start>=0); @@ -2514,11 +2515,15 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, if (p1[0] == L'.' && IS_SEP(&p1[1])) { // Skip leading '.\' - path = &path[2]; - while (IS_SEP(path)) { - path++; + p1 = &path[2]; + if (explicit_curdir) { + path[1] = SEP; + // TODO: Don't change minP2 & handle explicit curdir in main loop + p2 = minP2 = p1; + } + while (IS_SEP(p1)) { + p1++; } - p1 = p2 = minP2 = path; lastC = SEP; } else { @@ -2618,7 +2623,7 @@ wchar_t * _Py_normpath(wchar_t *path, Py_ssize_t size) { Py_ssize_t norm_length; - return _Py_normpath_and_size(path, size, 0, &norm_length); + return _Py_normpath_and_size(path, size, 0, &norm_length, 0); } From d83a580a601a4feceaf7ae471f61487d80ba9e08 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Thu, 25 Apr 2024 12:53:59 +0200 Subject: [PATCH 37/72] Add additional test --- Lib/test/test_ntpath.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 5573d0e21a06ac..37fdf415381e1b 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -837,6 +837,7 @@ def test_abspath(self): tester('ntpath.abspath(" ")', cwd_dir + "\\ ") tester('ntpath.abspath("?")', cwd_dir + "\\?") tester('ntpath.abspath("./con")', cwd_dir + "\\con") + tester('ntpath.abspath("./Z:spam")', cwd_dir + "\\Z:spam") drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") From 08b967402e2acd646991c7b4ea82cfae324c5035 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Thu, 25 Apr 2024 14:07:15 +0200 Subject: [PATCH 38/72] null in cwd? --- Lib/test/test_ntpath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 37fdf415381e1b..e1df00085fd33d 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -836,8 +836,8 @@ def test_abspath(self): tester('ntpath.abspath("")', cwd_dir) tester('ntpath.abspath(" ")', cwd_dir + "\\ ") tester('ntpath.abspath("?")', cwd_dir + "\\?") - tester('ntpath.abspath("./con")', cwd_dir + "\\con") - tester('ntpath.abspath("./Z:spam")', cwd_dir + "\\Z:spam") + self.assertEqual(nt._path_abspath("./con"), cwd_dir + "\\con") + self.assertEqual(nt._path_abspath("./Z:spam"), cwd_dir + "\\Z:spam") drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") From 6eb1442773b0594eb4c67f348fb486a62c6bc8ed Mon Sep 17 00:00:00 2001 From: nineteendo Date: Thu, 25 Apr 2024 17:25:00 +0200 Subject: [PATCH 39/72] Add Windows 11 check --- Lib/test/test_ntpath.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index e1df00085fd33d..91b676e8e2b955 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -836,8 +836,12 @@ def test_abspath(self): tester('ntpath.abspath("")', cwd_dir) tester('ntpath.abspath(" ")', cwd_dir + "\\ ") tester('ntpath.abspath("?")', cwd_dir + "\\?") - self.assertEqual(nt._path_abspath("./con"), cwd_dir + "\\con") - self.assertEqual(nt._path_abspath("./Z:spam"), cwd_dir + "\\Z:spam") + # bpo-45354: Windows 11 changed MS-DOS device name handling + if sys.getwindowsversion()[:3] < (10, 0, 22000): + tester('ntpath.abspath("./con")', "\\\\.\\con") + else: + tester('ntpath.abspath("./con")', cwd_dir + "\\con") + tester('ntpath.abspath("./Z:spam")', cwd_dir + "\\Z:spam") drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") From b7f9df6309a5f35a88464b7e1b7ebec73ee1f65b Mon Sep 17 00:00:00 2001 From: nineteendo Date: Thu, 25 Apr 2024 17:27:05 +0200 Subject: [PATCH 40/72] Add additional test --- Lib/test/test_ntpath.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 91b676e8e2b955..1f58f2e043cbfd 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -836,6 +836,7 @@ def test_abspath(self): tester('ntpath.abspath("")', cwd_dir) tester('ntpath.abspath(" ")', cwd_dir + "\\ ") tester('ntpath.abspath("?")', cwd_dir + "\\?") + tester('ntpath.abspath("con")', "\\\\.\\con") # bpo-45354: Windows 11 changed MS-DOS device name handling if sys.getwindowsversion()[:3] < (10, 0, 22000): tester('ntpath.abspath("./con")', "\\\\.\\con") From 3ad0d7f25aa0db4b423cf8e721b859e1f7d78f99 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Fri, 26 Apr 2024 21:48:54 +0200 Subject: [PATCH 41/72] Fix drive relative paths --- Lib/test/test_ntpath.py | 2 ++ Python/fileutils.c | 65 +++++++++++++++++++++++------------------ 2 files changed, 39 insertions(+), 28 deletions(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 1f58f2e043cbfd..bad0820ec65f47 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -347,6 +347,7 @@ def test_normpath(self): tester("ntpath.normpath('..')", r'..') tester("ntpath.normpath('.')", r'.') + tester("ntpath.normpath('c:.')", 'c:') tester("ntpath.normpath('')", r'.') tester("ntpath.normpath('/')", '\\') tester("ntpath.normpath('c:/')", 'c:\\') @@ -354,6 +355,7 @@ def test_normpath(self): tester("ntpath.normpath('c:/../../..')", 'c:\\') tester("ntpath.normpath('../.././..')", r'..\..\..') tester("ntpath.normpath('K:../.././..')", r'K:..\..\..') + tester("ntpath.normpath('./a/b')", r'a\b') tester("ntpath.normpath('C:////a/b')", r'C:\a\b') tester("ntpath.normpath('//machine/share//a/b')", r'\\machine\share\a\b') diff --git a/Python/fileutils.c b/Python/fileutils.c index 39a577cca3d5ee..36d506496e7d62 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2513,41 +2513,50 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, #endif #define SEP_OR_END(x) (IS_SEP(x) || IS_END(x)) - if (p1[0] == L'.' && IS_SEP(&p1[1])) { - // Skip leading '.\' - p1 = &path[2]; - if (explicit_curdir) { - path[1] = SEP; - // TODO: Don't change minP2 & handle explicit curdir in main loop - p2 = minP2 = p1; - } - while (IS_SEP(p1)) { - p1++; - } - lastC = SEP; - } - else { - Py_ssize_t drvsize, rootsize; - _Py_skiproot(path, size, &drvsize, &rootsize); - if (drvsize || rootsize) { - // Skip past root and update minP2 - p1 = &path[drvsize + rootsize]; + Py_ssize_t drvsize, rootsize; + _Py_skiproot(path, size, &drvsize, &rootsize); + if (drvsize || rootsize) { + // Skip past root and update minP2 + p1 = &path[drvsize + rootsize]; #ifndef ALTSEP - p2 = p1; + p2 = p1; #else - for (; p2 < p1; ++p2) { - if (*p2 == ALTSEP) { - *p2 = SEP; - } + for (; p2 < p1; ++p2) { + if (*p2 == ALTSEP) { + *p2 = SEP; } + } #endif - minP2 = p2 - 1; - lastC = *minP2; + minP2 = p2 - 1; + lastC = *minP2; #ifdef MS_WINDOWS - if (lastC != SEP) { - minP2++; + if (lastC != SEP) { + minP2++; + } +#endif + } + if (p1[0] == L'.' && SEP_OR_END(&p1[1])) { + // Skip leading '.\' + lastC = *(++p1); +#ifdef ALTSEP + if (lastC == ALTSEP) { + lastC = SEP; + if (explicit_curdir) { + p2[1] = SEP; } + } #endif + if (explicit_curdir) { + // TODO: Don't change minP2 & handle explicit curdir in main loop + if (!lastC) { + p2 = minP2 = &p2[1]; + } + else { + p2 = minP2 = &p2[2]; + } + } + while (IS_SEP(p1)) { + p1++; } } From 549fa4e67facfac0491937c40281ff670d91f877 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sat, 27 Apr 2024 12:41:20 +0200 Subject: [PATCH 42/72] Handle all qualified referencing --- Lib/test/test_ntpath.py | 7 ++++++ Python/fileutils.c | 48 +++++++++++++++++++++++++++-------------- 2 files changed, 39 insertions(+), 16 deletions(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index bad0820ec65f47..1d6b5799a21e92 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -842,9 +842,16 @@ def test_abspath(self): # bpo-45354: Windows 11 changed MS-DOS device name handling if sys.getwindowsversion()[:3] < (10, 0, 22000): tester('ntpath.abspath("./con")', "\\\\.\\con") + tester('ntpath.abspath("foo/../con")', "\\\\.\\con") + tester('ntpath.abspath("con/foo/..")', "\\\\.\\con") + tester('ntpath.abspath("con/.")', "\\\\.\\con") else: tester('ntpath.abspath("./con")', cwd_dir + "\\con") + tester('ntpath.abspath("foo/../con")', cwd_dir + "\\con") + tester('ntpath.abspath("con/foo/..")', cwd_dir + "\\con") + tester('ntpath.abspath("con/.")', cwd_dir + "\\con") tester('ntpath.abspath("./Z:spam")', cwd_dir + "\\Z:spam") + tester('ntpath.abspath("spam/../Z:eggs")', cwd_dir + "\\Z:eggs") drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") diff --git a/Python/fileutils.c b/Python/fileutils.c index 36d506496e7d62..a638e44a3b0064 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2482,8 +2482,8 @@ _Py_find_basename(const wchar_t *filename) the path, if known. If -1, the first null character will be assumed to be the end of the path. 'start' is the position where to start normalizing. 'normsize' will be set to contain the length of the - resulting normalized path. If 'explicit_curdir' is set, a leading - '.\' is preserved */ + resulting normalized path. If 'explicit_curdir' is set, an explicit + curdir will be used for qualified referencing in the cwd */ wchar_t * _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *normsize, int explicit_curdir) @@ -2504,6 +2504,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, wchar_t *p2 = path; // destination of a scanned character to be ljusted wchar_t *minP2 = path; // the beginning of the destination range wchar_t lastC = L'\0'; // the last ljusted character, p2[-1] in most cases + int explicit = 0; // uses qualified referencing in the cwd #define IS_END(x) (pEnd ? (x) == pEnd : !*(x)) #ifdef ALTSEP @@ -2541,27 +2542,16 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, #ifdef ALTSEP if (lastC == ALTSEP) { lastC = SEP; - if (explicit_curdir) { - p2[1] = SEP; - } } #endif - if (explicit_curdir) { - // TODO: Don't change minP2 & handle explicit curdir in main loop - if (!lastC) { - p2 = minP2 = &p2[1]; - } - else { - p2 = minP2 = &p2[2]; - } - } while (IS_SEP(p1)) { p1++; } + explicit = 1; } - // Skip past cwd, not allowed if size is unknown - if (size >= 0 && path + start > p1) { + // Skip past cwd, not allowed if size is unknown or explicit_curdir is set + if (size >= 0 && path + start > p1 && !explicit_curdir) { p1 = p2 = path + start; lastC = *(p1-1); } @@ -2594,9 +2584,11 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, p2 = p3 + 1; } else { p2 = p3; + explicit = 1; } p1 += 1; } else if (sep_at_1) { + explicit = 1; } else { *p2++ = lastC = c; } @@ -2616,6 +2608,30 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, } else { --p2; } + if (explicit_curdir && !rootsize && explicit) { + // Add explicit curdir + if (p2 == minP2 - 1) { + // '.' + p2++; + assert(p2 < p1); + *p2 = L'.'; + } + else if (minP2[0] != L'.' || minP2[1] != L'.' || + !SEP_OR_END(&minP2[2])) + { + // Not '..\' + p2 += 2; + assert(p2 < p1); + wchar_t *p3 = p2 - 2; + while (p3 != minP2) { + p3[2] = *p3--; + } + p3[2] = p3[0]; + p3[0] = L'.'; + p3[1] = SEP; + } + p2[1] = L'\0'; + } *normsize = p2 - path + 1; #undef SEP_OR_END #undef IS_SEP From a768039abf02e83064958b6b8fb45d7d510ce854 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sat, 27 Apr 2024 13:15:20 +0200 Subject: [PATCH 43/72] Fix ambiguous operation --- Lib/test/test_ntpath.py | 22 +++++++++++----------- Lib/test/test_posixpath.py | 1 + Python/fileutils.c | 9 +++++---- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 1d6b5799a21e92..9b736fad13bff9 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -838,20 +838,20 @@ def test_abspath(self): tester('ntpath.abspath("")', cwd_dir) tester('ntpath.abspath(" ")', cwd_dir + "\\ ") tester('ntpath.abspath("?")', cwd_dir + "\\?") - tester('ntpath.abspath("con")', "\\\\.\\con") + tester('ntpath.abspath("con")', r"\\.\con") # bpo-45354: Windows 11 changed MS-DOS device name handling if sys.getwindowsversion()[:3] < (10, 0, 22000): - tester('ntpath.abspath("./con")', "\\\\.\\con") - tester('ntpath.abspath("foo/../con")', "\\\\.\\con") - tester('ntpath.abspath("con/foo/..")', "\\\\.\\con") - tester('ntpath.abspath("con/.")', "\\\\.\\con") + tester('ntpath.abspath("./con")', r"\\.\con") + tester('ntpath.abspath("foo/../con")', r"\\.\con") + tester('ntpath.abspath("con/foo/..")', r"\\.\con") + tester('ntpath.abspath("con/.")', r"\\.\con") else: - tester('ntpath.abspath("./con")', cwd_dir + "\\con") - tester('ntpath.abspath("foo/../con")', cwd_dir + "\\con") - tester('ntpath.abspath("con/foo/..")', cwd_dir + "\\con") - tester('ntpath.abspath("con/.")', cwd_dir + "\\con") - tester('ntpath.abspath("./Z:spam")', cwd_dir + "\\Z:spam") - tester('ntpath.abspath("spam/../Z:eggs")', cwd_dir + "\\Z:eggs") + tester('ntpath.abspath("./con")', cwd_dir + r"\con") + tester('ntpath.abspath("foo/../con")', cwd_dir + r"\con") + tester('ntpath.abspath("con/foo/..")', cwd_dir + r"\con") + tester('ntpath.abspath("con/.")', cwd_dir + r"\con") + tester('ntpath.abspath("./Z:spam")', cwd_dir + r"\Z:spam") + tester('ntpath.abspath("spam/../Z:eggs")', cwd_dir + r"\Z:eggs") drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 701516b96421bd..e6e5c921d87e09 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -371,6 +371,7 @@ def test_expanduser_pwd2(self): ("///..//./foo/.//bar", "/foo/bar"), (".", "."), (".//.", "."), + ("./foo/bar", "foo/bar"), ("..", ".."), ("../", ".."), ("../foo", "../foo"), diff --git a/Python/fileutils.c b/Python/fileutils.c index a638e44a3b0064..6f4867973619b7 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2538,7 +2538,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, } if (p1[0] == L'.' && SEP_OR_END(&p1[1])) { // Skip leading '.\' - lastC = *(++p1); + lastC = *++p1; #ifdef ALTSEP if (lastC == ALTSEP) { lastC = SEP; @@ -2620,15 +2620,16 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, !SEP_OR_END(&minP2[2])) { // Not '..\' + wchar_t *p3 = p2; p2 += 2; assert(p2 < p1); - wchar_t *p3 = p2 - 2; while (p3 != minP2) { - p3[2] = *p3--; + p3[2] = *p3; + p3--; } p3[2] = p3[0]; - p3[0] = L'.'; p3[1] = SEP; + p3[0] = L'.'; } p2[1] = L'\0'; } From 3339c1914ce5573b90b298ebe55956f6f4e19b75 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sat, 27 Apr 2024 13:24:55 +0200 Subject: [PATCH 44/72] Special case for '.' --- Modules/posixmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 88afa3edce014e..72cb3f105956ac 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5535,7 +5535,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) } // Preserve `.\` for qualified referencing abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len, 1); - if (abs_len == 0) { + if (abs_len == 0 || (abs_len == 1 && abs[0] == L'.')) { result = posix_getcwd(0); goto exit; } @@ -5585,7 +5585,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) wchar_t *p = memcpy(abs_buf, cwd_buf, cwd_len * sizeof(wchar_t)); p += cwd_len; if (add_sep) { - *(p++) = SEP; + *p++ = SEP; } memcpy(p, path_buf, path_len * sizeof(wchar_t)); p[path_len] = '\0'; From 0a4059974d8fbfa39dffa1fc2694f1a2721fa3d7 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sun, 28 Apr 2024 15:06:18 +0200 Subject: [PATCH 45/72] Handle drive relative paths Co-authored-by: Eryk Sun --- Lib/ntpath.py | 48 ++++++++++++++++++++++++++++++------------------ 1 file changed, 30 insertions(+), 18 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 841dc1d8b68859..e6e5cace4f70b2 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -601,28 +601,20 @@ def normpath(path): return _path_normpath(path) or "." -def _abspath_fallback(path): - """Return the absolute version of a path as a fallback function in case - `nt._path_abspath` is not available or raises OSError. See bpo-31047 for - more. - - """ - - path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) - return normpath(path) - # Return an absolute path. try: from nt import _path_abspath except ImportError: # not running on Windows - mock up something sensible - abspath = _abspath_fallback + def abspath(path): + """Return the absolute version of a path.""" + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + path = join(os.getcwdb(), path) + else: + path = join(os.getcwd(), path) + return normpath(path) else: # use native Windows method on Windows def abspath(path): @@ -633,7 +625,27 @@ def abspath(path): return os.fsencode(_path_abspath(os.fsdecode(path))) return _path_abspath(path) except (OSError, ValueError): - return _abspath_fallback(path) + # See gh-75230, handle outside for cleaner traceback + pass + if not isabs(path): + if isinstance(path, bytes): + sep = b'/' + cwd = os.getcwdb() + else: + sep = '/' + cwd = os.getcwd() + drive, _, path = splitroot(path) + if drive and drive != splitroot(cwd)[0]: + try: + if isinstance(drive, bytes): + cwd = os.fsencode(_path_abspath(os.fsdecode(drive))) + else: + cwd = _path_abspath(drive) + except (OSError, ValueError): + # Invalid drive + cwd = join(drive, sep) + path = join(cwd, path) + return normpath(path) try: from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink From 2315bbb5d2f8f727f5327d6cf72dfc6767898d82 Mon Sep 17 00:00:00 2001 From: nineteendo Date: Sun, 28 Apr 2024 16:14:06 +0200 Subject: [PATCH 46/72] Add tests --- Lib/test/test_ntpath.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 9b736fad13bff9..47c189e87f0f06 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -808,6 +808,9 @@ def test_abspath(self): tester('ntpath.abspath("C:\\spam. . .")', "C:\\spam") tester('ntpath.abspath("C:/nul")', "\\\\.\\nul") tester('ntpath.abspath("C:\\nul")', "\\\\.\\nul") + tester('ntpath.abspath("Z:spam")', r"Z:\spam") + self.assertEqual(ntpath.abspath("Z:\x00"), "Z:\\\x00") + self.assertEqual(ntpath.abspath("\x00:spam"), "\x00:\\spam") tester('ntpath.abspath("//..")', "\\\\") tester('ntpath.abspath("//../")', "\\\\..\\") tester('ntpath.abspath("//../..")', "\\\\..\\") From b3d04ac495cb2ba75b9622c988bfcc1c2f2f29e8 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Sat, 4 May 2024 08:32:59 +0200 Subject: [PATCH 47/72] Direct C call for `nt.isdevdrive()` --- Lib/ntpath.py | 14 +---- Lib/test/test_ntpath.py | 2 + Modules/clinic/posixmodule.c.h | 15 ++--- Modules/posixmodule.c | 101 +++++++++++++++++++++++---------- 4 files changed, 80 insertions(+), 52 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index e6e5cace4f70b2..c0d1dd2e517daf 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -932,19 +932,7 @@ def commonpath(paths): from nt import _path_isfile as isfile from nt import _path_islink as islink from nt import _path_exists as exists + from nt import _path_isdevdrive as isdevdrive except ImportError: # Use genericpath.* as imported above pass - - -try: - from nt import _path_isdevdrive - def isdevdrive(path): - """Determines whether the specified path is on a Windows Dev Drive.""" - try: - return _path_isdevdrive(abspath(path)) - except OSError: - return False -except ImportError: - # Use genericpath.isdevdrive as imported above - pass diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 47c189e87f0f06..ddad3900f6a088 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1135,6 +1135,8 @@ def test_fast_paths_in_use(self): self.assertFalse(inspect.isfunction(os.path.islink)) self.assertTrue(os.path.exists is nt._path_exists) self.assertFalse(inspect.isfunction(os.path.exists)) + self.assertTrue(os.path.isdevdrive is nt._path_isdevdrive) + self.assertFalse(inspect.isfunction(os.path.isdevdrive)) @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") def test_isdevdrive(self): diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 7fdad8af7dff82..675d4d938a1e87 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1731,7 +1731,7 @@ PyDoc_STRVAR(os__path_isdevdrive__doc__, {"_path_isdevdrive", _PyCFunction_CAST(os__path_isdevdrive), METH_FASTCALL|METH_KEYWORDS, os__path_isdevdrive__doc__}, static PyObject * -os__path_isdevdrive_impl(PyObject *module, path_t *path); +os__path_isdevdrive_impl(PyObject *module, PyObject *path); static PyObject * os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1763,21 +1763,16 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_isdevdrive", "path", 0, 0); + PyObject *path; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - if (!path_converter(args[0], &path)) { - goto exit; - } - return_value = os__path_isdevdrive_impl(module, &path); + path = args[0]; + return_value = os__path_isdevdrive_impl(module, path); exit: - /* Cleanup for path */ - path_cleanup(&path); - return return_value; } @@ -12652,4 +12647,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=a5c86cf571db8e59 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=412b4e312064330d input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index f2554fd3174024..efa6f7195effec 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4720,16 +4720,38 @@ os_listmounts_impl(PyObject *module, path_t *volume) /*[clinic input] os._path_isdevdrive - path: path_t + path: object Determines whether the specified path is on a Windows Dev Drive. [clinic start generated code]*/ static PyObject * -os__path_isdevdrive_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=1f437ea6677433a2 input=ee83e4996a48e23d]*/ +os__path_isdevdrive_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=4f5a02fe80648dab input=cb26f0f438f49cb6]*/ { + path_t _path = PATH_T_INITIALIZE("isdevdrive", "path", 0, 0); + if (!path_converter(path, &_path)) { + path_cleanup(&_path); + if (PyErr_ExceptionMatches(PyExc_ValueError)) { + PyErr_Clear(); + Py_RETURN_FALSE; + } + return NULL; + } + + PyObject *abs_obj = posix_abspath((wchar_t *)_path.wide, _path.length, 0); + if (!abs_obj) { + path_cleanup(&_path); + return NULL; + } + wchar_t *abs_buf = PyUnicode_AsWideCharString(abs_obj, NULL); + Py_DECREF(abs_obj); + if (!abs_buf) { + path_cleanup(&_path); + return NULL; + } + #ifndef PERSISTENT_VOLUME_STATE_DEV_VOLUME /* This flag will be documented at https://learn.microsoft.com/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_file_fs_persistent_volume_information @@ -4743,7 +4765,7 @@ os__path_isdevdrive_impl(PyObject *module, path_t *path) wchar_t volume[MAX_PATH]; Py_BEGIN_ALLOW_THREADS - if (!GetVolumePathNameW(path->wide, volume, MAX_PATH)) { + if (!GetVolumePathNameW(abs_buf, volume, MAX_PATH)) { /* invalid path of some kind */ /* Note that this also includes the case where a volume is mounted in a path longer than 260 characters. This is likely to be rare @@ -4793,6 +4815,8 @@ os__path_isdevdrive_impl(PyObject *module, path_t *path) } Py_END_ALLOW_THREADS + path_cleanup(&_path); + PyMem_RawFree(abs_buf); if (err) { PyErr_SetFromWindowsErr(err); return NULL; @@ -5505,38 +5529,18 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return result; } -/*[clinic input] -os._path_abspath - - path: unicode - / - -Make path absolute. -[clinic start generated code]*/ - static PyObject * -os__path_abspath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ +posix_abspath(wchar_t *path_buf, Py_ssize_t path_len, int use_bytes) { - Py_ssize_t path_len, abs_len; + Py_ssize_t abs_len; wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; PyObject *result = NULL; - wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); - if (!path_buf) { - goto exit; - } - #ifdef MS_WINDOWS - if (wcslen(path_buf) != path_len) { - PyErr_Format(PyExc_ValueError, - "_path_abspath: embedded null character in path"); - goto exit; - } // Preserve `.\` for qualified referencing abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len, 1); if (abs_len == 0 || (abs_len == 1 && abs[0] == L'.')) { - result = posix_getcwd(0); + result = posix_getcwd(use_bytes); goto exit; } if (_PyOS_getfullpathname(abs, &abs_buf) < 0) { @@ -5547,7 +5551,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) abs_len = wcslen(abs_buf); #else if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { - result = posix_getcwd(0); + result = posix_getcwd(use_bytes); goto exit; } @@ -5594,14 +5598,53 @@ os__path_abspath_impl(PyObject *module, PyObject *path) #endif result = PyUnicode_FromWideChar(abs, abs_len); + if (use_bytes) { + Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); + } exit: - PyMem_Free(path_buf); PyMem_Free(cwd_buf); PyMem_RawFree(abs_buf); return result; } + +/*[clinic input] +os._path_abspath + + path: unicode + / + +Make path absolute. +[clinic start generated code]*/ + +static PyObject * +os__path_abspath_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ +{ + Py_ssize_t path_len; + PyObject *result = NULL; + + wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); + if (!path_buf) { + goto exit; + } + +#ifdef MS_WINDOWS + if (wcslen(path_buf) != path_len) { + PyErr_Format(PyExc_ValueError, + "_path_abspath: embedded null character in path"); + goto exit; + } +#endif + + result = posix_abspath(path_buf, path_len, 0); + +exit: + PyMem_Free(path_buf); + return result; +} + /*[clinic input] os.mkdir From 9c55537b0010029daf61319365eaf27dd1cfba9a Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Sat, 4 May 2024 09:38:19 +0200 Subject: [PATCH 48/72] Revert "Direct C call for `nt.isdevdrive()`" This reverts commit b3d04ac495cb2ba75b9622c988bfcc1c2f2f29e8. --- Lib/ntpath.py | 14 +- Lib/test/test_ntpath.py | 2 - Modules/clinic/posixmodule.c.h | 15 ++- Modules/posixmodule.c | 235 ++++----------------------------- 4 files changed, 49 insertions(+), 217 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 95b0c5ec7da3d1..b2f3e1b1049464 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -933,7 +933,19 @@ def commonpath(paths): from nt import _path_isfile as isfile from nt import _path_islink as islink from nt import _path_exists as exists - from nt import _path_isdevdrive as isdevdrive except ImportError: # Use genericpath.* as imported above pass + + +try: + from nt import _path_isdevdrive + def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive.""" + try: + return _path_isdevdrive(abspath(path)) + except OSError: + return False +except ImportError: + # Use genericpath.isdevdrive as imported above + pass diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index ddad3900f6a088..47c189e87f0f06 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1135,8 +1135,6 @@ def test_fast_paths_in_use(self): self.assertFalse(inspect.isfunction(os.path.islink)) self.assertTrue(os.path.exists is nt._path_exists) self.assertFalse(inspect.isfunction(os.path.exists)) - self.assertTrue(os.path.isdevdrive is nt._path_isdevdrive) - self.assertFalse(inspect.isfunction(os.path.isdevdrive)) @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") def test_isdevdrive(self): diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 675d4d938a1e87..7fdad8af7dff82 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1731,7 +1731,7 @@ PyDoc_STRVAR(os__path_isdevdrive__doc__, {"_path_isdevdrive", _PyCFunction_CAST(os__path_isdevdrive), METH_FASTCALL|METH_KEYWORDS, os__path_isdevdrive__doc__}, static PyObject * -os__path_isdevdrive_impl(PyObject *module, PyObject *path); +os__path_isdevdrive_impl(PyObject *module, path_t *path); static PyObject * os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1763,16 +1763,21 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_isdevdrive", "path", 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_isdevdrive_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + return_value = os__path_isdevdrive_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -12647,4 +12652,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=412b4e312064330d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a5c86cf571db8e59 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 016df7c42d5457..2c20b561cce935 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4723,38 +4723,16 @@ os_listmounts_impl(PyObject *module, path_t *volume) /*[clinic input] os._path_isdevdrive - path: object + path: path_t Determines whether the specified path is on a Windows Dev Drive. [clinic start generated code]*/ static PyObject * -os__path_isdevdrive_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=4f5a02fe80648dab input=cb26f0f438f49cb6]*/ +os__path_isdevdrive_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=1f437ea6677433a2 input=ee83e4996a48e23d]*/ { - path_t _path = PATH_T_INITIALIZE("isdevdrive", "path", 0, 0); - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; - } - - PyObject *abs_obj = posix_abspath((wchar_t *)_path.wide, _path.length, 0); - if (!abs_obj) { - path_cleanup(&_path); - return NULL; - } - wchar_t *abs_buf = PyUnicode_AsWideCharString(abs_obj, NULL); - Py_DECREF(abs_obj); - if (!abs_buf) { - path_cleanup(&_path); - return NULL; - } - #ifndef PERSISTENT_VOLUME_STATE_DEV_VOLUME /* This flag will be documented at https://learn.microsoft.com/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_file_fs_persistent_volume_information @@ -4768,7 +4746,7 @@ os__path_isdevdrive_impl(PyObject *module, PyObject *path) wchar_t volume[MAX_PATH]; Py_BEGIN_ALLOW_THREADS - if (!GetVolumePathNameW(abs_buf, volume, MAX_PATH)) { + if (!GetVolumePathNameW(path->wide, volume, MAX_PATH)) { /* invalid path of some kind */ /* Note that this also includes the case where a volume is mounted in a path longer than 260 characters. This is likely to be rare @@ -4818,8 +4796,6 @@ os__path_isdevdrive_impl(PyObject *module, PyObject *path) } Py_END_ALLOW_THREADS - path_cleanup(&_path); - PyMem_RawFree(abs_buf); if (err) { PyErr_SetFromWindowsErr(err); return NULL; @@ -5532,158 +5508,38 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return result; } -#ifdef MS_WINDOWS - -/* We centralise SECURITY_ATTRIBUTE initialization based around -templates that will probably mostly match common POSIX mode settings. -The _Py_SECURITY_ATTRIBUTE_DATA structure contains temporary data, as -a constructed SECURITY_ATTRIBUTE structure typically refers to memory -that has to be alive while it's being used. - -Typical use will look like: - SECURITY_ATTRIBUTES *pSecAttr = NULL; - struct _Py_SECURITY_ATTRIBUTE_DATA secAttrData; - int error, error2; - - Py_BEGIN_ALLOW_THREADS - switch (mode) { - case 0x1C0: // 0o700 - error = initializeMkdir700SecurityAttributes(&pSecAttr, &secAttrData); - break; - ... - default: - error = initializeDefaultSecurityAttributes(&pSecAttr, &secAttrData); - break; - } - - if (!error) { - // do operation, passing pSecAttr - } - - // Unconditionally clear secAttrData. - error2 = clearSecurityAttributes(&pSecAttr, &secAttrData); - if (!error) { - error = error2; - } - Py_END_ALLOW_THREADS - - if (error) { - PyErr_SetFromWindowsErr(error); - return NULL; - } -*/ - -struct _Py_SECURITY_ATTRIBUTE_DATA { - SECURITY_ATTRIBUTES securityAttributes; - PACL acl; - SECURITY_DESCRIPTOR sd; - EXPLICIT_ACCESS_W ea[4]; - char sid[64]; -}; - -static int -initializeDefaultSecurityAttributes( - PSECURITY_ATTRIBUTES *securityAttributes, - struct _Py_SECURITY_ATTRIBUTE_DATA *data -) { - assert(securityAttributes); - assert(data); - *securityAttributes = NULL; - memset(data, 0, sizeof(*data)); - return 0; -} - -static int -initializeMkdir700SecurityAttributes( - PSECURITY_ATTRIBUTES *securityAttributes, - struct _Py_SECURITY_ATTRIBUTE_DATA *data -) { - assert(securityAttributes); - assert(data); - *securityAttributes = NULL; - memset(data, 0, sizeof(*data)); - - if (!InitializeSecurityDescriptor(&data->sd, SECURITY_DESCRIPTOR_REVISION) - || !SetSecurityDescriptorGroup(&data->sd, NULL, TRUE)) { - return GetLastError(); - } - - int use_alias = 0; - DWORD cbSid = sizeof(data->sid); - if (!CreateWellKnownSid(WinCreatorOwnerRightsSid, NULL, (PSID)data->sid, &cbSid)) { - use_alias = 1; - } - - data->securityAttributes.nLength = sizeof(SECURITY_ATTRIBUTES); - data->ea[0].grfAccessPermissions = GENERIC_ALL; - data->ea[0].grfAccessMode = SET_ACCESS; - data->ea[0].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; - if (use_alias) { - data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_NAME; - data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; - data->ea[0].Trustee.ptstrName = L"CURRENT_USER"; - } else { - data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_SID; - data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP; - data->ea[0].Trustee.ptstrName = (LPWCH)(SID*)data->sid; - } - - data->ea[1].grfAccessPermissions = GENERIC_ALL; - data->ea[1].grfAccessMode = SET_ACCESS; - data->ea[1].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; - data->ea[1].Trustee.TrusteeForm = TRUSTEE_IS_NAME; - data->ea[1].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; - data->ea[1].Trustee.ptstrName = L"SYSTEM"; - - data->ea[2].grfAccessPermissions = GENERIC_ALL; - data->ea[2].grfAccessMode = SET_ACCESS; - data->ea[2].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; - data->ea[2].Trustee.TrusteeForm = TRUSTEE_IS_NAME; - data->ea[2].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; - data->ea[2].Trustee.ptstrName = L"ADMINISTRATORS"; - - int r = SetEntriesInAclW(3, data->ea, NULL, &data->acl); - if (r) { - return r; - } - if (!SetSecurityDescriptorDacl(&data->sd, TRUE, data->acl, FALSE)) { - return GetLastError(); - } - data->securityAttributes.lpSecurityDescriptor = &data->sd; - *securityAttributes = &data->securityAttributes; - return 0; -} +/*[clinic input] +os._path_abspath -static int -clearSecurityAttributes( - PSECURITY_ATTRIBUTES *securityAttributes, - struct _Py_SECURITY_ATTRIBUTE_DATA *data -) { - assert(securityAttributes); - assert(data); - *securityAttributes = NULL; - if (data->acl) { - if (LocalFree((void *)data->acl)) { - return GetLastError(); - } - } - return 0; -} + path: unicode + / -#endif +Make path absolute. +[clinic start generated code]*/ static PyObject * -posix_abspath(wchar_t *path_buf, Py_ssize_t path_len, int use_bytes) +os__path_abspath_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ { - Py_ssize_t abs_len; + Py_ssize_t path_len, abs_len; wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; PyObject *result = NULL; + wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); + if (!path_buf) { + goto exit; + } + #ifdef MS_WINDOWS + if (wcslen(path_buf) != path_len) { + PyErr_Format(PyExc_ValueError, + "_path_abspath: embedded null character in path"); + goto exit; + } // Preserve `.\` for qualified referencing abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len, 1); if (abs_len == 0 || (abs_len == 1 && abs[0] == L'.')) { - result = posix_getcwd(use_bytes); + result = posix_getcwd(0); goto exit; } if (_PyOS_getfullpathname(abs, &abs_buf) < 0) { @@ -5694,7 +5550,7 @@ posix_abspath(wchar_t *path_buf, Py_ssize_t path_len, int use_bytes) abs_len = wcslen(abs_buf); #else if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { - result = posix_getcwd(use_bytes); + result = posix_getcwd(0); goto exit; } @@ -5741,53 +5597,14 @@ posix_abspath(wchar_t *path_buf, Py_ssize_t path_len, int use_bytes) #endif result = PyUnicode_FromWideChar(abs, abs_len); - if (use_bytes) { - Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); - } exit: + PyMem_Free(path_buf); PyMem_Free(cwd_buf); PyMem_RawFree(abs_buf); return result; } - -/*[clinic input] -os._path_abspath - - path: unicode - / - -Make path absolute. -[clinic start generated code]*/ - -static PyObject * -os__path_abspath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ -{ - Py_ssize_t path_len; - PyObject *result = NULL; - - wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); - if (!path_buf) { - goto exit; - } - -#ifdef MS_WINDOWS - if (wcslen(path_buf) != path_len) { - PyErr_Format(PyExc_ValueError, - "_path_abspath: embedded null character in path"); - goto exit; - } -#endif - - result = posix_abspath(path_buf, path_len, 0); - -exit: - PyMem_Free(path_buf); - return result; -} - /*[clinic input] os.mkdir From 2bf2cd3459d0f5a5497e424532793fed6feb46c6 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Sat, 4 May 2024 09:41:10 +0200 Subject: [PATCH 49/72] Revert "Revert "Direct C call for `nt.isdevdrive()`"" This reverts commit 9c55537b0010029daf61319365eaf27dd1cfba9a. --- Lib/ntpath.py | 14 +- Lib/test/test_ntpath.py | 2 + Modules/clinic/posixmodule.c.h | 15 +-- Modules/posixmodule.c | 235 +++++++++++++++++++++++++++++---- 4 files changed, 217 insertions(+), 49 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index b2f3e1b1049464..95b0c5ec7da3d1 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -933,19 +933,7 @@ def commonpath(paths): from nt import _path_isfile as isfile from nt import _path_islink as islink from nt import _path_exists as exists + from nt import _path_isdevdrive as isdevdrive except ImportError: # Use genericpath.* as imported above pass - - -try: - from nt import _path_isdevdrive - def isdevdrive(path): - """Determines whether the specified path is on a Windows Dev Drive.""" - try: - return _path_isdevdrive(abspath(path)) - except OSError: - return False -except ImportError: - # Use genericpath.isdevdrive as imported above - pass diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 47c189e87f0f06..ddad3900f6a088 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1135,6 +1135,8 @@ def test_fast_paths_in_use(self): self.assertFalse(inspect.isfunction(os.path.islink)) self.assertTrue(os.path.exists is nt._path_exists) self.assertFalse(inspect.isfunction(os.path.exists)) + self.assertTrue(os.path.isdevdrive is nt._path_isdevdrive) + self.assertFalse(inspect.isfunction(os.path.isdevdrive)) @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") def test_isdevdrive(self): diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 7fdad8af7dff82..675d4d938a1e87 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1731,7 +1731,7 @@ PyDoc_STRVAR(os__path_isdevdrive__doc__, {"_path_isdevdrive", _PyCFunction_CAST(os__path_isdevdrive), METH_FASTCALL|METH_KEYWORDS, os__path_isdevdrive__doc__}, static PyObject * -os__path_isdevdrive_impl(PyObject *module, path_t *path); +os__path_isdevdrive_impl(PyObject *module, PyObject *path); static PyObject * os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1763,21 +1763,16 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - path_t path = PATH_T_INITIALIZE("_path_isdevdrive", "path", 0, 0); + PyObject *path; args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - if (!path_converter(args[0], &path)) { - goto exit; - } - return_value = os__path_isdevdrive_impl(module, &path); + path = args[0]; + return_value = os__path_isdevdrive_impl(module, path); exit: - /* Cleanup for path */ - path_cleanup(&path); - return return_value; } @@ -12652,4 +12647,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=a5c86cf571db8e59 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=412b4e312064330d input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2c20b561cce935..016df7c42d5457 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4723,16 +4723,38 @@ os_listmounts_impl(PyObject *module, path_t *volume) /*[clinic input] os._path_isdevdrive - path: path_t + path: object Determines whether the specified path is on a Windows Dev Drive. [clinic start generated code]*/ static PyObject * -os__path_isdevdrive_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=1f437ea6677433a2 input=ee83e4996a48e23d]*/ +os__path_isdevdrive_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=4f5a02fe80648dab input=cb26f0f438f49cb6]*/ { + path_t _path = PATH_T_INITIALIZE("isdevdrive", "path", 0, 0); + if (!path_converter(path, &_path)) { + path_cleanup(&_path); + if (PyErr_ExceptionMatches(PyExc_ValueError)) { + PyErr_Clear(); + Py_RETURN_FALSE; + } + return NULL; + } + + PyObject *abs_obj = posix_abspath((wchar_t *)_path.wide, _path.length, 0); + if (!abs_obj) { + path_cleanup(&_path); + return NULL; + } + wchar_t *abs_buf = PyUnicode_AsWideCharString(abs_obj, NULL); + Py_DECREF(abs_obj); + if (!abs_buf) { + path_cleanup(&_path); + return NULL; + } + #ifndef PERSISTENT_VOLUME_STATE_DEV_VOLUME /* This flag will be documented at https://learn.microsoft.com/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_file_fs_persistent_volume_information @@ -4746,7 +4768,7 @@ os__path_isdevdrive_impl(PyObject *module, path_t *path) wchar_t volume[MAX_PATH]; Py_BEGIN_ALLOW_THREADS - if (!GetVolumePathNameW(path->wide, volume, MAX_PATH)) { + if (!GetVolumePathNameW(abs_buf, volume, MAX_PATH)) { /* invalid path of some kind */ /* Note that this also includes the case where a volume is mounted in a path longer than 260 characters. This is likely to be rare @@ -4796,6 +4818,8 @@ os__path_isdevdrive_impl(PyObject *module, path_t *path) } Py_END_ALLOW_THREADS + path_cleanup(&_path); + PyMem_RawFree(abs_buf); if (err) { PyErr_SetFromWindowsErr(err); return NULL; @@ -5508,38 +5532,158 @@ os__path_normpath_impl(PyObject *module, PyObject *path) return result; } -/*[clinic input] -os._path_abspath +#ifdef MS_WINDOWS - path: unicode - / +/* We centralise SECURITY_ATTRIBUTE initialization based around +templates that will probably mostly match common POSIX mode settings. +The _Py_SECURITY_ATTRIBUTE_DATA structure contains temporary data, as +a constructed SECURITY_ATTRIBUTE structure typically refers to memory +that has to be alive while it's being used. -Make path absolute. -[clinic start generated code]*/ +Typical use will look like: + SECURITY_ATTRIBUTES *pSecAttr = NULL; + struct _Py_SECURITY_ATTRIBUTE_DATA secAttrData; + int error, error2; + + Py_BEGIN_ALLOW_THREADS + switch (mode) { + case 0x1C0: // 0o700 + error = initializeMkdir700SecurityAttributes(&pSecAttr, &secAttrData); + break; + ... + default: + error = initializeDefaultSecurityAttributes(&pSecAttr, &secAttrData); + break; + } + + if (!error) { + // do operation, passing pSecAttr + } + + // Unconditionally clear secAttrData. + error2 = clearSecurityAttributes(&pSecAttr, &secAttrData); + if (!error) { + error = error2; + } + Py_END_ALLOW_THREADS + + if (error) { + PyErr_SetFromWindowsErr(error); + return NULL; + } +*/ + +struct _Py_SECURITY_ATTRIBUTE_DATA { + SECURITY_ATTRIBUTES securityAttributes; + PACL acl; + SECURITY_DESCRIPTOR sd; + EXPLICIT_ACCESS_W ea[4]; + char sid[64]; +}; + +static int +initializeDefaultSecurityAttributes( + PSECURITY_ATTRIBUTES *securityAttributes, + struct _Py_SECURITY_ATTRIBUTE_DATA *data +) { + assert(securityAttributes); + assert(data); + *securityAttributes = NULL; + memset(data, 0, sizeof(*data)); + return 0; +} + +static int +initializeMkdir700SecurityAttributes( + PSECURITY_ATTRIBUTES *securityAttributes, + struct _Py_SECURITY_ATTRIBUTE_DATA *data +) { + assert(securityAttributes); + assert(data); + *securityAttributes = NULL; + memset(data, 0, sizeof(*data)); + + if (!InitializeSecurityDescriptor(&data->sd, SECURITY_DESCRIPTOR_REVISION) + || !SetSecurityDescriptorGroup(&data->sd, NULL, TRUE)) { + return GetLastError(); + } + + int use_alias = 0; + DWORD cbSid = sizeof(data->sid); + if (!CreateWellKnownSid(WinCreatorOwnerRightsSid, NULL, (PSID)data->sid, &cbSid)) { + use_alias = 1; + } + + data->securityAttributes.nLength = sizeof(SECURITY_ATTRIBUTES); + data->ea[0].grfAccessPermissions = GENERIC_ALL; + data->ea[0].grfAccessMode = SET_ACCESS; + data->ea[0].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; + if (use_alias) { + data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_NAME; + data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; + data->ea[0].Trustee.ptstrName = L"CURRENT_USER"; + } else { + data->ea[0].Trustee.TrusteeForm = TRUSTEE_IS_SID; + data->ea[0].Trustee.TrusteeType = TRUSTEE_IS_WELL_KNOWN_GROUP; + data->ea[0].Trustee.ptstrName = (LPWCH)(SID*)data->sid; + } + + data->ea[1].grfAccessPermissions = GENERIC_ALL; + data->ea[1].grfAccessMode = SET_ACCESS; + data->ea[1].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; + data->ea[1].Trustee.TrusteeForm = TRUSTEE_IS_NAME; + data->ea[1].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; + data->ea[1].Trustee.ptstrName = L"SYSTEM"; + + data->ea[2].grfAccessPermissions = GENERIC_ALL; + data->ea[2].grfAccessMode = SET_ACCESS; + data->ea[2].grfInheritance = SUB_CONTAINERS_AND_OBJECTS_INHERIT; + data->ea[2].Trustee.TrusteeForm = TRUSTEE_IS_NAME; + data->ea[2].Trustee.TrusteeType = TRUSTEE_IS_ALIAS; + data->ea[2].Trustee.ptstrName = L"ADMINISTRATORS"; + + int r = SetEntriesInAclW(3, data->ea, NULL, &data->acl); + if (r) { + return r; + } + if (!SetSecurityDescriptorDacl(&data->sd, TRUE, data->acl, FALSE)) { + return GetLastError(); + } + data->securityAttributes.lpSecurityDescriptor = &data->sd; + *securityAttributes = &data->securityAttributes; + return 0; +} + +static int +clearSecurityAttributes( + PSECURITY_ATTRIBUTES *securityAttributes, + struct _Py_SECURITY_ATTRIBUTE_DATA *data +) { + assert(securityAttributes); + assert(data); + *securityAttributes = NULL; + if (data->acl) { + if (LocalFree((void *)data->acl)) { + return GetLastError(); + } + } + return 0; +} + +#endif static PyObject * -os__path_abspath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ +posix_abspath(wchar_t *path_buf, Py_ssize_t path_len, int use_bytes) { - Py_ssize_t path_len, abs_len; + Py_ssize_t abs_len; wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; PyObject *result = NULL; - wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); - if (!path_buf) { - goto exit; - } - #ifdef MS_WINDOWS - if (wcslen(path_buf) != path_len) { - PyErr_Format(PyExc_ValueError, - "_path_abspath: embedded null character in path"); - goto exit; - } // Preserve `.\` for qualified referencing abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len, 1); if (abs_len == 0 || (abs_len == 1 && abs[0] == L'.')) { - result = posix_getcwd(0); + result = posix_getcwd(use_bytes); goto exit; } if (_PyOS_getfullpathname(abs, &abs_buf) < 0) { @@ -5550,7 +5694,7 @@ os__path_abspath_impl(PyObject *module, PyObject *path) abs_len = wcslen(abs_buf); #else if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { - result = posix_getcwd(0); + result = posix_getcwd(use_bytes); goto exit; } @@ -5597,14 +5741,53 @@ os__path_abspath_impl(PyObject *module, PyObject *path) #endif result = PyUnicode_FromWideChar(abs, abs_len); + if (use_bytes) { + Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); + } exit: - PyMem_Free(path_buf); PyMem_Free(cwd_buf); PyMem_RawFree(abs_buf); return result; } + +/*[clinic input] +os._path_abspath + + path: unicode + / + +Make path absolute. +[clinic start generated code]*/ + +static PyObject * +os__path_abspath_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ +{ + Py_ssize_t path_len; + PyObject *result = NULL; + + wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); + if (!path_buf) { + goto exit; + } + +#ifdef MS_WINDOWS + if (wcslen(path_buf) != path_len) { + PyErr_Format(PyExc_ValueError, + "_path_abspath: embedded null character in path"); + goto exit; + } +#endif + + result = posix_abspath(path_buf, path_len, 0); + +exit: + PyMem_Free(path_buf); + return result; +} + /*[clinic input] os.mkdir From 824f0d2ec59e0369878a0a0318ea2cba2e31afcf Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Sat, 4 May 2024 09:43:54 +0200 Subject: [PATCH 50/72] Restore lost code --- Lib/ntpath.py | 14 ++++- Lib/test/test_ntpath.py | 2 - Modules/clinic/posixmodule.c.h | 15 +++-- Modules/posixmodule.c | 101 ++++++++++----------------------- 4 files changed, 52 insertions(+), 80 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 95b0c5ec7da3d1..b2f3e1b1049464 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -933,7 +933,19 @@ def commonpath(paths): from nt import _path_isfile as isfile from nt import _path_islink as islink from nt import _path_exists as exists - from nt import _path_isdevdrive as isdevdrive except ImportError: # Use genericpath.* as imported above pass + + +try: + from nt import _path_isdevdrive + def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive.""" + try: + return _path_isdevdrive(abspath(path)) + except OSError: + return False +except ImportError: + # Use genericpath.isdevdrive as imported above + pass diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index ddad3900f6a088..47c189e87f0f06 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -1135,8 +1135,6 @@ def test_fast_paths_in_use(self): self.assertFalse(inspect.isfunction(os.path.islink)) self.assertTrue(os.path.exists is nt._path_exists) self.assertFalse(inspect.isfunction(os.path.exists)) - self.assertTrue(os.path.isdevdrive is nt._path_isdevdrive) - self.assertFalse(inspect.isfunction(os.path.isdevdrive)) @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") def test_isdevdrive(self): diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 675d4d938a1e87..7fdad8af7dff82 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -1731,7 +1731,7 @@ PyDoc_STRVAR(os__path_isdevdrive__doc__, {"_path_isdevdrive", _PyCFunction_CAST(os__path_isdevdrive), METH_FASTCALL|METH_KEYWORDS, os__path_isdevdrive__doc__}, static PyObject * -os__path_isdevdrive_impl(PyObject *module, PyObject *path); +os__path_isdevdrive_impl(PyObject *module, path_t *path); static PyObject * os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, PyObject *kwnames) @@ -1763,16 +1763,21 @@ os__path_isdevdrive(PyObject *module, PyObject *const *args, Py_ssize_t nargs, P }; #undef KWTUPLE PyObject *argsbuf[1]; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_isdevdrive", "path", 0, 0); args = _PyArg_UnpackKeywords(args, nargs, NULL, kwnames, &_parser, 1, 1, 0, argsbuf); if (!args) { goto exit; } - path = args[0]; - return_value = os__path_isdevdrive_impl(module, path); + if (!path_converter(args[0], &path)) { + goto exit; + } + return_value = os__path_isdevdrive_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -12647,4 +12652,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=412b4e312064330d input=a9049054013a1b77]*/ +/*[clinic end generated code: output=a5c86cf571db8e59 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 016df7c42d5457..12484f5963ff43 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -4723,38 +4723,16 @@ os_listmounts_impl(PyObject *module, path_t *volume) /*[clinic input] os._path_isdevdrive - path: object + path: path_t Determines whether the specified path is on a Windows Dev Drive. [clinic start generated code]*/ static PyObject * -os__path_isdevdrive_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=4f5a02fe80648dab input=cb26f0f438f49cb6]*/ +os__path_isdevdrive_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=1f437ea6677433a2 input=ee83e4996a48e23d]*/ { - path_t _path = PATH_T_INITIALIZE("isdevdrive", "path", 0, 0); - if (!path_converter(path, &_path)) { - path_cleanup(&_path); - if (PyErr_ExceptionMatches(PyExc_ValueError)) { - PyErr_Clear(); - Py_RETURN_FALSE; - } - return NULL; - } - - PyObject *abs_obj = posix_abspath((wchar_t *)_path.wide, _path.length, 0); - if (!abs_obj) { - path_cleanup(&_path); - return NULL; - } - wchar_t *abs_buf = PyUnicode_AsWideCharString(abs_obj, NULL); - Py_DECREF(abs_obj); - if (!abs_buf) { - path_cleanup(&_path); - return NULL; - } - #ifndef PERSISTENT_VOLUME_STATE_DEV_VOLUME /* This flag will be documented at https://learn.microsoft.com/windows-hardware/drivers/ddi/ntifs/ns-ntifs-_file_fs_persistent_volume_information @@ -4768,7 +4746,7 @@ os__path_isdevdrive_impl(PyObject *module, PyObject *path) wchar_t volume[MAX_PATH]; Py_BEGIN_ALLOW_THREADS - if (!GetVolumePathNameW(abs_buf, volume, MAX_PATH)) { + if (!GetVolumePathNameW(path->wide, volume, MAX_PATH)) { /* invalid path of some kind */ /* Note that this also includes the case where a volume is mounted in a path longer than 260 characters. This is likely to be rare @@ -4818,8 +4796,6 @@ os__path_isdevdrive_impl(PyObject *module, PyObject *path) } Py_END_ALLOW_THREADS - path_cleanup(&_path); - PyMem_RawFree(abs_buf); if (err) { PyErr_SetFromWindowsErr(err); return NULL; @@ -5672,18 +5648,38 @@ clearSecurityAttributes( #endif +/*[clinic input] +os._path_abspath + + path: unicode + / + +Make path absolute. +[clinic start generated code]*/ + static PyObject * -posix_abspath(wchar_t *path_buf, Py_ssize_t path_len, int use_bytes) +os__path_abspath_impl(PyObject *module, PyObject *path) +/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ { - Py_ssize_t abs_len; + Py_ssize_t path_len, abs_len; wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; PyObject *result = NULL; + wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); + if (!path_buf) { + goto exit; + } + #ifdef MS_WINDOWS + if (wcslen(path_buf) != path_len) { + PyErr_Format(PyExc_ValueError, + "_path_abspath: embedded null character in path"); + goto exit; + } // Preserve `.\` for qualified referencing abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len, 1); if (abs_len == 0 || (abs_len == 1 && abs[0] == L'.')) { - result = posix_getcwd(use_bytes); + result = posix_getcwd(0); goto exit; } if (_PyOS_getfullpathname(abs, &abs_buf) < 0) { @@ -5694,7 +5690,7 @@ posix_abspath(wchar_t *path_buf, Py_ssize_t path_len, int use_bytes) abs_len = wcslen(abs_buf); #else if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { - result = posix_getcwd(use_bytes); + result = posix_getcwd(0); goto exit; } @@ -5741,53 +5737,14 @@ posix_abspath(wchar_t *path_buf, Py_ssize_t path_len, int use_bytes) #endif result = PyUnicode_FromWideChar(abs, abs_len); - if (use_bytes) { - Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); - } exit: + PyMem_Free(path_buf); PyMem_Free(cwd_buf); PyMem_RawFree(abs_buf); return result; } - -/*[clinic input] -os._path_abspath - - path: unicode - / - -Make path absolute. -[clinic start generated code]*/ - -static PyObject * -os__path_abspath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ -{ - Py_ssize_t path_len; - PyObject *result = NULL; - - wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); - if (!path_buf) { - goto exit; - } - -#ifdef MS_WINDOWS - if (wcslen(path_buf) != path_len) { - PyErr_Format(PyExc_ValueError, - "_path_abspath: embedded null character in path"); - goto exit; - } -#endif - - result = posix_abspath(path_buf, path_len, 0); - -exit: - PyMem_Free(path_buf); - return result; -} - /*[clinic input] os.mkdir From 5850c3b1b83b46c5026876a3dfb2fc96e2af6e35 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Thu, 9 May 2024 17:22:53 +0200 Subject: [PATCH 51/72] Zero-valued flag enum has no name --- Doc/howto/enum.rst | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index 30be15230fc088..ec82d47d52830b 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -1126,6 +1126,11 @@ Using the following snippet for our examples:: ... PURPLE = RED | BLUE ... WHITE = RED | GREEN | BLUE ... + >>> class Perm(IntFlag): + ... R = 4 + ... W = 2 + ... X = 1 + ... the following are true: @@ -1149,6 +1154,8 @@ the following are true: >>> (Color.RED | Color.GREEN).name 'RED|GREEN' + >>> repr(Perm.R & Perm.W).name) + 'None' - multi-bit flags, aka aliases, can be returned from operations:: From eb26e222c1d0ed7c0d49f3c25410d67e46ade799 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Thu, 9 May 2024 17:26:13 +0200 Subject: [PATCH 52/72] Revert "Zero-valued flag enum has no name" This reverts commit 5850c3b1b83b46c5026876a3dfb2fc96e2af6e35. --- Doc/howto/enum.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Doc/howto/enum.rst b/Doc/howto/enum.rst index ec82d47d52830b..30be15230fc088 100644 --- a/Doc/howto/enum.rst +++ b/Doc/howto/enum.rst @@ -1126,11 +1126,6 @@ Using the following snippet for our examples:: ... PURPLE = RED | BLUE ... WHITE = RED | GREEN | BLUE ... - >>> class Perm(IntFlag): - ... R = 4 - ... W = 2 - ... X = 1 - ... the following are true: @@ -1154,8 +1149,6 @@ the following are true: >>> (Color.RED | Color.GREEN).name 'RED|GREEN' - >>> repr(Perm.R & Perm.W).name) - 'None' - multi-bit flags, aka aliases, can be returned from operations:: From d89e875c51f8ab91f5bfc2a67946ff8a1a4f82ec Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Tue, 28 May 2024 11:27:19 +0200 Subject: [PATCH 53/72] Direct C call for `posixpath.abspath()` --- Lib/ntpath.py | 7 +----- Lib/posixpath.py | 10 ++------- Lib/test/test_posixpath.py | 2 ++ Modules/clinic/posixmodule.c.h | 17 ++++++++------- Modules/posixmodule.c | 39 +++++++++++++++++----------------- 5 files changed, 34 insertions(+), 41 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 3615f900a7a074..435a45123dc256 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -574,8 +574,6 @@ def abspath(path): """Return the absolute version of a path.""" path = os.fspath(path) try: - if isinstance(path, bytes): - return os.fsencode(_path_abspath(os.fsdecode(path))) return _path_abspath(path) except (OSError, ValueError): # See gh-75230, handle outside for cleaner traceback @@ -590,10 +588,7 @@ def abspath(path): drive, _, path = splitroot(path) if drive and drive != splitroot(cwd)[0]: try: - if isinstance(drive, bytes): - cwd = os.fsencode(_path_abspath(os.fsdecode(drive))) - else: - cwd = _path_abspath(drive) + cwd = _path_abspath(drive) except (OSError, ValueError): # Invalid drive cwd = join(drive, sep) diff --git a/Lib/posixpath.py b/Lib/posixpath.py index 15f6c26753f4c3..b99df5a0162f88 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -373,7 +373,7 @@ def normpath(path): try: - from posix import _path_abspath + from posix import _path_abspath as abspath except ImportError: def abspath(path): """Return an absolute path.""" @@ -385,13 +385,7 @@ def abspath(path): if not path.startswith('/'): path = join(os.getcwd(), path) return normpath(path) -else: - def abspath(path): - """Return an absolute path.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_abspath(os.fsdecode(path))) - return _path_abspath(path) + # Return a canonical path (i.e. the absolute location of a file on the # filesystem). diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 9dcd28b65e9d5d..0071114a57dd2b 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -293,6 +293,8 @@ def test_fast_paths_in_use(self): self.assertFalse(inspect.isfunction(os.path.splitroot)) self.assertTrue(os.path.normpath is posix._path_normpath) self.assertFalse(inspect.isfunction(os.path.normpath)) + self.assertTrue(os.path.abspath is posix._path_abspath) + self.assertFalse(inspect.isfunction(os.path.abspath)) def test_expanduser(self): self.assertEqual(posixpath.expanduser("foo"), "foo") diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 51abbe304684dd..60731c78014104 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2459,28 +2459,29 @@ PyDoc_STRVAR(os__path_abspath__doc__, "_path_abspath($module, path, /)\n" "--\n" "\n" -"Make path absolute."); +"Return an absolute path."); #define OS__PATH_ABSPATH_METHODDEF \ {"_path_abspath", (PyCFunction)os__path_abspath, METH_O, os__path_abspath__doc__}, static PyObject * -os__path_abspath_impl(PyObject *module, PyObject *path); +os__path_abspath_impl(PyObject *module, path_t *path); static PyObject * os__path_abspath(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - PyObject *path; + path_t path = PATH_T_INITIALIZE("_path_abspath", "path", 0, !IS_MS_WINDOWS, 1, 0, 0); - if (!PyUnicode_Check(arg)) { - _PyArg_BadArgument("_path_abspath", "argument", "str", arg); + if (!path_converter(arg, &path)) { goto exit; } - path = arg; - return_value = os__path_abspath_impl(module, path); + return_value = os__path_abspath_impl(module, &path); exit: + /* Cleanup for path */ + path_cleanup(&path); + return return_value; } @@ -12787,4 +12788,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=47e788c9e8faa707 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5c39b00895e92770 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index acf6042d2f2094..18292f56cdccb0 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5520,49 +5520,48 @@ os__path_normpath_impl(PyObject *module, path_t *path) return result; } +#ifdef MS_WINDOWS +#define IS_MS_WINDOWS 1 +#else +#define IS_MS_WINDOWS 0 +#endif + /*[clinic input] os._path_abspath - path: unicode + path: path_t(make_wide=True, nonstrict="!IS_MS_WINDOWS") / -Make path absolute. +Return an absolute path. [clinic start generated code]*/ static PyObject * -os__path_abspath_impl(PyObject *module, PyObject *path) -/*[clinic end generated code: output=b58956d662b60be0 input=577ecb3473d22113]*/ +os__path_abspath_impl(PyObject *module, path_t *path) +/*[clinic end generated code: output=bb40fbf3be7251a4 input=6e209cdad4aa4d4d]*/ { - Py_ssize_t path_len, abs_len; + Py_ssize_t abs_len; wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; PyObject *result = NULL; - wchar_t *path_buf = PyUnicode_AsWideCharString(path, &path_len); - if (!path_buf) { - goto exit; - } - + wchar_t *path_buf = (wchar_t *)path->wide; + Py_ssize_t path_len = path->length; + int use_bytes = PyBytes_Check(path->object); #ifdef MS_WINDOWS - if (wcslen(path_buf) != path_len) { - PyErr_Format(PyExc_ValueError, - "_path_abspath: embedded null character in path"); - goto exit; - } // Preserve `.\` for qualified referencing abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len, 1); if (abs_len == 0 || (abs_len == 1 && abs[0] == L'.')) { - result = posix_getcwd(0); + result = posix_getcwd(use_bytes); goto exit; } if (_PyOS_getfullpathname(abs, &abs_buf) < 0) { - result = win32_error_object("GetFullPathNameW", path); + result = win32_error_object("GetFullPathNameW", path->object); goto exit; } abs = abs_buf; abs_len = wcslen(abs_buf); #else if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { - result = posix_getcwd(0); + result = posix_getcwd(use_bytes); goto exit; } @@ -5609,9 +5608,11 @@ os__path_abspath_impl(PyObject *module, PyObject *path) #endif result = PyUnicode_FromWideChar(abs, abs_len); + if (use_bytes) { + Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); + } exit: - PyMem_Free(path_buf); PyMem_Free(cwd_buf); PyMem_RawFree(abs_buf); return result; From 66c057fb29de14c2667981df946e7e46bf1180c8 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Tue, 28 May 2024 11:33:23 +0200 Subject: [PATCH 54/72] Optimisation --- Lib/ntpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 435a45123dc256..5e24c6e6380249 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -572,12 +572,12 @@ def abspath(path): else: # use native Windows method on Windows def abspath(path): """Return the absolute version of a path.""" - path = os.fspath(path) try: return _path_abspath(path) except (OSError, ValueError): # See gh-75230, handle outside for cleaner traceback pass + path = os.fspath(path) if not isabs(path): if isinstance(path, bytes): sep = b'/' From 8c1e11c87758fc58c4965ce19d1bfab290404d38 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Tue, 28 May 2024 16:50:29 +0200 Subject: [PATCH 55/72] Use root directory for other drives --- Lib/ntpath.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 5e24c6e6380249..d79f56326a58b6 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -564,9 +564,15 @@ def abspath(path): path = os.fspath(path) if not isabs(path): if isinstance(path, bytes): - path = join(os.getcwdb(), path) + sep = b'/' + cwd = os.getcwdb() else: - path = join(os.getcwd(), path) + sep = '/' + cwd = os.getcwd() + drive, _, path = splitroot(path) + if drive and drive != splitroot(cwd)[0]: + cwd = join(drive, sep) + path = join(cwd, path) return normpath(path) else: # use native Windows method on Windows From 50f5882c2788dd02f959201e89f6cc5b3d576379 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Tue, 28 May 2024 17:10:06 +0200 Subject: [PATCH 56/72] Fix drive relative paths --- Lib/ntpath.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index d79f56326a58b6..65b936b568af65 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -569,10 +569,10 @@ def abspath(path): else: sep = '/' cwd = os.getcwd() - drive, _, path = splitroot(path) + drive, root, path = splitroot(path) if drive and drive != splitroot(cwd)[0]: cwd = join(drive, sep) - path = join(cwd, path) + path = join(cwd, root + path) return normpath(path) else: # use native Windows method on Windows @@ -591,14 +591,14 @@ def abspath(path): else: sep = '/' cwd = os.getcwd() - drive, _, path = splitroot(path) + drive, root, path = splitroot(path) if drive and drive != splitroot(cwd)[0]: try: cwd = _path_abspath(drive) except (OSError, ValueError): # Invalid drive cwd = join(drive, sep) - path = join(cwd, path) + path = join(cwd, root + path) return normpath(path) try: From 74d7431ec0ac90cd8a70cd29c6265c97e800f37e Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Tue, 28 May 2024 17:21:06 +0200 Subject: [PATCH 57/72] Simplify implementation --- Lib/ntpath.py | 57 ++++++++++++++++++++++----------------------------- 1 file changed, 24 insertions(+), 33 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 65b936b568af65..00c76236349ba8 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -554,26 +554,33 @@ def normpath(path): return prefix + sep.join(comps) +def _abspath_fallback(path): + """Return the absolute version of a path as a fallback function in case + `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for + more. + + """ + + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + sep = b'/' + cwd = os.getcwdb() + else: + sep = '/' + cwd = os.getcwd() + drive, root, path = splitroot(path) + if drive and drive != splitroot(cwd)[0]: + cwd = join(drive, sep) + path = join(cwd, root + path) + return normpath(path) + # Return an absolute path. try: from nt import _path_abspath except ImportError: # not running on Windows - mock up something sensible - def abspath(path): - """Return the absolute version of a path.""" - path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - sep = b'/' - cwd = os.getcwdb() - else: - sep = '/' - cwd = os.getcwd() - drive, root, path = splitroot(path) - if drive and drive != splitroot(cwd)[0]: - cwd = join(drive, sep) - path = join(cwd, root + path) - return normpath(path) + abspath = _abspath_fallback else: # use native Windows method on Windows def abspath(path): @@ -581,25 +588,9 @@ def abspath(path): try: return _path_abspath(path) except (OSError, ValueError): - # See gh-75230, handle outside for cleaner traceback + # Handle outside for cleaner traceback pass - path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - sep = b'/' - cwd = os.getcwdb() - else: - sep = '/' - cwd = os.getcwd() - drive, root, path = splitroot(path) - if drive and drive != splitroot(cwd)[0]: - try: - cwd = _path_abspath(drive) - except (OSError, ValueError): - # Invalid drive - cwd = join(drive, sep) - path = join(cwd, root + path) - return normpath(path) + return _abspath_fallback(path) try: from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink From 807f83f50eea9bebc6e0c356250d7bf08cf23ea5 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Tue, 28 May 2024 17:29:24 +0200 Subject: [PATCH 58/72] Update Lib/ntpath.py --- Lib/ntpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 00c76236349ba8..22067f642ee74c 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -571,7 +571,7 @@ def _abspath_fallback(path): cwd = os.getcwd() drive, root, path = splitroot(path) if drive and drive != splitroot(cwd)[0]: - cwd = join(drive, sep) + cwd, root = drive, sep path = join(cwd, root + path) return normpath(path) From b19bbebd7bb73f6f207883a03c718f368eda59e1 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Tue, 28 May 2024 17:37:09 +0200 Subject: [PATCH 59/72] Improve readability --- Lib/ntpath.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 22067f642ee74c..8ba284e443e8f1 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -571,8 +571,9 @@ def _abspath_fallback(path): cwd = os.getcwd() drive, root, path = splitroot(path) if drive and drive != splitroot(cwd)[0]: - cwd, root = drive, sep - path = join(cwd, root + path) + path = join(drive, sep + path) + else: + path = join(cwd, root + path) return normpath(path) # Return an absolute path. From f934977513359233daea9cc4b71da7511a6f11a4 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Tue, 28 May 2024 20:35:18 +0200 Subject: [PATCH 60/72] Check embedded null manually --- Modules/clinic/posixmodule.c.h | 4 ++-- Modules/posixmodule.c | 15 +++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Modules/clinic/posixmodule.c.h b/Modules/clinic/posixmodule.c.h index 60731c78014104..f37d9086807e06 100644 --- a/Modules/clinic/posixmodule.c.h +++ b/Modules/clinic/posixmodule.c.h @@ -2471,7 +2471,7 @@ static PyObject * os__path_abspath(PyObject *module, PyObject *arg) { PyObject *return_value = NULL; - path_t path = PATH_T_INITIALIZE("_path_abspath", "path", 0, !IS_MS_WINDOWS, 1, 0, 0); + path_t path = PATH_T_INITIALIZE("_path_abspath", "path", 0, 1, 1, 0, 0); if (!path_converter(arg, &path)) { goto exit; @@ -12788,4 +12788,4 @@ os__supports_virtual_terminal(PyObject *module, PyObject *Py_UNUSED(ignored)) #ifndef OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #define OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF #endif /* !defined(OS__SUPPORTS_VIRTUAL_TERMINAL_METHODDEF) */ -/*[clinic end generated code: output=5c39b00895e92770 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=5a851e6712142787 input=a9049054013a1b77]*/ diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 18292f56cdccb0..2e113fdce211ef 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5520,16 +5520,10 @@ os__path_normpath_impl(PyObject *module, path_t *path) return result; } -#ifdef MS_WINDOWS -#define IS_MS_WINDOWS 1 -#else -#define IS_MS_WINDOWS 0 -#endif - /*[clinic input] os._path_abspath - path: path_t(make_wide=True, nonstrict="!IS_MS_WINDOWS") + path: path_t(make_wide=True, nonstrict=True) / Return an absolute path. @@ -5537,7 +5531,7 @@ Return an absolute path. static PyObject * os__path_abspath_impl(PyObject *module, path_t *path) -/*[clinic end generated code: output=bb40fbf3be7251a4 input=6e209cdad4aa4d4d]*/ +/*[clinic end generated code: output=bb40fbf3be7251a4 input=8870bbcaa7c920a1]*/ { Py_ssize_t abs_len; wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; @@ -5547,6 +5541,11 @@ os__path_abspath_impl(PyObject *module, path_t *path) Py_ssize_t path_len = path->length; int use_bytes = PyBytes_Check(path->object); #ifdef MS_WINDOWS + if (wcslen(path_buf) != path_len) { + PyErr_Format(PyExc_ValueError, + "_path_abspath: embedded null character in path"); + goto exit; + } // Preserve `.\` for qualified referencing abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len, 1); if (abs_len == 0 || (abs_len == 1 && abs[0] == L'.')) { From cf01b9043819efc4540551fca3baa72b0e041b19 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Wed, 29 May 2024 10:21:39 +0200 Subject: [PATCH 61/72] Revert fallback code on POSIX --- Lib/ntpath.py | 53 +++++++++++++++++++++++++++------------------------ 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 8ba284e443e8f1..2daef0f7a9d7bf 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -554,34 +554,20 @@ def normpath(path): return prefix + sep.join(comps) -def _abspath_fallback(path): - """Return the absolute version of a path as a fallback function in case - `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for - more. - - """ - - path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - sep = b'/' - cwd = os.getcwdb() - else: - sep = '/' - cwd = os.getcwd() - drive, root, path = splitroot(path) - if drive and drive != splitroot(cwd)[0]: - path = join(drive, sep + path) - else: - path = join(cwd, root + path) - return normpath(path) - # Return an absolute path. try: from nt import _path_abspath except ImportError: # not running on Windows - mock up something sensible - abspath = _abspath_fallback + def abspath(path): + """Return the absolute version of a path.""" + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + path = join(os.getcwdb(), path) + else: + path = join(os.getcwd(), path) + return normpath(path) else: # use native Windows method on Windows def abspath(path): @@ -589,9 +575,26 @@ def abspath(path): try: return _path_abspath(path) except (OSError, ValueError): - # Handle outside for cleaner traceback + # See gh-75230, handle outside for cleaner traceback pass - return _abspath_fallback(path) + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + sep = b'/' + cwd = os.getcwdb() + else: + sep = '/' + cwd = os.getcwd() + drive, root, path = splitroot(path) + if drive and drive != splitroot(cwd)[0]: + try: + path = join(_path_abspath(drive), path) + except (OSError, ValueError): + # Invalid drive \x00:Windows, assume root directory + path = drive + sep + path + else: + path = join(cwd, root + path) + return normpath(path) try: from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink From 29f9cfc90e7a8882863229fac109cd0db42e8a6b Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Wed, 29 May 2024 14:24:18 +0200 Subject: [PATCH 62/72] Update Lib/ntpath.py Co-authored-by: Eryk Sun --- Lib/ntpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index 2daef0f7a9d7bf..1349ff0ec190ad 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -590,7 +590,7 @@ def abspath(path): try: path = join(_path_abspath(drive), path) except (OSError, ValueError): - # Invalid drive \x00:Windows, assume root directory + # Invalid drive \x00: on Windows; assume root directory path = drive + sep + path else: path = join(cwd, root + path) From 374a39e9737e81938b0415b4023189aac32a47a8 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Wed, 29 May 2024 14:33:56 +0200 Subject: [PATCH 63/72] Update Lib/test/test_ntpath.py --- Lib/test/test_ntpath.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 7f326d6e611a3d..fb698c884251fd 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -808,8 +808,8 @@ def test_abspath(self): tester('ntpath.abspath("C:\\spam. . .")', "C:\\spam") tester('ntpath.abspath("C:/nul")', "\\\\.\\nul") tester('ntpath.abspath("C:\\nul")', "\\\\.\\nul") - tester('ntpath.abspath("Z:spam")', r"Z:\spam") - self.assertEqual(ntpath.abspath("Z:\x00"), "Z:\\\x00") + self.assertTrue(ntpath.isabs(ntpath.abspath("C:spam"))) + self.assertTrue(ntpath.isabs(ntpath.abspath("C:\x00"))) self.assertEqual(ntpath.abspath("\x00:spam"), "\x00:\\spam") tester('ntpath.abspath("//..")', "\\\\") tester('ntpath.abspath("//../")', "\\\\..\\") From adc708ded57fbb079d867b7632bc42d6c208d4d3 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Wed, 29 May 2024 23:11:26 +0200 Subject: [PATCH 64/72] Apply suggestions from code review Co-authored-by: Erlend E. Aasland --- Modules/posixmodule.c | 2 +- Python/fileutils.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2e113fdce211ef..dd6f5b4b206f73 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5590,7 +5590,7 @@ os__path_abspath_impl(PyObject *module, path_t *path) abs_buf = PyMem_RawMalloc(((size_t)abs_len + 1) * sizeof(wchar_t)); if (!abs_buf) { - result = PyErr_NoMemory(); + PyErr_NoMemory(); goto exit; } diff --git a/Python/fileutils.c b/Python/fileutils.c index 7c881a0d27d3e8..2e59233748ffb5 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2489,7 +2489,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *normsize, int explicit_curdir) { assert(path != NULL); - assert(start>=0); + assert(start >= 0); if ((size < 0 && !path[0]) || size == 0) { *normsize = 0; return path; From 4d0289aeced28f077df9007692dd735878ea665c Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Wed, 29 May 2024 23:15:13 +0200 Subject: [PATCH 65/72] Narrow scope --- Modules/posixmodule.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index dd6f5b4b206f73..788919814b8770 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5534,7 +5534,7 @@ os__path_abspath_impl(PyObject *module, path_t *path) /*[clinic end generated code: output=bb40fbf3be7251a4 input=8870bbcaa7c920a1]*/ { Py_ssize_t abs_len; - wchar_t *abs, *abs_buf = NULL, *cwd_buf = NULL; + wchar_t *abs, *abs_buf = NULL; PyObject *result = NULL; wchar_t *path_buf = (wchar_t *)path->wide; @@ -5559,6 +5559,7 @@ os__path_abspath_impl(PyObject *module, path_t *path) abs = abs_buf; abs_len = wcslen(abs_buf); #else + wchar_t *cwd_buf = NULL; if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { result = posix_getcwd(use_bytes); goto exit; @@ -5612,7 +5613,9 @@ os__path_abspath_impl(PyObject *module, path_t *path) } exit: +#ifndef MS_WINDOWS PyMem_Free(cwd_buf); +#endif PyMem_RawFree(abs_buf); return result; } From fcffc5353e3ec7216c4c09b00d75356fe5576bfb Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Wed, 29 May 2024 23:22:32 +0200 Subject: [PATCH 66/72] Improve comments --- Python/fileutils.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Python/fileutils.c b/Python/fileutils.c index 2e59233748ffb5..e8438fae184cc2 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2494,8 +2494,8 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, *normsize = 0; return path; } - // Start beyond end of path if (size >= 0 && start >= size) { + // Don't normalize path *normsize = size; return path; } @@ -2611,7 +2611,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, if (explicit_curdir && !rootsize && explicit) { // Add explicit curdir if (p2 == minP2 - 1) { - // '.' + // Set to '.' p2++; assert(p2 < p1); *p2 = L'.'; @@ -2619,7 +2619,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, else if (minP2[0] != L'.' || minP2[1] != L'.' || !SEP_OR_END(&minP2[2])) { - // Not '..\' + // Add leading '.\' wchar_t *p3 = p2; p2 += 2; assert(p2 < p1); From 1ebe5f4c9772efc98766610c54c114736f5e2df0 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Wed, 29 May 2024 23:24:47 +0200 Subject: [PATCH 67/72] Just check if path is absolute --- Lib/test/test_ntpath.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index fb698c884251fd..a0e5a8e66279be 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -810,7 +810,7 @@ def test_abspath(self): tester('ntpath.abspath("C:\\nul")', "\\\\.\\nul") self.assertTrue(ntpath.isabs(ntpath.abspath("C:spam"))) self.assertTrue(ntpath.isabs(ntpath.abspath("C:\x00"))) - self.assertEqual(ntpath.abspath("\x00:spam"), "\x00:\\spam") + self.assertTrue(ntpath.isabs(ntpath.abspath("\x00:spam"))) tester('ntpath.abspath("//..")', "\\\\") tester('ntpath.abspath("//../")', "\\\\..\\") tester('ntpath.abspath("//../..")', "\\\\..\\") From e047f03af01044bbf05870f8cf25971d109fad0c Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Thu, 30 May 2024 14:21:38 +0200 Subject: [PATCH 68/72] Narrow scope of `cwd_buf` --- Modules/posixmodule.c | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 788919814b8770..2932e0440dcd6c 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5559,7 +5559,6 @@ os__path_abspath_impl(PyObject *module, path_t *path) abs = abs_buf; abs_len = wcslen(abs_buf); #else - wchar_t *cwd_buf = NULL; if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { result = posix_getcwd(use_bytes); goto exit; @@ -5574,7 +5573,7 @@ os__path_abspath_impl(PyObject *module, path_t *path) goto exit; } Py_ssize_t cwd_len; - cwd_buf = PyUnicode_AsWideCharString(cwd_obj, &cwd_len); + wchar_t *cwd_buf = PyUnicode_AsWideCharString(cwd_obj, &cwd_len); Py_DECREF(cwd_obj); if (!cwd_buf) { goto exit; @@ -5586,17 +5585,20 @@ os__path_abspath_impl(PyObject *module, path_t *path) if ((size_t)abs_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { PyErr_SetString(PyExc_OverflowError, "path is too long"); + PyMem_Free(cwd_buf); goto exit; } abs_buf = PyMem_RawMalloc(((size_t)abs_len + 1) * sizeof(wchar_t)); if (!abs_buf) { PyErr_NoMemory(); + PyMem_Free(cwd_buf); goto exit; } // Join cwd & path wchar_t *p = memcpy(abs_buf, cwd_buf, cwd_len * sizeof(wchar_t)); + PyMem_Free(cwd_buf); p += cwd_len; if (add_sep) { *p++ = SEP; @@ -5613,9 +5615,6 @@ os__path_abspath_impl(PyObject *module, path_t *path) } exit: -#ifndef MS_WINDOWS - PyMem_Free(cwd_buf); -#endif PyMem_RawFree(abs_buf); return result; } From 625fc79181bd1417601edd8b4ec8e207b427e512 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Thu, 30 May 2024 14:43:54 +0200 Subject: [PATCH 69/72] Update Python/fileutils.c --- Python/fileutils.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/fileutils.c b/Python/fileutils.c index e8438fae184cc2..1d29af8ffcd996 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2550,8 +2550,8 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, explicit = 1; } - // Skip past cwd, not allowed if size is unknown or explicit_curdir is set - if (size >= 0 && path + start > p1 && !explicit_curdir) { + // Skip past cwd, not allowed if size is unknown + if (size >= 0 && path + start > p1) { p1 = p2 = path + start; lastC = *(p1-1); } From fc3f22747bfe69ff8e6ca496a6a5737b8f43914b Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Thu, 30 May 2024 14:45:41 +0200 Subject: [PATCH 70/72] Free memory before calling the `PyErr` APIs --- Modules/posixmodule.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index 2932e0440dcd6c..e232a8b37082c0 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5584,15 +5584,15 @@ os__path_abspath_impl(PyObject *module, path_t *path) abs_len = prefix_len + path_len; if ((size_t)abs_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { - PyErr_SetString(PyExc_OverflowError, "path is too long"); PyMem_Free(cwd_buf); + PyErr_SetString(PyExc_OverflowError, "path is too long"); goto exit; } abs_buf = PyMem_RawMalloc(((size_t)abs_len + 1) * sizeof(wchar_t)); if (!abs_buf) { - PyErr_NoMemory(); PyMem_Free(cwd_buf); + PyErr_NoMemory(); goto exit; } From 54c70214e60bd5c281125b2635d528b2188316d0 Mon Sep 17 00:00:00 2001 From: Nice Zombies Date: Thu, 30 May 2024 16:57:58 +0200 Subject: [PATCH 71/72] Apply suggestions from code review MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Python/fileutils.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Python/fileutils.c b/Python/fileutils.c index 1d29af8ffcd996..4dcaf8cfefc5cb 100644 --- a/Python/fileutils.c +++ b/Python/fileutils.c @@ -2483,7 +2483,7 @@ _Py_find_basename(const wchar_t *filename) to be the end of the path. 'start' is the position where to start normalizing. 'normsize' will be set to contain the length of the resulting normalized path. If 'explicit_curdir' is set, an explicit - curdir will be used for qualified referencing in the cwd */ + curdir will be used for qualified referencing in the cwd. */ wchar_t * _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, Py_ssize_t *normsize, int explicit_curdir) @@ -2553,7 +2553,7 @@ _Py_normpath_and_size(wchar_t *path, Py_ssize_t size, Py_ssize_t start, // Skip past cwd, not allowed if size is unknown if (size >= 0 && path + start > p1) { p1 = p2 = path + start; - lastC = *(p1-1); + lastC = *(p1 - 1); } /* if pEnd is specified, check that. Else, check for null terminator */ for (; !IS_END(p1); ++p1) { From dc66ef995a11ddf9a926a65772e0ce86a44eb1a6 Mon Sep 17 00:00:00 2001 From: Nineteendo Date: Thu, 30 May 2024 21:09:46 +0200 Subject: [PATCH 72/72] Use explicit return values --- Modules/posixmodule.c | 29 +++++++++++------------------ 1 file changed, 11 insertions(+), 18 deletions(-) diff --git a/Modules/posixmodule.c b/Modules/posixmodule.c index e232a8b37082c0..8abf7491c85c43 100644 --- a/Modules/posixmodule.c +++ b/Modules/posixmodule.c @@ -5535,33 +5535,28 @@ os__path_abspath_impl(PyObject *module, path_t *path) { Py_ssize_t abs_len; wchar_t *abs, *abs_buf = NULL; - PyObject *result = NULL; wchar_t *path_buf = (wchar_t *)path->wide; Py_ssize_t path_len = path->length; int use_bytes = PyBytes_Check(path->object); #ifdef MS_WINDOWS if (wcslen(path_buf) != path_len) { - PyErr_Format(PyExc_ValueError, - "_path_abspath: embedded null character in path"); - goto exit; + return PyErr_Format(PyExc_ValueError, + "_path_abspath: embedded null character in path"); } // Preserve `.\` for qualified referencing abs = _Py_normpath_and_size(path_buf, path_len, 0, &abs_len, 1); if (abs_len == 0 || (abs_len == 1 && abs[0] == L'.')) { - result = posix_getcwd(use_bytes); - goto exit; + return posix_getcwd(use_bytes); } if (_PyOS_getfullpathname(abs, &abs_buf) < 0) { - result = win32_error_object("GetFullPathNameW", path->object); - goto exit; + return win32_error_object("GetFullPathNameW", path->object); } abs = abs_buf; abs_len = wcslen(abs_buf); #else if (path_len == 0 || (path_len == 1 && path_buf[0] == L'.')) { - result = posix_getcwd(use_bytes); - goto exit; + return posix_getcwd(use_bytes); } if (_Py_isabs(path_buf)) { @@ -5570,13 +5565,13 @@ os__path_abspath_impl(PyObject *module, path_t *path) else { PyObject *cwd_obj = posix_getcwd(0); if (!cwd_obj) { - goto exit; + return NULL; } Py_ssize_t cwd_len; wchar_t *cwd_buf = PyUnicode_AsWideCharString(cwd_obj, &cwd_len); Py_DECREF(cwd_obj); if (!cwd_buf) { - goto exit; + return NULL; } int add_sep = cwd_buf[cwd_len - 1] != SEP; @@ -5586,14 +5581,13 @@ os__path_abspath_impl(PyObject *module, path_t *path) if ((size_t)abs_len + 1 > (size_t)PY_SSIZE_T_MAX / sizeof(wchar_t)) { PyMem_Free(cwd_buf); PyErr_SetString(PyExc_OverflowError, "path is too long"); - goto exit; + return NULL; } abs_buf = PyMem_RawMalloc(((size_t)abs_len + 1) * sizeof(wchar_t)); if (!abs_buf) { PyMem_Free(cwd_buf); - PyErr_NoMemory(); - goto exit; + return PyErr_NoMemory(); } // Join cwd & path @@ -5609,13 +5603,12 @@ os__path_abspath_impl(PyObject *module, path_t *path) } #endif - result = PyUnicode_FromWideChar(abs, abs_len); + PyObject *result = PyUnicode_FromWideChar(abs, abs_len); + PyMem_RawFree(abs_buf); if (use_bytes) { Py_SETREF(result, PyUnicode_EncodeFSDefault(result)); } -exit: - PyMem_RawFree(abs_buf); return result; }