From 3c3c29d6285fbdad633a7fab5935ed318889d3ed Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Thu, 7 Nov 2024 22:10:52 +0200 Subject: [PATCH 01/54] ctypes: Correctly handle NULL dlsym values For dlsym(), a return value of NULL does not necessarily indicate an error [1]. Therefore, to avoid using stale (or NULL) dlerror() values, we must: 1. clear the previous error state by calling dlerror() 2. call dlsym() 3. call dlerror() If the return value of dlerror() is not NULL, an error occured. In our case, we also subjectively treat a NULL return value from dlsym() as an error, so we format a special string for that. [1]: https://man7.org/linux/man-pages/man3/dlsym.3.html Signed-off-by: Georgios Alexopoulos --- ...-11-07-20-24-58.gh-issue-126554.ri12eb.rst | 1 + Modules/_ctypes/_ctypes.c | 42 +++++++++++++++---- 2 files changed, 35 insertions(+), 8 deletions(-) create mode 100644 Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst diff --git a/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst b/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst new file mode 100644 index 00000000000000..cfb06a528e913e --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst @@ -0,0 +1 @@ +Correctly handle NULL dlsym() values in ctypes. diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index eae69e484e1660..9b6f40de4d7677 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -967,18 +967,30 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, return NULL; } #else + dlerror(); address = (void *)dlsym(handle, name); - if (!address) { #ifdef __CYGWIN__ -/* dlerror() isn't very helpful on cygwin */ + if (!address) { + /* dlerror() isn't very helpful on cygwin */ PyErr_Format(PyExc_ValueError, "symbol '%s' not found", name); + return NULL; + } #else - PyErr_SetString(PyExc_ValueError, dlerror()); -#endif + char *dlerr; + dlerr = dlerror(); + if (dlerr) { + PyErr_SetString(PyExc_ValueError, dlerr); + return NULL; + } + else if (!address) { + PyErr_Format(PyExc_ValueError, + "symbol '%s' not found", + name); return NULL; } +#endif #endif ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); return PyCData_AtAddress(st, type, address); @@ -3774,19 +3786,33 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #else + dlerror(); address = (PPROC)dlsym(handle, name); - if (!address) { #ifdef __CYGWIN__ -/* dlerror() isn't very helpful on cygwin */ + if (!address) { + /* dlerror() isn't very helpful on cygwin */ PyErr_Format(PyExc_AttributeError, "function '%s' not found", name); + Py_DECREF(ftuple); + return NULL; + } #else - PyErr_SetString(PyExc_AttributeError, dlerror()); -#endif + char *dlerr; + dlerr = dlerror(); + if (dlerr) { + PyErr_SetString(PyExc_AttributeError, dlerr); + Py_DECREF(ftuple); + return NULL; + } + else if (!address) { + PyErr_Format(PyExc_AttributeError, + "function '%s' not found", + name); Py_DECREF(ftuple); return NULL; } +#endif #endif ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { From f71abe8dd0c7ae8ab528e7f891d63b2326835a3c Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Fri, 8 Nov 2024 15:15:56 +0200 Subject: [PATCH 02/54] Update Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst Co-authored-by: Peter Bierma --- .../next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst b/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst index cfb06a528e913e..1e41ceba2c2398 100644 --- a/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst +++ b/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst @@ -1 +1 @@ -Correctly handle NULL dlsym() values in ctypes. +Fix rare crash upon system failure in :class:`ctypes.CDLL`. From 64a739476cd5482e0ff3c89a32a5045b8e4089b2 Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Fri, 8 Nov 2024 15:17:05 +0200 Subject: [PATCH 03/54] Update Modules/_ctypes/_ctypes.c Co-authored-by: Peter Bierma --- Modules/_ctypes/_ctypes.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 9b6f40de4d7677..f6dbb1586909f4 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -978,8 +978,7 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, return NULL; } #else - char *dlerr; - dlerr = dlerror(); + char *dlerr = dlerror(); if (dlerr) { PyErr_SetString(PyExc_ValueError, dlerr); return NULL; From 512a084c4a64f0292389a4846df0f6379a2693df Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Fri, 8 Nov 2024 15:17:35 +0200 Subject: [PATCH 04/54] Update Modules/_ctypes/_ctypes.c Co-authored-by: Peter Bierma --- Modules/_ctypes/_ctypes.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index f6dbb1586909f4..ff4118f479ffd9 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3797,8 +3797,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #else - char *dlerr; - dlerr = dlerror(); + char *dlerr = dlerror(); if (dlerr) { PyErr_SetString(PyExc_AttributeError, dlerr); Py_DECREF(ftuple); From cd7d5d7190de42beef70ccf72a0053418441baeb Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Fri, 8 Nov 2024 15:52:06 +0200 Subject: [PATCH 05/54] Add dlerror() comment, handle callproc.c Signed-off-by: Georgios Alexopoulos --- Modules/_ctypes/_ctypes.c | 10 ++++++++++ Modules/_ctypes/callproc.c | 19 ++++++++++++++++--- 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index ff4118f479ffd9..8d270e5e21dcdd 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -967,6 +967,11 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, return NULL; } #else + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ dlerror(); address = (void *)dlsym(handle, name); #ifdef __CYGWIN__ @@ -3785,6 +3790,11 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #else + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ dlerror(); address = (PPROC)dlsym(handle, name); #ifdef __CYGWIN__ diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 5ac9cf16681645..c066d2a006fd2b 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1623,10 +1623,23 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) if (PySys_Audit("ctypes.dlsym/handle", "O", args) < 0) { return NULL; } + + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ + dlerror(); ptr = dlsym((void*)handle, name); - if (!ptr) { - PyErr_SetString(PyExc_OSError, - dlerror()); + char *dlerr = dlerror(); + if (dlerr) { + PyErr_SetString(PyExc_OSError, dlerr); + return NULL; + } + else if (!ptr) { + PyErr_Format(PyExc_OSError, + "symbol '%s' not found", + name); return NULL; } return PyLong_FromVoidPtr(ptr); From cf39b5b455cca8a3b2c65895d897993532fdacb5 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Fri, 8 Nov 2024 16:24:30 +0200 Subject: [PATCH 06/54] test: Add ctypes null dlsym case Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 69 ++++++++++++++++++++++++++++ 1 file changed, 69 insertions(+) create mode 100644 Lib/test/test_ctypes/test_dlerror.py diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py new file mode 100644 index 00000000000000..0f2d61d9a53bb5 --- /dev/null +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -0,0 +1,69 @@ +import os.path +import sys +import unittest +from ctypes import CDLL + +FOO_C = r""" +#include + +/* This is a 'GNU indirect function' (IFUNC) that will be called by + dlsym() to resolve the symbol "foo" to an address. Typically, such + a function would return the address of an actual function, but it + can also just return NULL. For some background on IFUNCs, see + https://willnewton.name/uncategorized/using-gnu-indirect-functions/ + + Adapted from Michael Kerrisk's answer: https://stackoverflow.com/a/53590014 +*/ + +asm (".type foo, @gnu_indirect_function"); + +void *foo(void) +{ + fprintf(stderr, "foo IFUNC called\n"); + return NULL; +} +""" + + +@unittest.skipUnless(sys.platform.startswith('linux'), + 'Test only valid for Linux') +class TestNullDlsym(unittest.TestCase): + def test_null_dlsym(self): + import subprocess + import tempfile + + try: + p = subprocess.Popen(['gcc', '--version'], stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL) + out, _ = p.communicate() + except OSError: + raise unittest.SkipTest('gcc, needed for test, not available') + with tempfile.TemporaryDirectory() as d: + # Create a source file foo.c, that uses + # a GNU Indirect Function. See FOO_C. + srcname = os.path.join(d, 'foo.c') + libname = 'py_ctypes_test_null_dlsym' + dstname = os.path.join(d, 'lib%s.so' % libname) + with open(srcname, 'w') as f: + f.write(FOO_C) + self.assertTrue(os.path.exists(srcname)) + # Compile the file to a shared library + cmd = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] + out = subprocess.check_output(cmd) + self.assertTrue(os.path.exists(dstname)) + # Load the shared library + L = CDLL(dstname) + + with self.assertRaises(AttributeError) as cm: + # Try accessing the 'foo' symbol. + # It should resolve via dlsym() to NULL, + # and since we subjectively treat NULL + # addresses as errors, we should get + # an error. + L.foo + + self.assertEqual(str(cm.exception), + "function 'foo' not found") + +if __name__ == "__main__": + unittest.main() From a72ce98f079202612cbdce928e6e85dbe5a2207d Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Fri, 8 Nov 2024 17:05:14 +0200 Subject: [PATCH 07/54] Add comment to test, minor wip-debugging prints Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 27 +++++++++++++++++++++++++-- Modules/_ctypes/_ctypes.c | 8 ++++++-- Modules/_ctypes/callproc.c | 4 +++- 3 files changed, 34 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 0f2d61d9a53bb5..8378abd304be44 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -28,6 +28,27 @@ @unittest.skipUnless(sys.platform.startswith('linux'), 'Test only valid for Linux') class TestNullDlsym(unittest.TestCase): + """GH-126554: Ensure that we catch NULL dlsym return values + + In rare cases, such as when using GNU IFUNCs, dlsym(), + the C function that ctypes' CDLL uses to get the address + of symbols, can return NULL. + + The objective way of telling if an error during symbol + lookup happened is to call glibc's dlerror() and check + for a non-NULL return value. + + However, there can be cases where dlsym() returns NULL + and dlerror() is also NULL, meaning that glibc did not + encounter any error. + + In the case of ctypes, we subjectively treat that as + an error, and throw a relevant exception. + + This test case ensures that we correctly enforce + this 'dlsym returned NULL -> throw Error' rule. + """ + def test_null_dlsym(self): import subprocess import tempfile @@ -62,8 +83,10 @@ def test_null_dlsym(self): # an error. L.foo - self.assertEqual(str(cm.exception), - "function 'foo' not found") + pred = "function 'foo' not found" in str(cm.exception) + self.assertTrue(pred) + # self.assertEqual(str(cm.exception), + # "function 'foo' not found") if __name__ == "__main__": unittest.main() diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8d270e5e21dcdd..fe9f031f5be9e7 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -985,12 +985,14 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, #else char *dlerr = dlerror(); if (dlerr) { + // XXX: This assumes that UTF-8 is the default locale. + // Investigate if this can cause problems. PyErr_SetString(PyExc_ValueError, dlerr); return NULL; } else if (!address) { PyErr_Format(PyExc_ValueError, - "symbol '%s' not found", + "[CDataType_in_dll_impl]: symbol '%s' not found", name); return NULL; } @@ -3809,13 +3811,15 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) #else char *dlerr = dlerror(); if (dlerr) { + // XXX: This assumes that UTF-8 is the default locale. + // Investigate if this can cause problems. PyErr_SetString(PyExc_AttributeError, dlerr); Py_DECREF(ftuple); return NULL; } else if (!address) { PyErr_Format(PyExc_AttributeError, - "function '%s' not found", + "[PyCFuncPtr_FromDll]: function '%s' not found", name); Py_DECREF(ftuple); return NULL; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index c066d2a006fd2b..a1daabd0efe243 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1633,12 +1633,14 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) ptr = dlsym((void*)handle, name); char *dlerr = dlerror(); if (dlerr) { + // XXX: This assumes that UTF-8 is the default locale. + // Investigate if this can cause problems. PyErr_SetString(PyExc_OSError, dlerr); return NULL; } else if (!ptr) { PyErr_Format(PyExc_OSError, - "symbol '%s' not found", + "[py_dl_sym]:symbol '%s' not found", name); return NULL; } From c820d16922520fb4df0ac21a060fd029e4cdc9d6 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Fri, 8 Nov 2024 18:32:00 +0200 Subject: [PATCH 08/54] Remove wip/debug prints Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 6 ++---- Modules/_ctypes/_ctypes.c | 4 ++-- Modules/_ctypes/callproc.c | 2 +- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 8378abd304be44..9a47d04cd2ca52 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -83,10 +83,8 @@ def test_null_dlsym(self): # an error. L.foo - pred = "function 'foo' not found" in str(cm.exception) - self.assertTrue(pred) - # self.assertEqual(str(cm.exception), - # "function 'foo' not found") + self.assertEqual(str(cm.exception), + "function 'foo' not found") if __name__ == "__main__": unittest.main() diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index fe9f031f5be9e7..4ff686f115dbb9 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -992,7 +992,7 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, } else if (!address) { PyErr_Format(PyExc_ValueError, - "[CDataType_in_dll_impl]: symbol '%s' not found", + "symbol '%s' not found", name); return NULL; } @@ -3819,7 +3819,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) } else if (!address) { PyErr_Format(PyExc_AttributeError, - "[PyCFuncPtr_FromDll]: function '%s' not found", + "function '%s' not found", name); Py_DECREF(ftuple); return NULL; diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index a1daabd0efe243..b38516aa32b700 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1640,7 +1640,7 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) } else if (!ptr) { PyErr_Format(PyExc_OSError, - "[py_dl_sym]:symbol '%s' not found", + "symbol '%s' not found", name); return NULL; } From 675112567f02f701e9dcae60a0769577d4038273 Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Fri, 8 Nov 2024 18:44:21 +0200 Subject: [PATCH 09/54] Update Modules/_ctypes/_ctypes.c Co-authored-by: Peter Bierma --- Modules/_ctypes/_ctypes.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 4ff686f115dbb9..0403ceace0d575 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -985,8 +985,8 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, #else char *dlerr = dlerror(); if (dlerr) { - // XXX: This assumes that UTF-8 is the default locale. - // Investigate if this can cause problems. + // XXX: This assumes that UTF-8 is the default locale. + // Investigate if this can cause problems. PyErr_SetString(PyExc_ValueError, dlerr); return NULL; } From e1f3b43dc4bd5271518fb1d714aaadfa5319739b Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Fri, 8 Nov 2024 18:44:36 +0200 Subject: [PATCH 10/54] Update Modules/_ctypes/callproc.c Co-authored-by: Peter Bierma --- Modules/_ctypes/callproc.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index b38516aa32b700..f15827704665c4 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1633,8 +1633,8 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) ptr = dlsym((void*)handle, name); char *dlerr = dlerror(); if (dlerr) { - // XXX: This assumes that UTF-8 is the default locale. - // Investigate if this can cause problems. + // XXX: This assumes that UTF-8 is the default locale. + // Investigate if this can cause problems. PyErr_SetString(PyExc_OSError, dlerr); return NULL; } From 8f7587d7fd111c1a743b3cefa01fcf0b4cb0209b Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Fri, 8 Nov 2024 18:44:48 +0200 Subject: [PATCH 11/54] Update Modules/_ctypes/_ctypes.c Co-authored-by: Peter Bierma --- Modules/_ctypes/_ctypes.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 0403ceace0d575..b8ffb05cdd2ee5 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3811,8 +3811,8 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) #else char *dlerr = dlerror(); if (dlerr) { - // XXX: This assumes that UTF-8 is the default locale. - // Investigate if this can cause problems. + // XXX: This assumes that UTF-8 is the default locale. + // Investigate if this can cause problems. PyErr_SetString(PyExc_AttributeError, dlerr); Py_DECREF(ftuple); return NULL; From 2b3a88cb7289bb9fe69806832aebdb2f1dde898c Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Fri, 8 Nov 2024 19:00:44 +0200 Subject: [PATCH 12/54] Explicitly fail if not AttributeError raised Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 9a47d04cd2ca52..dda5e9b34fd7b0 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -82,6 +82,7 @@ def test_null_dlsym(self): # addresses as errors, we should get # an error. L.foo + self.fail("AttributeError should have been raised!") self.assertEqual(str(cm.exception), "function 'foo' not found") From 9a1af296c7361d9dd6507dd238bdce1f64fec0b7 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Sat, 9 Nov 2024 00:33:51 +0200 Subject: [PATCH 13/54] Tidy up Assertion in test Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index dda5e9b34fd7b0..52eca8736ceba0 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -75,17 +75,14 @@ def test_null_dlsym(self): # Load the shared library L = CDLL(dstname) - with self.assertRaises(AttributeError) as cm: + with self.assertRaisesRegex(AttributeError, "function 'foo' not found"): # Try accessing the 'foo' symbol. # It should resolve via dlsym() to NULL, # and since we subjectively treat NULL # addresses as errors, we should get # an error. L.foo - self.fail("AttributeError should have been raised!") - self.assertEqual(str(cm.exception), - "function 'foo' not found") if __name__ == "__main__": unittest.main() From 3e02e09f328a93cd62df993f6376b8c7b8b74b78 Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Sat, 9 Nov 2024 23:47:29 +0200 Subject: [PATCH 14/54] Update Modules/_ctypes/_ctypes.c Co-authored-by: Peter Bierma --- Modules/_ctypes/_ctypes.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index b8ffb05cdd2ee5..29a57bec0b8bb7 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -985,8 +985,8 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, #else char *dlerr = dlerror(); if (dlerr) { - // XXX: This assumes that UTF-8 is the default locale. - // Investigate if this can cause problems. + // XXX: This assumes that UTF-8 is the default locale. + // Investigate if this can cause problems. PyErr_SetString(PyExc_ValueError, dlerr); return NULL; } From fbdbc26c723cb24867511214d62b0fee4987debe Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Sat, 9 Nov 2024 23:47:49 +0200 Subject: [PATCH 15/54] Update Modules/_ctypes/_ctypes.c Co-authored-by: Peter Bierma --- Modules/_ctypes/_ctypes.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 29a57bec0b8bb7..51c5ebd4307af3 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3811,8 +3811,6 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) #else char *dlerr = dlerror(); if (dlerr) { - // XXX: This assumes that UTF-8 is the default locale. - // Investigate if this can cause problems. PyErr_SetString(PyExc_AttributeError, dlerr); Py_DECREF(ftuple); return NULL; From fdf601302a32b34d044b0666492ac6c4cea4ed7c Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Sat, 9 Nov 2024 23:47:59 +0200 Subject: [PATCH 16/54] Update Modules/_ctypes/callproc.c Co-authored-by: Peter Bierma --- Modules/_ctypes/callproc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index f15827704665c4..c066d2a006fd2b 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1633,8 +1633,6 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) ptr = dlsym((void*)handle, name); char *dlerr = dlerror(); if (dlerr) { - // XXX: This assumes that UTF-8 is the default locale. - // Investigate if this can cause problems. PyErr_SetString(PyExc_OSError, dlerr); return NULL; } From 3aa29e5bde9b3aabcfe82201eb2ead20be652755 Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 12:05:12 +0200 Subject: [PATCH 17/54] Update Lib/test/test_ctypes/test_dlerror.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_ctypes/test_dlerror.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 52eca8736ceba0..ba7850ecd58b69 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -10,9 +10,9 @@ dlsym() to resolve the symbol "foo" to an address. Typically, such a function would return the address of an actual function, but it can also just return NULL. For some background on IFUNCs, see - https://willnewton.name/uncategorized/using-gnu-indirect-functions/ + https://willnewton.name/uncategorized/using-gnu-indirect-functions. - Adapted from Michael Kerrisk's answer: https://stackoverflow.com/a/53590014 + Adapted from Michael Kerrisk's answer: https://stackoverflow.com/a/53590014. */ asm (".type foo, @gnu_indirect_function"); From 0616214bd845d2b6d7848c55c4299f13d7fae3be Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 12:05:43 +0200 Subject: [PATCH 18/54] Update Lib/test/test_ctypes/test_dlerror.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_ctypes/test_dlerror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index ba7850ecd58b69..8b11c936209200 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -58,7 +58,7 @@ def test_null_dlsym(self): stderr=subprocess.DEVNULL) out, _ = p.communicate() except OSError: - raise unittest.SkipTest('gcc, needed for test, not available') + self.skipTest('gcc is missing') with tempfile.TemporaryDirectory() as d: # Create a source file foo.c, that uses # a GNU Indirect Function. See FOO_C. From 6d59ab750472593dcecc23f1d8b608ee7b5a4dd2 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Mon, 11 Nov 2024 12:32:52 +0200 Subject: [PATCH 19/54] test: Smoothly check for existence of GCC Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 8b11c936209200..2ec9c6064dad44 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -53,12 +53,13 @@ def test_null_dlsym(self): import subprocess import tempfile - try: - p = subprocess.Popen(['gcc', '--version'], stdout=subprocess.PIPE, - stderr=subprocess.DEVNULL) - out, _ = p.communicate() - except OSError: - self.skipTest('gcc is missing') + retcode = subprocess.call(["gcc --version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + shell=True) + if retcode != 0: + self.skipTest("gcc is missing") + with tempfile.TemporaryDirectory() as d: # Create a source file foo.c, that uses # a GNU Indirect Function. See FOO_C. From 31c621d7a05ede4a25c55c68c05e561c5138a363 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Mon, 11 Nov 2024 12:37:07 +0200 Subject: [PATCH 20/54] test: Clean up libfoo.so compilation command Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 2ec9c6064dad44..31ededb7eb4d79 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -61,18 +61,15 @@ def test_null_dlsym(self): self.skipTest("gcc is missing") with tempfile.TemporaryDirectory() as d: - # Create a source file foo.c, that uses - # a GNU Indirect Function. See FOO_C. + # Create a C file with a GNU Indirect Function (FOO_C) + # and compile it into a shared library. srcname = os.path.join(d, 'foo.c') - libname = 'py_ctypes_test_null_dlsym' - dstname = os.path.join(d, 'lib%s.so' % libname) + dstname = os.path.join(d, 'libfoo.so') with open(srcname, 'w') as f: f.write(FOO_C) - self.assertTrue(os.path.exists(srcname)) - # Compile the file to a shared library - cmd = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] - out = subprocess.check_output(cmd) - self.assertTrue(os.path.exists(dstname)) + args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] + p = subprocess.run(args, capture_output=True) + self.assertEqual(p.returncode, 0, p) # Load the shared library L = CDLL(dstname) From 53475a418f35b309463e2e0ad0cf190e5816f03b Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 12:42:45 +0200 Subject: [PATCH 21/54] Update Lib/test/test_ctypes/test_dlerror.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_ctypes/test_dlerror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 31ededb7eb4d79..556a60047d7197 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,4 +1,4 @@ -import os.path +import os import sys import unittest from ctypes import CDLL From 7fc538ef0823918d4fb92d52daec94e537bb378d Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 12:43:19 +0200 Subject: [PATCH 22/54] Update Modules/_ctypes/_ctypes.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/_ctypes/_ctypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 51c5ebd4307af3..524fd8519f4877 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -983,7 +983,7 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, return NULL; } #else - char *dlerr = dlerror(); + const char *dlerr = dlerror(); if (dlerr) { // XXX: This assumes that UTF-8 is the default locale. // Investigate if this can cause problems. From e20d97f162f1bf52294110e79ac0ef72c3b23fc8 Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 12:43:41 +0200 Subject: [PATCH 23/54] Update Modules/_ctypes/_ctypes.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/_ctypes/_ctypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 524fd8519f4877..22a6bac099dd9f 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3797,7 +3797,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) * Clear the previous value before calling dlsym(), * to ensure we can tell if our call resulted in an error. */ - dlerror(); + (void)dlerror(); address = (PPROC)dlsym(handle, name); #ifdef __CYGWIN__ if (!address) { From 95b072e7f33550d845d488754fb5a552270a2087 Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 12:44:03 +0200 Subject: [PATCH 24/54] Update Modules/_ctypes/_ctypes.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/_ctypes/_ctypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 22a6bac099dd9f..45902d5b34976a 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3809,7 +3809,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #else - char *dlerr = dlerror(); + const char *dlerr = dlerror(); if (dlerr) { PyErr_SetString(PyExc_AttributeError, dlerr); Py_DECREF(ftuple); From 048385c52a4198e6ccae0d9b0f1052acd0d4b34b Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 12:44:17 +0200 Subject: [PATCH 25/54] Update Modules/_ctypes/callproc.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/_ctypes/callproc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index c066d2a006fd2b..ff85c49b83d35b 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1629,7 +1629,7 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) * Clear the previous value before calling dlsym(), * to ensure we can tell if our call resulted in an error. */ - dlerror(); + (void)dlerror(); ptr = dlsym((void*)handle, name); char *dlerr = dlerror(); if (dlerr) { From 658098f9aaa5480fd6ef25b8bb8147230c1927ff Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 12:44:34 +0200 Subject: [PATCH 26/54] Update Modules/_ctypes/callproc.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/_ctypes/callproc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index ff85c49b83d35b..2629594db7679d 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1631,7 +1631,7 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) */ (void)dlerror(); ptr = dlsym((void*)handle, name); - char *dlerr = dlerror(); + const char *dlerr = dlerror(); if (dlerr) { PyErr_SetString(PyExc_OSError, dlerr); return NULL; From 6f4b921276401afd3ef8020f477174fc5cb1a6d6 Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 12:45:11 +0200 Subject: [PATCH 27/54] Update Modules/_ctypes/_ctypes.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/_ctypes/_ctypes.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 45902d5b34976a..e6f124abf4222e 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -972,7 +972,7 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, * Clear the previous value before calling dlsym(), * to ensure we can tell if our call resulted in an error. */ - dlerror(); + (void)dlerror(); address = (void *)dlsym(handle, name); #ifdef __CYGWIN__ if (!address) { From 25b6cf980214df07ee7d3241fa45c2f72f560694 Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 12:54:06 +0200 Subject: [PATCH 28/54] Update Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- .../next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst b/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst index 1e41ceba2c2398..6af89c7d4709ec 100644 --- a/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst +++ b/Misc/NEWS.d/next/C_API/2024-11-07-20-24-58.gh-issue-126554.ri12eb.rst @@ -1 +1,2 @@ -Fix rare crash upon system failure in :class:`ctypes.CDLL`. +Fix error handling in :class:`ctypes.CDLL` objects +which could result in a crash in rare situations. From 4bac25493117d2b0fcc2fb66072eed87bdf7a862 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Mon, 11 Nov 2024 13:00:00 +0200 Subject: [PATCH 29/54] test: Clarify string vs list in subprocess.run Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 556a60047d7197..9054fd380d9327 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -53,6 +53,17 @@ def test_null_dlsym(self): import subprocess import tempfile + # When using 'shell=True', Python invokes + # /bin/sh -c 'args[0]' args[1] ... + # + # The shell uses arg[0] as the + # command to execute, + # while the rest are arguments + # to the shell itself. + # + # To avoid mistakes, pass a single string + # (equivalent to a single-elem list), + # with the shell command to execute. retcode = subprocess.call(["gcc --version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, From ee6122cd661a82fd28e5d5004b6dfe14362428eb Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Mon, 11 Nov 2024 13:00:35 +0200 Subject: [PATCH 30/54] test: Use string in gcc --version Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 9054fd380d9327..715c70c692dca6 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -64,7 +64,7 @@ def test_null_dlsym(self): # To avoid mistakes, pass a single string # (equivalent to a single-elem list), # with the shell command to execute. - retcode = subprocess.call(["gcc --version"], + retcode = subprocess.call("gcc --version", stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True) From 08a7b8839fd8af4bb65b20f43a52716e13449c42 Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Mon, 11 Nov 2024 13:18:23 +0200 Subject: [PATCH 31/54] Update Lib/test/test_ctypes/test_dlerror.py MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Lib/test/test_ctypes/test_dlerror.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 715c70c692dca6..b61da1370af7ed 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -53,18 +53,9 @@ def test_null_dlsym(self): import subprocess import tempfile - # When using 'shell=True', Python invokes - # /bin/sh -c 'args[0]' args[1] ... - # - # The shell uses arg[0] as the - # command to execute, - # while the rest are arguments - # to the shell itself. - # - # To avoid mistakes, pass a single string - # (equivalent to a single-elem list), - # with the shell command to execute. - retcode = subprocess.call("gcc --version", + # Recall: using shell=True is equivalent to: /bin/sh -c 'args[0]' args[1] ..., + # so we need to pass a list with a single item, namely the command to run. + retcode = subprocess.call(["gcc --version"], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, shell=True) From 8c6b8aeacf65c2cad85d2847e9582a56ca6bfcf0 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 12 Nov 2024 10:18:59 +0100 Subject: [PATCH 32/54] Write to a pipe instead of stderr --- Lib/test/test_ctypes/test_dlerror.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index b61da1370af7ed..4be734e01f9d07 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -4,7 +4,7 @@ from ctypes import CDLL FOO_C = r""" -#include +#include /* This is a 'GNU indirect function' (IFUNC) that will be called by dlsym() to resolve the symbol "foo" to an address. Typically, such @@ -19,7 +19,7 @@ void *foo(void) { - fprintf(stderr, "foo IFUNC called\n"); + write($DESCRIPTOR, "OK", 2); return NULL; } """ @@ -62,13 +62,17 @@ def test_null_dlsym(self): if retcode != 0: self.skipTest("gcc is missing") + pipe_r, pipe_w = os.pipe() + self.addCleanup(os.close, pipe_r) + self.addCleanup(os.close, pipe_w) + with tempfile.TemporaryDirectory() as d: # Create a C file with a GNU Indirect Function (FOO_C) # and compile it into a shared library. srcname = os.path.join(d, 'foo.c') dstname = os.path.join(d, 'libfoo.so') with open(srcname, 'w') as f: - f.write(FOO_C) + f.write(FOO_C.replace('$DESCRIPTOR', str(pipe_w))) args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] p = subprocess.run(args, capture_output=True) self.assertEqual(p.returncode, 0, p) @@ -83,6 +87,9 @@ def test_null_dlsym(self): # an error. L.foo + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + if __name__ == "__main__": unittest.main() From 291bf1605c7e1bb32400432a1e678eb1d9b57376 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 12 Nov 2024 10:19:47 +0100 Subject: [PATCH 33/54] Reorganize dlerror() code - Only clear the error if we'll get it later - De-duplicate the default message --- Modules/_ctypes/_ctypes.c | 58 ++++++++++++++++++--------------------- 1 file changed, 27 insertions(+), 31 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index e6f124abf4222e..2a5875334a3521 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -956,50 +956,46 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, return NULL; } +#undef USE_DLERROR #ifdef MS_WIN32 Py_BEGIN_ALLOW_THREADS address = (void *)GetProcAddress(handle, name); Py_END_ALLOW_THREADS - if (!address) { - PyErr_Format(PyExc_ValueError, - "symbol '%s' not found", - name); - return NULL; - } #else - /* dlerror() always returns the latest error. - * - * Clear the previous value before calling dlsym(), - * to ensure we can tell if our call resulted in an error. - */ - (void)dlerror(); + #ifdef __CYGWIN__ + // dlerror() isn't very helpful on cygwin + #else + #define USE_DLERROR + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ + (void)dlerror(); + #endif address = (void *)dlsym(handle, name); -#ifdef __CYGWIN__ - if (!address) { - /* dlerror() isn't very helpful on cygwin */ - PyErr_Format(PyExc_ValueError, - "symbol '%s' not found", - name); - return NULL; +#endif + + if (address) { + ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); + return PyCData_AtAddress(st, type, address); } -#else + +#ifdef USE_DLERROR + // This assumes the error message is UTF-8 (or ASCII). + // Investigate if this can cause problems. const char *dlerr = dlerror(); if (dlerr) { - // XXX: This assumes that UTF-8 is the default locale. - // Investigate if this can cause problems. PyErr_SetString(PyExc_ValueError, dlerr); return NULL; } - else if (!address) { - PyErr_Format(PyExc_ValueError, - "symbol '%s' not found", - name); - return NULL; - } -#endif #endif - ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); - return PyCData_AtAddress(st, type, address); +#undef USE_DLERROR + + PyErr_Format(PyExc_ValueError, + "symbol '%s' not found", + name); + return NULL; } /*[clinic input] From 1cbffcc6571081a2a8b82c99a656d3a36e94350f Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Tue, 12 Nov 2024 10:55:56 +0100 Subject: [PATCH 34/54] Use PyUnicode_DecodeLocale for the error --- Modules/_ctypes/_ctypes.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 2a5875334a3521..97ba70b9d819ba 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -986,8 +986,13 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, // Investigate if this can cause problems. const char *dlerr = dlerror(); if (dlerr) { - PyErr_SetString(PyExc_ValueError, dlerr); - return NULL; + PyObject *message = PyUnicode_DecodeLocale(dlerr, "strict"); + if (message) { + PyErr_SetObject(PyExc_ValueError, message); + return NULL; + } + // Ignore errors from converting the message to str + PyErr_Clear(); } #endif #undef USE_DLERROR From 5748c13bbe6d9dc2f09ca2d23bd7133742988e83 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 16:26:35 +0200 Subject: [PATCH 35/54] Clean up PyCFuncPtr_FromDll, uses #define USE_DLERROR Signed-off-by: Georgios Alexopoulos --- Modules/_ctypes/_ctypes.c | 57 +++++++++++++++++++++++---------------- 1 file changed, 34 insertions(+), 23 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 97ba70b9d819ba..8a270593bee7a0 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3778,6 +3778,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } +#undef USE_DLERROR #ifdef MS_WIN32 address = FindAddress(handle, name, (PyObject *)type); if (!address) { @@ -3793,30 +3794,39 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #else - /* dlerror() always returns the latest error. - * - * Clear the previous value before calling dlsym(), - * to ensure we can tell if our call resulted in an error. - */ - (void)dlerror(); + #ifdef __CYGWIN__ + //dlerror() isn't very helpful on cygwin */ + #else + #define USE_DLERROR + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ + (void)dlerror(); + #endif address = (PPROC)dlsym(handle, name); -#ifdef __CYGWIN__ - if (!address) { - /* dlerror() isn't very helpful on cygwin */ - PyErr_Format(PyExc_AttributeError, - "function '%s' not found", - name); - Py_DECREF(ftuple); - return NULL; - } -#else - const char *dlerr = dlerror(); - if (dlerr) { - PyErr_SetString(PyExc_AttributeError, dlerr); - Py_DECREF(ftuple); - return NULL; + + if (address) { + goto dlsym_ok; } - else if (!address) { + else { +#ifdef USE_DLERROR + // This assumes the error message is UTF-8 (or ASCII). + // Investigate if this can cause problems. + const char *dlerr = dlerror(); + if (dlerr) { + PyObject *message = PyUnicode_DecodeLocale(dlerr, "strict"); + if (message) { + PyErr_SetObject(PyExc_AttributeError, message); + Py_DECREF(ftuple); + return NULL; + } + // Ignore errors from converting the message to str + PyErr_Clear(); + } +#endif +#undef USE_DLERROR PyErr_Format(PyExc_AttributeError, "function '%s' not found", name); @@ -3824,7 +3834,8 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) return NULL; } #endif -#endif + +dlsym_ok: ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { Py_DECREF(ftuple); From 65e2b7e12474ca44d7fe931bd5978ba9bba0b985 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 16:26:50 +0200 Subject: [PATCH 36/54] Clean up py_dl_sym, uses #define USE_DLERROR Signed-off-by: Georgios Alexopoulos --- Modules/_ctypes/callproc.c | 47 ++++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 17 deletions(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 2629594db7679d..d0bfc63c3e5f0c 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1623,26 +1623,39 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) if (PySys_Audit("ctypes.dlsym/handle", "O", args) < 0) { return NULL; } - - /* dlerror() always returns the latest error. - * - * Clear the previous value before calling dlsym(), - * to ensure we can tell if our call resulted in an error. - */ - (void)dlerror(); + #ifdef __CYGWIN__ + // dlerror() isn't very helpful on cygwin + #else + #define USE_DLERROR + /* dlerror() always returns the latest error. + * + * Clear the previous value before calling dlsym(), + * to ensure we can tell if our call resulted in an error. + */ + (void)dlerror(); + #endif ptr = dlsym((void*)handle, name); - const char *dlerr = dlerror(); + if (ptr) + return PyLong_FromVoidPtr(ptr); +#ifdef USE_DLERROR + // This assumes the error message is UTF-8 (or ASCII). + // Investigate if this can cause problems. + const char *dlerr = dlerror(); if (dlerr) { - PyErr_SetString(PyExc_OSError, dlerr); - return NULL; - } - else if (!ptr) { - PyErr_Format(PyExc_OSError, - "symbol '%s' not found", - name); - return NULL; + PyObject *message = PyUnicode_DecodeLocale(dlerr, "strict"); + if (message) { + PyErr_SetObject(PyExc_OSError, message); + return NULL; + } + // Ignore errors from converting the message to str + PyErr_Clear(); } - return PyLong_FromVoidPtr(ptr); +#endif +#undef USE_DLERROR + PyErr_Format(PyExc_OSError, + "symbol '%s' not found", + name); + return NULL; } #endif From 38fbaed6585c63b77913effb581d316b0d6534f3 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 16:27:10 +0200 Subject: [PATCH 37/54] test: Exercise 3 code paths, one for each C function we want to test Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 4be734e01f9d07..699f9f92fd39d9 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,7 +1,8 @@ import os import sys import unittest -from ctypes import CDLL +from ctypes import CDLL, c_int +from _ctypes import dlopen, dlsym FOO_C = r""" #include @@ -76,9 +77,9 @@ def test_null_dlsym(self): args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] p = subprocess.run(args, capture_output=True) self.assertEqual(p.returncode, 0, p) - # Load the shared library - L = CDLL(dstname) + # Case #1: Test 'PyCFuncPtr_FromDll' from Modules/_ctypes/_ctypes.c + L = CDLL(dstname) with self.assertRaisesRegex(AttributeError, "function 'foo' not found"): # Try accessing the 'foo' symbol. # It should resolve via dlsym() to NULL, @@ -87,8 +88,23 @@ def test_null_dlsym(self): # an error. L.foo - # Assert that the IFUNC was called - self.assertEqual(os.read(pipe_r, 2), b'OK') + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + # Case #2: Test 'CDataType_in_dll_impl' from Modules/_ctypes/_ctypes.c + with self.assertRaisesRegex(ValueError, "symbol 'foo' not found"): + c_int.in_dll(L, "foo") + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + # Case #3: Test 'py_dl_sym' from Modules/_ctypes/callproc.c + L = dlopen(dstname) + with self.assertRaisesRegex(OSError, "symbol 'foo' not found"): + dlsym(L, "foo") + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') if __name__ == "__main__": From cc8e3669854d3be412789f82a772337bf3b7b3cb Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 16:44:51 +0200 Subject: [PATCH 38/54] Add empty statement after label, placate C compiler Signed-off-by: Georgios Alexopoulos --- Modules/_ctypes/_ctypes.c | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8a270593bee7a0..c59e0ba933a58d 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3836,6 +3836,12 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) #endif dlsym_ok: + /* Add an empty statement (;) to placate some C compilers + that do not allow declarations after labels. + + See https://stackoverflow.com/a/18496437. + */ + ; ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { Py_DECREF(ftuple); From 4319332393fbf2485911e0be17847f9e4baf1f39 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 17:07:01 +0200 Subject: [PATCH 39/54] Move label to before #endif (WIN32) Signed-off-by: Georgios Alexopoulos --- Modules/_ctypes/_ctypes.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index c59e0ba933a58d..2e431f3fe61541 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -3833,8 +3833,6 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(ftuple); return NULL; } -#endif - dlsym_ok: /* Add an empty statement (;) to placate some C compilers that do not allow declarations after labels. @@ -3842,6 +3840,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) See https://stackoverflow.com/a/18496437. */ ; +#endif ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { Py_DECREF(ftuple); From 8356cf48be2789f6a9589728915d5fb137b74900 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 17:32:51 +0200 Subject: [PATCH 40/54] Import ctypes, _ctypes inside test, so we don't get ImportError in windows systems Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 699f9f92fd39d9..badf7e9dd7d1d4 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,8 +1,6 @@ import os import sys import unittest -from ctypes import CDLL, c_int -from _ctypes import dlopen, dlsym FOO_C = r""" #include @@ -49,10 +47,11 @@ class TestNullDlsym(unittest.TestCase): This test case ensures that we correctly enforce this 'dlsym returned NULL -> throw Error' rule. """ - def test_null_dlsym(self): import subprocess import tempfile + from ctypes import CDLL, c_int + from _ctypes import dlopen, dlsym # Recall: using shell=True is equivalent to: /bin/sh -c 'args[0]' args[1] ..., # so we need to pass a list with a single item, namely the command to run. From 1e1e78b139e2252a28a44d1898130eda8dd06410 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 17:40:57 +0200 Subject: [PATCH 41/54] Revert "Import ctypes, _ctypes inside test, so we don't get ImportError in windows systems" This reverts commit 8356cf48be2789f6a9589728915d5fb137b74900. --- Lib/test/test_ctypes/test_dlerror.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index badf7e9dd7d1d4..699f9f92fd39d9 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,6 +1,8 @@ import os import sys import unittest +from ctypes import CDLL, c_int +from _ctypes import dlopen, dlsym FOO_C = r""" #include @@ -47,11 +49,10 @@ class TestNullDlsym(unittest.TestCase): This test case ensures that we correctly enforce this 'dlsym returned NULL -> throw Error' rule. """ + def test_null_dlsym(self): import subprocess import tempfile - from ctypes import CDLL, c_int - from _ctypes import dlopen, dlsym # Recall: using shell=True is equivalent to: /bin/sh -c 'args[0]' args[1] ..., # so we need to pass a list with a single item, namely the command to run. From 4e870404900ee52c730947418b186defaa609d3a Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 17:47:54 +0200 Subject: [PATCH 42/54] Remove unnecessary goto, fix preprocessor cmd indentation Signed-off-by: Georgios Alexopoulos --- Modules/_ctypes/_ctypes.c | 22 ++++------------------ Modules/_ctypes/callproc.c | 2 -- 2 files changed, 4 insertions(+), 20 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 2e431f3fe61541..8f34267f1935b4 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -982,8 +982,6 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, } #ifdef USE_DLERROR - // This assumes the error message is UTF-8 (or ASCII). - // Investigate if this can cause problems. const char *dlerr = dlerror(); if (dlerr) { PyObject *message = PyUnicode_DecodeLocale(dlerr, "strict"); @@ -3807,13 +3805,8 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) #endif address = (PPROC)dlsym(handle, name); - if (address) { - goto dlsym_ok; - } - else { -#ifdef USE_DLERROR - // This assumes the error message is UTF-8 (or ASCII). - // Investigate if this can cause problems. + if (!address) { + #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { PyObject *message = PyUnicode_DecodeLocale(dlerr, "strict"); @@ -3825,22 +3818,15 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) // Ignore errors from converting the message to str PyErr_Clear(); } -#endif -#undef USE_DLERROR + #endif PyErr_Format(PyExc_AttributeError, "function '%s' not found", name); Py_DECREF(ftuple); return NULL; } -dlsym_ok: - /* Add an empty statement (;) to placate some C compilers - that do not allow declarations after labels. - - See https://stackoverflow.com/a/18496437. - */ - ; #endif +#undef USE_DLERROR ctypes_state *st = get_module_state_by_def(Py_TYPE(type)); if (!_validate_paramflags(st, type, paramflags)) { Py_DECREF(ftuple); diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index d0bfc63c3e5f0c..4e32d89fe25050 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1638,8 +1638,6 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) if (ptr) return PyLong_FromVoidPtr(ptr); #ifdef USE_DLERROR - // This assumes the error message is UTF-8 (or ASCII). - // Investigate if this can cause problems. const char *dlerr = dlerror(); if (dlerr) { PyObject *message = PyUnicode_DecodeLocale(dlerr, "strict"); From 6a3aea7b108d5c5047a6ec4d7abf9b46dedd7955 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 17:50:00 +0200 Subject: [PATCH 43/54] test: Remove shell=True from gcc check, use separate arguments Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 699f9f92fd39d9..2d0440e33be822 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -54,12 +54,9 @@ def test_null_dlsym(self): import subprocess import tempfile - # Recall: using shell=True is equivalent to: /bin/sh -c 'args[0]' args[1] ..., - # so we need to pass a list with a single item, namely the command to run. - retcode = subprocess.call(["gcc --version"], + retcode = subprocess.call(["gcc", "--version"], stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - shell=True) + stderr=subprocess.DEVNULL) if retcode != 0: self.skipTest("gcc is missing") From 73918bf75c65f396902b07ecf3f55402366aebec Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 17:52:03 +0200 Subject: [PATCH 44/54] Remove extra whitespace Signed-off-by: Georgios Alexopoulos --- Modules/_ctypes/_ctypes.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 8f34267f1935b4..dfa320f8432026 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -996,8 +996,8 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, #undef USE_DLERROR PyErr_Format(PyExc_ValueError, - "symbol '%s' not found", - name); + "symbol '%s' not found", + name); return NULL; } From 0ca32d4dcb9c5c39007b78574f39de782a506c22 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 21:30:20 +0200 Subject: [PATCH 45/54] test: Move imports to func, avoid ImportError on Windows Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 2d0440e33be822..1b91ef4a6f91a1 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,8 +1,6 @@ import os import sys import unittest -from ctypes import CDLL, c_int -from _ctypes import dlopen, dlsym FOO_C = r""" #include @@ -53,6 +51,8 @@ class TestNullDlsym(unittest.TestCase): def test_null_dlsym(self): import subprocess import tempfile + from ctypes import CDLL, c_int + from _ctypes import dlopen, dlsym retcode = subprocess.call(["gcc", "--version"], stdout=subprocess.DEVNULL, From 6db59637c0394e8b543e2cbb3b499b5f584ee69d Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 21:35:14 +0200 Subject: [PATCH 46/54] test: Explain why we moved imports to within test function Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 1b91ef4a6f91a1..35015627241cb7 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -51,6 +51,12 @@ class TestNullDlsym(unittest.TestCase): def test_null_dlsym(self): import subprocess import tempfile + + # To avoid ImportErrors on Windows, where _ctypes does not have + # dlopen and dlsym, + # import here, i.e., inside the test function. + # The skipUnless('linux') decorator ensures that we're on linux + # if we're executing these statements. from ctypes import CDLL, c_int from _ctypes import dlopen, dlsym From d179377df3a53b114101ae749d12f24aa4cd1fd9 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Tue, 12 Nov 2024 23:30:58 +0200 Subject: [PATCH 47/54] dlerror: Use surrogateescape error strategy for decoding locale Signed-off-by: Georgios Alexopoulos --- Modules/_ctypes/_ctypes.c | 8 ++------ Modules/_ctypes/callproc.c | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index dfa320f8432026..49080bfd7aa034 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -984,13 +984,11 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - PyObject *message = PyUnicode_DecodeLocale(dlerr, "strict"); + PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); if (message) { PyErr_SetObject(PyExc_ValueError, message); return NULL; } - // Ignore errors from converting the message to str - PyErr_Clear(); } #endif #undef USE_DLERROR @@ -3809,14 +3807,12 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - PyObject *message = PyUnicode_DecodeLocale(dlerr, "strict"); + PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); if (message) { PyErr_SetObject(PyExc_AttributeError, message); Py_DECREF(ftuple); return NULL; } - // Ignore errors from converting the message to str - PyErr_Clear(); } #endif PyErr_Format(PyExc_AttributeError, diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 4e32d89fe25050..53448e97e247bc 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1640,13 +1640,11 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { - PyObject *message = PyUnicode_DecodeLocale(dlerr, "strict"); + PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); if (message) { PyErr_SetObject(PyExc_OSError, message); return NULL; } - // Ignore errors from converting the message to str - PyErr_Clear(); } #endif #undef USE_DLERROR From 69333142d611e48c343734bf1070a111ba77dc3f Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Wed, 13 Nov 2024 16:33:11 +0200 Subject: [PATCH 48/54] Update Modules/_ctypes/callproc.c Co-authored-by: Petr Viktorin --- Modules/_ctypes/callproc.c | 1 + 1 file changed, 1 insertion(+) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 53448e97e247bc..604bd06f705fc0 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1623,6 +1623,7 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) if (PySys_Audit("ctypes.dlsym/handle", "O", args) < 0) { return NULL; } +#undef USE_DLERROR #ifdef __CYGWIN__ // dlerror() isn't very helpful on cygwin #else From 33cf8b775ccadc53fa62376ccb3789cb4a228ed5 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Wed, 13 Nov 2024 18:44:40 +0200 Subject: [PATCH 49/54] Clear DecodeLocale error, indent #ifdef, #undef Signed-off-by: Georgios Alexopoulos --- Modules/_ctypes/_ctypes.c | 11 ++++++++--- Modules/_ctypes/callproc.c | 9 ++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 49080bfd7aa034..011c00ce864abc 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -981,7 +981,7 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, return PyCData_AtAddress(st, type, address); } -#ifdef USE_DLERROR + #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); @@ -989,10 +989,12 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, PyErr_SetObject(PyExc_ValueError, message); return NULL; } + // Ignore errors from PyUnicode_DecodeLocale, + // fall back to the generic error below. + PyErr_Clear(); } -#endif + #endif #undef USE_DLERROR - PyErr_Format(PyExc_ValueError, "symbol '%s' not found", name); @@ -3813,6 +3815,9 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) Py_DECREF(ftuple); return NULL; } + // Ignore errors from PyUnicode_DecodeLocale, + // fall back to the generic error below. + PyErr_Clear(); } #endif PyErr_Format(PyExc_AttributeError, diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 604bd06f705fc0..c701ea578f44cc 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1638,7 +1638,7 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) ptr = dlsym((void*)handle, name); if (ptr) return PyLong_FromVoidPtr(ptr); -#ifdef USE_DLERROR + #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); @@ -1646,9 +1646,12 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) PyErr_SetObject(PyExc_OSError, message); return NULL; } + // Ignore errors from PyUnicode_DecodeLocale, + // fall back to the generic error below. + PyErr_Clear(); } -#endif -#undef USE_DLERROR + #endif + #undef USE_DLERROR PyErr_Format(PyExc_OSError, "symbol '%s' not found", name); From 94dfbaa07b469fa62ccf5bb72e18dc245d3cfb30 Mon Sep 17 00:00:00 2001 From: Petr Viktorin Date: Wed, 13 Nov 2024 18:03:17 +0100 Subject: [PATCH 50/54] Decref the message --- Modules/_ctypes/_ctypes.c | 2 ++ Modules/_ctypes/callproc.c | 1 + 2 files changed, 3 insertions(+) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 011c00ce864abc..6ce5fb79bfac1c 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -987,6 +987,7 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); if (message) { PyErr_SetObject(PyExc_ValueError, message); + Py_DECREF(message); return NULL; } // Ignore errors from PyUnicode_DecodeLocale, @@ -3813,6 +3814,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) if (message) { PyErr_SetObject(PyExc_AttributeError, message); Py_DECREF(ftuple); + Py_DECREF(message); return NULL; } // Ignore errors from PyUnicode_DecodeLocale, diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index c701ea578f44cc..a0e14b0d5e9846 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1644,6 +1644,7 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) PyObject *message = PyUnicode_DecodeLocale(dlerr, "surrogateescape"); if (message) { PyErr_SetObject(PyExc_OSError, message); + Py_DECREF(message); return NULL; } // Ignore errors from PyUnicode_DecodeLocale, From cda7771e64d8cdd479acbe1d57afe38f33d46998 Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Thu, 14 Nov 2024 15:43:16 +0200 Subject: [PATCH 51/54] Update Lib/test/test_ctypes/test_dlerror.py Co-authored-by: Petr Viktorin --- Lib/test/test_ctypes/test_dlerror.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index 35015627241cb7..ca07235a79aa10 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -14,7 +14,7 @@ Adapted from Michael Kerrisk's answer: https://stackoverflow.com/a/53590014. */ -asm (".type foo, @gnu_indirect_function"); +asm (".type foo STT_GNU_IFUNC"); void *foo(void) { From 4b1bb47832c81c110248d822b24fd834a2f37d05 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Thu, 14 Nov 2024 15:54:54 +0200 Subject: [PATCH 52/54] test: Handle different architectures, where IFUNC not supported Signed-off-by: Georgios Alexopoulos --- Lib/test/test_ctypes/test_dlerror.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py index ca07235a79aa10..4441e30cd7a2a7 100644 --- a/Lib/test/test_ctypes/test_dlerror.py +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -1,6 +1,7 @@ import os import sys import unittest +import platform FOO_C = r""" #include @@ -79,7 +80,15 @@ def test_null_dlsym(self): f.write(FOO_C.replace('$DESCRIPTOR', str(pipe_w))) args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] p = subprocess.run(args, capture_output=True) - self.assertEqual(p.returncode, 0, p) + + if p.returncode != 0: + # IFUNC is not supported on all architectures. + if platform.machine() == 'x86_64': + # It should be supported here. Something else went wrong. + p.check_returncode() + else: + # IFUNC might not be supported on this machine. + self.skipTest(f"could not compile indirect function: {p}") # Case #1: Test 'PyCFuncPtr_FromDll' from Modules/_ctypes/_ctypes.c L = CDLL(dstname) From 2290aa473369ef9886cc6b4ca4797e122aff9fdc Mon Sep 17 00:00:00 2001 From: George Alexopoulos Date: Thu, 14 Nov 2024 18:42:22 +0200 Subject: [PATCH 53/54] Update Modules/_ctypes/callproc.c MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Bénédikt Tran <10796600+picnixz@users.noreply.github.com> --- Modules/_ctypes/callproc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index a0e14b0d5e9846..9e22aa2c108aa5 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1636,8 +1636,9 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) (void)dlerror(); #endif ptr = dlsym((void*)handle, name); - if (ptr) + if (ptr) { return PyLong_FromVoidPtr(ptr); + } #ifdef USE_DLERROR const char *dlerr = dlerror(); if (dlerr) { From ab22349defe447d87ee4586fa72560bee5d1c027 Mon Sep 17 00:00:00 2001 From: Georgios Alexopoulos Date: Thu, 14 Nov 2024 18:59:21 +0200 Subject: [PATCH 54/54] Don't break when under 80 chars Signed-off-by: Georgios Alexopoulos --- Modules/_ctypes/_ctypes.c | 8 ++------ Modules/_ctypes/callproc.c | 4 +--- 2 files changed, 3 insertions(+), 9 deletions(-) diff --git a/Modules/_ctypes/_ctypes.c b/Modules/_ctypes/_ctypes.c index 6ce5fb79bfac1c..34529bce496d88 100644 --- a/Modules/_ctypes/_ctypes.c +++ b/Modules/_ctypes/_ctypes.c @@ -996,9 +996,7 @@ CDataType_in_dll_impl(PyObject *type, PyTypeObject *cls, PyObject *dll, } #endif #undef USE_DLERROR - PyErr_Format(PyExc_ValueError, - "symbol '%s' not found", - name); + PyErr_Format(PyExc_ValueError, "symbol '%s' not found", name); return NULL; } @@ -3822,9 +3820,7 @@ PyCFuncPtr_FromDll(PyTypeObject *type, PyObject *args, PyObject *kwds) PyErr_Clear(); } #endif - PyErr_Format(PyExc_AttributeError, - "function '%s' not found", - name); + PyErr_Format(PyExc_AttributeError, "function '%s' not found", name); Py_DECREF(ftuple); return NULL; } diff --git a/Modules/_ctypes/callproc.c b/Modules/_ctypes/callproc.c index 9e22aa2c108aa5..218c3a9c81e05f 100644 --- a/Modules/_ctypes/callproc.c +++ b/Modules/_ctypes/callproc.c @@ -1654,9 +1654,7 @@ static PyObject *py_dl_sym(PyObject *self, PyObject *args) } #endif #undef USE_DLERROR - PyErr_Format(PyExc_OSError, - "symbol '%s' not found", - name); + PyErr_Format(PyExc_OSError, "symbol '%s' not found", name); return NULL; } #endif