gh-132993: expose HASHLIB_GIL_MINSIZE to private extension modules (#132999)

Author: picnixz

Lib/test/support/hashlib_helper.py

@@ -1,5 +1,6 @@
 import functools
 import hashlib
+import importlib
 import unittest
 from test.support.import_helper import import_module
 
@@ -100,3 +101,22 @@ def wrapper(*args, **kwargs):
     def decorator(func_or_class):
         return _decorate_func_or_class(func_or_class, decorator_func)
     return decorator
+
+
+def find_gil_minsize(modules_names, default=2048):
+    """Get the largest GIL_MINSIZE value for the given cryptographic modules.
+
+    The valid module names are the following:
+
+    - _hashlib
+    - _md5, _sha1, _sha2, _sha3, _blake2
+    - _hmac
+    """
+    sizes = []
+    for module_name in modules_names:
+        try:
+            module = importlib.import_module(module_name)
+        except ImportError:
+            continue
+        sizes.append(getattr(module, '_GIL_MINSIZE', default))
+    return max(sizes, default=default)

Lib/test/test_hashlib.py

@@ -20,6 +20,7 @@
 import warnings
 from test import support
 from test.support import _4G, bigmemtest
+from test.support import hashlib_helper
 from test.support.import_helper import import_fresh_module
 from test.support import requires_resource
 from test.support import threading_helper
@@ -911,10 +912,13 @@ def test_case_shake256_vector(self):
 
     def test_gil(self):
         # Check things work fine with an input larger than the size required
-        # for multithreaded operation (which is hardwired to 2048).
-        gil_minsize = 2048
-
+        # for multithreaded operation. Currently, all cryptographic modules
+        # have the same constant value (2048) but in the future it might not
+        # be the case.
+        mods = ['_md5', '_sha1', '_sha2', '_sha3', '_blake2', '_hashlib']
+        gil_minsize = hashlib_helper.find_gil_minsize(mods)
         for cons in self.hash_constructors:
+            # constructors belong to one of the above modules
             m = cons(usedforsecurity=False)
             m.update(b'1')
             m.update(b'#' * gil_minsize)
@@ -923,6 +927,8 @@ def test_gil(self):
             m = cons(b'x' * gil_minsize, usedforsecurity=False)
             m.update(b'1')
 
+    def test_sha256_gil(self):
+        gil_minsize = hashlib_helper.find_gil_minsize(['_sha2', '_hashlib'])
         m = hashlib.sha256()
         m.update(b'1')
         m.update(b'#' * gil_minsize)

Lib/test/test_hmac.py

@@ -1100,6 +1100,11 @@ def HMAC(self, key, msg=None):
         """Create a HMAC object."""
         raise NotImplementedError
 
+    @property
+    def gil_minsize(self):
+        """Get the maximal input length for the GIL to be held."""
+        raise NotImplementedError
+
     def check_update(self, key, chunks):
         chunks = list(chunks)
         msg = b''.join(chunks)
@@ -1118,11 +1123,10 @@ def test_update(self):
             self.check_update(key, [msg])
 
     def test_update_large(self):
-        HASHLIB_GIL_MINSIZE = 2048
-
+        gil_minsize = self.gil_minsize
         key = random.randbytes(16)
-        top = random.randbytes(HASHLIB_GIL_MINSIZE + 1)
-        bot = random.randbytes(HASHLIB_GIL_MINSIZE + 1)
+        top = random.randbytes(gil_minsize + 1)
+        bot = random.randbytes(gil_minsize + 1)
         self.check_update(key, [top, bot])
 
     def test_update_exceptions(self):
@@ -1132,19 +1136,27 @@ def test_update_exceptions(self):
                 self.assertRaises(TypeError, h.update, msg)
 
 
-@hashlib_helper.requires_hashdigest('sha256')
+@requires_builtin_sha2()
 class PyUpdateTestCase(PyModuleMixin, UpdateTestCaseMixin, unittest.TestCase):
 
     def HMAC(self, key, msg=None):
         return self.hmac.HMAC(key, msg, digestmod='sha256')
 
+    @property
+    def gil_minsize(self):
+        return sha2._GIL_MINSIZE
+
 
 @hashlib_helper.requires_openssl_hashdigest('sha256')
 class OpenSSLUpdateTestCase(UpdateTestCaseMixin, unittest.TestCase):
 
     def HMAC(self, key, msg=None):
         return _hashlib.hmac_new(key, msg, digestmod='sha256')
 
+    @property
+    def gil_minsize(self):
+        return _hashlib._GIL_MINSIZE
+
 
 class BuiltinUpdateTestCase(BuiltinModuleMixin,
                             UpdateTestCaseMixin, unittest.TestCase):
@@ -1154,6 +1166,10 @@ def HMAC(self, key, msg=None):
         # are still built, making it possible to use SHA-2 hashes.
         return self.hmac.new(key, msg, digestmod='sha256')
 
+    @property
+    def gil_minsize(self):
+        return self.hmac._GIL_MINSIZE
+
 
 class CopyBaseTestCase:
 

Modules/_hashopenssl.c

@@ -2358,6 +2358,16 @@ hashlib_exception(PyObject *module)
     return 0;
 }
 
+static int
+hashlib_constants(PyObject *module)
+{
+    if (PyModule_AddIntConstant(module, "_GIL_MINSIZE",
+                                HASHLIB_GIL_MINSIZE) < 0)
+    {
+        return -1;
+    }
+    return 0;
+}
 
 static PyModuleDef_Slot hashlib_slots[] = {
     {Py_mod_exec, hashlib_init_hashtable},
@@ -2367,6 +2377,7 @@ static PyModuleDef_Slot hashlib_slots[] = {
     {Py_mod_exec, hashlib_md_meth_names},
     {Py_mod_exec, hashlib_init_constructors},
     {Py_mod_exec, hashlib_exception},
+    {Py_mod_exec, hashlib_constants},
     {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
     {Py_mod_gil, Py_MOD_GIL_NOT_USED},
     {0, NULL}

Modules/blake2module.c

@@ -229,6 +229,8 @@ blake2_exec(PyObject *m)
     // good a place as any to probe the CPU flags.
     detect_cpu_features(&st->flags);
 
+    ADD_INT_CONST("_GIL_MINSIZE", HASHLIB_GIL_MINSIZE);
+
     st->blake2b_type = (PyTypeObject *)PyType_FromModuleAndSpec(
         m, &blake2b_type_spec, NULL);
 

Modules/hmacmodule.c

@@ -1679,6 +1679,20 @@ hmacmodule_init_strings(hmacmodule_state *state)
     return 0;
 }
 
+static int
+hmacmodule_init_globals(PyObject *module, hmacmodule_state *state)
+{
+#define ADD_INT_CONST(NAME, VALUE)                                  \
+    do {                                                            \
+        if (PyModule_AddIntConstant(module, (NAME), (VALUE)) < 0) { \
+            return -1;                                              \
+        }                                                           \
+    } while (0)
+    ADD_INT_CONST("_GIL_MINSIZE", HASHLIB_GIL_MINSIZE);
+#undef ADD_INT_CONST
+    return 0;
+}
+
 static void
 hmacmodule_init_cpu_features(hmacmodule_state *state)
 {
@@ -1769,6 +1783,9 @@ hmacmodule_exec(PyObject *module)
     if (hmacmodule_init_strings(state) < 0) {
         return -1;
     }
+    if (hmacmodule_init_globals(module, state) < 0) {
+        return -1;
+    }
     hmacmodule_init_cpu_features(state);
     return 0;
 }

Modules/md5module.c

@@ -370,6 +370,9 @@ md5_exec(PyObject *m)
     if (PyModule_AddObjectRef(m, "MD5Type", (PyObject *)st->md5_type) < 0) {
         return -1;
     }
+    if (PyModule_AddIntConstant(m, "_GIL_MINSIZE", HASHLIB_GIL_MINSIZE) < 0) {
+        return -1;
+    }
 
     return 0;
 }
@@ -383,14 +386,14 @@ static PyModuleDef_Slot _md5_slots[] = {
 
 
 static struct PyModuleDef _md5module = {
-        PyModuleDef_HEAD_INIT,
-        .m_name = "_md5",
-        .m_size = sizeof(MD5State),
-        .m_methods = MD5_functions,
-        .m_slots = _md5_slots,
-        .m_traverse = _md5_traverse,
-        .m_clear = _md5_clear,
-        .m_free = _md5_free,
+    PyModuleDef_HEAD_INIT,
+    .m_name = "_md5",
+    .m_size = sizeof(MD5State),
+    .m_methods = MD5_functions,
+    .m_slots = _md5_slots,
+    .m_traverse = _md5_traverse,
+    .m_clear = _md5_clear,
+    .m_free = _md5_free,
 };
 
 PyMODINIT_FUNC

Modules/sha1module.c

@@ -362,8 +362,15 @@ _sha1_exec(PyObject *module)
     st->sha1_type = (PyTypeObject *)PyType_FromModuleAndSpec(
         module, &sha1_type_spec, NULL);
     if (PyModule_AddObjectRef(module,
-                           "SHA1Type",
-                           (PyObject *)st->sha1_type) < 0) {
+                              "SHA1Type",
+                              (PyObject *)st->sha1_type) < 0)
+    {
+        return -1;
+    }
+    if (PyModule_AddIntConstant(module,
+                                "_GIL_MINSIZE",
+                                HASHLIB_GIL_MINSIZE) < 0)
+    {
         return -1;
     }
 
@@ -381,14 +388,14 @@ static PyModuleDef_Slot _sha1_slots[] = {
 };
 
 static struct PyModuleDef _sha1module = {
-        PyModuleDef_HEAD_INIT,
-        .m_name = "_sha1",
-        .m_size = sizeof(SHA1State),
-        .m_methods = SHA1_functions,
-        .m_slots = _sha1_slots,
-        .m_traverse = _sha1_traverse,
-        .m_clear = _sha1_clear,
-        .m_free = _sha1_free
+    PyModuleDef_HEAD_INIT,
+    .m_name = "_sha1",
+    .m_size = sizeof(SHA1State),
+    .m_methods = SHA1_functions,
+    .m_slots = _sha1_slots,
+    .m_traverse = _sha1_traverse,
+    .m_clear = _sha1_clear,
+    .m_free = _sha1_free
 };
 
 PyMODINIT_FUNC

Modules/sha2module.c

@@ -894,6 +894,13 @@ static int sha2_exec(PyObject *module)
         return -1;
     }
 
+    if (PyModule_AddIntConstant(module,
+                                "_GIL_MINSIZE",
+                                HASHLIB_GIL_MINSIZE) < 0)
+    {
+        return -1;
+    }
+
     return 0;
 }
 

Modules/sha3module.c

@@ -639,8 +639,10 @@ _sha3_exec(PyObject *m)
     init_sha3type(shake_256_type, SHAKE256_spec);
 #undef init_sha3type
 
-    if (PyModule_AddStringConstant(m, "implementation",
-                                   "HACL") < 0) {
+    if (PyModule_AddStringConstant(m, "implementation", "HACL") < 0) {
+        return -1;
+    }
+    if (PyModule_AddIntConstant(m, "_GIL_MINSIZE", HASHLIB_GIL_MINSIZE) < 0) {
         return -1;
     }