Skip to content

Commit 0b13575

Browse files
gpsheadgpshead
authored
gh-99108: Refactor _sha256 & _sha512 into _sha2. (#101924)
This merges their code. They're backed by the same single HACL* static library, having them be a single module simplifies maintenance. This should unbreak the wasm enscripten builds that currently fail due to linking in --whole-archive mode and the HACL* library appearing twice. Long unnoticed error fixed: _sha512.SHA384Type was doubly assigned and was actually SHA512Type. Nobody depends on those internal names. Also rename LIBHACL_ make vars to LIBHACL_SHA2_ in preperation for other future HACL things.
1 parent 89ac665 commit 0b13575

18 files changed

+1310
-1493
lines changed

‎Lib/hashlib.py‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -92,13 +92,13 @@ def __get_builtin_constructor(name):
9292
import _md5
9393
cache['MD5'] = cache['md5'] = _md5.md5
9494
elif name in {'SHA256', 'sha256', 'SHA224', 'sha224'}:
95-
import _sha256
96-
cache['SHA224'] = cache['sha224'] = _sha256.sha224
97-
cache['SHA256'] = cache['sha256'] = _sha256.sha256
95+
import _sha2
96+
cache['SHA224'] = cache['sha224'] = _sha2.sha224
97+
cache['SHA256'] = cache['sha256'] = _sha2.sha256
9898
elif name in {'SHA512', 'sha512', 'SHA384', 'sha384'}:
99-
import _sha512
100-
cache['SHA384'] = cache['sha384'] = _sha512.sha384
101-
cache['SHA512'] = cache['sha512'] = _sha512.sha512
99+
import _sha2
100+
cache['SHA384'] = cache['sha384'] = _sha2.sha384
101+
cache['SHA512'] = cache['sha512'] = _sha2.sha512
102102
elif name in {'blake2b', 'blake2s'}:
103103
import _blake2
104104
cache['blake2b'] = _blake2.blake2b

‎Lib/test/test_hashlib.py‎

Lines changed: 9 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
1-
# Test hashlib module
2-
#
3-
# $Id$
1+
# Test the hashlib module.
42
#
53
# Copyright (C) 2005-2010 Gregory P. Smith ([email protected])
64
# Licensed to PSF under a Contributor Agreement.
@@ -28,7 +26,6 @@
2826
from http.client import HTTPException
2927

3028

31-
# default builtin hash module
3229
default_builtin_hashes = {'md5', 'sha1', 'sha256', 'sha512', 'sha3', 'blake2'}
3330
# --with-builtin-hashlib-hashes override
3431
builtin_hashes = sysconfig.get_config_var("PY_BUILTIN_HASHLIB_HASHES")
@@ -66,6 +63,7 @@ def get_fips_mode():
6663
requires_blake2 = unittest.skipUnless(_blake2, 'requires _blake2')
6764

6865
# bpo-46913: Don't test the _sha3 extension on a Python UBSAN build
66+
# TODO(gh-99108): Revisit this after _sha3 uses HACL*.
6967
SKIP_SHA3 = support.check_sanitizer(ub=True)
7068
requires_sha3 = unittest.skipUnless(not SKIP_SHA3, 'requires _sha3')
7169

@@ -107,7 +105,7 @@ class HashLibTestCase(unittest.TestCase):
107105

108106
shakes = {'shake_128', 'shake_256'}
109107

110-
# Issue #14693: fallback modules are always compiled under POSIX
108+
# gh-58898: Fallback modules are always compiled under POSIX.
111109
_warn_on_extension_import = (os.name == 'posix' or support.Py_DEBUG)
112110

113111
def _conditional_import_module(self, module_name):
@@ -116,7 +114,7 @@ def _conditional_import_module(self, module_name):
116114
return importlib.import_module(module_name)
117115
except ModuleNotFoundError as error:
118116
if self._warn_on_extension_import and module_name in builtin_hashes:
119-
warnings.warn('Did a C extension fail to compile? %s' % error)
117+
warnings.warn(f'Did a C extension fail to compile? {error}')
120118
return None
121119

122120
def __init__(self, *args, **kwargs):
@@ -147,7 +145,7 @@ def _test_algorithm_via_hashlib_new(data=None, _alg=algorithm, **kwargs):
147145
_hashlib = self._conditional_import_module('_hashlib')
148146
self._hashlib = _hashlib
149147
if _hashlib:
150-
# These two algorithms should always be present when this module
148+
# These algorithms should always be present when this module
151149
# is compiled. If not, something was compiled wrong.
152150
self.assertTrue(hasattr(_hashlib, 'openssl_md5'))
153151
self.assertTrue(hasattr(_hashlib, 'openssl_sha1'))
@@ -172,12 +170,10 @@ def add_builtin_constructor(name):
172170
_sha1 = self._conditional_import_module('_sha1')
173171
if _sha1:
174172
add_builtin_constructor('sha1')
175-
_sha256 = self._conditional_import_module('_sha256')
176-
if _sha256:
173+
_sha2 = self._conditional_import_module('_sha2')
174+
if _sha2:
177175
add_builtin_constructor('sha224')
178176
add_builtin_constructor('sha256')
179-
_sha512 = self._conditional_import_module('_sha512')
180-
if _sha512:
181177
add_builtin_constructor('sha384')
182178
add_builtin_constructor('sha512')
183179
if _blake2:
@@ -460,9 +456,9 @@ def check_blocksize_name(self, name, block_size=0, digest_size=0,
460456
self.assertEqual(len(m.hexdigest()), 2*digest_size)
461457
self.assertEqual(m.name, name)
462458
# split for sha3_512 / _sha3.sha3 object
463-
self.assertIn(name.split("_")[0], repr(m))
459+
self.assertIn(name.split("_")[0], repr(m).lower())
464460

465-
def test_blocksize_name(self):
461+
def test_blocksize_and_name(self):
466462
self.check_blocksize_name('md5', 64, 16)
467463
self.check_blocksize_name('sha1', 64, 20)
468464
self.check_blocksize_name('sha224', 64, 28)

‎Makefile.pre.in‎

Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -207,7 +207,7 @@ ENSUREPIP= @ENSUREPIP@
207207
# Internal static libraries
208208
LIBMPDEC_A= Modules/_decimal/libmpdec/libmpdec.a
209209
LIBEXPAT_A= Modules/expat/libexpat.a
210-
LIBHACL_A= Modules/_hacl/libHacl_Streaming_SHA2.a
210+
LIBHACL_SHA2_A= Modules/_hacl/libHacl_Streaming_SHA2.a
211211

212212
# Module state, compiler flags and linker flags
213213
# Empty CFLAGS and LDFLAGS are omitted.
@@ -575,10 +575,10 @@ LIBEXPAT_HEADERS= \
575575
##########################################################################
576576
# hashlib's HACL* library
577577

578-
LIBHACL_OBJS= \
578+
LIBHACL_SHA2_OBJS= \
579579
Modules/_hacl/Hacl_Streaming_SHA2.o
580580

581-
LIBHACL_HEADERS= \
581+
LIBHACL_SHA2_HEADERS= \
582582
Modules/_hacl/Hacl_Streaming_SHA2.h \
583583
Modules/_hacl/include/krml/FStar_UInt128_Verified.h \
584584
Modules/_hacl/include/krml/FStar_UInt_8_16_32_64.h \
@@ -912,12 +912,12 @@ $(LIBEXPAT_A): $(LIBEXPAT_OBJS)
912912
# Build HACL* static libraries for hashlib: libHacl_Streaming_SHA2.a
913913
LIBHACL_CFLAGS=-I$(srcdir)/Modules/_hacl/include -D_BSD_SOURCE -D_DEFAULT_SOURCE $(PY_STDMODULE_CFLAGS) $(CCSHARED)
914914

915-
Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_HEADERS)
915+
Modules/_hacl/Hacl_Streaming_SHA2.o: $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c $(LIBHACL_SHA2_HEADERS)
916916
$(CC) -c $(LIBHACL_CFLAGS) -o $@ $(srcdir)/Modules/_hacl/Hacl_Streaming_SHA2.c
917917

918-
$(LIBHACL_A): $(LIBHACL_OBJS)
918+
$(LIBHACL_SHA2_A): $(LIBHACL_SHA2_OBJS)
919919
-rm -f $@
920-
$(AR) $(ARFLAGS) $@ $(LIBHACL_OBJS)
920+
$(AR) $(ARFLAGS) $@ $(LIBHACL_SHA2_OBJS)
921921

922922
# create relative links from build/lib.platform/egg.so to Modules/egg.so
923923
# pybuilddir.txt is created too late. We cannot use it in Makefile
@@ -2635,9 +2635,8 @@ MODULE__HASHLIB_DEPS=$(srcdir)/Modules/hashlib.h
26352635
MODULE__IO_DEPS=$(srcdir)/Modules/_io/_iomodule.h
26362636
MODULE__MD5_DEPS=$(srcdir)/Modules/hashlib.h
26372637
MODULE__SHA1_DEPS=$(srcdir)/Modules/hashlib.h
2638-
MODULE__SHA256_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A)
2638+
MODULE__SHA2_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_SHA2_HEADERS) $(LIBHACL_SHA2_A)
26392639
MODULE__SHA3_DEPS=$(srcdir)/Modules/_sha3/sha3.c $(srcdir)/Modules/_sha3/sha3.h $(srcdir)/Modules/hashlib.h
2640-
MODULE__SHA512_DEPS=$(srcdir)/Modules/hashlib.h $(LIBHACL_HEADERS) $(LIBHACL_A)
26412640
MODULE__SOCKET_DEPS=$(srcdir)/Modules/socketmodule.h $(srcdir)/Modules/addrinfo.h $(srcdir)/Modules/getaddrinfo.c $(srcdir)/Modules/getnameinfo.c
26422641
MODULE__SSL_DEPS=$(srcdir)/Modules/_ssl.h $(srcdir)/Modules/_ssl/cert.c $(srcdir)/Modules/_ssl/debughelpers.c $(srcdir)/Modules/_ssl/misc.c $(srcdir)/Modules/_ssl_data.h $(srcdir)/Modules/_ssl_data_111.h $(srcdir)/Modules/_ssl_data_300.h $(srcdir)/Modules/socketmodule.h
26432642
MODULE__TESTCAPI_DEPS=$(srcdir)/Modules/_testcapi/testcapi_long.h $(srcdir)/Modules/_testcapi/parts.h

‎Misc/NEWS.d/next/Library/2023-02-15-01-54-06.gh-issue-99108.rjTSic.rst‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
The built-in extension modules for :mod:`hashlib` SHA2 algorithms, used when
2+
OpenSSL does not provide them, now live in a single internal ``_sha2`` module
3+
instead of separate ``_sha256`` and ``_sha512`` modules.

‎Modules/Setup‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -165,8 +165,7 @@ PYTHONPATH=$(COREPYTHONPATH)
165165
#_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
166166
#_md5 md5module.c
167167
#_sha1 sha1module.c
168-
#_sha256 sha256module.c
169-
#_sha512 sha512module.c
168+
#_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
170169
#_sha3 _sha3/sha3module.c
171170

172171
# text encodings and unicode

‎Modules/Setup.stdlib.in‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -79,8 +79,7 @@
7979
# hashing builtins, can be disabled with --without-builtin-hashlib-hashes
8080
@MODULE__MD5_TRUE@_md5 md5module.c
8181
@MODULE__SHA1_TRUE@_sha1 sha1module.c
82-
@MODULE__SHA256_TRUE@_sha256 sha256module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
83-
@MODULE__SHA512_TRUE@_sha512 sha512module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
82+
@MODULE__SHA2_TRUE@_sha2 sha2module.c -I$(srcdir)/Modules/_hacl/include Modules/_hacl/libHacl_Streaming_SHA2.a
8483
@MODULE__SHA3_TRUE@_sha3 _sha3/sha3module.c
8584
@MODULE__BLAKE2_TRUE@_blake2 _blake2/blake2module.c _blake2/blake2b_impl.c _blake2/blake2s_impl.c
8685

‎Modules/clinic/sha256module.c.h‎

Lines changed: 0 additions & 225 deletions
This file was deleted.

0 commit comments

Comments
 (0)