diff --git a/Doc/library/shutil.rst b/Doc/library/shutil.rst index 1819f708890c09..4c481bc8025d7f 100644 --- a/Doc/library/shutil.rst +++ b/Doc/library/shutil.rst @@ -601,8 +601,7 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. any format-specific extension. *format* is the archive format: one of - "zip" (if the :mod:`zlib` module is available), "tar", "gztar" (if the - :mod:`zlib` module is available), "bztar" (if the :mod:`bz2` module is + "zip", "tar", "gztar", "bztar" (if the :mod:`bz2` module is available), or "xztar" (if the :mod:`lzma` module is available). *root_dir* is a directory that will be the root directory of the @@ -653,9 +652,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. By default :mod:`shutil` provides these formats: - - *zip*: ZIP file (if the :mod:`zlib` module is available). + - *zip*: ZIP file. - *tar*: Uncompressed tar file. Uses POSIX.1-2001 pax format for new archives. - - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). + - *gztar*: gzip'ed tar-file. - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). @@ -768,10 +767,9 @@ provided. They rely on the :mod:`zipfile` and :mod:`tarfile` modules. By default :mod:`shutil` provides these formats: - - *zip*: ZIP file (unpacking compressed files works only if the corresponding - module is available). + - *zip*: ZIP file. - *tar*: uncompressed tar file. - - *gztar*: gzip'ed tar-file (if the :mod:`zlib` module is available). + - *gztar*: gzip'ed tar-file. - *bztar*: bzip2'ed tar-file (if the :mod:`bz2` module is available). - *xztar*: xz'ed tar-file (if the :mod:`lzma` module is available). diff --git a/Doc/library/test.rst b/Doc/library/test.rst index def22f8bb8ab2d..2e89944bff8442 100644 --- a/Doc/library/test.rst +++ b/Doc/library/test.rst @@ -748,7 +748,12 @@ The :mod:`test.support` module defines the following functions: .. decorator:: requires_zlib - Decorator for skipping tests if :mod:`zlib` doesn't exist. + Decorator for skipping tests if running on WASI as :mod:`zlib` is + unsupported. + +.. versionchanged:: next + Test is skipped only if it is running on WASI as builds without :mod:`zlib` on + other systems are unsupported. .. decorator:: requires_gzip diff --git a/Doc/library/zlib.rst b/Doc/library/zlib.rst index 965b82a3daffb9..e057ab16854168 100644 --- a/Doc/library/zlib.rst +++ b/Doc/library/zlib.rst @@ -9,14 +9,11 @@ For applications that require data compression, the functions in this module allow compression and decompression, using the zlib library. The zlib library -has its own home page at https://www.zlib.net. There are known -incompatibilities between the Python module and versions of the zlib library -earlier than 1.1.3; 1.1.3 has a `security vulnerability `_, so we recommend using -1.1.4 or later. +has its own `home page `_. zlib's functions have many options and often need to be used in a particular order. This documentation doesn't attempt to cover all of the permutations; -consult the zlib manual at http://www.zlib.net/manual.html for authoritative +consult the `zlib manual `_ for authoritative information. For reading and writing ``.gz`` files see the :mod:`gzip` module. diff --git a/Doc/using/configure.rst b/Doc/using/configure.rst index 72912cea2f0c28..c82aa6f5fa050a 100644 --- a/Doc/using/configure.rst +++ b/Doc/using/configure.rst @@ -32,6 +32,10 @@ Features and minimum versions required to build CPython: * Autoconf 2.72 and aclocal 1.16.5 are required to regenerate the :file:`configure` script. +* `zlib `_ 1.1.3 is the minium version and 1.1.4 is recommended due to + a `security vulnerability `_ for the :mod:`zlib` + extension module. + .. versionchanged:: 3.1 Tcl/Tk version 8.3.1 is now required. @@ -61,6 +65,9 @@ Features and minimum versions required to build CPython: .. versionchanged:: 3.14 Autoconf 2.72 is now required. +.. versionchanged:: next + zlib 1.1.3 is now required. + See also :pep:`7` "Style Guide for C Code" and :pep:`11` "CPython platform support". @@ -454,7 +461,7 @@ Options for third-party dependencies .. option:: ZLIB_CFLAGS .. option:: ZLIB_LIBS - C compiler and linker flags for ``libzlib``, used by :mod:`gzip` module, + C compiler and linker flags for ``libzlib``, used by the :mod:`zlib` module, overriding ``pkg-config``. diff --git a/Doc/whatsnew/3.14.rst b/Doc/whatsnew/3.14.rst index 05221a8edf4066..883695bd6a3d1a 100644 --- a/Doc/whatsnew/3.14.rst +++ b/Doc/whatsnew/3.14.rst @@ -1310,6 +1310,10 @@ Build changes * GNU Autoconf 2.72 is now required to generate :file:`configure`. (Contributed by Erlend Aasland in :gh:`115765`.) +* zlib is now required to build CPytho on all platforms except WASI. + For more information see :pep:`775`. + (Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`91246`.) + .. _whatsnew314-pep761: PEP 761: Discontinuation of PGP signatures diff --git a/Lib/encodings/zlib_codec.py b/Lib/encodings/zlib_codec.py index 95908a4b4a13a1..25acfba1c01483 100644 --- a/Lib/encodings/zlib_codec.py +++ b/Lib/encodings/zlib_codec.py @@ -6,7 +6,7 @@ """ import codecs -import zlib # this codec needs the optional zlib module ! +import zlib ### Codec APIs diff --git a/Lib/shutil.py b/Lib/shutil.py index 510ae8c6f22d59..4037c75c89fa2a 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -11,12 +11,6 @@ import collections import errno -try: - import zlib - del zlib - _ZLIB_SUPPORTED = True -except ImportError: - _ZLIB_SUPPORTED = False try: import bz2 @@ -1000,7 +994,7 @@ def _make_tarball(base_name, base_dir, compress="gzip", verbose=0, dry_run=0, """ if compress is None: tar_compression = '' - elif _ZLIB_SUPPORTED and compress == 'gzip': + elif compress == 'gzip': tar_compression = 'gz' elif _BZ2_SUPPORTED and compress == 'bzip2': tar_compression = 'bz2' @@ -1119,13 +1113,11 @@ def _make_zipfile(base_name, base_dir, verbose=0, dry_run=0, _ARCHIVE_FORMATS = { 'tar': (_make_tarball, [('compress', None)], "uncompressed tar file"), + 'gztar': (_make_tarball, [('compress', 'gzip')], + "gzip'ed tar-file"), + 'zip': (_make_zipfile, [], "ZIP file") } -if _ZLIB_SUPPORTED: - _ARCHIVE_FORMATS['gztar'] = (_make_tarball, [('compress', 'gzip')], - "gzip'ed tar-file") - _ARCHIVE_FORMATS['zip'] = (_make_zipfile, [], "ZIP file") - if _BZ2_SUPPORTED: _ARCHIVE_FORMATS['bztar'] = (_make_tarball, [('compress', 'bzip2')], "bzip2'ed tar-file") @@ -1345,12 +1337,9 @@ def _unpack_tarfile(filename, extract_dir, *, filter=None): _UNPACK_FORMATS = { 'tar': (['.tar'], _unpack_tarfile, [], "uncompressed tar file"), 'zip': (['.zip'], _unpack_zipfile, [], "ZIP file"), + 'gztar': (['.tar.gz', '.tgz'], _unpack_tarfile, [], "gzip'ed tar-file"), } -if _ZLIB_SUPPORTED: - _UNPACK_FORMATS['gztar'] = (['.tar.gz', '.tgz'], _unpack_tarfile, [], - "gzip'ed tar-file") - if _BZ2_SUPPORTED: _UNPACK_FORMATS['bztar'] = (['.tar.bz2', '.tbz2'], _unpack_tarfile, [], "bzip2'ed tar-file") diff --git a/Lib/tarfile.py b/Lib/tarfile.py index a0fab46b24e249..738d95cd1d15a7 100644 --- a/Lib/tarfile.py +++ b/Lib/tarfile.py @@ -364,10 +364,7 @@ def __init__(self, name, mode, comptype, fileobj, bufsize, try: if comptype == "gz": - try: - import zlib - except ImportError: - raise CompressionError("zlib module is not available") from None + import zlib self.zlib = zlib self.crc = zlib.crc32(b"") if mode == "r": diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index f31d98bf731d67..4f7e21afa409ee 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -493,12 +493,8 @@ def dec(*args, **kwargs): float.__getformat__("double").startswith("IEEE"), "test requires IEEE 754 doubles") -def requires_zlib(reason='requires zlib'): - try: - import zlib - except ImportError: - zlib = None - return unittest.skipUnless(zlib, reason) +def requires_zlib(reason='zlib is unsupported on WASI'): + return unittest.skipIf(sys.platform == "wasi", reason) def requires_gzip(reason='requires gzip'): try: diff --git a/Lib/test/test_zlib.py b/Lib/test/test_zlib.py index 4d97fe56f3a094..307438bd45192e 100644 --- a/Lib/test/test_zlib.py +++ b/Lib/test/test_zlib.py @@ -1,15 +1,27 @@ import unittest from test import support -from test.support import import_helper import binascii import copy import pickle import random import sys +import importlib from test.support import bigmemtest, _1G, _4G, is_s390x - -zlib = import_helper.import_module('zlib') +# Building CPython without zlib is not supported except WASI. +# +# Anyone who wants build CPython this way should be prepared to patch it, +# but the core team may help getting those patches to the main branch +# (as that’s the place where multiple third parties can cooperate). +# +# For tests to pass without zlib, this file needs to be removed. + +try: + zlib = importlib.import_module('zlib') +except ImportError as msg: + if sys.platform.startswith('wasi'): + raise unittest.SkipTest(str(msg)) + raise ImportError("Building CPython without zlib is not supported") requires_Compress_copy = unittest.skipUnless( hasattr(zlib.compressobj(), "copy"), diff --git a/Lib/zipfile/__init__.py b/Lib/zipfile/__init__.py index b8b496ad9471f4..ac5c4d980d5413 100644 --- a/Lib/zipfile/__init__.py +++ b/Lib/zipfile/__init__.py @@ -15,7 +15,7 @@ import time try: - import zlib # We may need its compression method + import zlib crc32 = zlib.crc32 except ImportError: zlib = None diff --git a/Lib/zipimport.py b/Lib/zipimport.py index 444c9dd11d8672..70e0988ba3b258 100644 --- a/Lib/zipimport.py +++ b/Lib/zipimport.py @@ -604,9 +604,7 @@ def _read_directory(archive): _importing_zlib = False -# Return the zlib.decompress function object, or NULL if zlib couldn't -# be imported. The function is cached when found, so subsequent calls -# don't import zlib again. +# Return the zlib.decompress function object or raise an import error. def _get_decompress_func(): global _importing_zlib if _importing_zlib: @@ -618,9 +616,6 @@ def _get_decompress_func(): _importing_zlib = True try: from zlib import decompress - except Exception: - _bootstrap._verbose_message('zipimport: zlib UNAVAILABLE') - raise ZipImportError("can't decompress data; zlib not available") finally: _importing_zlib = False diff --git a/Misc/NEWS.d/next/Build/2025-02-19-00-00-00.gh-issue-91246.ahsgd3.rst b/Misc/NEWS.d/next/Build/2025-02-19-00-00-00.gh-issue-91246.ahsgd3.rst new file mode 100644 index 00000000000000..a75a823a5d43c7 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2025-02-19-00-00-00.gh-issue-91246.ahsgd3.rst @@ -0,0 +1,2 @@ +Make zlib required to build CPython with the exception of WASI implementing :pep:`775`. +(Contributed by Stan Ulbrych and Gregory P. Smith in :gh:`130297`.) diff --git a/Modules/binascii.c b/Modules/binascii.c index 6bb01d148b6faa..839b193b866302 100644 --- a/Modules/binascii.c +++ b/Modules/binascii.c @@ -60,7 +60,7 @@ #include "Python.h" #include "pycore_long.h" // _PyLong_DigitValue #include "pycore_strhex.h" // _Py_strhex_bytes_with_sep() -#ifdef USE_ZLIB_CRC32 +#ifndef NO_ZLIB_CRC32 # include "zlib.h" #endif @@ -616,7 +616,7 @@ binascii_crc_hqx_impl(PyObject *module, Py_buffer *data, unsigned int crc) return PyLong_FromUnsignedLong(crc); } -#ifndef USE_ZLIB_CRC32 +#ifdef NO_ZLIB_CRC32 /* Crc - 32 BIT ANSI X3.66 CRC checksum files Also known as: ISO 3307 **********************************************************************| @@ -749,7 +749,7 @@ internal_crc32(const unsigned char *bin_data, Py_ssize_t len, unsigned int crc) result = (crc ^ 0xFFFFFFFF); return result & 0xffffffff; } -#endif /* USE_ZLIB_CRC32 */ +#endif /* NO_ZLIB_CRC32 */ /*[clinic input] binascii.crc32 -> unsigned_int @@ -765,9 +765,11 @@ static unsigned int binascii_crc32_impl(PyObject *module, Py_buffer *data, unsigned int crc) /*[clinic end generated code: output=52cf59056a78593b input=bbe340bc99d25aa8]*/ -#ifdef USE_ZLIB_CRC32 +#ifndef NO_ZLIB_CRC32 /* This is the same as zlibmodule.c zlib_crc32_impl. It exists in two - * modules for historical reasons. */ + * modules for historical reasons. They should be consolidated in the future + * once WASI supports zlib. + */ { /* Releasing the GIL for very small buffers is inefficient and may lower performance */ @@ -798,7 +800,7 @@ binascii_crc32_impl(PyObject *module, Py_buffer *data, unsigned int crc) } return crc & 0xffffffff; } -#else /* USE_ZLIB_CRC32 */ +#else /* NO_ZLIB_CRC32 */ { const unsigned char *bin_data = data->buf; Py_ssize_t len = data->len; @@ -815,7 +817,7 @@ binascii_crc32_impl(PyObject *module, Py_buffer *data, unsigned int crc) return internal_crc32(bin_data, len, crc); } } -#endif /* USE_ZLIB_CRC32 */ +#endif /* NO_ZLIB_CRC32 */ /*[clinic input] binascii.b2a_hex diff --git a/PCbuild/pythoncore.vcxproj b/PCbuild/pythoncore.vcxproj index 9ebf58ae8a9bc4..b963c6b4cff495 100644 --- a/PCbuild/pythoncore.vcxproj +++ b/PCbuild/pythoncore.vcxproj @@ -457,9 +457,7 @@ - - USE_ZLIB_CRC32;%(PreprocessorDefinitions) - + diff --git a/configure b/configure index 453b0123ded0a4..6e06be6b029b50 100755 --- a/configure +++ b/configure @@ -21450,20 +21450,6 @@ fi - - - if test "$ac_sys_system" = "Emscripten" -a -z "$ZLIB_CFLAGS" -a -z "$ZLIB_LIBS" -then : - - ZLIB_CFLAGS="-sUSE_ZLIB" - ZLIB_LIBS="-sUSE_ZLIB" - -fi - - - - - pkg_failed=no { printf "%s\n" "$as_me:${as_lineno-$LINENO}: checking for zlib >= 1.2.0" >&5 printf %s "checking for zlib >= 1.2.0... " >&6; } @@ -21820,18 +21806,29 @@ else printf "%s\n" "yes" >&6; } have_zlib=yes - printf "%s\n" "#define HAVE_ZLIB_COPY 1" >>confdefs.h + printf "%s\n" "#define HAVE_ZLIB_COPY 1" >>confdefs.h fi +case $ac_sys_system in #( + WASI) : -if test "x$have_zlib" = xyes -then : + BINASCII_CFLAGS="-DNO_ZLIB_CRC32" + BINASCII_LIBS="" + ;; #( + *) : - BINASCII_CFLAGS="-DUSE_ZLIB_CRC32 $ZLIB_CFLAGS" - BINASCII_LIBS="$ZLIB_LIBS" + if test "$have_zlib" != "yes"; then + { printf "%s\n" "$as_me:${as_lineno-$LINENO}: WARNING: zlib (zlib.h and libz) was not found. Building CPython without zlib is not supported, although it might work for limited use cases. Install your OS's zlib-devel or zlib1g-dev equivalent library or get it from https://zlib.net/." >&5 +printf "%s\n" "$as_me: WARNING: zlib (zlib.h and libz) was not found. Building CPython without zlib is not supported, although it might work for limited use cases. Install your OS's zlib-devel or zlib1g-dev equivalent library or get it from https://zlib.net/." >&2;} + fi + BINASCII_CFLAGS="$ZLIB_CFLAGS" + BINASCII_LIBS="$ZLIB_LIBS" -fi + ;; #( + *) : + ;; +esac diff --git a/configure.ac b/configure.ac index 234ae90616af62..a931e52f52cfc4 100644 --- a/configure.ac +++ b/configure.ac @@ -5323,14 +5323,10 @@ if test "$ac_cv_have_lchflags" = yes ; then fi dnl Check for compression libraries -AH_TEMPLATE([HAVE_ZLIB_COPY], [Define if the zlib library has inflateCopy]) - -dnl detect zlib from Emscripten emport -PY_CHECK_EMSCRIPTEN_PORT([ZLIB], [-sUSE_ZLIB]) +AH_TEMPLATE([HAVE_ZLIB_COPY], [Define if the zlib library has inflateCopy; zlib 1.2.0 (2003) added inflateCopy.]) PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.0], [ have_zlib=yes - dnl zlib 1.2.0 (2003) added inflateCopy AC_DEFINE([HAVE_ZLIB_COPY], [1]) ], [ WITH_SAVE_ENV([ @@ -5346,12 +5342,26 @@ PKG_CHECK_MODULES([ZLIB], [zlib >= 1.2.0], [ ]) ]) ]) - -dnl binascii can use zlib for optimized crc32. -AS_VAR_IF([have_zlib], [yes], [ - BINASCII_CFLAGS="-DUSE_ZLIB_CRC32 $ZLIB_CFLAGS" - BINASCII_LIBS="$ZLIB_LIBS" -]) +dnl temporarily throw only errors on builds other than wasi until wasi supports zlib +AS_CASE([$ac_sys_system], + [WASI], [ + BINASCII_CFLAGS="-DNO_ZLIB_CRC32" + BINASCII_LIBS="" + ], + [*], [ + if test "$have_zlib" != "yes"; then + AC_MSG_WARN([m4_normalize([ + zlib (zlib.h and libz) was not found. Building CPython + without zlib is not supported, although it might work + for limited use cases. + Install your OS's zlib-devel or zlib1g-dev equivalent + library or get it from https://zlib.net/. + ])]) + fi + BINASCII_CFLAGS="$ZLIB_CFLAGS" + BINASCII_LIBS="$ZLIB_LIBS" + ] +) dnl detect bzip2 from Emscripten emport PY_CHECK_EMSCRIPTEN_PORT([BZIP2], [-sUSE_BZIP2]) diff --git a/pyconfig.h.in b/pyconfig.h.in index 4295b4f5ea5fbd..ee4cac2ce8472a 100644 --- a/pyconfig.h.in +++ b/pyconfig.h.in @@ -1609,7 +1609,8 @@ /* Define to 1 if you have the 'writev' function. */ #undef HAVE_WRITEV -/* Define if the zlib library has inflateCopy */ +/* Define if the zlib library has inflateCopy; zlib 1.2.0 (2003) added + inflateCopy. */ #undef HAVE_ZLIB_COPY /* Define to 1 if you have the header file. */