From 1728982baabf3fa8ae3961836a8f831b61998ffd Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 25 Jan 2022 22:01:10 +0000 Subject: [PATCH 1/3] Refactor sanitiser skip tests into test.support (GH-30889) * Refactor sanitizer skip tests into test.support * fixup! Refactor sanitizer skip tests into test.support * fixup! fixup! Refactor sanitizer skip tests into test.support (cherry picked from commit b1cb8430504931f7854eac5d32cba74770078a4e) --- Lib/test/support/__init__.py | 32 +++++++++++++++++++++++++++++++- Lib/test/test_faulthandler.py | 20 +++++--------------- Lib/test/test_io.py | 25 +++++++------------------ 3 files changed, 43 insertions(+), 34 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 44c5bd5820a75e..b5fe01841f5b21 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -41,7 +41,7 @@ "requires_IEEE_754", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", "check__all__", "skip_if_buggy_ucrt_strfptime", - "check_disallow_instantiation", + "check_disallow_instantiation", "skip_if_sanitizer", # sys "is_jython", "is_android", "check_impl_detail", "unix_shell", "setswitchinterval", @@ -367,6 +367,36 @@ def wrapper(*args, **kw): return decorator +def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): + """Decorator raising SkipTest if running with a sanitizer active.""" + if not (address or memory or ub): + raise ValueError('At least one of address, memory, or ub must be True') + + if not reason: + reason = 'not working with sanitizers active' + + _cflags = sysconfig.get_config_var('CFLAGS') or '' + _config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + memory_sanitizer = ( + '-fsanitize=memory' in _cflags or + '--with-memory-sanitizer' in _config_args + ) + address_sanitizer = ( + '-fsanitize=address' in _cflags or + '--with-memory-sanitizer' in _config_args + ) + ub_sanitizer = ( + '-fsanitize=undefined' in _cflags or + '--with-undefined-behavior-sanitizer' in _config_args + ) + skip = ( + (memory and memory_sanitizer) or + (address and address_sanitizer) or + (ub and ub_sanitizer) + ) + return unittest.skipIf(skip, reason) + + def system_must_validate_cert(f): """Skip the test on TLS certificate validation failures.""" @functools.wraps(f) diff --git a/Lib/test/test_faulthandler.py b/Lib/test/test_faulthandler.py index ee3f41a108a14c..e0f09e821da0c9 100644 --- a/Lib/test/test_faulthandler.py +++ b/Lib/test/test_faulthandler.py @@ -6,10 +6,10 @@ import signal import subprocess import sys -import sysconfig from test import support from test.support import os_helper from test.support import script_helper, is_android +from test.support import skip_if_sanitizer import tempfile import unittest from textwrap import dedent @@ -21,16 +21,6 @@ TIMEOUT = 0.5 MS_WINDOWS = (os.name == 'nt') -_cflags = sysconfig.get_config_var('CFLAGS') or '' -_config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' -UB_SANITIZER = ( - '-fsanitize=undefined' in _cflags or - '--with-undefined-behavior-sanitizer' in _config_args -) -MEMORY_SANITIZER = ( - '-fsanitize=memory' in _cflags or - '--with-memory-sanitizer' in _config_args -) def expected_traceback(lineno1, lineno2, header, min_count=1): @@ -310,8 +300,8 @@ def test_gil_released(self): 3, 'Segmentation fault') - @unittest.skipIf(UB_SANITIZER or MEMORY_SANITIZER, - "sanitizer builds change crashing process output.") + @skip_if_sanitizer(memory=True, ub=True, reason="sanitizer " + "builds change crashing process output.") @skip_segfault_on_android def test_enable_file(self): with temporary_filename() as filename: @@ -327,8 +317,8 @@ def test_enable_file(self): @unittest.skipIf(sys.platform == "win32", "subprocess doesn't support pass_fds on Windows") - @unittest.skipIf(UB_SANITIZER or MEMORY_SANITIZER, - "sanitizer builds change crashing process output.") + @skip_if_sanitizer(memory=True, ub=True, reason="sanitizer " + "builds change crashing process output.") @skip_segfault_on_android def test_enable_fd(self): with tempfile.TemporaryFile('wb+') as fp: diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index 35013b6a090cbb..fb83762cb6af41 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -28,7 +28,6 @@ import random import signal import sys -import sysconfig import textwrap import threading import time @@ -44,6 +43,7 @@ from test.support import os_helper from test.support import threading_helper from test.support import warnings_helper +from test.support import skip_if_sanitizer from test.support.os_helper import FakePath import codecs @@ -66,17 +66,6 @@ def byteslike(*pos, **kw): class EmptyStruct(ctypes.Structure): pass -_cflags = sysconfig.get_config_var('CFLAGS') or '' -_config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' -MEMORY_SANITIZER = ( - '-fsanitize=memory' in _cflags or - '--with-memory-sanitizer' in _config_args -) - -ADDRESS_SANITIZER = ( - '-fsanitize=address' in _cflags -) - # Does io.IOBase finalizer log the exception if the close() method fails? # The exception is ignored silently by default in release build. IOBASE_EMITS_UNRAISABLE = (hasattr(sys, "gettotalrefcount") or sys.flags.dev_mode) @@ -1550,8 +1539,8 @@ def test_truncate_on_read_only(self): class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader - @unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") + @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " + "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedReaderTest.test_constructor(self) # The allocation can succeed on 32-bit builds, e.g. with more @@ -1915,8 +1904,8 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter - @unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") + @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " + "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedWriterTest.test_constructor(self) # The allocation can succeed on 32-bit builds, e.g. with more @@ -2414,8 +2403,8 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") + @skip_if_sanitizer(memory=True, address=True, reason= "sanitizer defaults to crashing " + "instead of returning NULL for malloc failure.") def test_constructor(self): BufferedRandomTest.test_constructor(self) # The allocation can succeed on 32-bit builds, e.g. with more From cd983cc750a266c502fb54ce62578595d50b5379 Mon Sep 17 00:00:00 2001 From: Pablo Galindo Salgado Date: Tue, 25 Jan 2022 23:14:03 +0000 Subject: [PATCH 2/3] Add skips to crashing tests under sanitizers instead of manually skipping them (GH-30897) (cherry picked from commit a27505345e34d462139f5f8b6b5e7c9a59955150) --- Lib/test/support/__init__.py | 17 +++++++++++------ Lib/test/test_crypt.py | 3 +++ Lib/test/test_idle.py | 4 ++++ Lib/test/test_tix.py | 5 +++++ Lib/test/test_tk.py | 6 ++++++ Lib/test/test_ttk_guionly.py | 4 ++++ 6 files changed, 33 insertions(+), 6 deletions(-) diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index b5fe01841f5b21..ed6f1979732d3b 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -41,7 +41,7 @@ "requires_IEEE_754", "requires_zlib", "anticipate_failure", "load_package_tests", "detect_api_mismatch", "check__all__", "skip_if_buggy_ucrt_strfptime", - "check_disallow_instantiation", "skip_if_sanitizer", + "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer", # sys "is_jython", "is_android", "check_impl_detail", "unix_shell", "setswitchinterval", @@ -367,13 +367,11 @@ def wrapper(*args, **kw): return decorator -def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): - """Decorator raising SkipTest if running with a sanitizer active.""" +def check_sanitizer(*, address=False, memory=False, ub=False): + """Returns True if Python is compiled with sanitizer support""" if not (address or memory or ub): raise ValueError('At least one of address, memory, or ub must be True') - if not reason: - reason = 'not working with sanitizers active' _cflags = sysconfig.get_config_var('CFLAGS') or '' _config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' @@ -389,11 +387,18 @@ def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): '-fsanitize=undefined' in _cflags or '--with-undefined-behavior-sanitizer' in _config_args ) - skip = ( + return ( (memory and memory_sanitizer) or (address and address_sanitizer) or (ub and ub_sanitizer) ) + + +def skip_if_sanitizer(reason=None, *, address=False, memory=False, ub=False): + """Decorator raising SkipTest if running with a sanitizer active.""" + if not reason: + reason = 'not working with sanitizers active' + skip = check_sanitizer(address=address, memory=memory, ub=ub) return unittest.skipIf(skip, reason) diff --git a/Lib/test/test_crypt.py b/Lib/test/test_crypt.py index 5dc83b4ecbfa00..877c575c5534ae 100644 --- a/Lib/test/test_crypt.py +++ b/Lib/test/test_crypt.py @@ -1,8 +1,11 @@ import sys import unittest +from test.support import check_sanitizer try: + if check_sanitizer(address=True, memory=True): + raise unittest.SkipTest("The crypt module SEGFAULTs on ASAN/MSAN builds") import crypt IMPORT_ERROR = None except ImportError as ex: diff --git a/Lib/test/test_idle.py b/Lib/test/test_idle.py index 8756b766334e83..b94b18a541a701 100644 --- a/Lib/test/test_idle.py +++ b/Lib/test/test_idle.py @@ -1,5 +1,9 @@ import unittest from test.support.import_helper import import_module +from test.support import check_sanitizer + +if check_sanitizer(address=True, memory=True): + raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds") # Skip test_idle if _tkinter wasn't built, if tkinter is missing, # if tcl/tk is not the 8.5+ needed for ttk widgets, diff --git a/Lib/test/test_tix.py b/Lib/test/test_tix.py index 8a60c7c8e1fbd5..454baeb38a9342 100644 --- a/Lib/test/test_tix.py +++ b/Lib/test/test_tix.py @@ -2,6 +2,11 @@ import unittest from test import support from test.support import import_helper +from test.support import check_sanitizer + +if check_sanitizer(address=True, memory=True): + raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds") + # Skip this test if the _tkinter module wasn't built. _tkinter = import_helper.import_module('_tkinter') diff --git a/Lib/test/test_tk.py b/Lib/test/test_tk.py index 69cc2322cd9aa8..8f90cbaba9f7c4 100644 --- a/Lib/test/test_tk.py +++ b/Lib/test/test_tk.py @@ -1,5 +1,11 @@ +import unittest from test import support from test.support import import_helper +from test.support import check_sanitizer + +if check_sanitizer(address=True, memory=True): + raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds") + # Skip test if _tkinter wasn't built. import_helper.import_module('_tkinter') diff --git a/Lib/test/test_ttk_guionly.py b/Lib/test/test_ttk_guionly.py index 8f59839d066e68..c4919045d75cb7 100644 --- a/Lib/test/test_ttk_guionly.py +++ b/Lib/test/test_ttk_guionly.py @@ -1,6 +1,10 @@ import unittest from test import support from test.support import import_helper +from test.support import check_sanitizer + +if check_sanitizer(address=True, memory=True): + raise unittest.SkipTest("Tests involvin libX11 can SEGFAULT on ASAN/MSAN builds") # Skip this test if _tkinter wasn't built. import_helper.import_module('_tkinter') From a2734a58b5b4386918a1d6682c23071605fd5a1e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 1 Mar 2022 15:44:08 +0100 Subject: [PATCH 3/3] bpo-46633: Skip tests on ASAN and/or MSAN builds (GH-31632) Skip tests on ASAN and/or MSAN builds: * multiprocessing tests * test___all__ * test_concurrent_futures * test_decimal * test_peg_generator * test_tools (cherry picked from commit 9204bb72a2da5885facc747e63d2bd2d654606fe) --- Lib/test/_test_multiprocessing.py | 6 ++++++ Lib/test/test___all__.py | 7 +++++++ Lib/test/test_concurrent_futures.py | 6 ++++++ Lib/test/test_decimal.py | 16 +++------------- Lib/test/test_peg_generator/__init__.py | 12 ++++++++++-- Lib/test/test_tools/__init__.py | 7 +++++++ 6 files changed, 39 insertions(+), 15 deletions(-) diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 3bc5b8f3d79b02..04657c08db92c0 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -73,6 +73,12 @@ msvcrt = None +if support.check_sanitizer(address=True): + # bpo-45200: Skip multiprocessing tests if Python is built with ASAN to + # work around a libasan race condition: dead lock in pthread_create(). + raise unittest.SkipTest("libasan has a pthread_create() dead lock") + + def latin(s): return s.encode('latin') diff --git a/Lib/test/test___all__.py b/Lib/test/test___all__.py index 15f42d2d114a68..6368ef02253b6d 100644 --- a/Lib/test/test___all__.py +++ b/Lib/test/test___all__.py @@ -5,6 +5,13 @@ import sys +if support.check_sanitizer(address=True, memory=True): + # bpo-46633: test___all__ is skipped because importing some modules + # directly can trigger known problems with ASAN (like tk or crypt). + raise unittest.SkipTest("workaround ASAN build issues on loading tests " + "like tk or crypt") + + class NoAll(RuntimeError): pass diff --git a/Lib/test/test_concurrent_futures.py b/Lib/test/test_concurrent_futures.py index 29e041deeca57f..50fa1f189b174f 100644 --- a/Lib/test/test_concurrent_futures.py +++ b/Lib/test/test_concurrent_futures.py @@ -32,6 +32,12 @@ import multiprocessing.util +if support.check_sanitizer(address=True, memory=True): + # bpo-46633: Skip the test because it is too slow when Python is built + # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. + raise unittest.SkipTest("test too slow on ASAN/MSAN build") + + def create_future(state=PENDING, exception=None, result=None): f = Future() f._state = state diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index b6173a5ffec96c..310c105d9a0e8f 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -34,7 +34,7 @@ import locale from test.support import (run_unittest, run_doctest, is_resource_enabled, requires_IEEE_754, requires_docstrings, - requires_legacy_unicode_capi) + requires_legacy_unicode_capi, check_sanitizer) from test.support import (TestFailed, run_with_locale, cpython_only, darwin_malloc_err_warning) @@ -43,17 +43,6 @@ import random import inspect import threading -import sysconfig -_cflags = sysconfig.get_config_var('CFLAGS') or '' -_config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' -MEMORY_SANITIZER = ( - '-fsanitize=memory' in _cflags or - '--with-memory-sanitizer' in _config_args -) - -ADDRESS_SANITIZER = ( - '-fsanitize=address' in _cflags -) if sys.platform == 'darwin': @@ -5511,7 +5500,8 @@ def __abs__(self): # Issue 41540: @unittest.skipIf(sys.platform.startswith("aix"), "AIX: default ulimit: test is flaky because of extreme over-allocation") - @unittest.skipIf(MEMORY_SANITIZER or ADDRESS_SANITIZER, "sanitizer defaults to crashing " + @unittest.skipIf(check_sanitizer(address=True, memory=True), + "ASAN/MSAN sanitizer defaults to crashing " "instead of returning NULL for malloc failure.") def test_maxcontext_exact_arith(self): diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py index fa855f2104c586..77f72fcc7c6e3b 100644 --- a/Lib/test/test_peg_generator/__init__.py +++ b/Lib/test/test_peg_generator/__init__.py @@ -1,7 +1,15 @@ -import os - +import os.path +import unittest +from test import support from test.support import load_package_tests + +if support.check_sanitizer(address=True, memory=True): + # bpo-46633: Skip the test because it is too slow when Python is built + # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. + raise unittest.SkipTest("test too slow on ASAN/MSAN build") + + # Load all tests in package def load_tests(*args): return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py index 61af6578e09530..34b0d3b8fb3eb0 100644 --- a/Lib/test/test_tools/__init__.py +++ b/Lib/test/test_tools/__init__.py @@ -6,6 +6,13 @@ from test import support from test.support import import_helper + +if support.check_sanitizer(address=True, memory=True): + # bpo-46633: Skip the test because it is too slow when Python is built + # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. + raise unittest.SkipTest("test too slow on ASAN/MSAN build") + + basepath = os.path.normpath( os.path.dirname( # os.path.dirname( # Lib