CI: Fix issues caused by pytest v9 (#2904)

Issues with pyproject.toml

Since the recent pytest v9 release, PEFT's pyproject.toml does not work
anymore. This is because it mixes both new and old format new formats
(ini_options), which is no longer allowed.

This PR fixes the issue by switching fully to the new format. Note that
if a pytest version < 9.0.0 is used, the added pytest options will be
ignored. Thus please update to 9.0.0+.

Context: https://github.com/pytest-dev/pytest/releases/tag/9.0.0

Issues with unittest.skip

Since pytest v9, unittest.skip raises an error instead of just skipping
the corresponding test (see these errors). As of writing this, it is
unclear if this will be fixed or not:

pytest-dev/pytest#13895

To avoid the error, we now use pytest.mark.skip insted of unittest.skip
/ unittest.skipUnless.

Tangential changes

1. Remove a few inheritances from unittest.TestCase that were
unnecessary
2. Skip the whole test_gpu_examples.py module if no accelerator is
available (faster than checking each class/function individually)

Author: BenjaminBossan

pyproject.toml

@@ -41,13 +41,13 @@ doctest_optionflags = [
     "NUMBER",
 ]
 
-[tool.pytest.ini_options]
-addopts = "--cov=src/peft --cov-report=term-missing --durations=10"
+addopts = ["--cov=src/peft", "--cov-report=term-missing", "--durations=10"]
+
 markers = [
     "single_gpu_tests: tests that run on a single GPU",
     "multi_gpu_tests: tests that run on multiple GPUs",
     "regression: whether to run regression suite test",
-    "bitsandbytes: select bitsandbytes integration tests"
+    "bitsandbytes: select bitsandbytes integration tests",
 ]
 
 filterwarnings = [

tests/test_gpu_examples.py

@@ -27,7 +27,7 @@
 import pytest
 import torch
 from accelerate import infer_auto_device_map
-from accelerate.test_utils.testing import run_command
+from accelerate.test_utils.testing import get_backend, run_command
 from accelerate.utils import patch_environment
 from accelerate.utils.imports import is_bf16_available
 from accelerate.utils.memory import clear_device_cache
@@ -108,6 +108,11 @@
 )
 
 
+device, _, _ = get_backend()
+if device == "cpu":
+    pytest.skip(allow_module_level=True, reason="GPU tests require hardware accelerator, got CPU only")
+
+
 # Some tests with multi GPU require specific device maps to ensure that the models are loaded in two devices
 DEVICE_MAP_MAP: dict[str, dict[str, int]] = {
     "facebook/opt-6.7b": {
@@ -3460,9 +3465,9 @@ def test_load_adapter_using_float16_autocast_dtype(self):
 @require_non_xpu
 @require_torch_gpu
 @require_aqlm
-@unittest.skipUnless(
-    version.parse(importlib.metadata.version("transformers")) >= version.parse("4.38.0"),
-    "test requires `transformers>=4.38.0`",
+@pytest.mark.skipif(
+    not version.parse(importlib.metadata.version("transformers")) >= version.parse("4.38.0"),
+    reason="test requires `transformers>=4.38.0`",
 )
 class PeftAqlmGPUTests(unittest.TestCase):
     r"""
@@ -3545,9 +3550,9 @@ def test_causal_lm_training_aqlm(self):
 @require_non_xpu
 @require_torch_gpu
 @require_hqq
-@unittest.skipUnless(
-    version.parse(importlib.metadata.version("transformers")) >= version.parse("4.36.1"),
-    "test requires `transformers>=4.36.1`",
+@pytest.mark.skipif(
+    not version.parse(importlib.metadata.version("transformers")) >= version.parse("4.36.1"),
+    reason="test requires `transformers>=4.36.1`",
 )
 class PeftHqqGPUTests(unittest.TestCase):
     r"""

tests/test_tuners_utils.py

@@ -16,7 +16,6 @@
 # limitations under the License.
 import dataclasses
 import re
-import unittest
 from copy import deepcopy
 
 import pytest
@@ -179,7 +178,7 @@
 BNB_TEST_CASES = [(x + y) for x in MAYBE_INCLUDE_ALL_LINEAR_LAYERS_TEST_CASES for y in BNB_QUANTIZATIONS]
 
 
-class PeftCustomKwargsTester(unittest.TestCase):
+class PeftCustomKwargsTester:
     r"""
     Test if the PeftModel is instantiated with correct behaviour for custom kwargs. This includes:
     - test if regex matching works correctly
@@ -444,7 +443,7 @@ def __init__(self, bias=True):
         self.sm = nn.LogSoftmax(dim=-1)
 
 
-class TestTargetedModuleNames(unittest.TestCase):
+class TestTargetedModuleNames:
     """Check that the attribute targeted_module_names is correctly set.
 
     This checks LoRA and IA³, but this should be sufficient, testing all other tuners is not necessary.
@@ -492,7 +491,7 @@ def test_realistic_example(self):
         assert model.targeted_module_names == expected
 
 
-class TestTargetedParameterNames(unittest.TestCase):
+class TestTargetedParameterNames:
     """Check that the attribute targeted_parameter_names (via target_parameters) is correctly set.
 
     This is only implemented for LoRA. Regex matching is currently not implemented.
@@ -520,7 +519,7 @@ def test_realistic_example(self):
         assert model.targeted_parameter_names == expected
 
 
-class TestExcludedModuleNames(unittest.TestCase):
+class TestExcludedModuleNames:
     """Check that the attribute exclude_module is correctly set.
 
     This checks LoRA and IA³, but this should be sufficient, testing all other tuners is not necessary.
@@ -1527,7 +1526,7 @@ class ModelWithNoConfig(nn.Module):
     pass
 
 
-class TestBaseTunerGetModelConfig(unittest.TestCase):
+class TestBaseTunerGetModelConfig:
     def test_get_model_config_use_to_dict(self):
         config = BaseTuner.get_model_config(ModelWithConfig())
         assert config == MockModelConfig.config

tests/testing_utils.py

@@ -12,7 +12,6 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 import os
-import unittest
 from contextlib import contextmanager
 from functools import lru_cache, wraps
 from unittest import mock
@@ -55,44 +54,40 @@ def require_non_cpu(test_case):
     Decorator marking a test that requires a hardware accelerator backend. These tests are skipped when there are no
     hardware accelerator available.
     """
-    return unittest.skipUnless(torch_device != "cpu", "test requires a hardware accelerator")(test_case)
+    return pytest.mark.skipif(torch_device == "cpu", reason="test requires a hardware accelerator")(test_case)
 
 
 def require_non_xpu(test_case):
     """
     Decorator marking a test that should be skipped for XPU.
     """
-    return unittest.skipUnless(torch_device != "xpu", "test requires a non-XPU")(test_case)
+    return pytest.mark.skipif(torch_device == "xpu", reason="test requires a non-XPU")(test_case)
 
 
 def require_torch_gpu(test_case):
     """
     Decorator marking a test that requires a GPU. Will be skipped when no GPU is available.
     """
-    if not torch.cuda.is_available():
-        return unittest.skip("test requires GPU")(test_case)
-    else:
-        return test_case
+    return pytest.mark.skipif(not torch.cuda.is_available(), reason="test requires GPU")(test_case)
 
 
 def require_torch_multi_gpu(test_case):
     """
     Decorator marking a test that requires multiple GPUs. Will be skipped when less than 2 GPUs are available.
     """
-    if not torch.cuda.is_available() or torch.cuda.device_count() < 2:
-        return unittest.skip("test requires multiple GPUs")(test_case)
-    else:
-        return test_case
+    multi_cuda_unavailable = not torch.cuda.is_available() or (device_count < 2)
+    return pytest.mark.skipif(multi_cuda_unavailable, reason="test requires multiple GPUs")(test_case)
 
 
 def require_torch_multi_accelerator(test_case):
     """
     Decorator marking a test that requires multiple hardware accelerators. These tests are skipped on a machine without
     multiple accelerators.
     """
-    return unittest.skipUnless(
-        torch_device != "cpu" and device_count > 1, "test requires multiple hardware accelerators"
-    )(test_case)
+    multi_device_unavailable = (torch_device == "cpu") or (device_count < 2)
+    return pytest.mark.skipif(multi_device_unavailable, reason="test requires multiple hardware accelerators")(
+        test_case
+    )
 
 
 def require_bitsandbytes(test_case):
@@ -112,58 +107,57 @@ def require_auto_gptq(test_case):
     """
     Decorator marking a test that requires auto-gptq. These tests are skipped when auto-gptq isn't installed.
     """
-    return unittest.skipUnless(is_gptqmodel_available() or is_auto_gptq_available(), "test requires auto-gptq")(
-        test_case
-    )
+    is_gptq_avaiable = is_gptqmodel_available() or is_auto_gptq_available()
+    return pytest.mark.skipif(not is_gptq_avaiable, reason="test requires auto-gptq")(test_case)
 
 
 def require_gptqmodel(test_case):
     """
     Decorator marking a test that requires gptqmodel. These tests are skipped when gptqmodel isn't installed.
     """
-    return unittest.skipUnless(is_gptqmodel_available(), "test requires gptqmodel")(test_case)
+    return pytest.mark.skipif(not is_gptqmodel_available(), reason="test requires gptqmodel")(test_case)
 
 
 def require_aqlm(test_case):
     """
     Decorator marking a test that requires aqlm. These tests are skipped when aqlm isn't installed.
     """
-    return unittest.skipUnless(is_aqlm_available(), "test requires aqlm")(test_case)
+    return pytest.mark.skipif(not is_aqlm_available(), reason="test requires aqlm")(test_case)
 
 
 def require_hqq(test_case):
     """
     Decorator marking a test that requires aqlm. These tests are skipped when aqlm isn't installed.
     """
-    return unittest.skipUnless(is_hqq_available(), "test requires hqq")(test_case)
+    return pytest.mark.skipif(not is_hqq_available(), reason="test requires hqq")(test_case)
 
 
 def require_auto_awq(test_case):
     """
     Decorator marking a test that requires auto-awq. These tests are skipped when auto-awq isn't installed.
     """
-    return unittest.skipUnless(is_auto_awq_available(), "test requires auto-awq")(test_case)
+    return pytest.mark.skipif(not is_auto_awq_available(), reason="test requires auto-awq")(test_case)
 
 
 def require_eetq(test_case):
     """
     Decorator marking a test that requires eetq. These tests are skipped when eetq isn't installed.
     """
-    return unittest.skipUnless(is_eetq_available(), "test requires eetq")(test_case)
+    return pytest.mark.skipif(not is_eetq_available(), reason="test requires eetq")(test_case)
 
 
 def require_optimum(test_case):
     """
     Decorator marking a test that requires optimum. These tests are skipped when optimum isn't installed.
     """
-    return unittest.skipUnless(is_optimum_available(), "test requires optimum")(test_case)
+    return pytest.mark.skipif(not is_optimum_available(), reason="test requires optimum")(test_case)
 
 
 def require_torchao(test_case):
     """
     Decorator marking a test that requires torchao. These tests are skipped when torchao isn't installed.
     """
-    return unittest.skipUnless(is_torchao_available(), "test requires torchao")(test_case)
+    return pytest.mark.skipif(not is_torchao_available(), reason="test requires torchao")(test_case)
 
 
 def require_deterministic_for_xpu(test_case):