diff --git a/test/test_models.py b/test/test_models.py
index 79591357bf3..97494d64971 100644
--- a/test/test_models.py
+++ b/test/test_models.py
@@ -682,6 +682,10 @@ def test_classification_model(model_fn, dev):
     model_name = model_fn.__name__
     if SKIP_BIG_MODEL and is_skippable(model_name, dev):
         pytest.skip("Skipped to reduce memory usage. Set env var SKIP_BIG_MODEL=0 to enable test for this model")
+    if model_name == "vit_h_14" and dev == "cuda":
+        # TODO: investigate why this fail on CI. It doesn't fail on AWS cluster with CUDA 11.6
+        # (can't test with later versions ATM)
+        pytest.xfail("https://github.com/pytorch/vision/issues/7143")
     kwargs = {**defaults, **_model_params.get(model_name, {})}
     num_classes = kwargs.get("num_classes")
     input_shape = kwargs.pop("input_shape")