Run third-party stubtest under xvfb-run (#8719)

Author: Akuli

.github/workflows/daily.yml

@@ -56,7 +56,7 @@ jobs:
           sudo apt update
           sudo apt install -y $(python tests/get_apt_packages.py)
       - name: Run stubtest
-        run: python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
+        run: xvfb-run python tests/stubtest_third_party.py --num-shards 4 --shard-index ${{ matrix.shard-index }}
 
   # https://github.community/t/run-github-actions-job-only-if-previous-job-has-failed/174786/2
   create-issue-on-failure:

.github/workflows/tests.yml

@@ -153,7 +153,7 @@ jobs:
               echo "Installing apt packages: $APT_PACKAGES"
               sudo apt update && sudo apt install -y $APT_PACKAGES
             fi
-            python tests/stubtest_third_party.py $STUBS
+            xvfb-run python tests/stubtest_third_party.py $STUBS
           else
             echo "Nothing to test"
           fi

stubs/PyAutoGUI/METADATA.toml

@@ -1,7 +1,2 @@
 version = "0.9.*"
 requires = ["types-Pillow"]
-
-[tool.stubtest]
-# pyautogui requires a display, resulting in the following error on the CI:
-# failed to import, KeyError: 'DISPLAY'
-skip = true

stubs/pynput/@tests/stubtest_allowlist.txt

@@ -0,0 +1,9 @@
+# TODO: go through this allowlist, figure out which of them are false positives
+pynput.keyboard.Controller._Key
+pynput.keyboard.Controller._KeyCode
+pynput.keyboard.Controller.__init__
+pynput.keyboard._base.Controller._Key
+pynput.keyboard._base.Controller._KeyCode
+pynput.keyboard._dummy.Controller._Key
+pynput.keyboard._dummy.Controller._KeyCode
+pynput.mouse.Controller.__init__

stubs/pynput/METADATA.toml

@@ -1,4 +1 @@
 version = "1.7.*"
-
-[tool.stubtest]
-skip = true  # A display server (e.g. X11) is required to import pynput

tests/stubtest_third_party.py

@@ -5,6 +5,7 @@
 
 import argparse
 import functools
+import os
 import subprocess
 import sys
 import tempfile
@@ -83,12 +84,21 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool:
             *packages_to_check,
             *modules_to_check,
         ]
+
+        # For packages that need a display, we need to pass at least $DISPLAY
+        # to stubtest. $DISPLAY is set by xvfb-run in CI.
+        #
+        # It seems that some other environment variables are needed too,
+        # because the CI fails if we pass only os.environ["DISPLAY"]. I didn't
+        # "bisect" to see which variables are actually needed.
+        stubtest_env = os.environ | {"MYPYPATH": str(dist), "MYPY_FORCE_COLOR": "1"}
+
         allowlist_path = dist / "@tests/stubtest_allowlist.txt"
         if allowlist_path.exists():
             stubtest_cmd.extend(["--allowlist", str(allowlist_path)])
 
         try:
-            subprocess.run(stubtest_cmd, env={"MYPYPATH": str(dist), "MYPY_FORCE_COLOR": "1"}, check=True, capture_output=True)
+            subprocess.run(stubtest_cmd, env=stubtest_env, check=True, capture_output=True)
         except subprocess.CalledProcessError as e:
             print_error("fail")
             print_commands(dist, pip_cmd, stubtest_cmd)
@@ -105,7 +115,7 @@ def run_stubtest(dist: Path, *, verbose: bool = False) -> bool:
                 print(file=sys.stderr)
             else:
                 print(f"Re-running stubtest with --generate-allowlist.\nAdd the following to {allowlist_path}:", file=sys.stderr)
-                ret = subprocess.run(stubtest_cmd + ["--generate-allowlist"], env={"MYPYPATH": str(dist)}, capture_output=True)
+                ret = subprocess.run(stubtest_cmd + ["--generate-allowlist"], env=stubtest_env, capture_output=True)
                 print_command_output(ret)
 
             return False