restore debug llvm CI coverage

and reduce redundant coverage in slow runs to save time

Author: andrewrk

.github/workflows/ci.yaml

@@ -20,6 +20,14 @@ jobs:
         uses: actions/checkout@v4
       - name: Build and Test
         run: sh ci/x86_64-linux-debug.sh
+  x86_64-linux-debug-llvm:
+    timeout-minutes: 540
+    runs-on: [self-hosted, Linux, x86_64]
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Build and Test
+        run: sh ci/x86_64-linux-debug-llvm.sh
   x86_64-linux-release:
     timeout-minutes: 540
     runs-on: [self-hosted, Linux, x86_64]

build.zig

@@ -92,6 +92,11 @@ pub fn build(b: *std.Build) !void {
     const skip_single_threaded = b.option(bool, "skip-single-threaded", "Main test suite skips tests that are single-threaded") orelse false;
     const skip_translate_c = b.option(bool, "skip-translate-c", "Main test suite skips translate-c tests") orelse false;
     const skip_run_translated_c = b.option(bool, "skip-run-translated-c", "Main test suite skips run-translated-c tests") orelse false;
+    const skip_freebsd = b.option(bool, "skip-freebsd", "Main test suite skips targets with freebsd OS") orelse false;
+    const skip_netbsd = b.option(bool, "skip-netbsd", "Main test suite skips targets with netbsd OS") orelse false;
+    const skip_windows = b.option(bool, "skip-windows", "Main test suite skips targets with windows OS") orelse false;
+    const skip_macos = b.option(bool, "skip-macos", "Main test suite skips targets with macos OS") orelse false;
+    const skip_linux = b.option(bool, "skip-linux", "Main test suite skips targets with linux OS") orelse false;
 
     const only_install_lib_files = b.option(bool, "lib-files-only", "Only install library files") orelse false;
 
@@ -435,6 +440,11 @@ pub fn build(b: *std.Build) !void {
         .include_paths = &.{},
         .skip_single_threaded = skip_single_threaded,
         .skip_non_native = skip_non_native,
+        .skip_freebsd = skip_freebsd,
+        .skip_netbsd = skip_netbsd,
+        .skip_windows = skip_windows,
+        .skip_macos = skip_macos,
+        .skip_linux = skip_linux,
         .skip_libc = skip_libc,
         .use_llvm = use_llvm,
         // 2923515904 was observed on an x86_64-linux-gnu host.
@@ -452,6 +462,11 @@ pub fn build(b: *std.Build) !void {
         .include_paths = &.{"test/c_import"},
         .skip_single_threaded = true,
         .skip_non_native = skip_non_native,
+        .skip_freebsd = skip_freebsd,
+        .skip_netbsd = skip_netbsd,
+        .skip_windows = skip_windows,
+        .skip_macos = skip_macos,
+        .skip_linux = skip_linux,
         .skip_libc = skip_libc,
         .use_llvm = use_llvm,
     }));
@@ -467,6 +482,11 @@ pub fn build(b: *std.Build) !void {
         .include_paths = &.{},
         .skip_single_threaded = true,
         .skip_non_native = skip_non_native,
+        .skip_freebsd = skip_freebsd,
+        .skip_netbsd = skip_netbsd,
+        .skip_windows = skip_windows,
+        .skip_macos = skip_macos,
+        .skip_linux = skip_linux,
         .skip_libc = true,
         .use_llvm = use_llvm,
         .no_builtin = true,
@@ -483,6 +503,11 @@ pub fn build(b: *std.Build) !void {
         .include_paths = &.{},
         .skip_single_threaded = true,
         .skip_non_native = skip_non_native,
+        .skip_freebsd = skip_freebsd,
+        .skip_netbsd = skip_netbsd,
+        .skip_windows = skip_windows,
+        .skip_macos = skip_macos,
+        .skip_linux = skip_linux,
         .skip_libc = true,
         .use_llvm = use_llvm,
         .no_builtin = true,
@@ -499,6 +524,11 @@ pub fn build(b: *std.Build) !void {
         .include_paths = &.{},
         .skip_single_threaded = skip_single_threaded,
         .skip_non_native = skip_non_native,
+        .skip_freebsd = skip_freebsd,
+        .skip_netbsd = skip_netbsd,
+        .skip_windows = skip_windows,
+        .skip_macos = skip_macos,
+        .skip_linux = skip_linux,
         .skip_libc = skip_libc,
         .use_llvm = use_llvm,
         // I observed a value of 5605064704 on the M2 CI.
@@ -536,6 +566,11 @@ pub fn build(b: *std.Build) !void {
     test_step.dependOn(tests.addCAbiTests(b, .{
         .test_target_filters = test_target_filters,
         .skip_non_native = skip_non_native,
+        .skip_freebsd = skip_freebsd,
+        .skip_netbsd = skip_netbsd,
+        .skip_windows = skip_windows,
+        .skip_macos = skip_macos,
+        .skip_linux = skip_linux,
         .skip_release = skip_release,
     }));
     test_step.dependOn(tests.addLinkTests(b, enable_macos_sdk, enable_ios_sdk, enable_symlinks_windows));
@@ -549,7 +584,6 @@ pub fn build(b: *std.Build) !void {
         .lldb = b.option([]const u8, "lldb", "path to lldb binary"),
         .optimize_modes = optimization_modes,
         .skip_single_threaded = skip_single_threaded,
-        .skip_non_native = skip_non_native,
         .skip_libc = skip_libc,
     })) |test_debugger_step| test_step.dependOn(test_debugger_step);
     if (tests.addLlvmIrTests(b, .{

ci/x86_64-linux-debug-llvm.sh

@@ -0,0 +1,70 @@
+#!/bin/sh
+
+# Requires cmake ninja-build
+
+set -x
+set -e
+
+ARCH="$(uname -m)"
+TARGET="$ARCH-linux-musl"
+MCPU="baseline"
+CACHE_BASENAME="zig+llvm+lld+clang-$TARGET-0.15.0-dev.233+7c85dc460"
+PREFIX="$HOME/deps/$CACHE_BASENAME"
+ZIG="$PREFIX/bin/zig"
+
+export PATH="$HOME/deps/wasmtime-v29.0.0-$ARCH-linux:$HOME/deps/qemu-linux-x86_64-9.2.0-rc1/bin:$HOME/local/bin:$PATH"
+
+# Make the `zig version` number consistent.
+# This will affect the cmake command below.
+git fetch --unshallow || true
+git fetch --tags
+
+# Override the cache directories because they won't actually help other CI runs
+# which will be testing alternate versions of zig, and ultimately would just
+# fill up space on the hard drive for no reason.
+export ZIG_GLOBAL_CACHE_DIR="$PWD/zig-global-cache"
+export ZIG_LOCAL_CACHE_DIR="$PWD/zig-local-cache"
+
+mkdir build-debug-llvm
+cd build-debug-llvm
+
+export CC="$ZIG cc -target $TARGET -mcpu=$MCPU"
+export CXX="$ZIG c++ -target $TARGET -mcpu=$MCPU"
+
+cmake .. \
+  -DCMAKE_INSTALL_PREFIX="stage3-debug" \
+  -DCMAKE_PREFIX_PATH="$PREFIX" \
+  -DCMAKE_BUILD_TYPE=Debug \
+  -DZIG_TARGET_TRIPLE="$TARGET" \
+  -DZIG_TARGET_MCPU="$MCPU" \
+  -DZIG_STATIC=ON \
+  -DZIG_NO_LIB=ON \
+  -DZIG_EXTRA_BUILD_ARGS="-Duse-llvm=true" \
+  -GNinja
+
+# Now cmake will use zig as the C/C++ compiler. We reset the environment variables
+# so that installation and testing do not get affected by them.
+unset CC
+unset CXX
+
+ninja install
+
+# simultaneously test building self-hosted without LLVM and with 32-bit arm
+stage3-debug/bin/zig build \
+  -Dtarget=arm-linux-musleabihf \
+  -Dno-lib
+
+stage3-debug/bin/zig build test docs \
+  --maxrss 21000000000 \
+  -Dlldb=$HOME/deps/lldb-zig/Debug-e0a42bb34/bin/lldb \
+  -fqemu \
+  -fwasmtime \
+  -Dstatic-llvm \
+  -Dskip-freebsd \
+  -Dskip-netbsd \
+  -Dskip-windows \
+  -Dskip-macos \
+  -Dtarget=native-native-musl \
+  --search-prefix "$PREFIX" \
+  --zig-lib-dir "$PWD/../lib" \
+  -Denable-superhtml

ci/x86_64-linux-debug.sh

@@ -25,12 +25,6 @@ git fetch --tags
 export ZIG_GLOBAL_CACHE_DIR="$PWD/zig-global-cache"
 export ZIG_LOCAL_CACHE_DIR="$PWD/zig-local-cache"
 
-# Test building from source without LLVM.
-cc -o bootstrap bootstrap.c
-./bootstrap
-./zig2 build -Dno-lib
-./zig-out/bin/zig test test/behavior.zig
-
 mkdir build-debug
 cd build-debug
 
@@ -65,39 +59,11 @@ stage3-debug/bin/zig build test docs \
   -fqemu \
   -fwasmtime \
   -Dstatic-llvm \
+  -Dskip-freebsd \
+  -Dskip-netbsd \
+  -Dskip-windows \
+  -Dskip-macos \
   -Dtarget=native-native-musl \
   --search-prefix "$PREFIX" \
   --zig-lib-dir "$PWD/../lib" \
   -Denable-superhtml
-
-# Ensure that updating the wasm binary from this commit will result in a viable build.
-stage3-debug/bin/zig build update-zig1
-
-mkdir ../build-new
-cd ../build-new
-
-export CC="$ZIG cc -target $TARGET -mcpu=$MCPU"
-export CXX="$ZIG c++ -target $TARGET -mcpu=$MCPU"
-
-cmake .. \
-  -DCMAKE_PREFIX_PATH="$PREFIX" \
-  -DCMAKE_BUILD_TYPE=Debug \
-  -DZIG_TARGET_TRIPLE="$TARGET" \
-  -DZIG_TARGET_MCPU="$MCPU" \
-  -DZIG_STATIC=ON \
-  -DZIG_NO_LIB=ON \
-  -GNinja
-
-unset CC
-unset CXX
-
-ninja install
-
-stage3/bin/zig test ../test/behavior.zig
-stage3/bin/zig build -p stage4 \
-  -Dstatic-llvm \
-  -Dtarget=native-native-musl \
-  -Dno-lib \
-  --search-prefix "$PREFIX" \
-  --zig-lib-dir "$PWD/../lib"
-stage4/bin/zig test ../test/behavior.zig

test/src/Debugger.zig

@@ -9,7 +9,6 @@ pub const Options = struct {
     lldb: ?[]const u8,
     optimize_modes: []const std.builtin.OptimizeMode,
     skip_single_threaded: bool,
-    skip_non_native: bool,
     skip_libc: bool,
 };
 

test/tests.zig

@@ -2273,6 +2273,11 @@ const ModuleTestOptions = struct {
     include_paths: []const []const u8,
     skip_single_threaded: bool,
     skip_non_native: bool,
+    skip_freebsd: bool,
+    skip_netbsd: bool,
+    skip_windows: bool,
+    skip_macos: bool,
+    skip_linux: bool,
     skip_libc: bool,
     use_llvm: ?bool = null,
     max_rss: usize = 0,
@@ -2295,6 +2300,12 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
         if (options.skip_non_native and !test_target.target.isNative())
             continue;
 
+        if (options.skip_freebsd and test_target.target.os_tag == .freebsd) continue;
+        if (options.skip_netbsd and test_target.target.os_tag == .netbsd) continue;
+        if (options.skip_windows and test_target.target.os_tag == .windows) continue;
+        if (options.skip_macos and test_target.target.os_tag == .macos) continue;
+        if (options.skip_linux and test_target.target.os_tag == .linux) continue;
+
         const resolved_target = b.resolveTargetQuery(test_target.target);
         const target = resolved_target.result;
         const triple_txt = target.zigTriple(b.allocator) catch @panic("OOM");
@@ -2501,6 +2512,11 @@ pub fn addModuleTests(b: *std.Build, options: ModuleTestOptions) *Step {
 const CAbiTestOptions = struct {
     test_target_filters: []const []const u8,
     skip_non_native: bool,
+    skip_freebsd: bool,
+    skip_netbsd: bool,
+    skip_windows: bool,
+    skip_macos: bool,
+    skip_linux: bool,
     skip_release: bool,
 };
 
@@ -2514,6 +2530,11 @@ pub fn addCAbiTests(b: *std.Build, options: CAbiTestOptions) *Step {
 
         for (c_abi_targets) |c_abi_target| {
             if (options.skip_non_native and !c_abi_target.target.isNative()) continue;
+            if (options.skip_freebsd and c_abi_target.target.os_tag == .freebsd) continue;
+            if (options.skip_netbsd and c_abi_target.target.os_tag == .netbsd) continue;
+            if (options.skip_windows and c_abi_target.target.os_tag == .windows) continue;
+            if (options.skip_macos and c_abi_target.target.os_tag == .macos) continue;
+            if (options.skip_linux and c_abi_target.target.os_tag == .linux) continue;
 
             const resolved_target = b.resolveTargetQuery(c_abi_target.target);
             const target = resolved_target.result;