ElementsProject · rustyrussell · Nov 12, 2025 · Nov 12, 2025 · Nov 12, 2025 · Nov 13, 2025
diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml
@@ -71,32 +71,6 @@ jobs:
             echo "Not a PR event."
           fi
 
-      - name: Set up Python 3.10
-        uses: actions/setup-python@v5
-        with:
-          python-version: "3.10"
-
-      - name: Install uv
-        uses: astral-sh/setup-uv@v5
-
-      - name: Install dependencies
-        run: |
-          bash -x .github/scripts/setup.sh
-          # We're going to check BOLT quotes, so get the latest version
-          git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
-      - name: Configure
-        run: ./configure --enable-debugbuild --enable-rust
-      - name: Check source
-        env:
-          VALGRIND: 0
-          PYTEST_OPTS: --timeout=1200 --durations=10
-        run: |
-          uv run make check-source BASE_REF="origin/${{ github.base_ref }}"
-      - name: Check Generated Files have been updated
-        run: uv run make check-gen-updated
-      - name: Check docs
-        run: uv run make check-doc
-
   compile:
     name: Compile CLN ${{ matrix.cfg }}
     runs-on: ubuntu-22.04
@@ -151,16 +125,74 @@ jobs:
           set -e
           ./configure --enable-debugbuild CC="$COMPILER" ${{ matrix.COPTFLAGS_VAR }}
 
-          uv run make -j $(nproc) testpack.tar.bz2
+          uv run make -j $(nproc) testpack.tar.gz
 
           # Rename now so we don't clash
-          mv testpack.tar.bz2 cln-${CFG}.tar.bz2
+          mv testpack.tar.gz cln-${CFG}.tar.gz
       - name: Check rust packages
         run: cargo test --all
       - uses: actions/upload-artifact@v4
         with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
-          path: cln-${{ matrix.CFG }}.tar.bz2
+          name: cln-${{ matrix.CFG }}.tar.gz
+          path: cln-${{ matrix.CFG }}.tar.gz
+
+  postbuild:
+    name: Postbuild checks
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    env:
+      BOLTDIR: bolts
+    needs:
+      - compile
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - CFG: compile-gcc
+            VALGRIND: 1
+          - CFG: compile-clang-sanitizers
+            VALGRIND: 0
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Set up Python 3.10
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.10"
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+
+      - name: Install dependencies
+        run: |
+          bash -x .github/scripts/setup.sh
+          sudo apt-get update -qq
+          # We're going to check BOLT quotes, so get the latest version
+          git clone https://github.com/lightning/bolts.git ../${BOLTDIR}
+
+      - name: Download
+        uses: actions/download-artifact@v4
+        with:
+          name: cln-${{ matrix.CFG }}.tar.gz
+
+      - name: Unpack prebuilt binaries
+        run: |
+          git submodule sync && git submodule update --init --recursive
+          # Make sure source appears older than what we're about to unpack
+          find . -type f -print0 | xargs -0 touch -d yesterday
+          tar xaf cln-${{ matrix.CFG }}.tar.gz
+
+      - name: Check source
+        env:
+          VALGRIND: 0
+          PYTEST_OPTS: --timeout=1200 --durations=10
+        run: |
+          uv run make check-source BASE_REF="origin/${{ github.base_ref }}" CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1
+      - name: Check Generated Files have been updated
+        run: uv run make check-gen-updated CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1
+      - name: Check docs
+        run: uv run make check-doc CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1
 
   check-units:
     # The unit test checks are not in the critical path (not dependent
@@ -202,12 +234,18 @@ jobs:
       - name: Download build
         uses: actions/download-artifact@v4
         with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
+          name: cln-${{ matrix.CFG }}.tar.gz
+
+      - name: Unpack prebuilt binaries
+        run: |
+          git submodule sync && git submodule update --init --recursive
+          # Make sure source appears older than what we're about to unpack
+          find . -type f -print0 | xargs -0 touch -d yesterday
+          tar xaf cln-${{ matrix.CFG }}.tar.gz
 
       - name: Check
         run: |
-          tar -xaf cln-${{ matrix.CFG }}.tar.bz2
-          uv run eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }}
+          uv run eatmydata make -j $(nproc) check-units installcheck VALGRIND=${{ matrix.VALGRIND }} CARGO=false CC=devtools/cc-nobuild SUPPRESS_GENERATION=1
 
   check-fuzz:
     name: Run fuzz regression tests
@@ -310,20 +348,22 @@ jobs:
       - name: Download build
         uses: actions/download-artifact@v4
         with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
+          name: cln-${{ matrix.CFG }}.tar.gz
 
-      - name: Unpack pre-built CLN
-        env:
-          CFG: ${{ matrix.CFG }}
+      - name: Unpack prebuilt binaries
         run: |
-          tar -xaf cln-${CFG}.tar.bz2
+          git submodule sync && git submodule update --init --recursive
+          # Make sure source appears older than what we're about to unpack
+          find . -type f -print0 | xargs -0 touch -d yesterday
+          tar xaf cln-${{ matrix.CFG }}.tar.gz
 
       - name: Switch network
         if: ${{ matrix.TEST_NETWORK == 'liquid-regtest' }}
         run: |
           # Loading the network from config.vars rather than the envvar is a terrible idea...
           sed -i 's/TEST_NETWORK=regtest/TEST_NETWORK=liquid-regtest/g' config.vars
           cat config.vars
+          touch -d yesterday config.vars
 
       - name: Test
         env:
@@ -401,10 +441,14 @@ jobs:
       - name: Download build
         uses: actions/download-artifact@v4
         with:
-          name: cln-compile-gcc.tar.bz2
+          name: cln-compile-gcc.tar.gz
 
-      - name: Unpack build
-        run: tar -xvjf cln-compile-gcc.tar.bz2
+      - name: Unpack prebuilt binaries
+        run: |
+          git submodule sync && git submodule update --init --recursive
+          # Make sure source appears older than what we're about to unpack
+          find . -type f -print0 | xargs -0 touch -d yesterday
+          tar xaf cln-compile-gcc.tar.gz
 
       - name: Test
         env:
@@ -470,10 +514,14 @@ jobs:
       - name: Download build
         uses: actions/download-artifact@v4
         with:
-          name: cln-compile-clang-sanitizers.tar.bz2
+          name: cln-compile-clang-sanitizers.tar.gz
 
-      - name: Unpack build
-        run: tar -xvjf cln-compile-clang-sanitizers.tar.bz2
+      - name: Unpack prebuilt binaries
+        run: |
+          git submodule sync && git submodule update --init --recursive
+          # Make sure source appears older than what we're about to unpack
+          find . -type f -print0 | xargs -0 touch -d yesterday
+          tar xaf cln-compile-clang-sanitizers.tar.gz
 
       - name: Test
         run: |
@@ -514,10 +562,14 @@ jobs:
       - name: Download build
         uses: actions/download-artifact@v4
         with:
-          name: cln-compile-gcc.tar.bz2
-      - name: Unpack pre-built CLN
+          name: cln-compile-gcc.tar.gz
+      - name: Unpack prebuilt binaries
         run: |
-          tar -xaf cln-compile-gcc.tar.bz2
+          git submodule sync && git submodule update --init --recursive
+          # Make sure source appears older than what we're about to unpack
+          find . -type f -print0 | xargs -0 touch -d yesterday
+          tar xaf cln-compile-gcc.tar.gz
+
       - name: Test
         run: |
           uv run eatmydata make -j $(nproc) check-doc-examples
@@ -572,13 +624,14 @@ jobs:
       - name: Download build
         uses: actions/download-artifact@v4
         with:
-          name: cln-${{ matrix.CFG }}.tar.bz2
+          name: cln-${{ matrix.CFG }}.tar.gz
 
-      - name: Unpack pre-built CLN
-        env:
-          CFG: ${{ matrix.CFG }}
+      - name: Unpack prebuilt binaries
         run: |
-          tar -xaf cln-${CFG}.tar.bz2
+          git submodule sync && git submodule update --init --recursive
+          # Make sure source appears older than what we're about to unpack
+          find . -type f -print0 | xargs -0 touch -d yesterday
+          tar xaf cln-${{ matrix.CFG }}.tar.gz
 
       - name: Test
         env:
@@ -608,17 +661,19 @@ jobs:
       - integration-valgrind
       - integration-sanitizers
       - min-btc-support
+      - postbuild
     if: ${{ always() }}
     steps:
       - name: Complete
         env:
-          JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC"
+          JOB_NAMES: "INTEGRATION CHECK_UNITS VALGRIND SANITIZERS BTC POSTBUILD"
           INTEGRATION: ${{ needs.integration.result }}
           CHECK_UNITS: ${{ needs['check-units'].result }}
           VALGRIND: ${{ needs['integration-valgrind'].result }}
           SANITIZERS: ${{ needs['integration-sanitizers'].result }}
           DOCS: ${{ needs['update-docs-examples'].result }}
           BTC: ${{ needs['min-btc-support'].result }}
+          POSTBUILD: ${{ needs['postbuild'].result }}
         run: |
           failed=""
           for name in $JOB_NAMES; do

diff --git a/Makefile b/Makefile
@@ -15,6 +15,7 @@ ECHO := echo
 SUPPRESS_OUTPUT :=
 endif
 
+CARGO := cargo
 DISTRO=$(shell lsb_release -is 2>/dev/null || echo unknown)-$(shell lsb_release -rs 2>/dev/null || echo unknown)
 OS=$(shell uname -s)
 ARCH=$(shell uname -m)
@@ -98,6 +99,7 @@ CCAN_OBJS :=					\
 	ccan-bitmap.o				\
 	ccan-bitops.o				\
 	ccan-breakpoint.o			\
+	ccan-cdump.o				\
 	ccan-closefrom.o			\
 	ccan-crc32c.o				\
 	ccan-crypto-hmac.o			\
@@ -223,8 +225,6 @@ CCAN_HEADERS :=						\
 	$(CCANDIR)/ccan/typesafe_cb/typesafe_cb.h	\
 	$(CCANDIR)/ccan/utf8/utf8.h
 
-CDUMP_OBJS := ccan-cdump.o ccan-strmap.o
-
 BOLT_GEN := tools/generate-wire.py
 WIRE_GEN := $(BOLT_GEN)
 
@@ -472,10 +472,6 @@ mkdocs.yml: $(MANPAGES:=.md)
 	)
 
 
-
-# Don't delete these intermediaries.
-.PRECIOUS: $(ALL_GEN_HEADERS) $(ALL_GEN_SOURCES) $(PYTHON_GENERATED)
-
 # Every single object file.
 ALL_OBJS := $(ALL_C_SOURCES:.c=.o)
 
@@ -680,7 +676,7 @@ TAGS:
 tags:
 	$(RM) tags; find * -name test -type d -prune -o \( -name '*.[ch]' -o -name '*.py' \) -print0 | xargs -0 ctags --append
 
-ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o $(CDUMP_OBJS) $(CCAN_OBJS)
+ccan/ccan/cdump/tools/cdump-enumstr: ccan/ccan/cdump/tools/cdump-enumstr.o libccan.a
 
 ALL_PROGRAMS += ccan/ccan/cdump/tools/cdump-enumstr
 # Can't add to ALL_OBJS, as that makes a circular dep.
@@ -702,6 +698,9 @@ endif
 header_versions_gen.h: tools/headerversions $(FORCE)
 	@tools/headerversions $@
 
+# Once you have libccan.a, you don't need these.
+.INTERMEDIATE: $(CCAN_OBJS)
+
 # We make a static library, this way linker can discard unused parts.
 libccan.a: $(CCAN_OBJS)
 	@$(call VERBOSE, "ar $@", $(AR) r $@ $(CCAN_OBJS))
@@ -923,26 +922,20 @@ install-data: installdirs $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(
 
 install: install-program install-data
 
-# Non-artifacts that are needed for testing. These are added to the
-# testpack.tar, used to transfer things between builder and tester
-# phase. If you get a missing file/executable while testing on CI it
-# is likely missing from this variable.
-TESTBINS = \
-	$(CLN_PLUGIN_EXAMPLES) \
-	tests/plugins/test_libplugin \
-	tests/plugins/channeld_fakenet \
-	tests/plugins/test_selfdisable_after_getmanifest \
-	tools/hsmtool
+# We exclude most of target/ and external, but we need:
+# 1. config files (we only tar up files *newer* than these)
+# 2. $(DEFAULT_TARGETS) for rust stuff.
+# 3. $(EXTERNAL_LIBS) for prebuild external libraries.
+TESTPACK_EXTRAS :=			\
+	config.vars ccan/config.h	\
+	$(DEFAULT_TARGETS)		\
+	$(EXTERNAL_LIBS)
 
 # The testpack is used in CI to transfer built artefacts between the
-# build and the test phase. This is necessary because the fixtures in
-# `tests/` explicitly use the binaries built in the current directory
-# rather than using `$PATH`, as that may pick up some other installed
-# version of `lightningd` leading to bogus results. We bundle up all
-# built artefacts here, and will unpack them on the tester (overlaying
-# on top of the checked out repo as if we had just built it in place).
-testpack.tar.bz2: $(BIN_PROGRAMS) $(PKGLIBEXEC_PROGRAMS) $(PLUGINS) $(PY_PLUGINS) $(MAN1PAGES) $(MAN5PAGES) $(MAN7PAGES) $(MAN8PAGES) $(DOC_DATA) config.vars $(TESTBINS) $(DEVTOOLS)
-	tar -caf $@ $^
+# build and the test phase.  Only useful on a freshly build tree!
+# We use Posix format for timestamps with subsecond accuracy.
+testpack.tar.gz: all-programs all-fuzz-programs all-test-programs default-targets
+	(find * -path external -prune -o -path target -prune -o -newer config.vars -print; ls $(TESTPACK_EXTRAS)) | tar --verbatim-files-from -T- -c --format=posix -f - | gzip -5 > $@
 
 uninstall:
 	@$(NORMAL_UNINSTALL)

@@ -9,13 +9,13 @@ DEFAULT_TARGETS += $(CLN_RPC_EXAMPLES) $(CLN_RPC_GENALL)
 MSGGEN_GENALL += $(CLN_RPC_GENALL)
 
 target/${RUST_PROFILE}/examples/cln-rpc-getinfo: ${CLN_RPC_SOURCES} cln-rpc/examples/getinfo.rs
-	cargo build ${CARGO_OPTS} --example cln-rpc-getinfo
+	$(CARGO) build ${CARGO_OPTS} --example cln-rpc-getinfo
 
 target/${RUST_PROFILE}/examples/cln-plugin-startup: ${CLN_RPC_SOURCES} plugins/examples/cln-plugin-startup.rs
-	cargo build ${CARGO_OPTS} --example cln-plugin-startup
+	$(CARGO) build ${CARGO_OPTS} --example cln-plugin-startup
 
 target/${RUST_PROFILE}/examples/cln-plugin-reentrant: ${CLN_RPC_SOURCES} plugins/examples/cln-plugin-reentrant.rs
-	cargo build ${CARGO_OPTS} --example cln-plugin-reentrant
+	$(CARGO) build ${CARGO_OPTS} --example cln-plugin-reentrant
 
 
 cln-rpc-all: ${CLN_RPC_GENALL} ${CLN_RPC_EXAMPLES}
diff --git a/devtools/cc-nobuild b/devtools/cc-nobuild
@@ -0,0 +1,9 @@
+#! /bin/sh
+# Version of CC which only supports -dumpmachine (for external/Makefile), and fails otherwise
+set -e
+
+if [ x"$*" = x"-dumpmachine" ]; then
+    CC="$(grep ^CC= config.vars | cut -d= -f2-)"
+    exec ${CC:=cc} "$@"
+fi
+exit 1