Merge bitcoin#901: ci: Switch all Linux builds to Debian and more improvements

9361f36 ci: Select number of parallel make jobs depending on CI environment (Tim Ruffing)
28eccdf ci: Split output of logs into multiple sections (Tim Ruffing)
c7f754f ci: Run PRs on merge result instead of on the source branch (Tim Ruffing)
b994a8b ci: Print information about binaries using "file" (Tim Ruffing)
f24e122 ci: Switch all Linux builds to Debian (Tim Ruffing)

Pull request description:

  Best reviewed commit by commit

ACKs for top commit:
  jonasnick:
    ACK 9361f36
  sipa:
    utACK 9361f36

Tree-SHA512: fc754e8b57dc58058cebbf63a60ca76e08dbaefea1508ea27b7f962bce697c10033da3f57a35f731bc7cf3e210eb00e3b8985ae8b729d7bd83faee085b878b9c

Author: real-or-random

.cirrus.yml

@@ -16,26 +16,38 @@ env:
   CTIMETEST: yes
   BENCH: yes
   ITERS: 2
-  # We only need the top commit
-  CIRRUS_CLONE_DEPTH: 1
+  MAKEFLAGS: -j2
 
 cat_logs_snippet: &CAT_LOGS
   always:
-    test_logs_script:
+    cat_tests_log_script:
       - cat tests.log || true
+    cat_exhaustive_tests_log_script:
       - cat exhaustive_tests.log || true
+    cat_valgrind_ctime_test_log_script:
       - cat valgrind_ctime_test.log || true
+    cat_bench_log_script:
       - cat bench.log || true
   on_failure:
-    debug_output_script:
+    cat_config_log_script:
       - cat config.log || true
+    cat_test_env_script:
       - cat test_env.log || true
+    cat_ci_env_script:
       - env
 
+merge_base_script_snippet: &MERGE_BASE
+  merge_base_script:
+    - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
+    - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH
+    - git config --global user.email "[email protected]"
+    - git config --global user.name "ci"
+    - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
+
 task:
-  name: "x86_64: Linux (Alpine Linux, Nix Shell)"
+  name: "x86_64: Linux (Debian stable)"
   container:
-    dockerfile: ci/linux-nixos.Dockerfile
+    dockerfile: ci/linux-debian.Dockerfile
     # Reduce number of CPUs to be able to do more builds in parallel.
     cpu: 1
     # More than enough for our scripts.
@@ -81,14 +93,15 @@ task:
         CC: gcc
     - env:
         CC: clang
+  << : *MERGE_BASE
   test_script:
-    - nix-shell ci/shell.nix --run ./ci/cirrus.sh
+    - ./ci/cirrus.sh
   << : *CAT_LOGS
 
 task:
-  name: "i686: Linux (Alpine Linux, Nix Shell)"
+  name: "i686: Linux (Debian stable)"
   container:
-    dockerfile: ci/linux-nixos.Dockerfile
+    dockerfile: ci/linux-debian.Dockerfile
     cpu: 1
     memory: 1G
   env:
@@ -99,16 +112,17 @@ task:
     SCHNORRSIG: yes
   matrix:
     - env:
-        CC: gcc
+        CC: i686-linux-gnu-gcc
     - env:
-        CC: clang
+        CC: clang --target=i686-pc-linux-gnu -isystem /usr/i686-linux-gnu/include
   matrix:
     - env:
         BIGNUM: gmp
     - env:
         BIGNUM: no
+  << : *MERGE_BASE
   test_script:
-    - nix-shell ci/shell-i686.nix --run ./ci/cirrus.sh
+    - ./ci/cirrus.sh
   << : *CAT_LOGS
 
 task:
@@ -118,6 +132,8 @@ task:
   env:
     HOMEBREW_NO_AUTO_UPDATE: 1
     HOMEBREW_NO_INSTALL_CLEANUP: 1
+    # Cirrus gives us a fixed number of 12 virtual CPUs. Not that we even have that many jobs at the moment...
+    MAKEFLAGS: -j13
   matrix:
     << : *ENV_MATRIX
   matrix:
@@ -163,14 +179,15 @@ task:
     - brew link valgrind
   brew_script:
     - brew install automake libtool gmp gcc@9
+  << : *MERGE_BASE
   test_script:
     - ./ci/cirrus.sh
   << : *CAT_LOGS
 
 task:
-  name: "s390x (big-endian): Linux (Debian QEMU)"
+  name: "s390x (big-endian): Linux (Debian stable, QEMU)"
   container:
-    dockerfile: ci/linux-debian-s390-qemu.Dockerfile
+    dockerfile: ci/linux-debian.Dockerfile
     cpu: 1
     memory: 1G
   env:
@@ -184,6 +201,7 @@ task:
     EXPERIMENTAL: yes
     SCHNORRSIG: yes
     CTIMETEST: no
+  << : *MERGE_BASE
   test_script:
     # https://sourceware.org/bugzilla/show_bug.cgi?id=27008
     - rm /etc/ld.so.cache

ci/cirrus.sh

@@ -12,10 +12,6 @@ valgrind --version || true
 
 ./autogen.sh
 
-# Nix doesn't store GNU file in /usr/bin, see https://lists.gnu.org/archive/html/bug-libtool/2015-09/msg00000.html .
-# The -i'' is necessary for macOS portability, see https://stackoverflow.com/a/4247319 .
-sed -i'' -e 's@/usr/bin/file@$(which file)@g' configure
-
 ./configure \
     --enable-experimental="$EXPERIMENTAL" \
     --with-test-override-wide-multiply="$WIDEMUL" --with-bignum="$BIGNUM" --with-asm="$ASM" \
@@ -25,22 +21,28 @@ sed -i'' -e 's@/usr/bin/file@$(which file)@g' configure
     --with-valgrind="$WITH_VALGRIND" \
     --host="$HOST" $EXTRAFLAGS
 
+# We have set "-j<n>" in MAKEFLAGS.
+make
+
+# Print information about binaries so that we can see that the architecture is correct
+file *tests || true
+file bench_* || true
+file .libs/* || true
+
 if [ -n "$BUILD" ]
 then
-    make -j2 "$BUILD"
+    make "$BUILD"
 fi
 
 if [ "$RUN_VALGRIND" = "yes" ]
 then
-    make -j2
     # the `--error-exitcode` is required to make the test fail if valgrind found errors, otherwise it'll return 0 (https://www.valgrind.org/docs/manual/manual-core.html)
     valgrind --error-exitcode=42 ./tests 16
     valgrind --error-exitcode=42 ./exhaustive_tests
 fi
 
 if [ -n "$QEMU_CMD" ]
 then
-    make -j2
     $QEMU_CMD ./tests 16
     $QEMU_CMD ./exhaustive_tests
 fi

ci/linux-debian-s390-qemu.Dockerfile

@@ -0,0 +1,13 @@
+FROM debian:stable
+
+RUN dpkg --add-architecture i386
+RUN dpkg --add-architecture s390x
+RUN apt-get update
+
+# dkpg-dev: to make pkg-config work in cross-builds
+RUN apt-get install --no-install-recommends --no-upgrade -y \
+        git ca-certificates \
+        make automake libtool pkg-config dpkg-dev valgrind qemu-user \
+        gcc clang libc6-dbg libgmp-dev \
+        gcc-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 libgmp-dev:i386 \
+        gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x