Merge branch 'master' into trim-headers-test

Author: delta1

.appveyor.yml

@@ -1,6 +1,4 @@
-### Global defaults
-
-env:
+env:  # Global defaults
   SECP256K1_TEST_ITERS: 16 # ELEMENTS: avoid test timeouts on arm
   PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y"
   MAKEJOBS: "-j3" # ELEMENTS: reduced from j4
@@ -19,57 +17,47 @@ persistent_worker_template: &PERSISTENT_WORKER_TEMPLATE
   persistent_worker: {}  # https://cirrus-ci.org/guide/persistent-workers/
 
 # https://cirrus-ci.org/guide/tips-and-tricks/#sharing-configuration-between-tasks
-base_template: &BASE_TEMPLATE
+filter_template: &FILTER_TEMPLATE
   skip: $CIRRUS_REPO_FULL_NAME == "bitcoin-core/gui" && $CIRRUS_PR == ""  # No need to run on the read-only mirror, unless it is a PR. https://cirrus-ci.org/guide/writing-tasks/#conditional-task-execution
+  stateful: false  # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks
+
+base_template: &BASE_TEMPLATE
+  << : *FILTER_TEMPLATE
   merge_base_script:
-    - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
+    # Unconditionally install git (used in fingerprint_script) and set the
+    # default git author name (used in verify-commits.py)
     - bash -c "$PACKAGE_MANAGER_INSTALL git"
-    - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH
     - git config --global user.email "[email protected]"
     - git config --global user.name "ci"
+    - if [ "$CIRRUS_PR" = "" ]; then exit 0; fi
+    - git fetch $CIRRUS_REPO_CLONE_URL $CIRRUS_BASE_BRANCH
     - git merge FETCH_HEAD  # Merge base to detect silent merge conflicts
-  stateful: false  # https://cirrus-ci.org/guide/writing-tasks/#stateful-tasks
 
-global_task_template: &GLOBAL_TASK_TEMPLATE
-  << : *BASE_TEMPLATE
+main_template: &MAIN_TEMPLATE
   timeout_in: 120m  # https://cirrus-ci.org/faq/#instance-timed-out
   container:
     # https://cirrus-ci.org/faq/#are-there-any-limits
     # Each project has 16 CPU in total, assign 2 to each container, so that 8 tasks run in parallel
     cpu: 2
+    greedy: true
     memory: 8G  # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers
   ccache_cache:
     folder: "/tmp/ccache_dir"
   depends_built_cache:
     folder: "depends/built"
+    fingerprint_script: echo $CIRRUS_TASK_NAME $(git rev-list -1 HEAD ./depends)
   ci_script:
     - ./ci/test_run_all.sh
 
-depends_sdk_cache_template: &DEPENDS_SDK_CACHE_TEMPLATE
-  depends_sdk_cache:
-    folder: "depends/sdk-sources"
+global_task_template: &GLOBAL_TASK_TEMPLATE
+  << : *BASE_TEMPLATE
+  << : *MAIN_TEMPLATE
 
 compute_credits_template: &CREDITS_TEMPLATE
   # https://cirrus-ci.org/pricing/#compute-credits
   # Only use credits for pull requests to the main repo
   use_compute_credits: $CIRRUS_REPO_FULL_NAME == 'ElementsProject/elements' && $CIRRUS_PR != ""
 
-#task:
-#  name: "Windows"
-#  windows_container:
-#    image: cirrusci/windowsservercore:2019
-#  env:
-#    CIRRUS_SHELL: powershell
-#    PATH: 'C:\Python37;C:\Python37\Scripts;%PATH%'
-#    PYTHONUTF8: 1
-#    QT_DOWNLOAD_URL: 'https://github.com/sipsorcery/qt_win_binary/releases/download/v1.6/Qt5.9.8_x64_static_vs2019.zip'
-#    QT_DOWNLOAD_HASH: '9a8c6eb20967873785057fdcd329a657c7f922b0af08c5fde105cc597dd37e21'
-#    QT_LOCAL_PATH: 'C:\Qt5.9.8_x64_static_vs2019'
-#    VCPKG_INSTALL_PATH: 'C:\tools\vcpkg\installed'
-#    VCPKG_COMMIT_ID: 'ed0df8ecc4ed7e755ea03e18aaf285fd9b4b4a74'
-#  install_script:
-#    - choco install python --version=3.7.7 -y
-
 task:
   name: 'lint [bionic]'
   << : *BASE_TEMPLATE
@@ -85,19 +73,114 @@ task:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
 
 task:
-  name: 'ARM [unit tests, no functional tests] [buster]'
+  name: "Win64 native [msvc]"
+  << : *FILTER_TEMPLATE
+  windows_container:
+    cpu: 4
+    memory: 8G
+    image: cirrusci/windowsservercore:visualstudio2019
+  timeout_in: 120m
+  env:
+    PATH: 'C:\jom;C:\Python39;C:\Python39\Scripts;C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\MSBuild\Current\Bin;%PATH%'
+    PYTHONUTF8: 1
+    CI_VCPKG_TAG: '2021.05.12'
+    VCPKG_DOWNLOADS: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\downloads'
+    VCPKG_DEFAULT_BINARY_CACHE: 'C:\Users\ContainerAdministrator\AppData\Local\vcpkg\archives'
+    QT_DOWNLOAD_URL: 'https://download.qt.io/official_releases/qt/5.15/5.15.2/single/qt-everywhere-src-5.15.2.zip'
+    QT_LOCAL_PATH: 'C:\qt-everywhere-src-5.15.2.zip'
+    QT_SOURCE_DIR: 'C:\qt-everywhere-src-5.15.2'
+    QTBASEDIR: 'C:\Qt_static'
+    x64_NATIVE_TOOLS: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\BuildTools\VC\Auxiliary\Build\vcvars64.bat"'
+    IgnoreWarnIntDirInTempDetected: 'true'
+  merge_script:
+    - git config --global user.email "[email protected]"
+    - git config --global user.name "ci"
+    # Windows filesystem loses the executable bit, and all of the executable
+    # files are considered "modified" now. It will break the following `git merge`
+    # command. The next two commands make git ignore this issue.
+    - git config core.filemode false
+    - git reset --hard
+    - PowerShell -NoLogo -Command if ($env:CIRRUS_PR -ne $null) { git fetch $env:CIRRUS_REPO_CLONE_URL $env:CIRRUS_BASE_BRANCH; git merge FETCH_HEAD; }
+  msvc_qt_built_cache:
+    folder: "%QTBASEDIR%"
+    reupload_on_changes: false
+    fingerprint_script:
+      - echo %QT_DOWNLOAD_URL%
+      - msbuild -version
+    populate_script:
+      - curl -L -o C:\jom.zip http://download.qt.io/official_releases/jom/jom.zip
+      - mkdir C:\jom
+      - tar -xf C:\jom.zip -C C:\jom
+      - curl -L -o %QT_LOCAL_PATH% %QT_DOWNLOAD_URL%
+      - tar -xf %QT_LOCAL_PATH% -C C:\
+      - '%x64_NATIVE_TOOLS%'
+      - cd %QT_SOURCE_DIR%
+      - mkdir build
+      - cd build
+      - ..\configure -release -silent -opensource -confirm-license -opengl desktop -static -static-runtime -mp -qt-zlib -qt-pcre -qt-libpng -nomake examples -nomake tests -nomake tools -no-angle -no-dbus -no-gif -no-gtk -no-ico -no-icu -no-libjpeg -no-libudev -no-sql-sqlite -no-sql-odbc -no-sqlite -no-vulkan -skip qt3d -skip qtactiveqt -skip qtandroidextras -skip qtcharts -skip qtconnectivity -skip qtdatavis3d -skip qtdeclarative -skip doc -skip qtdoc -skip qtgamepad -skip qtgraphicaleffects -skip qtimageformats -skip qtlocation -skip qtlottie -skip qtmacextras -skip qtmultimedia -skip qtnetworkauth -skip qtpurchasing -skip qtquick3d -skip qtquickcontrols -skip qtquickcontrols2 -skip qtquicktimeline -skip qtremoteobjects -skip qtscript -skip qtscxml -skip qtsensors -skip qtserialbus -skip qtserialport -skip qtspeech -skip qtsvg -skip qtvirtualkeyboard -skip qtwayland -skip qtwebchannel -skip qtwebengine -skip qtwebglplugin -skip qtwebsockets -skip qtwebview -skip qtx11extras -skip qtxmlpatterns -no-openssl -no-feature-bearermanagement -no-feature-printdialog -no-feature-printer -no-feature-printpreviewdialog -no-feature-printpreviewwidget -no-feature-sql -no-feature-sqlmodel -no-feature-textbrowser -no-feature-textmarkdownwriter -no-feature-textodfwriter -no-feature-xml -prefix %QTBASEDIR%
+      - jom
+      - jom install
+  vcpkg_tools_cache:
+    folder: '%VCPKG_DOWNLOADS%\tools'
+    reupload_on_changes: false
+    fingerprint_script:
+      - echo %CI_VCPKG_TAG%
+      - msbuild -version
+  vcpkg_binary_cache:
+    folder: '%VCPKG_DEFAULT_BINARY_CACHE%'
+    reupload_on_changes: true
+    fingerprint_script:
+      - echo %CI_VCPKG_TAG%
+      - msbuild -version
+    populate_script:
+      - mkdir %VCPKG_DEFAULT_BINARY_CACHE%
+  install_python_script:
+    - choco install --yes --no-progress python3 --version=3.9.6
+    - pip install zmq
+    - python -VV
+  install_vcpkg_script:
+    - cd ..
+    - git clone --quiet https://github.com/microsoft/vcpkg.git
+    - cd vcpkg
+    - git -c advice.detachedHead=false checkout %CI_VCPKG_TAG%
+    - .\bootstrap-vcpkg -disableMetrics
+    - echo set(VCPKG_BUILD_TYPE release) >> triplets\x64-windows-static.cmake
+    - .\vcpkg integrate install
+    - .\vcpkg version
+  build_script:
+    - cd %CIRRUS_WORKING_DIR%
+    - python build_msvc\msvc-autogen.py
+    - msbuild build_msvc\bitcoin.sln -property:Configuration=Release -maxCpuCount -verbosity:minimal -noLogo
+  unit_tests_script:
+    - src\test_bitcoin.exe -l test_suite
+    - src\bench_bitcoin.exe > NUL
+    - python test\util\test_runner.py
+    - python test\util\rpcauth-test.py
+  functional_tests_script:
+    # Increase the dynamic port range to the maximum allowed value to mitigate "OSError: [WinError 10048] Only one usage of each socket address (protocol/network address/port) is normally permitted".
+    # See: https://docs.microsoft.com/en-us/biztalk/technical-guides/settings-that-can-be-modified-to-improve-network-performance
+    - netsh int ipv4 set dynamicport tcp start=1025 num=64511
+    - netsh int ipv6 set dynamicport tcp start=1025 num=64511
+    # Exclude feature_dbcrash for now due to timeout
+    - python test\functional\test_runner.py --nocleanup --ci --quiet --combinedlogslen=4000 --jobs=4 --timeout-factor=8 --extended --exclude feature_dbcrash
+
+task:
+  name: 'ARM [unit tests, no functional tests] [bullseye]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: debian:buster
+    image: debian:bullseye
+    docker_arguments:
+      CI_IMAGE_NAME_TAG: debian:bullseye
+  << : *CREDITS_TEMPLATE
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_arm.sh"
 
 task:
-  name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [focal]'
+  name: 'Win64 [unit tests, no gui tests, no boost::process, no functional tests] [jammy]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:focal
+    image: ubuntu:jammy
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_win64.sh"
@@ -123,51 +206,51 @@ task:
     FILE_ENV: "./ci/test/00_setup_env_native_qt5.sh"
 
 task:
-  name: '[depends, sanitizers: thread (TSan), no gui] [hirsute]'
+  name: '[TSan, depends, gui] [jammy]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:hirsute
+    image: ubuntu:jammy
     cpu: 6  # Increase CPU and Memory to avoid timeout
     memory: 24G
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    MAKEJOBS: "-j6" # ELEMENTS: reduced from -j8
     FILE_ENV: "./ci/test/00_setup_env_native_tsan.sh"
 
 task:
-  name: '[depends, sanitizers: memory (MSan)] [focal]'
+  name: '[MSan, depends] [focal]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
     image: ubuntu:focal
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_native_msan.sh"
+    MAKEJOBS: "-j4"  # Avoid excessive memory use due to MSan
 
 task:
-  name: '[no depends, sanitizers: address/leak (ASan + LSan) + undefined (UBSan) + integer] [hirsute]'
+  name: '[ASan + LSan + UBSan + integer, no depends] [jammy]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:hirsute
+    image: ubuntu:jammy
     memory: 16G # ELEMENTS: need more memory
     cpu: 4  # ELEMENTS: cirrus wants more CPUs if you want more memory
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_native_asan.sh"
+    MAKEJOBS: "-j4"  # Avoid excessive memory use
 
 task:
-  name: '[no depends, sanitizers: fuzzer,address,undefined,integer] [focal]'
+  name: '[fuzzer,address,undefined,integer, no depends] [jammy]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
-    image: ubuntu:focal
+    image: ubuntu:jammy
     cpu: 8  # Increase CPU and memory to avoid timeout
     memory: 16G
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    MAKEJOBS: "-j8"
     FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh"
 
 task:
-  name: '[multiprocess, DEBUG] [focal]'
+  name: '[multiprocess, i686, DEBUG] [focal]'
   # Disable for Elements for now; Multiprocess build is not supported or tested and fails CI.
   only_if: false
   << : *GLOBAL_TASK_TEMPLATE
@@ -177,36 +260,39 @@ task:
     memory: 16G  # The default memory is sometimes just a bit too small, so double everything
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    MAKEJOBS: "-j8"
-    FILE_ENV: "./ci/test/00_setup_env_native_multiprocess.sh"
+    FILE_ENV: "./ci/test/00_setup_env_i686_multiprocess.sh"
 
 task:
-  name: '[no wallet] [bionic]'
+  name: '[no wallet, libbitcoinkernel] [bionic]'
   << : *GLOBAL_TASK_TEMPLATE
   container:
     image: ubuntu:bionic
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
-    FILE_ENV: "./ci/test/00_setup_env_native_nowallet.sh"
+    FILE_ENV: "./ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh"
 
 task:
-  name: 'macOS 10.14 [gui, no tests] [focal]'
-  << : *DEPENDS_SDK_CACHE_TEMPLATE
-  << : *GLOBAL_TASK_TEMPLATE
+  name: 'macOS 10.15 [gui, no tests] [focal]'
+  << : *BASE_TEMPLATE
+  macos_sdk_cache:
+    folder: "depends/SDKs/$MACOS_SDK"
+    fingerprint_key: "$MACOS_SDK"
+  << : *MAIN_TEMPLATE
   container:
     image: ubuntu:focal
   env:
+    MACOS_SDK: "Xcode-12.2-12B45b-extracted-SDK-with-libcxx-headers"
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     FILE_ENV: "./ci/test/00_setup_env_mac.sh"
 
 task:
-  name: 'macOS 11 native [gui] [no depends]'
+  name: 'macOS 12 native [gui, system sqlite only] [no depends]'
   brew_install_script:
-    - brew install boost libevent berkeley-db4 qt@5 miniupnpc libnatpmp ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt
+    - brew install boost libevent qt@5 miniupnpc libnatpmp ccache zeromq qrencode libtool automake gnu-getopt
   << : *GLOBAL_TASK_TEMPLATE
-  osx_instance:
+  macos_instance:
     # Use latest image, but hardcode version to avoid silent upgrades (and breaks)
-    image: big-sur-xcode-12.5  # https://cirrus-ci.org/guide/macOS
+    image: monterey-xcode-13.2  # https://cirrus-ci.org/guide/macOS
   env:
     << : *CIRRUS_EPHEMERAL_WORKER_TEMPLATE_ENV
     CI_USE_APT_INSTALL: "no"
@@ -217,10 +303,13 @@ task:
   name: 'ARM64 Android APK [focal]'
   # Disable for Elements; Android build is broken and unsupported.
   only_if: false
-  << : *DEPENDS_SDK_CACHE_TEMPLATE
+  android_sdk_cache:
+    folder: "depends/SDKs/android"
+    fingerprint_key: "ANDROID_API_LEVEL=28 ANDROID_BUILD_TOOLS_VERSION=28.0.3 ANDROID_NDK_VERSION=23.1.7779620"
   depends_sources_cache:
     folder: "depends/sources"
-  << : *GLOBAL_TASK_TEMPLATE
+    fingerprint_script: git rev-list -1 HEAD ./depends
+  << : *MAIN_TEMPLATE
   container:
     image: ubuntu:focal
   env: