diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index e84cfa174f3..d15932b5cdb 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -22,7 +22,7 @@ jobs: runs-on: ${{ matrix.platform }} steps: - name: Checkout source code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Rust ${{ matrix.toolchain }} toolchain run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ matrix.toolchain }} @@ -32,14 +32,38 @@ jobs: run: | rustup target add thumbv7m-none-eabi sudo apt-get -y install gcc-arm-none-eabi - - name: shellcheck the CI script + - name: shellcheck the CI and `contrib` scripts if: "matrix.platform == 'ubuntu-latest'" run: | sudo apt-get -y install shellcheck - shellcheck ci/ci-tests.sh + shellcheck ci/*.sh -aP ci + shellcheck contrib/*.sh -aP contrib - name: Set RUSTFLAGS to deny warnings if: "matrix.toolchain == '1.63.0'" run: echo "RUSTFLAGS=-D warnings" >> "$GITHUB_ENV" + - name: Enable caching for bitcoind + id: cache-bitcoind + uses: actions/cache@v4 + with: + path: bin/bitcoind-${{ runner.os }}-${{ runner.arch }} + key: bitcoind-${{ runner.os }}-${{ runner.arch }} + - name: Enable caching for electrs + id: cache-electrs + uses: actions/cache@v4 + with: + path: bin/electrs-${{ runner.os }}-${{ runner.arch }} + key: electrs-${{ runner.os }}-${{ runner.arch }} + - name: Download bitcoind/electrs + if: "matrix.platform != 'windows-latest' && (steps.cache-bitcoind.outputs.cache-hit != 'true' || steps.cache-electrs.outputs.cache-hit != 'true')" + run: | + source ./contrib/download_bitcoind_electrs.sh + mkdir bin + mv "$BITCOIND_EXE" bin/bitcoind-${{ runner.os }}-${{ runner.arch }} + mv "$ELECTRS_EXE" bin/electrs-${{ runner.os }}-${{ runner.arch }} + - name: Set bitcoind/electrs environment variables + run: | + echo "BITCOIND_EXE=$( pwd )/bin/bitcoind-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" + echo "ELECTRS_EXE=$( pwd )/bin/electrs-${{ runner.os }}-${{ runner.arch }}" >> "$GITHUB_ENV" - name: Run CI script shell: bash # Default on Winblows is powershell run: CI_MINIMIZE_DISK_USAGE=1 ./ci/ci-tests.sh @@ -50,7 +74,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout source code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install Rust stable toolchain @@ -72,14 +96,14 @@ jobs: TOOLCHAIN: stable steps: - name: Checkout source code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Rust ${{ env.TOOLCHAIN }} toolchain run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }} rustup override set ${{ env.TOOLCHAIN }} - name: Cache routing graph snapshot id: cache-graph - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: lightning/net_graph-2023-12-10.bin key: ldk-net_graph-v0.0.118-2023-12-10.bin @@ -96,7 +120,7 @@ jobs: EXPECTED_ROUTING_GRAPH_SNAPSHOT_SHASUM: e94b38ef4b3ce683893bf6a3ee28d60cb37c73b059403ff77b7e7458157968c2 - name: Cache scorer snapshot id: cache-scorer - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: lightning/scorer-2023-12-10.bin key: ldk-scorer-v0.0.118-2023-12-10.bin @@ -142,7 +166,7 @@ jobs: TOOLCHAIN: stable steps: - name: Checkout source code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install Rust ${{ env.TOOLCHAIN }} toolchain @@ -165,7 +189,7 @@ jobs: TOOLCHAIN: stable steps: - name: Checkout source code - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 - name: Install Rust ${{ env.TOOLCHAIN }} toolchain @@ -194,7 +218,7 @@ jobs: TOOLCHAIN: 1.63 steps: - name: Checkout source code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Rust ${{ env.TOOLCHAIN }} toolchain run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }} @@ -217,7 +241,7 @@ jobs: TOOLCHAIN: stable steps: - name: Checkout source code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Rust ${{ env.TOOLCHAIN }} toolchain run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }} @@ -235,7 +259,7 @@ jobs: TOOLCHAIN: 1.63.0 steps: - name: Checkout source code - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Install Rust ${{ env.TOOLCHAIN }} toolchain run: | curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y --profile=minimal --default-toolchain ${{ env.TOOLCHAIN }} @@ -250,7 +274,7 @@ jobs: runs-on: ubuntu-latest if: github.ref_name != 'main' # `main` has no diff with itself steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 with: fetch-depth: 0 - name: Relative diff diff --git a/ci/check-compiles.sh b/ci/check-compiles.sh index 2fe62be787e..7ad9f4df196 100755 --- a/ci/check-compiles.sh +++ b/ci/check-compiles.sh @@ -1,7 +1,7 @@ #!/bin/sh set -e set -x -echo Testing $(git log -1 --oneline) +echo "Testing $(git log -1 --oneline)" cargo check cargo doc cargo doc --document-private-items diff --git a/ci/check-each-commit.sh b/ci/check-each-commit.sh index e4723c837ed..42e3d0e76e2 100755 --- a/ci/check-each-commit.sh +++ b/ci/check-each-commit.sh @@ -12,4 +12,4 @@ if [ "$(git log --pretty="%H %D" | grep "^[0-9a-f]*.* $1")" = "" ]; then echo "It seems like the current checked-out commit is not based on $1" exit 1 fi -git rebase --exec ci/check-compiles.sh $1 +git rebase --exec ci/check-compiles.sh "$1" diff --git a/ci/ci-tests.sh b/ci/ci-tests.sh index b99ae820ade..74bf427d613 100755 --- a/ci/ci-tests.sh +++ b/ci/ci-tests.sh @@ -14,50 +14,6 @@ function PIN_RELEASE_DEPS { return 0 # Don't fail the script if our rustc is higher than the last check } -# The tests of `lightning-transaction-sync` require `electrs` and `bitcoind` -# binaries. Here, we download the binaries, validate them, and export their -# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the -# `electrsd`/`bitcoind` crates in our tests. -function DOWNLOAD_ELECTRS_AND_BITCOIND { - ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases" - ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254" - BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/" - BITCOIND_VERSION="25.1" - if [[ "$HOST_PLATFORM" == *linux* ]]; then - ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip - ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1" - BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz - BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b" - elif [[ "$HOST_PLATFORM" == *darwin* ]]; then - ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip - ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd" - BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz - BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1" - else - echo -e "\n\nUnsupported platform. Exiting.." - exit 1 - fi - - DL_TMP_DIR=$(mktemp -d) - trap 'rm -rf -- "$DL_TMP_DIR"' EXIT - - pushd "$DL_TMP_DIR" - ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME" - curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL" - echo "$ELECTRS_DL_HASH $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c - unzip "$ELECTRS_DL_FILE_NAME" - export ELECTRS_EXE="$DL_TMP_DIR"/electrs - chmod +x "$ELECTRS_EXE" - - BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME" - curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL" - echo "$BITCOIND_DL_HASH $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c - tar xzf "$BITCOIND_DL_FILE_NAME" - export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind - chmod +x "$BITCOIND_EXE" - popd -} - PIN_RELEASE_DEPS # pin the release dependencies in our main workspace # Starting with version 1.10.0, the `regex` crate has an MSRV of rustc 1.65.0. @@ -71,9 +27,6 @@ PIN_RELEASE_DEPS # pin the release dependencies in our main workspace export RUST_BACKTRACE=1 -# Build `lightning-transaction-sync` in no_download mode. -export RUSTFLAGS="$RUSTFLAGS --cfg no_download" - echo -e "\n\nBuilding and testing all workspace crates..." cargo test --verbose --color always cargo check --verbose --color always @@ -91,20 +44,22 @@ cargo check --verbose --color always --features rpc-client,rest-client,tokio popd if [[ "$HOST_PLATFORM" != *windows* ]]; then - echo -e "\n\nBuilding and testing Transaction Sync Clients with features" - pushd lightning-transaction-sync - - DOWNLOAD_ELECTRS_AND_BITCOIND - - cargo test --verbose --color always --features esplora-blocking - cargo check --verbose --color always --features esplora-blocking - cargo test --verbose --color always --features esplora-async - cargo check --verbose --color always --features esplora-async - cargo test --verbose --color always --features esplora-async-https - cargo check --verbose --color always --features esplora-async-https - cargo test --verbose --color always --features electrum - cargo check --verbose --color always --features electrum - popd + if [ -z "$BITCOIND_EXE" ] || [ -z "$ELECTRS_EXE" ]; then + echo -e "\n\nSkipping testing Transaction Sync Clients due to BITCOIND_EXE or ELECTRS_EXE being unset." + else + echo -e "\n\nBuilding and testing Transaction Sync Clients with features" + pushd lightning-transaction-sync + + cargo test --verbose --color always --features esplora-blocking + cargo check --verbose --color always --features esplora-blocking + cargo test --verbose --color always --features esplora-async + cargo check --verbose --color always --features esplora-async + cargo test --verbose --color always --features esplora-async-https + cargo check --verbose --color always --features esplora-async-https + cargo test --verbose --color always --features electrum + cargo check --verbose --color always --features electrum + popd + fi fi echo -e "\n\nTest futures builds" diff --git a/ci/rustfmt.sh b/ci/rustfmt.sh index 56384aae3c5..1a9053a28dd 100755 --- a/ci/rustfmt.sh +++ b/ci/rustfmt.sh @@ -14,8 +14,8 @@ VERS="" # Run fmt TMP_FILE=$(mktemp) -find . -name '*.rs' -type f |sort >$TMP_FILE -for file in $(comm -23 $TMP_FILE rustfmt_excluded_files); do +find . -name '*.rs' -type f |sort >"$TMP_FILE" +for file in $(comm -23 "$TMP_FILE" rustfmt_excluded_files); do echo "Checking formatting of $file" - rustfmt $VERS --edition 2021 --check $file + rustfmt $VERS --edition 2021 --check "$file" done diff --git a/contrib/download_bitcoind_electrs.sh b/contrib/download_bitcoind_electrs.sh new file mode 100755 index 00000000000..8b938a563d3 --- /dev/null +++ b/contrib/download_bitcoind_electrs.sh @@ -0,0 +1,47 @@ +#!/bin/bash +set -eox pipefail + +# The tests of `lightning-transaction-sync` require `electrs` and `bitcoind` +# binaries. Here, we download the binaries, validate them, and export their +# location via `ELECTRS_EXE`/`BITCOIND_EXE` which will be used by the +# `electrsd`/`bitcoind` crates in our tests. + +HOST_PLATFORM="$(rustc --version --verbose | grep "host:" | awk '{ print $2 }')" +ELECTRS_DL_ENDPOINT="https://github.com/RCasatta/electrsd/releases/download/electrs_releases" +ELECTRS_VERSION="esplora_a33e97e1a1fc63fa9c20a116bb92579bbf43b254" +BITCOIND_DL_ENDPOINT="https://bitcoincore.org/bin/" +BITCOIND_VERSION="25.1" +if [[ "$HOST_PLATFORM" == *linux* ]]; then + ELECTRS_DL_FILE_NAME=electrs_linux_"$ELECTRS_VERSION".zip + ELECTRS_DL_HASH="865e26a96e8df77df01d96f2f569dcf9622fc87a8d99a9b8fe30861a4db9ddf1" + BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-linux-gnu.tar.gz + BITCOIND_DL_HASH="a978c407b497a727f0444156e397b50491ce862d1f906fef9b521415b3611c8b" +elif [[ "$HOST_PLATFORM" == *darwin* ]]; then + ELECTRS_DL_FILE_NAME=electrs_macos_"$ELECTRS_VERSION".zip + ELECTRS_DL_HASH="2d5ff149e8a2482d3658e9b386830dfc40c8fbd7c175ca7cbac58240a9505bcd" + BITCOIND_DL_FILE_NAME=bitcoin-"$BITCOIND_VERSION"-x86_64-apple-darwin.tar.gz + BITCOIND_DL_HASH="1acfde0ec3128381b83e3e5f54d1c7907871d324549129592144dd12a821eff1" +else + printf "\n\n" + echo "Unsupported platform: $HOST_PLATFORM Exiting.." + exit 1 +fi + +DL_TMP_DIR=$(mktemp -d) +trap 'rm -rf -- "$DL_TMP_DIR"' EXIT + +pushd "$DL_TMP_DIR" +ELECTRS_DL_URL="$ELECTRS_DL_ENDPOINT"/"$ELECTRS_DL_FILE_NAME" +curl -L -o "$ELECTRS_DL_FILE_NAME" "$ELECTRS_DL_URL" +echo "$ELECTRS_DL_HASH $ELECTRS_DL_FILE_NAME"|shasum -a 256 -c +unzip "$ELECTRS_DL_FILE_NAME" +export ELECTRS_EXE="$DL_TMP_DIR"/electrs +chmod +x "$ELECTRS_EXE" + +BITCOIND_DL_URL="$BITCOIND_DL_ENDPOINT"/bitcoin-core-"$BITCOIND_VERSION"/"$BITCOIND_DL_FILE_NAME" +curl -L -o "$BITCOIND_DL_FILE_NAME" "$BITCOIND_DL_URL" +echo "$BITCOIND_DL_HASH $BITCOIND_DL_FILE_NAME"|shasum -a 256 -c +tar xzf "$BITCOIND_DL_FILE_NAME" +export BITCOIND_EXE="$DL_TMP_DIR"/bitcoin-"$BITCOIND_VERSION"/bin/bitcoind +chmod +x "$BITCOIND_EXE" +popd diff --git a/contrib/run-rustfmt.sh b/contrib/run-rustfmt.sh index 823e9e025b6..ffce4242f3e 100755 --- a/contrib/run-rustfmt.sh +++ b/contrib/run-rustfmt.sh @@ -14,8 +14,8 @@ VERS="" # Run fmt TMP_FILE=$(mktemp) -find . -name '*.rs' -type f |sort >$TMP_FILE -for file in $(comm -23 $TMP_FILE rustfmt_excluded_files); do +find . -name '*.rs' -type f |sort >"$TMP_FILE" +for file in $(comm -23 "$TMP_FILE" rustfmt_excluded_files); do echo "Formatting $file..." - rustfmt $VERS --edition 2021 $file + rustfmt $VERS --edition 2021 "$file" done diff --git a/lightning-transaction-sync/Cargo.toml b/lightning-transaction-sync/Cargo.toml index ab796749954..5ef9ef3e46f 100644 --- a/lightning-transaction-sync/Cargo.toml +++ b/lightning-transaction-sync/Cargo.toml @@ -34,10 +34,7 @@ electrum-client = { version = "0.19.0", optional = true } lightning = { version = "0.0.123-beta", path = "../lightning", default-features = false, features = ["std", "_test_utils"] } tokio = { version = "1.35.0", features = ["full"] } -[target.'cfg(all(not(target_os = "windows"), not(no_download)))'.dev-dependencies] -electrsd = { version = "0.27.3", default-features = false, features = ["legacy", "esplora_a33e97e1", "bitcoind_25_0"] } - -[target.'cfg(all(not(target_os = "windows"), no_download))'.dev-dependencies] +[target.'cfg(not(target_os = "windows"))'.dev-dependencies] electrsd = { version = "0.27.3", default-features = false, features = ["legacy"] } [lints]