diff --git a/ci/Jenkinsfile.linux b/ci/Jenkinsfile.linux new file mode 100644 index 0000000000..b07131e2be --- /dev/null +++ b/ci/Jenkinsfile.linux @@ -0,0 +1,189 @@ +#!/usr/bin/env groovy +/* beacon_chain + * Copyright (c) 2025 Status Research & Development GmbH + * Licensed and distributed under either of + * * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). + * * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). + * at your option. This file may not be copied, modified, or distributed except according to those terms. + */ +library 'status-jenkins-lib@v1.9.2' + +pipeline { + agent { + dockerfile { + label 'linuxcontainer' + filename 'linux.Dockerfile' + dir 'ci' + } + } + + parameters { + choice( + name: 'VERBOSITY', + description: 'Value for the V make flag to increase log verbosity', + choices: [0, 1, 2] + ) + string( + name: 'NIM_COMMIT', + description: 'Value for the NIM_COMMIT make flag to choose Nim commit', + defaultValue: nimCommitForJob(), + ) + } + + options { + disableRestartFromStage() + timestamps() + ansiColor('xterm') + /* This also includes wait time in the queue. */ + timeout(time: 24, unit: 'HOURS') + /* Limit builds retained. */ + buildDiscarder(logRotator( + numToKeepStr: '5', + daysToKeepStr: '30', + artifactNumToKeepStr: '3', + )) + /* Throttle number of concurrent builds. */ + throttleJobProperty( + throttleEnabled: true, + throttleOption: 'category', + categories: ['nimbus-eth2'], + maxConcurrentPerNode: 1, + maxConcurrentTotal: 9 + ) + /* Abort old builds for non-main branches. */ + disableConcurrentBuilds( + abortPrevious: !isMainBranch() + ) + } + + environment { + NPROC = Runtime.getRuntime().availableProcessors() + MAKEFLAGS = "V=${params.VERBOSITY} NIM_COMMIT=${params.NIM_COMMIT} -j${env.NPROC}" + XDG_CACHE_HOME = "${env.WORKSPACE_TMP}/.cache" + } + + stages { + stage('Deps') { + steps { + timeout(20) { + script { + /* To allow the following parallel stages. */ + sh 'make QUICK_AND_DIRTY_COMPILER=1 update' + /* Allow the following parallel stages. */ + sh 'make deps' + /* Download test vectors. */ + sh './scripts/setup_scenarios.sh' + } + } + } + } + + stage('Build') { + steps { + timeout(50) { + script { + sh 'make LOG_LEVEL=TRACE' + } + } + } + } + + stage('Check Docs') { + steps { + script { + sh './scripts/check_docs_help_msg.sh' + } + } + } + + stage('Tests') { + parallel { + stage('General') { + steps { + timeout(60) { + script { + sh 'make DISABLE_TEST_FIXTURES_SCRIPT=1 NIMFLAGS="--passC:\"-fno-lto\" --passL:\"-fno-lto\"" test' + sh 'git diff --exit-code --ignore-submodules=all' + } + } + } + } + + stage('REST') { + steps { + timeout(5) { + script { + sh 'make restapi-test' + } + } + } + post { always { + sh 'tar cjf restapi-test.tar.gz resttest0_data/*.txt' + } } + } + } + post { always { timeout(5) { + archiveArtifacts(artifacts: '*.tar.gz', allowEmptyArchive: true) + } } } + } + + stage('Finalizations') { + stages { /* parallel builds of minimal / mainnet not yet supported */ + stage('minimal') { + steps { + timeout(26) { + script { + sh 'make local-testnet-minimal' + } + } + } + post { always { + sh 'tar cjf local-testnet-minimal.tar.gz local-testnet-minimal/logs/*' + } } + } + + stage('mainnet') { + steps { + timeout(62) { + script { + sh 'make local-testnet-mainnet' + } + } + } + post { always { + sh 'tar cjf local-testnet-mainnet.tar.gz local-testnet-mainnet/logs/*' + } } + } + } + post { + always { timeout(10) { + /* DEBUG: Show file sizes to catch too big ones. */ + sh 'ls -hl *.tar.gz' + archiveArtifacts( + artifacts: '*.tar.gz', + excludes: '**/geth-*.tar.gz', /* `scripts/geth_binaries.sh` */ + allowEmptyArchive: true + ) + } } + } + } + } + + post { + always { + cleanWs( + disableDeferredWipeout: true, + deleteDirs: true + ) + dir("${env.WORKSPACE}@tmp") { deleteDir() } + } + } +} + +def isMainBranch() { + return ['stable', 'testing', 'unstable'].contains(env.BRANCH_NAME) +} + +def nimCommitForJob() { + return JOB_NAME.contains('nimv2_2') ? 'upstream/version-2-2' : '' +} diff --git a/ci/Jenkinsfile.macos b/ci/Jenkinsfile.macos new file mode 100644 index 0000000000..8fbd885e3e --- /dev/null +++ b/ci/Jenkinsfile.macos @@ -0,0 +1,173 @@ +#!/usr/bin/env groovy +/* beacon_chain + * Copyright (c) 2019-2025 Status Research & Development GmbH + * Licensed and distributed under either of + * * MIT license (license terms in the root directory or at https://opensource.org/licenses/MIT). + * * Apache v2 license (license terms in the root directory or at https://www.apache.org/licenses/LICENSE-2.0). + * at your option. This file may not be copied, modified, or distributed except according to those terms. + */ +library 'status-jenkins-lib@v1.9.2' + +pipeline { + /* This way we run the same Jenkinsfile on different platforms. */ + agent { label 'macos && aarch64' } + + parameters { + choice( + name: 'VERBOSITY', + description: 'Value for the V make flag to increase log verbosity', + choices: [0, 1, 2] + ) + string( + name: 'NIM_COMMIT', + description: 'Value for the NIM_COMMIT make flag to choose Nim commit', + defaultValue: nimCommitForJob(), + ) + } + + options { + disableRestartFromStage() + timestamps() + ansiColor('xterm') + /* This also includes wait time in the queue. */ + timeout(time: 24, unit: 'HOURS') + /* Limit builds retained. */ + buildDiscarder(logRotator( + numToKeepStr: '5', + daysToKeepStr: '30', + artifactNumToKeepStr: '3', + )) + /* Throttle number of concurrent builds. */ + throttleJobProperty( + throttleEnabled: true, + throttleOption: 'category', + categories: ['nimbus-eth2'], + maxConcurrentPerNode: 1, + maxConcurrentTotal: 9 + ) + /* Abort old builds for non-main branches. */ + disableConcurrentBuilds( + abortPrevious: !isMainBranch() + ) + } + + environment { + NPROC = Runtime.getRuntime().availableProcessors() + MAKEFLAGS = "V=${params.VERBOSITY} NIM_COMMIT=${params.NIM_COMMIT} -j${env.NPROC}" + XDG_CACHE_HOME = "${env.WORKSPACE_TMP}/.cache" + } + + stages { + stage('Setup') { + steps { script { + def brew_prefix = brew.prefix() + /* Explicit PATH to avoid using HomeBrew LLVM. */ + env.PATH = "/usr/local/bin:/usr/sbin:/usr/bin:/bin:${brew_prefix}/bin" + /* Newer Clang 18.0 from Homebrew on macOS, XCode provides 15.0. + * Temp fix for BLST issue: https://github.com/supranational/blst/issues/209 */ + if (utils.arch() == 'arm64') { + env.PATH = "${brew_prefix}/opt/llvm/bin:$PATH" + env.LDFLAGS = "-L${brew_prefix}/opt/llvm/lib" + env.CPPFLAGS = "-I${brew_prefix}/opt/llvm/include" + } + } } + } + + stage('Deps') { + steps { timeout(20) { + /* To allow the following parallel stages. */ + sh 'make QUICK_AND_DIRTY_COMPILER=1 update' + /* Allow the following parallel stages. */ + sh 'make deps' + /* Download test vectors. */ + sh './scripts/setup_scenarios.sh' + } } + } + + stage('Build') { + steps { timeout(50) { + sh 'make LOG_LEVEL=TRACE' + } } + } + + stage('Check Docs') { + steps { + sh './scripts/check_docs_help_msg.sh' + } + } + + stage('Tests') { + parallel { + stage('General') { + steps { timeout(60) { + sh 'make DISABLE_TEST_FIXTURES_SCRIPT=1 test' + sh 'git diff --exit-code --ignore-submodules=all' /* Check no uncommitted changes. */ + } } + } + + stage('REST') { + steps { timeout(5) { + sh 'make restapi-test' + } } + post { always { + sh 'tar cjf restapi-test.tar.gz resttest0_data/*.txt' + } } + } + } + post { always { timeout(5) { + archiveArtifacts(artifacts: '*.tar.gz', allowEmptyArchive: true) + } } } + } + + stage('Finalizations') { + stages { /* parallel builds of minimal / mainnet not yet supported */ + stage('minimal') { + steps { timeout(26) { + sh 'make local-testnet-minimal' + } } + post { always { + sh 'tar cjf local-testnet-minimal.tar.gz local-testnet-minimal/logs/*' + } } + } + + stage('mainnet') { + steps { timeout(62) { + sh 'make local-testnet-mainnet' + } } + post { always { + sh 'tar cjf local-testnet-mainnet.tar.gz local-testnet-mainnet/logs/*' + } } + } + } + post { + always { timeout(10) { + /* DEBUG: Show file sizes to catch too big ones. */ + sh 'ls -hl *.tar.gz' + archiveArtifacts( + artifacts: '*.tar.gz', + excludes: '**/geth-*.tar.gz', /* `scripts/geth_binaries.sh` */ + allowEmptyArchive: true + ) + } } + } + } + } + + post { + always { + cleanWs( + disableDeferredWipeout: true, + deleteDirs: true + ) + dir("${env.WORKSPACE}@tmp") { deleteDir() } + } + } +} + +def isMainBranch() { + return ['stable', 'testing', 'unstable'].contains(env.BRANCH_NAME) +} + +def nimCommitForJob() { + return JOB_NAME.contains('nimv2_2') ? 'upstream/version-2-2' : '' +} diff --git a/ci/linux.Dockerfile b/ci/linux.Dockerfile new file mode 100644 index 0000000000..6f8c139726 --- /dev/null +++ b/ci/linux.Dockerfile @@ -0,0 +1,37 @@ +FROM harbor.status.im/infra/ci-build-containers:linux-base-1.0.0 +USER root + +RUN apt-get update && apt-get install -yq --no-install-recommends \ + figlet \ + git \ + git-lfs \ + make \ + openssl \ + lsof \ + psmisc \ + procps \ + curl \ + jq \ + openjdk-17-jre-headless \ + python3 \ + python3-pip \ + python3-venv \ + gcc-11 \ + g++-11 \ + lsb-release \ + && rm -rf /var/lib/apt/lists/* + +RUN update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-11 100 \ + && update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-11 100 + +RUN ln -sf /usr/bin/gcc /usr/bin/cc \ + && ln -sf /usr/bin/g++ /usr/bin/c++ + +RUN pip3 install --no-cache-dir --break-system-packages \ + mkdocs \ + mkdocs-material \ + mkdocs-material-extensions \ + pymdown-extensions + +USER jenkins +ENTRYPOINT [""] \ No newline at end of file diff --git a/nix/shell.nix b/nix/shell.nix index 8b5fe3c511..6c36f0f0bb 100644 --- a/nix/shell.nix +++ b/nix/shell.nix @@ -21,6 +21,7 @@ in pkgs.mkShell { openssl # for generating the JWT file lsof # for killing processes by port killall # for killing processes manually + procps # for killing processes with pkill curl # for working with the node APIs jq # for parsing beacon API for LC start openjdk # for running web3signer