From b91b0b082e53236c86e9e685dceb6566ecd0328d Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 13 Jul 2023 18:56:15 -0700 Subject: [PATCH 01/13] Add base Docker image for Linux CI jobs --- .ci/docker/README.md | 1 + .ci/docker/build.sh | 20 +++++++++++ .ci/docker/common/install_base.sh | 36 +++++++++++++++++++ .ci/docker/common/install_clang.sh | 41 +++++++++++++++++++++ .ci/docker/common/install_user.sh | 19 ++++++++++ .ci/docker/ubuntu/Dockerfile | 22 ++++++++++++ .github/workflows/docker-builds.yml | 55 +++++++++++++++++++++++++++++ 7 files changed, 194 insertions(+) create mode 100644 .ci/docker/README.md create mode 100755 .ci/docker/build.sh create mode 100755 .ci/docker/common/install_base.sh create mode 100755 .ci/docker/common/install_clang.sh create mode 100644 .ci/docker/common/install_user.sh create mode 100644 .ci/docker/ubuntu/Dockerfile create mode 100644 .github/workflows/docker-builds.yml diff --git a/.ci/docker/README.md b/.ci/docker/README.md new file mode 100644 index 00000000000..1333ed77b7e --- /dev/null +++ b/.ci/docker/README.md @@ -0,0 +1 @@ +TODO diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh new file mode 100755 index 00000000000..e6b73ea8880 --- /dev/null +++ b/.ci/docker/build.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -ex + +IMAGE_NAME="$1" +shift + +OS="ubuntu" +OS_VERSION=22.04 +CLANG_VERSION=12 + +docker build \ + --no-cache \ + --progress=plain \ + --build-arg "IMAGE_NAME=${IMAGE_NAME}" \ + --build-arg "OS_VERSION=${OS_VERSION}" \ + --build-arg "CLANG_VERSION=${CLANG_VERSION}" \ + -f "${OS}"/Dockerfile \ + "$@" \ + . diff --git a/.ci/docker/common/install_base.sh b/.ci/docker/common/install_base.sh new file mode 100755 index 00000000000..34e4d383585 --- /dev/null +++ b/.ci/docker/common/install_base.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -ex + +install_ubuntu() { + apt-get update + + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + curl \ + git \ + wget \ + sudo \ + vim \ + jq \ + vim \ + unzip \ + gdb + + # Cleanup package manager + apt-get autoclean && apt-get clean + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +} + +# Install base packages depending on the base OS +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +case "$ID" in + ubuntu) + install_ubuntu + ;; + *) + echo "Unable to determine OS..." + exit 1 + ;; +esac diff --git a/.ci/docker/common/install_clang.sh b/.ci/docker/common/install_clang.sh new file mode 100755 index 00000000000..d5f3f1a54c1 --- /dev/null +++ b/.ci/docker/common/install_clang.sh @@ -0,0 +1,41 @@ +#!/bin/bash + +set -ex + +install_ubuntu() { + apt-get update + + apt-get install -y --no-install-recommends clang-"$CLANG_VERSION" + apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION" + + # Use update-alternatives to make this version the default + update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50 + update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-"$CLANG_VERSION" 50 + # Override cc/c++ to clang as well + update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50 + update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50 + + # CLANG's packaging is a little messed up (the runtime libs aren't + # added into the linker path), so give it a little help + CLANG_LIB=("/usr/lib/llvm-$CLANG_VERSION/lib/clang/"*"/lib/linux") + echo "$CLANG_LIB" > /etc/ld.so.conf.d/clang.conf + ldconfig + + # Cleanup package manager + apt-get autoclean && apt-get clean + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +} + +if [ -n "$CLANG_VERSION" ]; then + # Install base packages depending on the base OS + ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') + case "$ID" in + ubuntu) + install_ubuntu + ;; + *) + echo "Unable to determine OS..." + exit 1 + ;; + esac +fi diff --git a/.ci/docker/common/install_user.sh b/.ci/docker/common/install_user.sh new file mode 100644 index 00000000000..112c89a64df --- /dev/null +++ b/.ci/docker/common/install_user.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -ex + +# Same as ec2-user +echo "ci-user:x:1000:1000::/var/lib/ci-user:" >> /etc/passwd +echo "ci-user:x:1000:" >> /etc/group +# Needed on Focal or newer +echo "ci-user:*:19110:0:99999:7:::" >> /etc/shadow + +# Create $HOME +mkdir -p /var/lib/ci-user +chown ci-user:ci-user /var/lib/ci-user + +# Allow sudo +echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user + +# Test that sudo works +$SUDO -u ci-user $SUDO -v diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile new file mode 100644 index 00000000000..f803b577e9f --- /dev/null +++ b/.ci/docker/ubuntu/Dockerfile @@ -0,0 +1,22 @@ +ARG OS_VERSION + +FROM ubuntu:${OS_VERSION} + +ARG OS_VERSION + +ENV DEBIAN_FRONTEND noninteractive + +# Install common dependencies +COPY ./common/install_base.sh install_base.sh +RUN bash ./install_base.sh && rm install_base.sh + +# Install clang +COPY ./common/install_clang.sh install_clang.sh +RUN bash ./install_clang.sh && rm install_clang.sh + +# Setup user +COPY ./common/install_user.sh install_user.sh +RUN bash ./install_user.sh && rm install_user.sh + +USER ci-user +CMD ["bash"] diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml new file mode 100644 index 00000000000..440c81f69ff --- /dev/null +++ b/.github/workflows/docker-builds.yml @@ -0,0 +1,55 @@ +name: docker-builds + +on: + workflow_dispatch: + pull_request: + paths: + - .ci/docker/** + - .github/workflows/docker-builds.yml + push: + branches: + - main + schedule: + - cron: 1 3 * * 3 + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} + cancel-in-progress: true + +env: + ALPINE_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine + AWS_DEFAULT_REGION: us-east-1 + +jobs: + docker-build: + runs-on: [self-hosted, linux.2xlarge] + timeout-minutes: 240 + strategy: + fail-fast: false + matrix: + include: + - docker-image-name: executorch-ubuntu-22.04-clang12 + env: + DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/${{ matrix.docker-image-name }} + steps: + - name: Clean workspace + shell: bash + run: | + echo "${GITHUB_WORKSPACE}" + sudo rm -rf "${GITHUB_WORKSPACE}" + mkdir "${GITHUB_WORKSPACE}" + + - name: Checkout Executorch + uses: actions/checkout@v3 + + - name: Setup Linux + uses: pytorch/test-infra/.github/actions/setup-linux@main + + # TODO: Move calculate Docker image to test-infra as a shareable GHA + - name: Build docker image + id: build-docker-image + shell: bash + env: + DOCKER_IMAGE_NAME: ${{ matrix.docker-image-name }} + run: | + .ci/docker/build.sh From d48e1d6108b125a8baf89f631af94b22f82a40f7 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Thu, 13 Jul 2023 19:30:24 -0700 Subject: [PATCH 02/13] Use executorch instead of pytorch on ECR --- .github/workflows/docker-builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index 440c81f69ff..e9fba26d896 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -30,7 +30,7 @@ jobs: include: - docker-image-name: executorch-ubuntu-22.04-clang12 env: - DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/pytorch/${{ matrix.docker-image-name }} + DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }} steps: - name: Clean workspace shell: bash From 990883f4e227186aa01bb5c3f23e2fbef3697617 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 14 Jul 2023 11:43:12 -0700 Subject: [PATCH 03/13] Add setup-ssh --- .github/workflows/docker-builds.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index e9fba26d896..3abbad89edf 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -39,6 +39,11 @@ jobs: sudo rm -rf "${GITHUB_WORKSPACE}" mkdir "${GITHUB_WORKSPACE}" + - name: Setup SSH (Click me for login details) + uses: pytorch/test-infra/.github/actions/setup-ssh@main + with: + github-secret: ${{ secrets.GITHUB_TOKEN }} + - name: Checkout Executorch uses: actions/checkout@v3 @@ -52,4 +57,4 @@ jobs: env: DOCKER_IMAGE_NAME: ${{ matrix.docker-image-name }} run: | - .ci/docker/build.sh + .ci/docker/build.sh "${DOCKER_IMAGE_NAME}" From d3097f20338205cbb639ad165a966e12715d2101 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 14 Jul 2023 11:44:53 -0700 Subject: [PATCH 04/13] Add a sleep cause checkout ends too quickly --- .github/workflows/docker-builds.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index 3abbad89edf..e899df3678e 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -57,4 +57,7 @@ jobs: env: DOCKER_IMAGE_NAME: ${{ matrix.docker-image-name }} run: | + # DEBUG + sleep 300 + .ci/docker/build.sh "${DOCKER_IMAGE_NAME}" From 9c24c6c943aa34de6ff411e689d63665426fac54 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 14 Jul 2023 13:04:02 -0700 Subject: [PATCH 05/13] Keep only the minimum while getting calculate docker image ready on test-infra --- .ci/docker/build.sh | 4 ---- .ci/docker/common/install_user.sh | 2 +- .ci/docker/ubuntu/Dockerfile | 1 + .github/workflows/docker-builds.yml | 18 ++++++------------ 4 files changed, 8 insertions(+), 17 deletions(-) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index e6b73ea8880..fe41d5d1054 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -2,9 +2,6 @@ set -ex -IMAGE_NAME="$1" -shift - OS="ubuntu" OS_VERSION=22.04 CLANG_VERSION=12 @@ -12,7 +9,6 @@ CLANG_VERSION=12 docker build \ --no-cache \ --progress=plain \ - --build-arg "IMAGE_NAME=${IMAGE_NAME}" \ --build-arg "OS_VERSION=${OS_VERSION}" \ --build-arg "CLANG_VERSION=${CLANG_VERSION}" \ -f "${OS}"/Dockerfile \ diff --git a/.ci/docker/common/install_user.sh b/.ci/docker/common/install_user.sh index 112c89a64df..fdc0a55202b 100644 --- a/.ci/docker/common/install_user.sh +++ b/.ci/docker/common/install_user.sh @@ -16,4 +16,4 @@ chown ci-user:ci-user /var/lib/ci-user echo 'ci-user ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/ci-user # Test that sudo works -$SUDO -u ci-user $SUDO -v +sudo -u ci-user sudo -v diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile index f803b577e9f..16c91c571d0 100644 --- a/.ci/docker/ubuntu/Dockerfile +++ b/.ci/docker/ubuntu/Dockerfile @@ -11,6 +11,7 @@ COPY ./common/install_base.sh install_base.sh RUN bash ./install_base.sh && rm install_base.sh # Install clang +ARG CLANG_VERSION COPY ./common/install_clang.sh install_clang.sh RUN bash ./install_clang.sh && rm install_clang.sh diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index e899df3678e..e8d78f2a097 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -16,10 +16,6 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }} cancel-in-progress: true -env: - ALPINE_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/tool/alpine - AWS_DEFAULT_REGION: us-east-1 - jobs: docker-build: runs-on: [self-hosted, linux.2xlarge] @@ -30,7 +26,7 @@ jobs: include: - docker-image-name: executorch-ubuntu-22.04-clang12 env: - DOCKER_IMAGE_BASE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }} + DOCKER_IMAGE: 308535385114.dkr.ecr.us-east-1.amazonaws.com/executorch/${{ matrix.docker-image-name }} steps: - name: Clean workspace shell: bash @@ -50,14 +46,12 @@ jobs: - name: Setup Linux uses: pytorch/test-infra/.github/actions/setup-linux@main - # TODO: Move calculate Docker image to test-infra as a shareable GHA - name: Build docker image id: build-docker-image shell: bash - env: - DOCKER_IMAGE_NAME: ${{ matrix.docker-image-name }} + working-directory: .ci/docker run: | - # DEBUG - sleep 300 - - .ci/docker/build.sh "${DOCKER_IMAGE_NAME}" + # TODO: Move calculate Docker image to test-infra as a shareable GHA + # and use that here instead. + DOCKER_TAG=$(git rev-parse HEAD:./) + ./build.sh "${DOCKER_IMAGE_NAME}" -t "${DOCKER_IMAGE}:${DOCKER_TAG}" From ce377a1ee1bb4f32454592e201bd559eb3d197d2 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Fri, 14 Jul 2023 13:11:25 -0700 Subject: [PATCH 06/13] Clean up unused DOCKER_IMAGE_NAME --- .github/workflows/docker-builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index e8d78f2a097..5afb102090a 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -54,4 +54,4 @@ jobs: # TODO: Move calculate Docker image to test-infra as a shareable GHA # and use that here instead. DOCKER_TAG=$(git rev-parse HEAD:./) - ./build.sh "${DOCKER_IMAGE_NAME}" -t "${DOCKER_IMAGE}:${DOCKER_TAG}" + ./build.sh -t "${DOCKER_IMAGE}:${DOCKER_TAG}" From 70705acf3105902fd42ed8414421c758a6eaf018 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 17 Jul 2023 18:07:15 -0700 Subject: [PATCH 07/13] Use shareable calculate docker image GHA --- .ci/docker/build.sh | 3 +++ .github/workflows/docker-builds.yml | 16 +++++++++------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index fe41d5d1054..2a70eee8ea4 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -2,6 +2,9 @@ set -ex +IMAGE_NAME="$1" +shift + OS="ubuntu" OS_VERSION=22.04 CLANG_VERSION=12 diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index 5afb102090a..29df21f6760 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -48,10 +48,12 @@ jobs: - name: Build docker image id: build-docker-image - shell: bash - working-directory: .ci/docker - run: | - # TODO: Move calculate Docker image to test-infra as a shareable GHA - # and use that here instead. - DOCKER_TAG=$(git rev-parse HEAD:./) - ./build.sh -t "${DOCKER_IMAGE}:${DOCKER_TAG}" + uses: huydhn/test-infra/.github/actions/calculate-docker-image@calculate-docker-image + with: + docker-image-name: ${{ matrix.docker-image-name }} + always-rebuild: true + push: true + + - name: Teardown Linux + uses: pytorch/test-infra/.github/actions/teardown-linux@main + if: always() From 025772432256b4aa61b680c590b79a26cba3fba0 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Mon, 17 Jul 2023 19:33:56 -0700 Subject: [PATCH 08/13] Update README.md --- .ci/docker/README.md | 25 ++++++++++++++++++++++++- 1 file changed, 24 insertions(+), 1 deletion(-) diff --git a/.ci/docker/README.md b/.ci/docker/README.md index 1333ed77b7e..ff045f9de49 100644 --- a/.ci/docker/README.md +++ b/.ci/docker/README.md @@ -1 +1,24 @@ -TODO +# Docker images for Executorch CI + +This directory contains everything needed to build the Docker images +that are used in Executorch CI. + +## Contents + +* `build.sh` -- dispatch script to launch all builds +* `common` -- scripts used to execute individual Docker build stages +* `ubuntu` -- Dockerfile for Ubuntu image for CPU build and test jobs + +## Usage + +```bash +# Generic usage +./build.sh "${IMAGE_NAME}" "${DOCKER_BUILD_PARAMETERS}" + +# Build a specific image +./build.sh executorch-ubuntu-22.04-clang12 -t myimage:latest + +# Set CLANG version (see build.sh) and build image +CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest +``` + From d4c3ad01d72e4d7d4a400ccc4655f06e190626f0 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 18 Jul 2023 17:36:42 -0700 Subject: [PATCH 09/13] Use upstream calculate docker image --- .github/workflows/docker-builds.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index 29df21f6760..447db0ca309 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -48,7 +48,7 @@ jobs: - name: Build docker image id: build-docker-image - uses: huydhn/test-infra/.github/actions/calculate-docker-image@calculate-docker-image + uses: pytorch/test-infra/.github/actions/calculate-docker-image@calculate-docker-image with: docker-image-name: ${{ matrix.docker-image-name }} always-rebuild: true From 56bcaf0e83b5e1258b0d646746f2e859d10047d3 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 18 Jul 2023 17:50:05 -0700 Subject: [PATCH 10/13] Install buck and fix lint error --- .ci/docker/README.md | 1 - .ci/docker/build.sh | 2 ++ .ci/docker/common/install_buck.sh | 28 ++++++++++++++++++++++++++++ .ci/docker/common/install_clang.sh | 7 +++++-- .ci/docker/common/install_user.sh | 0 .ci/docker/ubuntu/Dockerfile | 4 ++++ .github/workflows/docker-builds.yml | 2 +- 7 files changed, 40 insertions(+), 4 deletions(-) create mode 100755 .ci/docker/common/install_buck.sh mode change 100644 => 100755 .ci/docker/common/install_user.sh diff --git a/.ci/docker/README.md b/.ci/docker/README.md index ff045f9de49..b74bb835aff 100644 --- a/.ci/docker/README.md +++ b/.ci/docker/README.md @@ -21,4 +21,3 @@ that are used in Executorch CI. # Set CLANG version (see build.sh) and build image CLANG_VERSION=11 ./build.sh executorch-ubuntu-22.04-clang11 -t myimage:latest ``` - diff --git a/.ci/docker/build.sh b/.ci/docker/build.sh index 2a70eee8ea4..260b3e23407 100755 --- a/.ci/docker/build.sh +++ b/.ci/docker/build.sh @@ -5,6 +5,8 @@ set -ex IMAGE_NAME="$1" shift +echo "Building ${IMAGE_NAME} Docker image" + OS="ubuntu" OS_VERSION=22.04 CLANG_VERSION=12 diff --git a/.ci/docker/common/install_buck.sh b/.ci/docker/common/install_buck.sh new file mode 100755 index 00000000000..66fcbcff95f --- /dev/null +++ b/.ci/docker/common/install_buck.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -ex + +install_ubuntu() { + apt-get update + + # TODO: Setup buck will come later in a separate PR + wget -q https://github.com/facebook/buck/releases/download/v2021.01.12.01/buck.2021.01.12.01_all.deb + apt install ./buck.2021.01.12.01_all.deb + + rm buck.2021.01.12.01_all.deb + # Cleanup package manager + apt-get autoclean && apt-get clean + rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* +} + +# Install base packages depending on the base OS +ID=$(grep -oP '(?<=^ID=).+' /etc/os-release | tr -d '"') +case "$ID" in + ubuntu) + install_ubuntu + ;; + *) + echo "Unable to determine OS..." + exit 1 + ;; +esac diff --git a/.ci/docker/common/install_clang.sh b/.ci/docker/common/install_clang.sh index d5f3f1a54c1..5457a982cc9 100755 --- a/.ci/docker/common/install_clang.sh +++ b/.ci/docker/common/install_clang.sh @@ -17,8 +17,11 @@ install_ubuntu() { # CLANG's packaging is a little messed up (the runtime libs aren't # added into the linker path), so give it a little help - CLANG_LIB=("/usr/lib/llvm-$CLANG_VERSION/lib/clang/"*"/lib/linux") - echo "$CLANG_LIB" > /etc/ld.so.conf.d/clang.conf + CLANG_LIBS=("/usr/lib/llvm-$CLANG_VERSION/lib/clang/"*"/lib/linux") + for CLANG_LIB in "${CLANG_LIB[@]}" + do + echo "${CLANG_LIB}" > /etc/ld.so.conf.d/clang.conf + done ldconfig # Cleanup package manager diff --git a/.ci/docker/common/install_user.sh b/.ci/docker/common/install_user.sh old mode 100644 new mode 100755 diff --git a/.ci/docker/ubuntu/Dockerfile b/.ci/docker/ubuntu/Dockerfile index 16c91c571d0..5ad25de1af7 100644 --- a/.ci/docker/ubuntu/Dockerfile +++ b/.ci/docker/ubuntu/Dockerfile @@ -15,6 +15,10 @@ ARG CLANG_VERSION COPY ./common/install_clang.sh install_clang.sh RUN bash ./install_clang.sh && rm install_clang.sh +# Setup buck +COPY ./common/install_buck.sh install_buck.sh +RUN bash ./install_buck.sh && rm install_buck.sh + # Setup user COPY ./common/install_user.sh install_user.sh RUN bash ./install_user.sh && rm install_user.sh diff --git a/.github/workflows/docker-builds.yml b/.github/workflows/docker-builds.yml index 447db0ca309..4e7f105a4a8 100644 --- a/.github/workflows/docker-builds.yml +++ b/.github/workflows/docker-builds.yml @@ -48,7 +48,7 @@ jobs: - name: Build docker image id: build-docker-image - uses: pytorch/test-infra/.github/actions/calculate-docker-image@calculate-docker-image + uses: pytorch/test-infra/.github/actions/calculate-docker-image@main with: docker-image-name: ${{ matrix.docker-image-name }} always-rebuild: true From 1a0c7bbf13b2067eb95ae0f4b4efe78e764ce412 Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 18 Jul 2023 17:54:50 -0700 Subject: [PATCH 11/13] Yes --- .ci/docker/common/install_buck.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.ci/docker/common/install_buck.sh b/.ci/docker/common/install_buck.sh index 66fcbcff95f..2f497c04ec4 100755 --- a/.ci/docker/common/install_buck.sh +++ b/.ci/docker/common/install_buck.sh @@ -7,7 +7,7 @@ install_ubuntu() { # TODO: Setup buck will come later in a separate PR wget -q https://github.com/facebook/buck/releases/download/v2021.01.12.01/buck.2021.01.12.01_all.deb - apt install ./buck.2021.01.12.01_all.deb + apt install -y ./buck.2021.01.12.01_all.deb rm buck.2021.01.12.01_all.deb # Cleanup package manager From e34a203adabc8bf1ce3a576c050c70f91f890dca Mon Sep 17 00:00:00 2001 From: Huy Do Date: Tue, 18 Jul 2023 18:33:08 -0700 Subject: [PATCH 12/13] Remove clang hack --- .ci/docker/common/install_clang.sh | 9 --------- 1 file changed, 9 deletions(-) diff --git a/.ci/docker/common/install_clang.sh b/.ci/docker/common/install_clang.sh index 5457a982cc9..3713ca71c8e 100755 --- a/.ci/docker/common/install_clang.sh +++ b/.ci/docker/common/install_clang.sh @@ -15,15 +15,6 @@ install_ubuntu() { update-alternatives --install /usr/bin/cc cc /usr/bin/clang 50 update-alternatives --install /usr/bin/c++ c++ /usr/bin/clang++ 50 - # CLANG's packaging is a little messed up (the runtime libs aren't - # added into the linker path), so give it a little help - CLANG_LIBS=("/usr/lib/llvm-$CLANG_VERSION/lib/clang/"*"/lib/linux") - for CLANG_LIB in "${CLANG_LIB[@]}" - do - echo "${CLANG_LIB}" > /etc/ld.so.conf.d/clang.conf - done - ldconfig - # Cleanup package manager apt-get autoclean && apt-get clean rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* From 5b8fab967cdb415dd6f9280ed2548c3ec0af07eb Mon Sep 17 00:00:00 2001 From: Huy Do Date: Wed, 19 Jul 2023 17:15:15 -0700 Subject: [PATCH 13/13] Install Buck2 --- .ci/docker/common/install_buck.sh | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/.ci/docker/common/install_buck.sh b/.ci/docker/common/install_buck.sh index 2f497c04ec4..7fa90fa7597 100755 --- a/.ci/docker/common/install_buck.sh +++ b/.ci/docker/common/install_buck.sh @@ -4,12 +4,15 @@ set -ex install_ubuntu() { apt-get update + apt-get install -y zstd - # TODO: Setup buck will come later in a separate PR - wget -q https://github.com/facebook/buck/releases/download/v2021.01.12.01/buck.2021.01.12.01_all.deb - apt install -y ./buck.2021.01.12.01_all.deb + wget -q https://github.com/facebook/buck2/releases/download/2023-07-18/buck2-x86_64-unknown-linux-gnu.zst + zstd -d buck2-x86_64-unknown-linux-gnu.zst -o buck2 - rm buck.2021.01.12.01_all.deb + chmod +x buck2 + mv buck2 /usr/bin/ + + rm buck2-x86_64-unknown-linux-gnu.zst # Cleanup package manager apt-get autoclean && apt-get clean rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*