remote cache with leeway and GCP buckets

Author: akosyakov

.gitpod.Dockerfile

@@ -1,5 +1,13 @@
 FROM gitpod/workspace-full:latest
 
+USER root
+
+# leeway
+ENV LEEWAY_NESTED_WORKSPACE=true
+RUN cd /usr/bin && curl -fsSL https://github.com/gitpod-io/leeway/releases/download/v0.2.5/leeway_0.2.5_Linux_x86_64.tar.gz | tar xz
+
+USER gitpod
+
 # We use latest major version of Node.js distributed VS Code. (see about dialog in your local VS Code)
 RUN bash -c ". .nvm/nvm.sh \
     && nvm install 14 \
@@ -17,3 +25,33 @@ RUN sudo apt-get update \
         libasound2 libgbm1 xfonts-base xfonts-terminus fonts-noto fonts-wqy-microhei \
         fonts-droid-fallback vim-tiny nano libgconf2-dev libgtk-3-dev twm \
     && sudo apt-get clean && sudo rm -rf /var/cache/apt/* && sudo rm -rf /var/lib/apt/lists/* && sudo rm -rf /tmp/*
+
+## Register leeway autocompletion in bashrc
+RUN bash -c "echo . \<\(leeway bash-completion\) >> ~/.bashrc"
+
+### Google Cloud ###
+# not installed via repository as then 'docker-credential-gcr' is not available
+ARG GCS_DIR=/opt/google-cloud-sdk
+ENV PATH=$GCS_DIR/bin:$PATH
+RUN sudo chown gitpod: /opt \
+    && mkdir $GCS_DIR \
+    && curl -fsSL https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-344.0.0-linux-x86_64.tar.gz \
+    | tar -xzvC /opt \
+    && /opt/google-cloud-sdk/install.sh --quiet --usage-reporting=false --bash-completion=true \
+    --additional-components docker-credential-gcr alpha beta \
+    # needed for access to our private registries
+    && docker-credential-gcr configure-docker
+
+# Install tools for gsutil
+RUN sudo install-packages \
+    gcc \
+    python-dev \
+    python-setuptools
+
+RUN bash -c "pip uninstall crcmod; pip install --no-cache-dir -U crcmod"
+
+# Set Application Default Credentials (ADC) based on user-provided env var
+RUN echo ". /workspace/vscode/scripts/setup-google-adc.sh" >> ~/.bashrc
+
+ENV LEEWAY_WORKSPACE_ROOT=/workspace/vscode
+ENV LEEWAY_REMOTE_CACHE_BUCKET=gitpod-core-leeway-cache-branch

.gitpod.yml

@@ -5,10 +5,9 @@ ports:
     onOpen: open-browser
 tasks:
   - init: |
-      yarn
-      yarn --cwd ./build compile
-      yarn compile
-      yarn download-builtin-extensions
+      export VSCODE_INIT_BUILD_DIR=$(leeway describe "//:init" -t "/tmp/build/{{ .Metadata.Name }}.{{ .Metadata.Version }}")
+      leeway build
+      sudo cp -rup "$VSCODE_INIT_BUILD_DIR/install/." . | true
     command: |
       gp sync-done init
       export NODE_ENV=development

.leewayignore

@@ -0,0 +1,4 @@
+/node_modules/
+/out/
+/.build/
+/.git/

BUILD.yaml

@@ -0,0 +1,17 @@
+packages:
+  - name: install
+    type: generic
+    srcs:
+    - "**"
+    config:
+      commands:
+        - ["yarn"]
+  - name: init
+    type: generic
+    deps:
+    - ":install"
+    config:
+      commands:
+        - ["yarn", "--cwd", "./install/build", "compile"]
+        - ["yarn", "--cwd", "./install", "compile"]
+        - ["yarn", "--cwd", "./install", "download-builtin-extensions"]

WORKSPACE.yaml

@@ -0,0 +1 @@
+defaultTarget: "//:init"

build/npm/postinstall.js

@@ -88,4 +88,8 @@ runtime "${runtime}"`;
 	yarnInstall(watchPath);
 }
 
-cp.execSync('git config pull.rebase merges');
+try {
+	cp.execSync('git config pull.rebase merges');
+} catch (e) {
+	console.error(e)
+}

scripts/setup-google-adc.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+# #### Instructions to fill the GCP_ADC_FILE env var
+#  1. `gcloud auth login <gcp-email>` and authenticate
+#  2. `gcloud auth application-default login` and authenticate
+#  3. `cat ~/.config/gcloud/application_default_credentials.json` and copy the output
+#  4. Go to https://gitpod.io/settings/ and create:
+#     - name: GCP_ADC_FILE
+#     - value: paste-the-output
+#     - repo: gitpod-io/vscode
+
+GCLOUD_ADC_PATH="/home/gitpod/.config/gcloud/application_default_credentials.json"
+
+if [ ! -f "$GCLOUD_ADC_PATH" ]; then
+    if [ -z "$GCP_ADC_FILE" ]; then
+        echo "GCP_ADC_FILE not set, doing nothing."
+        return;
+    fi
+    echo "$GCP_ADC_FILE" > "$GCLOUD_ADC_PATH"
+    echo "Set GOOGLE_APPLICATION_CREDENTIALS value based on contents from GCP_ADC_FILE"
+fi
+export GOOGLE_APPLICATION_CREDENTIALS="$GCLOUD_ADC_PATH"
+