Merge branch 'main' into release/2022.06.1

Author: felladrin

.github/CODEOWNERS

@@ -41,6 +41,7 @@
 /install/installer/pkg/components/server @gitpod-io/engineering-webapp
 /install/installer/pkg/components/server/ide @gitpod-io/engineering-ide
 /install/installer/pkg/components/usage @gitpod-io/engineering-webapp
+/install/installer/pkg/components/usage-api @gitpod-io/engineering-webapp
 /install/installer/pkg/components/workspace @gitpod-io/engineering-workspace
 /install/installer/pkg/components/workspace/ide @gitpod-io/engineering-ide
 /install/installer/pkg/components/ws-daemon @gitpod-io/engineering-workspace

.github/workflows/configcat.yml

@@ -0,0 +1,15 @@
+on: [push]
+name: Configcat code references
+jobs:
+  scan-repo:
+    runs-on: ubuntu-latest
+    name: Scan repository for configcat code references
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+    - name: Scan & upload
+      uses: configcat/scan-repository@v1
+      with:
+        api-user: ${{ secrets.CONFIGCAT_API_USER }}
+        api-pass: ${{ secrets.CONFIGCAT_API_PASS }}
+        config-id: 08da1258-6541-4fc7-8b61-c8b47f82f3a0

.gitpod.yml

@@ -1,4 +1,4 @@
-image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:ljb-werft-cli-grpc-changes.2
+image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:aledbf-go184.0
 workspaceLocation: gitpod/gitpod-ws.code-workspace
 checkoutLocation: gitpod
 ports:
@@ -30,11 +30,14 @@ ports:
     onOpen: ignore
   # Dev Theia
   - port: 13444
+  # Used when using port-forwarding to SSH to preview environment VMs
+  - port: 8022
+    onOpen: ignore
 tasks:
   - name: Install Preview Environment kube-context
     command: |
       (cd dev/preview/previewctl && go install .)
-      previewctl install-context
+      previewctl install-context --watch
       exit
   - name: Add Harvester kubeconfig
     command: |

.prettierrc.json

@@ -2,5 +2,13 @@
     "printWidth": 120,
     "tabWidth": 4,
     "endOfLine": "auto",
-    "trailingComma": "all"
+    "trailingComma": "all",
+    "overrides": [
+        {
+          "files": [".werft/**/*.yaml"],
+          "options": {
+            "tabWidth": 2
+          }
+        }
+      ]
 }

.werft/.prettierignore

@@ -0,0 +1,2 @@
+vm/charts/**
+vm/manifests/**

.werft/aks-installer-tests.yaml

@@ -0,0 +1,82 @@
+# debug using `werft run github -f -s .werft/installer-tests.ts -j .werft/aks-installer-tests.yaml -a debug=true`
+pod:
+  serviceAccount: werft
+  affinity:
+  nodeAffinity:
+    requiredDuringSchedulingIgnoredDuringExecution:
+      nodeSelectorTerms:
+        - matchExpressions:
+            - key: dev/workload
+              operator: In
+              values:
+                - "builds"
+  securityContext:
+    runAsUser: 0
+  volumes:
+  - name: sh-playground-sa-perm
+    secret:
+      secretName: sh-playground-sa-perm
+  - name: sh-playground-dns-perm
+    secret:
+      secretName: sh-playground-dns-perm
+  containers:
+  - name: nightly-test
+    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:aledbf-go184.0
+    workingDir: /workspace
+    imagePullPolicy: Always
+    volumeMounts:
+    - name: sh-playground-sa-perm
+      mountPath: /mnt/secrets/sh-playground-sa-perm
+    - name: sh-playground-dns-perm # this sa is used for the DNS management
+      mountPath: /mnt/secrets/sh-playground-dns-perm
+    env:
+    - name: ARM_SUBSCRIPTION_ID
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: subscriptionid
+    - name: ARM_TENANT_ID
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: tenantid
+    - name: ARM_CLIENT_ID
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: clientid
+    - name: ARM_CLIENT_SECRET
+      valueFrom:
+        secretKeyRef:
+          name: aks-credentials
+          key: clientsecret
+    - name: GOOGLE_APPLICATION_CREDENTIALS
+      value: "/mnt/secrets/sh-playground-sa-perm/sh-sa.json"
+    - name: TF_VAR_dns_sa_creds
+      value: "/mnt/secrets/sh-playground-dns-perm/sh-dns-sa.json"
+    - name: NODENAME
+      valueFrom:
+        fieldRef:
+          fieldPath: spec.nodeName
+    command:
+      - bash
+      - -c
+      - |
+        sleep 1
+        set -Eeuo pipefail
+
+          sudo chown -R gitpod:gitpod /workspace
+          sudo apt update && apt install gettext-base
+
+          curl -sL https://aka.ms/InstallAzureCLIDeb | sudo bash
+
+          export TF_VAR_TEST_ID=$(echo $RANDOM | md5sum | head -c 5; echo)
+
+          (cd .werft && yarn install && mv node_modules ..) | werft log slice prep
+          printf '{{ toJson . }}' > context.json
+
+          npx ts-node .werft/installer-tests.ts "STANDARD_AKS_TEST"
+
+# The bit below makes this a cron job
+plugins:
+  cron: "15 3 * * *"

.werft/build.ts

@@ -63,7 +63,7 @@ async function run(context: any) {
     if (!config.withPreview || config.publishRelease) {
         werft.phase("deploy", "not deploying");
         console.log("running without preview environment or publish-release is set");
-        return
+        return;
     }
 
     try {

.werft/build.yaml

@@ -23,9 +23,6 @@ pod:
   - name: gcp-sa-release
     secret:
       secretName: gcp-sa-gitpod-release-deployer
-  - name: gpsh-coredev-license
-    secret:
-      secretName: gpsh-coredev-license
   - name: prometheus-remote-write-auth
     secret:
       secretName: prometheus-remote-write-auth
@@ -54,6 +51,10 @@ pod:
   - name: fluent-bit-external
     secret:
       secretName: fluent-bit-external
+  - name: github-token-gitpod-bot
+    secret:
+      defaultMode: 420
+      secretName: github-token-gitpod-bot
   # - name: deploy-key
   #   secret:
   #     secretName: deploy-key
@@ -74,7 +75,7 @@ pod:
     - name: MYSQL_TCP_PORT
       value: 23306
   - name: build
-    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:ljb-werft-cli-grpc-changes.2
+    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:aledbf-go184.0
     workingDir: /workspace
     imagePullPolicy: IfNotPresent
     resources:
@@ -95,9 +96,6 @@ pod:
     - name: gcp-sa-release
       mountPath: /mnt/secrets/gcp-sa-release
       readOnly: true
-    - name: gpsh-coredev-license
-      mountPath: /mnt/secrets/gpsh-coredev
-      readOnly: true
     - name: gpsh-harvester-license
       mountPath: /mnt/secrets/gpsh-harvester
       readOnly: true
@@ -118,6 +116,8 @@ pod:
       mountPath: /mnt/secrets/harvester-k3s-dockerhub-pull-account
     - name: fluent-bit-external
       mountPath: /mnt/fluent-bit-external
+    - mountPath: /mnt/secrets/github-token-gitpod-bot
+      name: github-token-gitpod-bot
     # - name: deploy-key
     #   mountPath: /mnt/secrets/deploy-key
     #   readOnly: true
@@ -137,8 +137,6 @@ pod:
       value: http://athens-athens-proxy.athens.svc.cluster.local:9999
     - name: GOCACHE
       value: /go-build-cache
-    - name: WERFT_HOST
-      value: "werft.werft.svc.cluster.local:7777"
     - name: NODENAME
       valueFrom:
         fieldRef:
@@ -218,6 +216,11 @@ pod:
         secretKeyRef:
           name: replicated
           key: token
+    # Used by the Werft CLI through werft-credential-helper.sh
+    - name: WERFT_GITHUB_TOKEN_PATH
+      value: "/mnt/secrets/github-token-gitpod-bot/token"
+    - name: WERFT_CREDENTIAL_HELPER
+      value: "/workspace/dev/preview/werft-credential-helper.sh"
     command:
       - bash
       - -c

.werft/config.yaml

@@ -1,5 +1,5 @@
 rules:
-- path: ".werft/build.yaml"
-  matchesAll:
-  - or: ["repo.ref ~= refs/heads/"]
-  - or: ["trigger !== deleted"]
+  - path: ".werft/build.yaml"
+    matchesAll:
+      - or: ["repo.ref ~= refs/heads/"]
+      - or: ["trigger !== deleted"]

.werft/debug.yaml

@@ -53,7 +53,7 @@ pod:
     - name: MYSQL_TCP_PORT
       value: 23306
   - name: build
-    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:ljb-werft-cli-grpc-changes.2
+    image: eu.gcr.io/gitpod-core-dev/dev/dev-environment:aledbf-go184.0
     workingDir: /workspace
     imagePullPolicy: IfNotPresent
     volumeMounts:
@@ -96,8 +96,6 @@ pod:
       value: http://athens-athens-proxy.athens.svc.cluster.local:9999
     - name: GOCACHE
       value: /go-build-cache
-    - name: WERFT_HOST
-      value: "werft.werft.svc.cluster.local:7777"
     - name: NODENAME
       valueFrom:
         fieldRef: