Add opt-in to use Harvester k3s cluster for development

Author: mads-hartmann

.werft/build.ts

@@ -52,6 +52,9 @@ Tracing.initialize()
             // Explicitly not using process.exit as we need to flush tracing, see tracing.js
             process.exitCode = 1
         }
+
+        // TODO: Find a better place for this. This terminates background proxies.
+        exec(`sudo killall kubectl || true`)
     })
 
 // Werft phases
@@ -77,7 +80,10 @@ const installerSlices = {
 }
 
 const vmSlices = {
-    BOOT_VM: 'Booting VM'
+    BOOT_VM: 'Booting VM',
+    SSH_PROXY: 'SSH Proxy',
+    COPY_CERT_MANAGER_RESOURCES: 'Copy CertManager resources from core-dev',
+    KUBECONFIG: 'Getting kubeconfig'
 }
 
 export function parseVersion(context) {
@@ -272,7 +278,7 @@ export async function build(context, version) {
 
     const destname = version.split(".")[0];
     const namespace = `staging-${destname}`;
-    const domain = `${destname}.staging.gitpod-dev.com`;
+    const domain = withVM ? `${destname}.preview.gitpod-dev.com` : `${destname}.staging.gitpod-dev.com`;
     const monitoringDomain = `${destname}.preview.gitpod-dev.com`;
     const url = `https://${domain}`;
     const deploymentConfig: DeploymentConfig = {
@@ -293,7 +299,12 @@ export async function build(context, version) {
     if (withVM) {
         werft.phase(phases.VM, "Start VM");
 
-        if (!VM.vmExists({ name: destname })) {
+        werft.log(vmSlices.COPY_CERT_MANAGER_RESOURCES, 'Copy over CertManager resources from core-dev')
+        exec(`kubectl get secret clouddns-dns01-solver-svc-acct -n certmanager -o yaml | sed 's/namespace: certmanager/namespace: cert-manager/g' > clouddns-dns01-solver-svc-acct.yaml`)
+        exec(`kubectl get clusterissuer letsencrypt-issuer-gitpod-core-dev -o yaml | sed 's/letsencrypt-issuer-gitpod-core-dev/letsencrypt-issuer/g' > letsencrypt-issuer.yaml`)
+
+        const existingVM = VM.vmExists({ name: destname })
+        if (!existingVM) {
             werft.log(vmSlices.BOOT_VM, 'Starting VM')
             VM.startVM({ name: destname })
         } else {
@@ -303,8 +314,29 @@ export async function build(context, version) {
         werft.log(vmSlices.BOOT_VM, 'Waiting for VM to be ready')
         VM.waitForVM({ name: destname, timeoutMS: 1000 * 60 * 3 })
 
-        werft.done(phases.VM)
-        return
+        // TODO: We're hoping to have the proxy in the Harvester cluster be able to forward
+        //       both SSH and Kube API so we don't need these two. Currently this will cause
+        //       the Werft job to never exit as these two proxies will keep the job from
+        //       terminating.
+        werft.log(vmSlices.SSH_PROXY, 'Starting SSH proxy')
+        VM.startSSHProxy({ name: destname })
+
+        werft.log(vmSlices.SSH_PROXY, 'Starting k8s API proxy')
+        VM.startKubeAPIProxy({ name: destname })
+
+        werft.log(vmSlices.KUBECONFIG, 'Copying k3s kubeconfig')
+        // TODO: This is currently flaky and sometimes fails if the VM isn't ready in time.
+        //       instead of just sleeping we should periodically check if SSH access is ready.
+        exec(`sleep 10`)
+        exec(`ssh -i /workspace/.ssh/id_rsa_harvester_vm [email protected] -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 'sudo cat /etc/rancher/k3s/k3s.yaml' > k3s.yml`)
+
+        // TODO: This was a quick have to override the existing kubeconfig so all future kubectl commands use the k3s cluster.
+        //       We might want to keep both kubeconfigs around and be explicit about which one we're using
+        exec(`mv k3s.yml /home/gitpod/.kube/config`)
+
+        if (!existingVM) {
+            exec(`kubectl apply -f clouddns-dns01-solver-svc-acct.yaml -f letsencrypt-issuer.yaml`)
+        }
     }
 
     werft.phase(phases.PREDEPLOY, "Checking for existing installations...");

.werft/build.yaml

@@ -36,6 +36,9 @@ pod:
   - name: harvester-kubeconfig
     secret:
       secretName: harvester-kubeconfig
+  - name: harvester-vm-ssh-keys
+    secret:
+      secretName: harvester-vm-ssh-keys
   # - name: deploy-key
   #   secret:
   #     secretName: deploy-key
@@ -82,6 +85,8 @@ pod:
       readOnly: false
     - name: harvester-kubeconfig
       mountPath: /mnt/secrets/harvester-kubeconfig
+    - name: harvester-vm-ssh-keys
+      mountPath: /mnt/secrets/harvester-vm-ssh-keys
     # - name: deploy-key
     #   mountPath: /mnt/secrets/deploy-key
     #   readOnly: true
@@ -163,6 +168,12 @@ pod:
         export DOCKER_HOST=tcp://$NODENAME:2475
         sudo chown -R gitpod:gitpod /workspace
 
+        mkdir /workspace/.ssh
+        cp /mnt/secrets/harvester-vm-ssh-keys/id_rsa /workspace/.ssh/id_rsa_harvester_vm
+        cp /mnt/secrets/harvester-vm-ssh-keys/id_rsa.pub /workspace/.ssh/id_rsa_harvester_vm.pub
+        sudo chmod 600 /workspace/.ssh/id_rsa_harvester_vm
+        sudo chmod 644 /workspace/.ssh/id_rsa_harvester_vm.pub
+
         (cd .werft && yarn install && mv node_modules ..) | werft log slice prep
         printf '{{ toJson . }}' > context.json
 

.werft/vm/manifests.ts

@@ -15,10 +15,9 @@ type VirtualMachineManifestArguments = {
   vmName: string
   namespace: string
   claimName: string,
-  userDataSecretName: string
 }
 
-export function VirtualMachineManifest({ vmName, namespace, claimName, userDataSecretName }: VirtualMachineManifestArguments) {
+export function VirtualMachineManifest({ vmName, namespace, claimName }: VirtualMachineManifestArguments) {
   return `
 apiVersion: kubevirt.io/v1
 type: kubevirt.io.virtualmachine
@@ -46,7 +45,7 @@ spec:
         machine:
           type: q35
         cpu:
-          cores: 1
+          cores: 4
           sockets: 1
           threads: 1
         devices:
@@ -64,8 +63,8 @@ spec:
                 bus: virtio
         resources:
           limits:
-            memory: 2Gi
-            cpu: 1
+            memory: 8Gi
+            cpu: 4
       evictionStrategy: LiveMigrate
       networks:
         - pod: {}
@@ -76,10 +75,23 @@ spec:
             claimName: ${claimName}
         - name: cloudinitdisk
           cloudInitNoCloud:
-            networkDataSecretRef:
-              name: ${userDataSecretName}
-            secretRef:
-              name: ${userDataSecretName}
+            userData: |-
+              #cloud-config
+              users:
+                - name: ubuntu
+                  sudo: "ALL=(ALL) NOPASSWD: ALL"
+                  ssh_authorized_keys:
+                    - ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAACAQC/aB/HYsb56V0NBOEab6j33v3LIxRiGqG4fmidAryAXevLyTANJPF8m44KSzSQg7AI7PMy6egxQp/JqH2b+3z1cItWuHZSU+klsKNuf5HxK7AOrND3ahbejZfyYewtKFQ3X9rv5Sk8TAR5gw5oPbkTR61jiLa58Sw7UkhLm2EDguGASb6mBal8iboiF8Wpl8QIvPmJaGIOY2YwXLepwFA3S3kVqW88eh2WFmjTMre5ASLguYNkHXjyb/TuhVFzAvphzpl84RAaEyjKYnk45fh4xRXx+oKqlfKRJJ/Owxa7SmGO+/4rWb3chdnpodHeu7XjERmjYLY+r46sf6n6ySgEht1xAWjMb1uqZqkDx+fDDsjFSeaN3ncX6HSoDOrphFmXYSwaMpZ8v67A791fuUPrMLC+YMckhTuX2g4i3XUdumIWvhaMvKhy/JRRMsfUH0h+KAkBLI6tn5ozoXiQhgM4SAE5HsMr6CydSIzab0yY3sq0avmZgeoc78+8PKPkZG1zRMEspV/hKKBC8hq7nm0bu4IgzuEIYHowOD8svqA0ufhDWxTt6A4Jo0xDzhFyKme7KfmW7SIhpejf3T1Wlf+QINs1hURr8LSOZEyY2SzYmAoQ49N0SSPb5xyG44cptpKcj0WCAJjBJoZqz0F5x9TjJ8XToB5obyJfRHD1JjxoMQ== [email protected]
+              chpasswd:
+                list: |
+                  ubuntu:ubuntu
+                expire: False
+              runcmd:
+                - curl -sfL https://get.k3s.io | sh -
+                - sleep 10
+                - kubectl label nodes ${vmName} gitpod.io/workload_meta=true gitpod.io/workload_ide=true gitpod.io/workload_workspace_services=true gitpod.io/workload_workspace_regular=true gitpod.io/workload_workspace_headless=true gitpod.io/workspace_0=true gitpod.io/workspace_1=true gitpod.io/workspace_2=true
+                - kubectl create ns certs
+                - kubectl apply -f https://github.com/jetstack/cert-manager/releases/download/v1.6.1/cert-manager.yaml
 `
 }
 
@@ -123,23 +135,3 @@ type UserDataSecretManifestOptions = {
   namespace: string,
   secretName: string
 }
-
-export function UserDataSecretManifest({ namespace, secretName }: UserDataSecretManifestOptions) {
-  const userdata = Buffer.from(`#cloud-config
-users:
-  - name: ubuntu
-    lock_passwd: false
-    sudo: "ALL=(ALL) NOPASSWD: ALL"
-    passwd: "$6$exDY1mhS4KUYCE/2$zmn9ToZwTKLhCw.b4/b.ZRTIZM30JZ4QrOQ2aOXJ8yk96xpcCof0kxKwuX1kqLG/ygbJ1f8wxED22bTL4F46P0"`).toString("base64")
-  return `
-apiVersion: v1
-type: secret
-kind: Secret
-data:
-  networkdata: ""
-  userdata: ${userdata}
-metadata:
-  name: ${secretName}
-  namespace: ${namespace}
-`
-}

.werft/vm/vm.ts

@@ -21,27 +21,18 @@ EOF
  */
 export function startVM(options: { name: string }) {
     const namespace = `preview-${options.name}`
-    const userDataSecretName = `userdata-${options.name}`
 
     kubectlApplyManifest(
         Manifests.NamespaceManifest({
             namespace
         })
     )
 
-    kubectlApplyManifest(
-        Manifests.UserDataSecretManifest({
-            namespace,
-            secretName: userDataSecretName,
-        })
-    )
-
     kubectlApplyManifest(
         Manifests.VirtualMachineManifest({
             namespace,
             vmName: options.name,
-            claimName: `${options.name}-${Date.now()}`,
-            userDataSecretName
+            claimName: `${options.name}-${Date.now()}`
         }),
         { validate: false }
     )
@@ -87,5 +78,20 @@ export function waitForVM(options: { name: string, timeoutMS: number }) {
         console.log(`VM is not yet running. Current status is ${status}. Sleeping 5 seconds`)
         exec('sleep 5', { silent: true })
     }
+}
+
+/**
+ * Proxy 127.0.0.1:22 to :22 in the VM through the k8s service
+ */
+export function startSSHProxy(options: { name: string }) {
+    const namespace = `preview-${options.name}`
+    exec(`sudo kubectl --kubeconfig=${KUBECONFIG_PATH} -n ${namespace} port-forward service/proxy 22:22`, { async: true, silent: true })
+}
 
+/**
+ * Proxy 127.0.0.1:6443 to :6443 in the VM through the k8s service
+ */
+export function startKubeAPIProxy(options: { name: string }) {
+    const namespace = `preview-${options.name}`
+    exec(`sudo kubectl --kubeconfig=${KUBECONFIG_PATH} -n ${namespace} port-forward service/proxy 6443:6443`, { async: true, silent: true })
 }