Ensure install is actually non-blocking

Author: mads-hartmann

.werft/jobs/build/deploy-to-preview-environment.ts

@@ -161,8 +161,19 @@ export async function deployToPreviewEnvironment(werft: Werft, jobConfig: JobCon
         // success rate or installing monitoring satellite, but we can at least count and debug errors.
         // In the future we can consider not closing spans when closing phases, or restructuring our phases
         // based on parallelism boundaries
+        const monitoringSatelliteInstaller = new MonitoringSatelliteInstaller({
+            kubeconfigPath: PREVIEW_K3S_KUBECONFIG_PATH,
+            branch: jobConfig.observability.branch,
+            satelliteNamespace: deploymentConfig.namespace,
+            clusterName: deploymentConfig.namespace,
+            nodeExporterPort: 9100,
+            previewDomain: deploymentConfig.domain,
+            stackdriverServiceAccount: STACKDRIVER_SERVICEACCOUNT,
+            withVM: withVM,
+            werft: werft
+        });
         const sliceID = "observability"
-        installMonitoring(werft, PREVIEW_K3S_KUBECONFIG_PATH, deploymentConfig.namespace, 9100, deploymentConfig.domain, STACKDRIVER_SERVICEACCOUNT, withVM, jobConfig.observability.branch)
+        monitoringSatelliteInstaller.install()
             .then(() => {
                 werft.log(sliceID, "Succeeded installing monitoring satellite")
             })
@@ -485,7 +496,18 @@ async function deployToDevWithHelm(werft: Werft, jobConfig: JobConfig, deploymen
     werft.log(`observability`, "Installing monitoring-satellite...")
     if (deploymentConfig.withObservability) {
         try {
-            await installMonitoring(werft, CORE_DEV_KUBECONFIG_PATH, namespace, nodeExporterPort, monitoringDomain, STACKDRIVER_SERVICEACCOUNT, false, jobConfig.observability.branch);
+            const installMonitoringSatellite = new MonitoringSatelliteInstaller({
+                kubeconfigPath: CORE_DEV_KUBECONFIG_PATH,
+                branch: jobConfig.observability.branch,
+                satelliteNamespace: namespace,
+                clusterName: namespace,
+                nodeExporterPort: nodeExporterPort,
+                previewDomain: domain,
+                stackdriverServiceAccount: STACKDRIVER_SERVICEACCOUNT,
+                withVM: false,
+                werft: werft
+            });
+            await installMonitoringSatellite.install()
         } catch (err) {
             if (!jobConfig.mainBuild) {
                 werft.fail('observability', err);
@@ -794,21 +816,6 @@ async function installMetaCertificates(werft: Werft, branch: string, withVM: boo
     await installCertificate(werft, metaInstallCertParams, { ...metaEnv(), slice: slice });
 }
 
-async function installMonitoring(werft: Werft, kubeconfig: string, namespace: string, nodeExporterPort: number, domain: string, stackdriverServiceAccount: any, withVM: boolean, observabilityBranch: string) {
-    const installMonitoringSatellite = new MonitoringSatelliteInstaller({
-        kubeconfigPath: kubeconfig,
-        branch: observabilityBranch,
-        satelliteNamespace: namespace,
-        clusterName: namespace,
-        nodeExporterPort: nodeExporterPort,
-        previewDomain: domain,
-        stackdriverServiceAccount: stackdriverServiceAccount,
-        withVM: withVM,
-        werft: werft
-    });
-    installMonitoringSatellite.install()
-}
-
 // returns the static IP address
 function getCoreDevIngressIP(): string {
     return "104.199.27.246";

.werft/observability/monitoring-satellite.ts

@@ -22,7 +22,7 @@ const sliceName = "observability";
 export class MonitoringSatelliteInstaller {
     constructor(private readonly options: MonitoringSatelliteInstallerOptions) {}
 
-    public install() {
+    public async install() {
         const {
             werft,
             branch,
@@ -89,50 +89,61 @@ export class MonitoringSatelliteInstaller {
             this.postProcessManifests();
         }
 
-        // The correct kubectl context should already be configured prior to this step
-        // Only checks node-exporter readiness for harvester
-        this.ensureCorrectInstallationOrder();
+        this.ensureCorrectInstallationOrder()
+        this.deployGitpodServiceMonitors();
+        await this.waitForReadiness()
     }
 
     private ensureCorrectInstallationOrder() {
         const { werft, kubeconfigPath } = this.options;
 
         werft.log(sliceName, "installing monitoring-satellite");
         exec(`cd observability && hack/deploy-satellite.sh --kubeconfig ${kubeconfigPath}`, { slice: sliceName });
-
-        this.deployGitpodServiceMonitors();
-        this.checkReadiness();
     }
 
-    private checkReadiness() {
+    private async waitForReadiness() {
         const { kubeconfigPath, satelliteNamespace } = this.options;
 
+        const checks: Promise<any>[] = [];
         // For some reason prometheus' statefulset always take quite some time to get created
         // Therefore we wait a couple of seconds
-        exec(
-            `sleep 30 && kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} statefulset prometheus-k8s`,
-            { slice: sliceName },
+        checks.push(
+            exec(
+                `sleep 30 && kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} statefulset prometheus-k8s`,
+                { slice: sliceName, async: true },
+            ),
         );
-        exec(`kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} deployment grafana`, {
-            slice: sliceName,
-        });
-        exec(
-            `kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} deployment kube-state-metrics`,
-            { slice: sliceName },
+        checks.push(
+            exec(`kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} deployment grafana`, {
+                slice: sliceName,
+                async: true,
+            }),
         );
-        exec(
-            `kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} deployment otel-collector`,
-            { slice: sliceName },
+        checks.push(
+            exec(
+                `kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} deployment kube-state-metrics`,
+                { slice: sliceName, async: true },
+            ),
+        );
+        checks.push(
+            exec(
+                `kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} deployment otel-collector`,
+                { slice: sliceName, async: true },
+            ),
         );
 
         // core-dev is just too unstable for node-exporter
         // we don't guarantee that it will run at all
         if (this.options.withVM) {
-            exec(
-                `kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} daemonset node-exporter`,
-                { slice: sliceName },
+            checks.push(
+                exec(
+                    `kubectl --kubeconfig ${kubeconfigPath} rollout status -n ${satelliteNamespace} daemonset node-exporter`,
+                    { slice: sliceName, async: true },
+                ),
             );
         }
+
+        await Promise.all(checks);
     }
 
     private deployGitpodServiceMonitors() {