werft/jobs/build: Add observability support to installer

Signed-off-by: ArthurSens <[email protected]>

Author: ArthurSens

.werft/jobs/build/deploy-to-preview-environment.ts

@@ -148,7 +148,7 @@ export async function deployToPreviewEnvironment(werft: Werft, jobConfig: JobCon
 
         issueMetaCerts(werft, PROXY_SECRET_NAME, "default", domain, withVM)
         werft.done('certificate')
-        installMonitoring(deploymentConfig.namespace, 9100, deploymentConfig.domain, STACKDRIVER_SERVICEACCOUNT, true, jobConfig.observability.branch);
+        installMonitoring(deploymentConfig.namespace, 9100, deploymentConfig.domain, STACKDRIVER_SERVICEACCOUNT, withVM, jobConfig.observability.branch);
         werft.done('observability')
     }
 
@@ -200,12 +200,14 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
     werft.log(installerSlices.FIND_FREE_HOST_PORTS, "Find last ports");
     let wsdaemonPortMeta = findLastHostPort(namespace, 'ws-daemon', metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }))
     let registryNodePortMeta = findLastHostPort(namespace, 'registry-facade', metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }))
+    let nodeExporterPort = findLastHostPort(namespace, 'node-exporter', metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }))
 
-    if (isNaN(wsdaemonPortMeta) || isNaN(wsdaemonPortMeta)) {
+    if (isNaN(wsdaemonPortMeta) || isNaN(wsdaemonPortMeta) || isNaN(nodeExporterPort)) {
         werft.log(installerSlices.FIND_FREE_HOST_PORTS, "Can't reuse, check for some free ports.");
-        [wsdaemonPortMeta, registryNodePortMeta] = findFreeHostPorts([
+        [wsdaemonPortMeta, registryNodePortMeta, nodeExporterPort] = findFreeHostPorts([
             { start: 10000, end: 11000 },
             { start: 30000, end: 31000 },
+            { start: 31001, end: 32000 },
         ], metaEnv({ slice: installerSlices.FIND_FREE_HOST_PORTS, silent: true }));
     }
     werft.log(installerSlices.FIND_FREE_HOST_PORTS,
@@ -320,10 +322,22 @@ async function deployToDevWithInstaller(werft: Werft, jobConfig: JobConfig, depl
             exec(`yq w -i config.yaml analytics.writer ${deploymentConfig.analytics!}`, { slice: installerSlices.INSTALLER_RENDER });
         }
 
-        if (withObservability) {
+        if (withVM || withObservability) {
             // TODO: there's likely more to do...
             const tracingEndpoint = exec(`yq r ./.werft/jobs/build/helm/values.tracing.yaml tracing.endpoint`, { slice: installerSlices.INSTALLER_RENDER }).stdout.trim();
             exec(`yq w -i config.yaml observability.tracing.endpoint ${tracingEndpoint}`, { slice: installerSlices.INSTALLER_RENDER });
+
+            // If the preview is running on Harvester, we've already deployed monitoring-satellite during 'VM' phase.
+            // Therefore, we want to skip installing it here.
+            if(!withVM) {
+                try {
+                    installMonitoring(deploymentConfig.namespace, nodeExporterPort, monitoringDomain, STACKDRIVER_SERVICEACCOUNT, withVM, jobConfig.observability.branch);
+                } catch (err) {
+                    werft.fail('observability', err)
+                } finally {
+                    werft.done('observability')
+                }
+            }
         }
 
         werft.log("authProviders", "copy authProviders from secret")

.werft/observability/monitoring-satellite.ts

@@ -74,27 +74,33 @@ export async function installMonitoringSatellite(params: InstallMonitoringSatell
     }
 
     // The correct kubectl context should already be configured prior to this step
-    ensureCorrectInstallationOrder(params.satelliteNamespace)
+    // Only checks node-exporter readiness for harvester
+    ensureCorrectInstallationOrder(params.satelliteNamespace, params.withVM)
 }
 
-async function ensureCorrectInstallationOrder(namespace: string){
+async function ensureCorrectInstallationOrder(namespace: string, checkNodeExporterStatus: boolean){
     const werft = getGlobalWerftInstance()
 
     werft.log(sliceName, 'installing monitoring-satellite')
     exec('cd observability && hack/deploy-satellite.sh', {slice: sliceName})
 
     deployGitpodServiceMonitors()
-    checkReadiness(namespace)
+    checkReadiness(namespace, checkNodeExporterStatus)
 }
 
-async function checkReadiness(namespace: string) {
+async function checkReadiness(namespace: string, checkNodeExporterStatus: boolean) {
     // For some reason prometheus' statefulset always take quite some time to get created
     // Therefore we wait a couple of seconds
     exec(`sleep 30 && kubectl rollout status -n ${namespace} statefulset prometheus-k8s`, {slice: sliceName, async: true})
     exec(`kubectl rollout status -n ${namespace} deployment grafana`, {slice: sliceName, async: true})
     exec(`kubectl rollout status -n ${namespace} deployment kube-state-metrics`, {slice: sliceName, async: true})
     exec(`kubectl rollout status -n ${namespace} deployment otel-collector`, {slice: sliceName, async: true})
-    exec(`kubectl rollout status -n ${namespace} daemonset node-exporter`, {slice: sliceName, async: true})
+
+    // core-dev is just too unstable for node-exporter
+    // we don't guarantee that it will run at all
+    if(checkNodeExporterStatus) {
+        exec(`kubectl rollout status -n ${namespace} daemonset node-exporter`, {slice: sliceName, async: true})
+    }
 }
 
 async function deployGitpodServiceMonitors() {