Skip to content

Commit 9a0afff

Browse files
author
Prince Rachit Sinha
authored
[werft] Support installation of gitpod workspace component in a separate k3s cluster (#4664)
* Support workspace deployment in a separate k3s cluster using flag k3s-ws
1 parent 8f0c24a commit 9a0afff

File tree

6 files changed

+470
-169
lines changed

6 files changed

+470
-169
lines changed

‎.werft/build.ts

Lines changed: 220 additions & 141 deletions
Large diffs are not rendered by default.

‎.werft/util/certs.ts

Lines changed: 29 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,7 @@ export class IssueCertificateParams {
99
dnsZoneDomain: string
1010
domain: string
1111
ip: string
12-
additionalWsSubdomains: string[]
13-
includeDefaults: boolean
12+
additionalSubdomains: string[]
1413
pathToKubeConfig: string
1514
bucketPrefixTail: string
1615
certNamespace: string
@@ -24,14 +23,11 @@ export class InstallCertificateParams {
2423
destinationNamespace: string
2524
}
2625

27-
function getDefaultSubDomains(): string[] {
28-
return ["", "*.", "*.ws-dev."];
29-
}
30-
3126
export async function issueCertficate(werft, params: IssueCertificateParams) {
32-
const subdomains = params.includeDefaults ? getDefaultSubDomains() : [];
33-
for (const sd of params.additionalWsSubdomains) {
34-
subdomains.push(`*.ws-${sd}.`);
27+
var subdomains = [];
28+
werft.log("certificate", `Subdomains: ${params.additionalSubdomains}`)
29+
for (const sd of params.additionalSubdomains) {
30+
subdomains.push(sd);
3531
}
3632

3733
// sanity: check if there is a "SAN short enough to fit into CN (63 characters max)"
@@ -63,10 +59,10 @@ export async function issueCertficate(werft, params: IssueCertificateParams) {
6359

6460
werft.log('certificate', `waiting until certificate ${params.certNamespace}/${params.namespace} is ready...`)
6561
let notReadyYet = true;
66-
while (notReadyYet) {
67-
werft.log('certificate', `polling state of certs/${params.namespace}...`)
62+
for (let i = 0; i < 90 && notReadyYet; i++) {
63+
werft.log('certificate', `polling state of ${params.certNamespace}/${params.namespace}...`)
6864
const result = exec(`export KUBECONFIG=${params.pathToKubeConfig} && kubectl -n ${params.certNamespace} get certificate ${params.namespace} -o jsonpath="{.status.conditions[?(@.type == 'Ready')].status}"`, { silent: true, dontCheckRc: true });
69-
if (result.code === 0 && result.stdout === "True") {
65+
if (result != undefined && result.code === 0 && result.stdout === "True") {
7066
notReadyYet = false;
7167
break;
7268
}
@@ -76,13 +72,26 @@ export async function issueCertficate(werft, params: IssueCertificateParams) {
7672
}
7773

7874
export async function installCertficate(werft, params: InstallCertificateParams) {
75+
let notReadyYet = true;
7976
werft.log('certificate', `copying certificate from "${params.certNamespace}/${params.certName}" to "${params.destinationNamespace}/${params.certSecretName}"`);
80-
// certmanager is configured to create a secret in the namespace "certs" with the name "${namespace}".
81-
exec(`export KUBECONFIG=${params.pathToKubeConfig} && kubectl get secret ${params.certName} --namespace=${params.certNamespace} -o yaml \
82-
| yq d - 'metadata.namespace' \
83-
| yq d - 'metadata.uid' \
84-
| yq d - 'metadata.resourceVersion' \
85-
| yq d - 'metadata.creationTimestamp' \
86-
| sed 's/${params.certName}/${params.certSecretName}/g' \
87-
| kubectl apply --namespace=${params.destinationNamespace} -f -`);
77+
const cmd = `export KUBECONFIG=${params.pathToKubeConfig} && kubectl get secret ${params.certName} --namespace=${params.certNamespace} -o yaml \
78+
| yq d - 'metadata.namespace' \
79+
| yq d - 'metadata.uid' \
80+
| yq d - 'metadata.resourceVersion' \
81+
| yq d - 'metadata.creationTimestamp' \
82+
| sed 's/${params.certName}/${params.certSecretName}/g' \
83+
| kubectl apply --namespace=${params.destinationNamespace} -f -`
84+
85+
for (let i = 0; i < 60 && notReadyYet; i++) {
86+
const result = exec(cmd, { silent: true, dontCheckRc: true });
87+
if (result != undefined && result.code === 0) {
88+
notReadyYet = false;
89+
break;
90+
}
91+
werft.log('certificate', `Could not copy "${params.certNamespace}/${params.certName}", will retry`);
92+
await sleep(5000);
93+
}
94+
if (!notReadyYet) {
95+
werft.log('certificate', `copied certificate from "${params.certNamespace}/${params.certName}" to "${params.destinationNamespace}/${params.certSecretName}"`);
96+
}
8897
}

‎.werft/util/gpctl.ts

Lines changed: 43 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
import * as shell from 'shelljs';
2+
3+
function buildRequiredFlags(pathToKubeConfig: string): string {
4+
if (pathToKubeConfig != "") {
5+
return ` --kubeconfig=${pathToKubeConfig}`
6+
}
7+
return ""
8+
}
9+
10+
export function buildGpctlBinary() {
11+
shell.exec(`cd /workspace/dev/gpctl && go build && cd -`)
12+
}
13+
14+
export function printClustersList(pathToKubeConfig: string): string {
15+
let cmd = `/workspace/dev/gpctl/gpctl clusters list` + buildRequiredFlags(pathToKubeConfig)
16+
const result = shell.exec(cmd).trim()
17+
return result
18+
}
19+
20+
export function uncordonCluster(pathToKubeConfig: string, name: string): string {
21+
let cmd = `/workspace/dev/gpctl/gpctl clusters uncordon --name=${name}` + buildRequiredFlags(pathToKubeConfig)
22+
const result = shell.exec(cmd).trim()
23+
return result
24+
}
25+
26+
export function registerCluster(pathToKubeConfig: string, name: string, url: string): string {
27+
let cmd = `/workspace/dev/gpctl/gpctl clusters register \
28+
--name ${name} \
29+
--hint-cordoned \
30+
--hint-govern \
31+
--tls-path ./wsman-tls \
32+
--url ${url}` + buildRequiredFlags(pathToKubeConfig)
33+
34+
const result = shell.exec(cmd).trim()
35+
return result
36+
}
37+
38+
export function getClusterTLS(pathToKubeConfig: string): string {
39+
let cmd = `/workspace/dev/gpctl/gpctl clusters get-tls-config` + buildRequiredFlags(pathToKubeConfig)
40+
const result = shell.exec(cmd).trim()
41+
return result
42+
}
43+

‎.werft/util/kubectl.ts

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,10 @@ import { exec, ExecOptions, werft } from './shell';
44

55
export const IS_PREVIEW_APP_LABEL: string = "isPreviewApp";
66

7-
export function setKubectlContextNamespace(namespace, shellOpts) {
7+
export function setKubectlContextNamespace(pathToKubeConfig, namespace, shellOpts) {
88
[
9-
"kubectl config current-context",
10-
`kubectl config set-context --current --namespace=${namespace}`
9+
`export KUBECONFIG=${pathToKubeConfig} && kubectl config current-context`,
10+
`export KUBECONFIG=${pathToKubeConfig} && kubectl config set-context --current --namespace=${namespace}`
1111
].forEach(cmd => exec(cmd, shellOpts));
1212
}
1313

‎.werft/values.wsCluster.yaml renamed to ‎.werft/values.disableMeta.yaml

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,6 @@ components:
33
proxy:
44
disabled: true
55

6-
wsProxy:
7-
hostHeader: Host
8-
96
wsManagerBridge:
107
disabled: true
118

@@ -35,5 +32,3 @@ minio:
3532
mysql:
3633
enabled: false
3734

38-
registry-facade:
39-
enabled: false

‎.werft/values.k3sWsCluster.yaml

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
installation:
2+
stage: devstaging
3+
tenant: gitpod-core
4+
region: europe-west1
5+
cluster: "00"
6+
shortname: "k3s"
7+
hostname: staging.gitpod-dev.com
8+
imagePrefix: eu.gcr.io/gitpod-core-dev/build/
9+
certificatesSecret:
10+
secretName: proxy-config-certificates
11+
version: not-set
12+
imagePullPolicy: Always
13+
affinity:
14+
nodeAffinity:
15+
requiredDuringSchedulingIgnoredDuringExecution:
16+
nodeSelectorTerms:
17+
- matchExpressions:
18+
- key: gitpod.io/workload_services
19+
operator: In
20+
values:
21+
- "true"
22+
authProviders: []
23+
tracing:
24+
endoint: http://jaeger-collector:14268/api/traces
25+
samplerType: const
26+
samplerParam: "1"
27+
28+
components:
29+
wsManagerBridge:
30+
disabled: true
31+
32+
agentSmith:
33+
name: "agent-smith"
34+
disabled: false
35+
# in preview envs, we never want DaemonSets not to be scheduled (because they don't trigger scaleup)
36+
resources:
37+
cpu: 1m
38+
memory: 32Mi
39+
40+
registryFacade:
41+
daemonSet: true
42+
# in preview envs, we never want DaemonSets not to be scheduled (because they don't trigger scaleup)
43+
resources:
44+
cpu: 1m
45+
memory: 32Mi
46+
47+
contentService:
48+
remoteStorage:
49+
blobQuota: 1073741824 # 1 GiB
50+
51+
workspace:
52+
# configure GCP registry
53+
pullSecret:
54+
secretName: gcp-sa-registry-auth
55+
affinity:
56+
default: "gitpod.io/workload_workspace"
57+
templates:
58+
default:
59+
spec:
60+
dnsConfig:
61+
nameservers:
62+
- 1.1.1.1
63+
- 8.8.8.8
64+
dnsPolicy: None # do NOT query against K8s DNS (https://kubernetes.io/docs/concepts/services-networking/dns-pod-service/)
65+
env:
66+
- name: THEIA_PREVENT_METADATA_ACCESS
67+
value: true
68+
regular:
69+
spec:
70+
containers:
71+
- name: "workspace"
72+
env:
73+
- name: THEIA_RATELIMIT_LOG
74+
value: "50"
75+
- name: SUPERVISOR_DEBUG_ENABLE
76+
value: "true"
77+
prebuild:
78+
spec:
79+
containers:
80+
- name: workspace
81+
# Intended to reduce the density for prebuilds
82+
resources:
83+
limits:
84+
cpu: "5"
85+
memory: 12Gi
86+
requests:
87+
cpu: 1m
88+
ephemeral-storage: 5Gi
89+
memory: 4608Mi # = 2 * 2304Mi
90+
91+
# Enable events trace
92+
wsManager:
93+
eventTraceLogLocation: "/tmp/evts.json"
94+
95+
wsDaemon:
96+
hostWorkspaceArea: /mnt/disks/ssd0/workspaces
97+
volumes:
98+
- name: gcloud-tmp
99+
hostPath:
100+
path: /mnt/disks/ssd0/sync-tmp
101+
type: DirectoryOrCreate
102+
volumeMounts:
103+
- mountPath: /mnt/sync-tmp
104+
name: gcloud-tmp
105+
userNamespaces:
106+
fsShift: fuse
107+
shiftfsModuleLoader:
108+
enabled: false
109+
seccompProfileInstaller:
110+
enabled: true
111+
# in preview envs, we never want DaemonSets not to be scheduled (because they don't trigger scaleup)
112+
resources:
113+
cpu: 1m
114+
memory: 32Mi
115+
116+
wsScheduler:
117+
scaler:
118+
enabled: true
119+
controller:
120+
kind: "constant"
121+
constant:
122+
setpoint: 1
123+
124+
# Enable ws-proxy in dev
125+
wsProxy:
126+
useHTTPS: true
127+
hostHeader: Host
128+
name: "ws-proxy"
129+
disabled: false
130+
replicas: 1
131+
serviceType: "LoadBalancer"
132+
wsManagerProxy:
133+
enabled: true
134+
ports:
135+
wsManagerProxy:
136+
expose: true
137+
containerPort: 8081
138+
servicePort: 8081
139+
httpsProxy:
140+
expose: true
141+
containerPort: 9090
142+
servicePort: 443
143+
httpProxy:
144+
expose: true
145+
containerPort: 8080
146+
servicePort: 80
147+
148+
imageBuilder:
149+
disabled: true
150+
hostDindData: "/mnt/disks/ssd0/builder"
151+
# configure GCP registry
152+
registry:
153+
name: eu.gcr.io/gitpod-core-dev/registry
154+
secretName: gcp-sa-registry-auth
155+
path: gcp-sa-registry-auth
156+
registryCerts: []
157+
158+
159+
# configure GCP registry
160+
docker-registry:
161+
enabled: false
162+
163+
rabbitmq:
164+
# ensure shovels are configured on boot
165+
disabled: true
166+
enabled: false
167+
168+
cert-manager:
169+
enabled: true
170+
171+
dbMigrations:
172+
disabled: true
173+
174+
db:
175+
autoMigrate: false

0 commit comments

Comments
 (0)