From f80fd43d34cf4a24aa1613ad63739f6ff545e817 Mon Sep 17 00:00:00 2001 From: Tarun Pothulapati Date: Thu, 7 Jul 2022 09:13:20 +0000 Subject: [PATCH] [local-preview] Support `127-0-0-1.nip.io` for `DOMAIN` Due to the way docker works in non-native platforms, It is very hard to have a consistent experience across all platforms as we can't just use the [docker bridge netwrok IP's in non-native platforms](https://docs.docker.com/desktop/networking/). This means that users have to search their Host IP, and use It to get up and working [which we tried, but understand that it's not a good UX](https://github.com/gitpod-io/website/pull/2349). But users can use `127-0-0-1.nip.io` as the DOMAIN which resolves to `127.0.0.1` and is available in all platforms as its `localhost`. This works well and good for all user communication but internal communication fails as `127-0-0-1.nip.io` for them is something else. So, This PR fixes that by adding new coredns `gitpod.db` coredns config essentially asking to route all `127-0-0-1.nip.io` to `proxy.default.svc.cluster.local`. [As k3s does not yet support overriding coredns config in a sane-way](https://github.com/k3s-io/k3s/issues/462) ,We instead skip the default coredns by adding `coredns.yaml.skip` file, and adding our own `custom-coredns.yaml` which is just plain `coredns.yaml` that comes with `k3s`, added with gitpod config. Signed-off-by: Tarun Pothulapati --- install/preview/entrypoint.sh | 9 +- install/preview/manifests/coredns.yaml | 214 +++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 4 deletions(-) create mode 100644 install/preview/manifests/coredns.yaml diff --git a/install/preview/entrypoint.sh b/install/preview/entrypoint.sh index 78a70eef1d08df..75440f6c65bad6 100755 --- a/install/preview/entrypoint.sh +++ b/install/preview/entrypoint.sh @@ -26,11 +26,9 @@ if [ "${total_cores}" -lt "${REQUIRED_CORES}" ]; then exit 1 fi -# Get container's IP address +# Set Domain to `127-0-0-1.nip.io` if not set if [ -z "${DOMAIN}" ]; then - NODE_IP=$(hostname -i) - DOMAIN_STRING=$(echo "${NODE_IP}" | sed "s/\./-/g") - DOMAIN="${DOMAIN_STRING}.nip.io" + DOMAIN="127-0-0-1.nip.io" fi echo "Gitpod Domain: $DOMAIN" @@ -133,6 +131,9 @@ for f in /var/lib/rancher/k3s/server/manifests/gitpod/*StatefulSet*.yaml; do yq # removing init container from ws-daemon (systemd and Ubuntu) yq eval-all -i 'del(.spec.template.spec.initContainers[0])' /var/lib/rancher/k3s/server/manifests/gitpod/*_DaemonSet_ws-daemon.yaml +touch /var/lib/rancher/k3s/server/manifests/coredns.yaml.skip +mv -f /app/manifests/coredns.yaml /var/lib/rancher/k3s/server/manifests/custom-coredns.yaml + for f in /var/lib/rancher/k3s/server/manifests/gitpod/*.yaml; do (cat "$f"; echo) >> /var/lib/rancher/k3s/server/manifests/gitpod.yaml; done rm -rf /var/lib/rancher/k3s/server/manifests/gitpod diff --git a/install/preview/manifests/coredns.yaml b/install/preview/manifests/coredns.yaml new file mode 100644 index 00000000000000..05478653cb2361 --- /dev/null +++ b/install/preview/manifests/coredns.yaml @@ -0,0 +1,214 @@ +# Copyright (c) 2022 Gitpod GmbH. All rights reserved. +# Licensed under the MIT License. See License-MIT.txt in the project root for license information. + +apiVersion: v1 +kind: ServiceAccount +metadata: + name: coredns + namespace: kube-system +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +rules: +- apiGroups: + - "" + resources: + - endpoints + - services + - pods + - namespaces + verbs: + - list + - watch +- apiGroups: + - discovery.k8s.io + resources: + - endpointslices + verbs: + - list + - watch +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + annotations: + rbac.authorization.kubernetes.io/autoupdate: "true" + labels: + kubernetes.io/bootstrapping: rbac-defaults + name: system:coredns +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: system:coredns +subjects: +- kind: ServiceAccount + name: coredns + namespace: kube-system +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: coredns + namespace: kube-system +data: + gitpod.db: | + ; 127-0-0-1.nip.io test file + 127-0-0-1.nip.io. IN SOA sns.dns.icann.org. noc.dns.icann.org. 2015082541 7200 3600 1209600 3600 + 127-0-0-1.nip.io. IN CNAME proxy.default.svc.cluster.local. + *.127-0-0-1.nip.io. IN CNAME proxy.default.svc.cluster.local. + *.ws.127-0-0-1.nip.io. IN CNAME proxy.default.svc.cluster.local. + Corefile: | + .:53 { + errors + health + ready + # extra configuration for `127-0-0-1.nip.io` + file /etc/coredns/gitpod.db 127-0-0-1.nip.io + kubernetes cluster.local in-addr.arpa ip6.arpa { + pods insecure + fallthrough in-addr.arpa ip6.arpa + } + hosts /etc/coredns/NodeHosts { + ttl 60 + reload 15s + fallthrough + } + prometheus :9153 + forward . /etc/resolv.conf + cache 30 + loop + reload + loadbalance + } +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system + labels: + k8s-app: kube-dns + kubernetes.io/name: "CoreDNS" +spec: + #replicas: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 1 + selector: + matchLabels: + k8s-app: kube-dns + template: + metadata: + labels: + k8s-app: kube-dns + spec: + priorityClassName: "system-cluster-critical" + serviceAccountName: coredns + tolerations: + - key: "CriticalAddonsOnly" + operator: "Exists" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + nodeSelector: + beta.kubernetes.io/os: linux + containers: + - name: coredns + image: rancher/mirrored-coredns-coredns:1.9.1 + imagePullPolicy: IfNotPresent + resources: + limits: + memory: 170Mi + requests: + cpu: 100m + memory: 70Mi + args: [ "-conf", "/etc/coredns/Corefile" ] + volumeMounts: + - name: config-volume + mountPath: /etc/coredns + readOnly: true + ports: + - containerPort: 53 + name: dns + protocol: UDP + - containerPort: 53 + name: dns-tcp + protocol: TCP + - containerPort: 9153 + name: metrics + protocol: TCP + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - NET_BIND_SERVICE + drop: + - all + readOnlyRootFilesystem: true + livenessProbe: + httpGet: + path: /health + port: 8080 + scheme: HTTP + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: 8181 + scheme: HTTP + initialDelaySeconds: 0 + periodSeconds: 2 + timeoutSeconds: 1 + successThreshold: 1 + failureThreshold: 3 + dnsPolicy: Default + volumes: + - name: config-volume + configMap: + name: coredns + items: + - key: gitpod.db + path: gitpod.db + - key: Corefile + path: Corefile + - key: NodeHosts + path: NodeHosts +--- +apiVersion: v1 +kind: Service +metadata: + name: kube-dns + namespace: kube-system + annotations: + prometheus.io/port: "9153" + prometheus.io/scrape: "true" + labels: + k8s-app: kube-dns + kubernetes.io/cluster-service: "true" + kubernetes.io/name: "CoreDNS" +spec: + selector: + k8s-app: kube-dns + clusterIP: 10.43.0.10 + ports: + - name: dns + port: 53 + protocol: UDP + - name: dns-tcp + port: 53 + protocol: TCP + - name: metrics + port: 9153 + protocol: TCP