From af008813048482d2fb39af043c1dea245c03a0a2 Mon Sep 17 00:00:00 2001 From: Tarun Pothulapati Date: Thu, 29 Sep 2022 11:30:38 +0000 Subject: [PATCH 1/4] [single-cluster/aws] Install Calico as the CNI Fixes https://github.com/gitpod-io/gitpod/issues/12953 This PR updates the EKS single cluster reference guides to install calico as the CNI. This is important for the network policies to work. For GKE and AKS, There are already options to do the same which we use already instead of doing it manually. Signed-off-by: Tarun Pothulapati --- install/infra/modules/eks/kubernetes.tf | 20 ------------------- install/infra/modules/eks/local.tf | 12 +++++------ install/infra/modules/eks/providers.tf | 6 +++--- install/infra/modules/eks/variables.tf | 2 +- .../infra/modules/tools/aws-calico/main.tf | 19 ++++++++++++++++++ .../modules/tools/azure-external-dns/main.tf | 8 ++++---- install/infra/single-cluster/aws/Makefile | 18 ++++++++++++++--- install/infra/single-cluster/aws/output.tf | 2 +- install/infra/single-cluster/aws/tools.tf | 5 +++++ 9 files changed, 54 insertions(+), 38 deletions(-) create mode 100644 install/infra/modules/tools/aws-calico/main.tf diff --git a/install/infra/modules/eks/kubernetes.tf b/install/infra/modules/eks/kubernetes.tf index c1a3e7ffd84ccd..8f2268127170ab 100644 --- a/install/infra/modules/eks/kubernetes.tf +++ b/install/infra/modules/eks/kubernetes.tf @@ -89,10 +89,6 @@ module "eks" { resolve_conflicts = "OVERWRITE" } kube-proxy = {} - vpc-cni = { - resolve_conflicts = "OVERWRITE" - service_account_role_arn = module.vpc_cni_irsa.iam_role_arn - } } eks_managed_node_group_defaults = { @@ -245,22 +241,6 @@ module "eks" { } } -module "vpc_cni_irsa" { - source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" - version = "~> 4.12" - - role_name_prefix = "VPC-CNI-IRSA" - attach_vpc_cni_policy = true - vpc_cni_enable_ipv4 = true - - oidc_providers = { - main = { - provider_arn = module.eks.oidc_provider_arn - namespace_service_accounts = ["kube-system:aws-node"] - } - } -} - resource "null_resource" "kubeconfig" { depends_on = [module.eks] provisioner "local-exec" { diff --git a/install/infra/modules/eks/local.tf b/install/infra/modules/eks/local.tf index 9e718f685cd27c..240d3594f71db9 100644 --- a/install/infra/modules/eks/local.tf +++ b/install/infra/modules/eks/local.tf @@ -1,6 +1,6 @@ - locals { - aws_cert_manager_enabled = local.domain_name_enabled && var.use_aws_cert_manager == true - aws_cert_manager_count = local.aws_cert_manager_enabled ? 1 : 0 - domain_name_enabled = var.domain_name != "" - domain_name_count = local.domain_name_enabled ? 1 : 0 - } +locals { + aws_cert_manager_enabled = local.domain_name_enabled && var.use_aws_cert_manager == true + aws_cert_manager_count = local.aws_cert_manager_enabled ? 1 : 0 + domain_name_enabled = var.domain_name != "" + domain_name_count = local.domain_name_enabled ? 1 : 0 +} diff --git a/install/infra/modules/eks/providers.tf b/install/infra/modules/eks/providers.tf index 7e1676c2f73eca..756bbed2c2fbba 100644 --- a/install/infra/modules/eks/providers.tf +++ b/install/infra/modules/eks/providers.tf @@ -1,8 +1,8 @@ terraform { required_providers { aws = { - version = " ~> 3.0" - source = "registry.terraform.io/hashicorp/aws" + version = " ~> 3.0" + source = "registry.terraform.io/hashicorp/aws" } helm = { source = "hashicorp/helm" @@ -12,5 +12,5 @@ terraform { } provider "aws" { - region = var.region + region = var.region } diff --git a/install/infra/modules/eks/variables.tf b/install/infra/modules/eks/variables.tf index d20b6b585b2ac2..e12c7ea0b2b7b4 100644 --- a/install/infra/modules/eks/variables.tf +++ b/install/infra/modules/eks/variables.tf @@ -43,7 +43,7 @@ variable "vpc_availability_zones" { } variable "domain_name" { - default = "" + default = "" description = "Domain name to associate with the route53 zone" } diff --git a/install/infra/modules/tools/aws-calico/main.tf b/install/infra/modules/tools/aws-calico/main.tf new file mode 100644 index 00000000000000..f31316cae91ea3 --- /dev/null +++ b/install/infra/modules/tools/aws-calico/main.tf @@ -0,0 +1,19 @@ +variable "kubeconfig" { + description = "Path to the KUBECONFIG file to connect to the cluster" + default = "./kubeconfig" +} + +provider "helm" { + kubernetes { + config_path = var.kubeconfig + } +} + +resource "helm_release" "calico" { + name = "tigera-operator" + repository = "https://projectcalico.docs.tigera.io/charts" + chart = "tigera-operator" + namespace = "tigera-operator" + version = "v3.24.1" + create_namespace = true +} diff --git a/install/infra/modules/tools/azure-external-dns/main.tf b/install/infra/modules/tools/azure-external-dns/main.tf index bc8a1d89dc845f..2be0a2d27aa2f7 100644 --- a/install/infra/modules/tools/azure-external-dns/main.tf +++ b/install/infra/modules/tools/azure-external-dns/main.tf @@ -1,7 +1,7 @@ -variable settings {} -variable domain_name { default = "test"} -variable kubeconfig { default = "conf"} -variable txt_owner_id { default = "nightly-test"} +variable "settings" {} +variable "domain_name" { default = "test" } +variable "kubeconfig" { default = "conf" } +variable "txt_owner_id" { default = "nightly-test" } provider "helm" { kubernetes { diff --git a/install/infra/single-cluster/aws/Makefile b/install/infra/single-cluster/aws/Makefile index 557aba55732dc5..19de38543b4f77 100644 --- a/install/infra/single-cluster/aws/Makefile +++ b/install/infra/single-cluster/aws/Makefile @@ -26,12 +26,16 @@ plan-cluster: @terraform plan -target=module.eks .PHONY: plan-tools -plan-tools: plan-cm-edns plan-cluster-issuer plan-cluster-autoscaler +plan-tools: plan-calico plan-cm-edns plan-cluster-issuer plan-cluster-autoscaler .PHONY: plan-cluster-autoscaler plan-cluster-autoscaler: @terraform plan -target=module.cluster-autoscaler +.PHONY: plan-calico +plan-calico: + @terraform plan -target=module.calico + .PHONY: plan-cm-edns plan-cm-edns: @terraform plan -target=module.certmanager -target=module.externaldns @@ -45,7 +49,11 @@ apply-cluster: @terraform apply -target=module.eks --auto-approve .PHONY: apply-tools -apply-tools: install-cm-edns install-cluster-issuer install-cluster-autoscaler +apply-tools: install-calico install-cm-edns install-cluster-issuer install-cluster-autoscaler + +.PHONY: install-calico +install-calico: + @terraform apply -target=module.calico --auto-approve .PHONY: install-cluster-autoscaler install-cluster-autoscaler: @@ -64,7 +72,11 @@ destroy-cluster: @terraform destroy -target=module.eks --auto-approve .PHONY: destroy-tools -destroy-tools: destroy-cluster-issuer destroy-cm-edns destroy-cluster-autoscaler +destroy-tools: destroy-calico destroy-cluster-issuer destroy-cm-edns destroy-cluster-autoscaler + +.PHONY: destroy-calico +destroy-calico: + @terraform destroy -target=module.calico --auto-approve .PHONY: destroy-cluster-autoscaler destroy-cluster-autoscaler: diff --git a/install/infra/single-cluster/aws/output.tf b/install/infra/single-cluster/aws/output.tf index b4c4942b4551d2..7da5efd99473b1 100644 --- a/install/infra/single-cluster/aws/output.tf +++ b/install/infra/single-cluster/aws/output.tf @@ -3,7 +3,7 @@ output "url" { } output "cluster_name" { - value = var.cluster_name + value = var.cluster_name } output "registry_backend" { diff --git a/install/infra/single-cluster/aws/tools.tf b/install/infra/single-cluster/aws/tools.tf index 657804c084a79c..6ce16a4d12dce9 100644 --- a/install/infra/single-cluster/aws/tools.tf +++ b/install/infra/single-cluster/aws/tools.tf @@ -29,3 +29,8 @@ module "cluster-autoscaler" { cluster_id = module.eks.cluster_id oidc_provider_arn = module.eks.oidc_provider_arn } + +module "calico" { + source = "../../modules/tools/aws-calico" + kubeconfig = var.kubeconfig +} From a6c18908b47e39f9498037afad92302b66e3f6cb Mon Sep 17 00:00:00 2001 From: Tarun Pothulapati Date: Mon, 3 Oct 2022 06:16:13 +0000 Subject: [PATCH 2/4] move them to infra tests Signed-off-by: Tarun Pothulapati --- .werft/installer-tests.ts | 6 ++++++ install/tests/Makefile | 10 ++++++++-- install/tests/main.tf | 31 ++++++++++++++++++------------- 3 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.werft/installer-tests.ts b/.werft/installer-tests.ts index f9235c40528602..60c88158ee6278 100644 --- a/.werft/installer-tests.ts +++ b/.werft/installer-tests.ts @@ -99,6 +99,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = { DESCRIPTION: `${op} an EKS cluster(version ${k8s_version})`, PHASES: [ "STANDARD_EKS_CLUSTER", + "CALICO", "CERT_MANAGER", "EXTERNALDNS", "CLUSTER_ISSUER", @@ -168,6 +169,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = { makeTarget: "external-dns", description: `Deploys external-dns with ${cloud} provider`, }, + CALICO: { + phase: "calico", + makeTarget: "calico", + description: `Deploys Calico`, + }, ADD_NS_RECORD: { phase: "add-ns-record", makeTarget: "add-ns-record", diff --git a/install/tests/Makefile b/install/tests/Makefile index 8910caa2a92901..dad6a520f455e0 100644 --- a/install/tests/Makefile +++ b/install/tests/Makefile @@ -180,9 +180,15 @@ k3s-standard-cluster: check-env-cluster-version @echo "Done creating k3s cluster" .PHONY: -## cert-manager: Installs cert-manager, optionally create secret for cloud-dns access -cert-manager: +## calico: Installs calico +calico: $(MAKE) select-workspace && \ + terraform apply -target=module.aws-calico -var kubeconfig=${KUBECONFIG} --auto-approve + @echo "Done installing Calico" + +.PHONY: +## cert-manager: Installs cert-manager, optionally create secret for cloud-dns access +cert-manager: check-env-cloud select-workspace terraform apply -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve @echo "Done installing cert-manager" diff --git a/install/tests/main.tf b/install/tests/main.tf index 4f47071e3d0705..22c6b9a808c523 100644 --- a/install/tests/main.tf +++ b/install/tests/main.tf @@ -13,7 +13,7 @@ variable "project" { default = "sh-automated-tests" } variable "sa_creds" { default = null } variable "dns_sa_creds" { default = null } -data local_file "dns_credentials" { +data "local_file" "dns_credentials" { filename = var.dns_sa_creds } @@ -62,10 +62,10 @@ module "k3s" { } module "gcp-issuer" { - source = "../infra/modules/tools/issuer" - kubeconfig = var.kubeconfig + source = "../infra/modules/tools/issuer" + kubeconfig = var.kubeconfig gcp_credentials = data.local_file.dns_credentials.content - issuer_name = "cloudDNS" + issuer_name = "cloudDNS" cert_manager_issuer = { project = "dns-for-playgrounds" serviceAccountSecretRef = { @@ -91,14 +91,14 @@ module "aks" { } module "eks" { - source = "../infra/modules/eks" - domain_name = "${var.TEST_ID}.${var.domain}" - cluster_name = var.TEST_ID - region = "eu-west-1" - vpc_availability_zones = ["eu-west-1c", "eu-west-1b"] - image_id = var.eks_node_image_id - kubeconfig = var.kubeconfig - cluster_version = var.cluster_version + source = "../infra/modules/eks" + domain_name = "${var.TEST_ID}.${var.domain}" + cluster_name = var.TEST_ID + region = "eu-west-1" + vpc_availability_zones = ["eu-west-1c", "eu-west-1b"] + image_id = var.eks_node_image_id + kubeconfig = var.kubeconfig + cluster_version = var.cluster_version create_external_registry = true create_external_database = true create_external_storage = true @@ -110,7 +110,7 @@ module "certmanager" { # source = "github.com/gitpod-io/gitpod//install/infra/terraform/tools/cert-manager?ref=main" source = "../infra/modules/tools/cert-manager" - kubeconfig = var.kubeconfig + kubeconfig = var.kubeconfig } module "clouddns-externaldns" { @@ -161,6 +161,11 @@ module "azure-add-dns-record" { domain_name = "${var.TEST_ID}.${var.domain}" } +module "aws-calico" { + source = "../infra/modules/tools/aws-calico" + kubeconfig = var.kubeconfig +} + module "aws-add-dns-record" { source = "../infra/modules/tools/cloud-dns-ns" credentials = var.dns_sa_creds From 1ee6db99b998a15345fbeb20e03c04bb27a79f23 Mon Sep 17 00:00:00 2001 From: Tarun Pothulapati Date: Mon, 3 Oct 2022 14:14:56 +0000 Subject: [PATCH 3/4] separate cluster creation and node creation Signed-off-by: Tarun Pothulapati --- install/infra/modules/eks/kubernetes.tf | 194 +++++++++--------------- 1 file changed, 72 insertions(+), 122 deletions(-) diff --git a/install/infra/modules/eks/kubernetes.tf b/install/infra/modules/eks/kubernetes.tf index 8f2268127170ab..8fca418642a611 100644 --- a/install/infra/modules/eks/kubernetes.tf +++ b/install/infra/modules/eks/kubernetes.tf @@ -88,6 +88,10 @@ module "eks" { coredns = { resolve_conflicts = "OVERWRITE" } + vpc-cni = { + resolve_conflicts = "OVERWRITE" + service_account_role_arn = module.vpc_cni_irsa.iam_role_arn + } kube-proxy = {} } @@ -112,122 +116,65 @@ module "eks" { service containerd restart EOT } +} - eks_managed_node_groups = { - Services = { - enable_bootstrap_user_data = true - instance_types = [var.service_machine_type] - name = "service-${var.cluster_name}" - iam_role_name = format("%s-%s", substr("${var.cluster_name}-svc-ng", 0, 58), random_string.ng_role_suffix.result) - subnet_ids = module.vpc.public_subnets - min_size = 1 - max_size = 4 - desired_size = 2 - block_device_mappings = [{ - device_name = "/dev/sda1" - - ebs = [{ - volume_size = 300 - volume_type = "gp3" - throughput = 500 - iops = 6000 - delete_on_termination = true - }] - }] - labels = { - "gitpod.io/workload_meta" = true - "gitpod.io/workload_ide" = true - "gitpod.io/workload_workspace_services" = true - } - - tags = { - "k8s.io/cluster-autoscaler/enabled" = true - "k8s.io/cluster-autoscaler/gitpod" = "owned" - } - - pre_bootstrap_user_data = <<-EOT - #!/bin/bash - set -ex - cat <<-EOF > /etc/profile.d/bootstrap.sh - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - EOF - # Source extra environment variables in bootstrap script - sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh - EOT - } +resource "null_resource" "kubeconfig" { + depends_on = [module.eks] + provisioner "local-exec" { + command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}" + } - RegularWorkspaces = { - instance_types = [var.workspace_machine_type] - name = "ws-regular-${var.cluster_name}" - iam_role_name = format("%s-%s", substr("${var.cluster_name}-regular-ws-ng", 0, 58), random_string.ng_role_suffix.result) - subnet_ids = module.vpc.public_subnets - min_size = 1 - max_size = 50 - block_device_mappings = [{ - device_name = "/dev/sda1" - - ebs = [{ - volume_size = 512 - volume_type = "gp3" - throughput = 500 - iops = 6000 - delete_on_termination = true - }] - }] - desired_size = 2 - enable_bootstrap_user_data = true - labels = { - "gitpod.io/workload_workspace_regular" = true - } - - tags = { - "k8s.io/cluster-autoscaler/enabled" = true - "k8s.io/cluster-autoscaler/gitpod" = "owned" - } - - pre_bootstrap_user_data = <<-EOT - #!/bin/bash - set -ex - cat <<-EOF > /etc/profile.d/bootstrap.sh - export CONTAINER_RUNTIME="containerd" - export USE_MAX_PODS=false - EOF - # Source extra environment variables in bootstrap script - sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh - EOT - } + lifecycle { + create_before_destroy = true + } +} + +// Install Calico Here + +module "service-nodes" { + depends_on = [module.eks] + + source = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group" + version = "18.30.0" + + cluster_name = var.cluster_name + cluster_version = var.cluster_version + cluster_endpoint = module.eks.cluster_endpoint + cluster_auth_base64 = module.eks.cluster_certificate_authority_data + + enable_bootstrap_user_data = true + instance_types = [var.service_machine_type] + name = "service-${var.cluster_name}" + create_iam_role = false + iam_role_arn = module.vpc_cni_irsa.iam_role_arn + iam_role_name = format("%s-%s", substr("${var.cluster_name}-svc-ng", 0, 58), random_string.ng_role_suffix.result) + subnet_ids = module.vpc.public_subnets + min_size = 1 + max_size = 4 + desired_size = 2 + block_device_mappings = [{ + device_name = "/dev/sda1" + + ebs = [{ + volume_size = 300 + volume_type = "gp3" + throughput = 500 + iops = 6000 + delete_on_termination = true + }] + }] + labels = { + "gitpod.io/workload_meta" = true + "gitpod.io/workload_ide" = true + "gitpod.io/workload_workspace_services" = true + } + + tags = { + "k8s.io/cluster-autoscaler/enabled" = true + "k8s.io/cluster-autoscaler/gitpod" = "owned" + } - HeadlessWorkspaces = { - instance_types = [var.workspace_machine_type] - name = "ws-headless-${var.cluster_name}" - iam_role_name = format("%s-%s", substr("${var.cluster_name}-headless-ws-ng", 0, 58), random_string.ng_role_suffix.result) - subnet_ids = module.vpc.public_subnets - min_size = 1 - max_size = 50 - block_device_mappings = [{ - device_name = "/dev/sda1" - - ebs = [{ - volume_size = 512 - volume_type = "gp3" - throughput = 500 - iops = 6000 - delete_on_termination = true - }] - }] - desired_size = 2 - enable_bootstrap_user_data = true - labels = { - "gitpod.io/workload_workspace_headless" = true - } - - tags = { - "k8s.io/cluster-autoscaler/enabled" = true - "k8s.io/cluster-autoscaler/gitpod" = "owned" - } - - pre_bootstrap_user_data = <<-EOT + pre_bootstrap_user_data = <<-EOT #!/bin/bash set -ex cat <<-EOF > /etc/profile.d/bootstrap.sh @@ -237,18 +184,21 @@ module "eks" { # Source extra environment variables in bootstrap script sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh EOT - } - } } -resource "null_resource" "kubeconfig" { - depends_on = [module.eks] - provisioner "local-exec" { - command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}" - } +module "vpc_cni_irsa" { + source = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks" + version = "~> 4.12" - lifecycle { - create_before_destroy = true + role_name_prefix = "VPC-CNI-IRSA" + attach_vpc_cni_policy = true + vpc_cni_enable_ipv4 = true + + oidc_providers = { + main = { + provider_arn = module.eks.oidc_provider_arn + namespace_service_accounts = ["kube-system:aws-node"] + } } } From 981ba0920b82f5b6d48198924a474f7948524802 Mon Sep 17 00:00:00 2001 From: Tarun Pothulapati Date: Mon, 3 Oct 2022 19:02:03 +0000 Subject: [PATCH 4/4] assign a iam_arn for node Signed-off-by: Tarun Pothulapati --- install/infra/modules/eks/kubernetes.tf | 29 +++++++++++-------------- 1 file changed, 13 insertions(+), 16 deletions(-) diff --git a/install/infra/modules/eks/kubernetes.tf b/install/infra/modules/eks/kubernetes.tf index 8fca418642a611..8290f4de6c069e 100644 --- a/install/infra/modules/eks/kubernetes.tf +++ b/install/infra/modules/eks/kubernetes.tf @@ -85,9 +85,6 @@ module "eks" { subnet_ids = module.vpc.public_subnets cluster_addons = { - coredns = { - resolve_conflicts = "OVERWRITE" - } vpc-cni = { resolve_conflicts = "OVERWRITE" service_account_role_arn = module.vpc_cni_irsa.iam_role_arn @@ -132,26 +129,26 @@ resource "null_resource" "kubeconfig" { // Install Calico Here module "service-nodes" { - depends_on = [module.eks] source = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group" - version = "18.30.0" + version = "18.8.1" cluster_name = var.cluster_name cluster_version = var.cluster_version cluster_endpoint = module.eks.cluster_endpoint cluster_auth_base64 = module.eks.cluster_certificate_authority_data - - enable_bootstrap_user_data = true - instance_types = [var.service_machine_type] - name = "service-${var.cluster_name}" - create_iam_role = false - iam_role_arn = module.vpc_cni_irsa.iam_role_arn - iam_role_name = format("%s-%s", substr("${var.cluster_name}-svc-ng", 0, 58), random_string.ng_role_suffix.result) - subnet_ids = module.vpc.public_subnets - min_size = 1 - max_size = 4 - desired_size = 2 + vpc_id = module.vpc.vpc_id + iam_role_name = format("%s-%s", substr("${var.cluster_name}-svc-ng", 0, 58), random_string.ng_role_suffix.result) + subnet_ids = module.vpc.public_subnets + vpc_security_group_ids = [ + module.eks.cluster_security_group_id, + ] + + instance_types = [var.service_machine_type] + name = "service-${var.cluster_name}" + min_size = 1 + max_size = 4 + desired_size = 2 block_device_mappings = [{ device_name = "/dev/sda1"