[single-cluster/aws] Install Calico as the CNI

.werft/installer-tests.ts

@@ -99,6 +99,7 @@ const TEST_CONFIGURATIONS: { [name: string]: TestConfig } = {
         DESCRIPTION: `${op} an EKS cluster(version ${k8s_version})`,
         PHASES: [
             "STANDARD_EKS_CLUSTER",
+            "CALICO",
             "CERT_MANAGER",
             "EXTERNALDNS",
             "CLUSTER_ISSUER",
@@ -168,6 +169,11 @@ const INFRA_PHASES: { [name: string]: InfraConfig } = {
         makeTarget: "external-dns",
         description: `Deploys external-dns with ${cloud} provider`,
     },
+    CALICO: {
+        phase: "calico",
+        makeTarget: "calico",
+        description: `Deploys Calico`,
+    },
     ADD_NS_RECORD: {
         phase: "add-ns-record",
         makeTarget: "add-ns-record",

install/infra/modules/eks/kubernetes.tf

@@ -85,14 +85,11 @@ module "eks" {
   subnet_ids = module.vpc.public_subnets
 
   cluster_addons = {
-    coredns = {
-      resolve_conflicts = "OVERWRITE"
-    }
-    kube-proxy = {}
     vpc-cni = {
       resolve_conflicts        = "OVERWRITE"
       service_account_role_arn = module.vpc_cni_irsa.iam_role_arn
     }
+    kube-proxy = {}
   }
 
   eks_managed_node_group_defaults = {
@@ -116,122 +113,65 @@ module "eks" {
       service containerd restart
       EOT
   }
+}
 
-  eks_managed_node_groups = {
-    Services = {
-      enable_bootstrap_user_data = true
-      instance_types             = [var.service_machine_type]
-      name                       = "service-${var.cluster_name}"
-      iam_role_name              = format("%s-%s", substr("${var.cluster_name}-svc-ng", 0, 58), random_string.ng_role_suffix.result)
-      subnet_ids                 = module.vpc.public_subnets
-      min_size                   = 1
-      max_size                   = 4
-      desired_size               = 2
-      block_device_mappings = [{
-        device_name = "/dev/sda1"
-
-        ebs = [{
-          volume_size           = 300
-          volume_type           = "gp3"
-          throughput            = 500
-          iops                  = 6000
-          delete_on_termination = true
-        }]
-      }]
-      labels = {
-        "gitpod.io/workload_meta"               = true
-        "gitpod.io/workload_ide"                = true
-        "gitpod.io/workload_workspace_services" = true
-      }
-
-      tags = {
-        "k8s.io/cluster-autoscaler/enabled" = true
-        "k8s.io/cluster-autoscaler/gitpod"  = "owned"
-      }
-
-      pre_bootstrap_user_data = <<-EOT
-        #!/bin/bash
-        set -ex
-        cat <<-EOF > /etc/profile.d/bootstrap.sh
-        export CONTAINER_RUNTIME="containerd"
-        export USE_MAX_PODS=false
-        EOF
-        # Source extra environment variables in bootstrap script
-        sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh
-        EOT
-    }
+resource "null_resource" "kubeconfig" {
+  depends_on = [module.eks]
+  provisioner "local-exec" {
+    command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}"
+  }
 
-    RegularWorkspaces = {
-      instance_types = [var.workspace_machine_type]
-      name           = "ws-regular-${var.cluster_name}"
-      iam_role_name  = format("%s-%s", substr("${var.cluster_name}-regular-ws-ng", 0, 58), random_string.ng_role_suffix.result)
-      subnet_ids     = module.vpc.public_subnets
-      min_size       = 1
-      max_size       = 50
-      block_device_mappings = [{
-        device_name = "/dev/sda1"
-
-        ebs = [{
-          volume_size           = 512
-          volume_type           = "gp3"
-          throughput            = 500
-          iops                  = 6000
-          delete_on_termination = true
-        }]
-      }]
-      desired_size               = 2
-      enable_bootstrap_user_data = true
-      labels = {
-        "gitpod.io/workload_workspace_regular" = true
-      }
-
-      tags = {
-        "k8s.io/cluster-autoscaler/enabled" = true
-        "k8s.io/cluster-autoscaler/gitpod"  = "owned"
-      }
-
-      pre_bootstrap_user_data = <<-EOT
-        #!/bin/bash
-        set -ex
-        cat <<-EOF > /etc/profile.d/bootstrap.sh
-        export CONTAINER_RUNTIME="containerd"
-        export USE_MAX_PODS=false
-        EOF
-        # Source extra environment variables in bootstrap script
-        sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh
-        EOT
-    }
+  lifecycle {
+    create_before_destroy = true
+  }
+}
+
+// Install Calico Here
+
+module "service-nodes" {
+
+  source  = "terraform-aws-modules/eks/aws//modules/eks-managed-node-group"
+  version = "18.8.1"
+
+  cluster_name        = var.cluster_name
+  cluster_version     = var.cluster_version
+  cluster_endpoint    = module.eks.cluster_endpoint
+  cluster_auth_base64 = module.eks.cluster_certificate_authority_data
+  vpc_id              = module.vpc.vpc_id
+  iam_role_name       = format("%s-%s", substr("${var.cluster_name}-svc-ng", 0, 58), random_string.ng_role_suffix.result)
+  subnet_ids          = module.vpc.public_subnets
+  vpc_security_group_ids = [
+    module.eks.cluster_security_group_id,
+  ]
+
+  instance_types = [var.service_machine_type]
+  name           = "service-${var.cluster_name}"
+  min_size       = 1
+  max_size       = 4
+  desired_size   = 2
+  block_device_mappings = [{
+    device_name = "/dev/sda1"
+
+    ebs = [{
+      volume_size           = 300
+      volume_type           = "gp3"
+      throughput            = 500
+      iops                  = 6000
+      delete_on_termination = true
+    }]
+  }]
+  labels = {
+    "gitpod.io/workload_meta"               = true
+    "gitpod.io/workload_ide"                = true
+    "gitpod.io/workload_workspace_services" = true
+  }
+
+  tags = {
+    "k8s.io/cluster-autoscaler/enabled" = true
+    "k8s.io/cluster-autoscaler/gitpod"  = "owned"
+  }
 
-    HeadlessWorkspaces = {
-      instance_types = [var.workspace_machine_type]
-      name           = "ws-headless-${var.cluster_name}"
-      iam_role_name  = format("%s-%s", substr("${var.cluster_name}-headless-ws-ng", 0, 58), random_string.ng_role_suffix.result)
-      subnet_ids     = module.vpc.public_subnets
-      min_size       = 1
-      max_size       = 50
-      block_device_mappings = [{
-        device_name = "/dev/sda1"
-
-        ebs = [{
-          volume_size           = 512
-          volume_type           = "gp3"
-          throughput            = 500
-          iops                  = 6000
-          delete_on_termination = true
-        }]
-      }]
-      desired_size               = 2
-      enable_bootstrap_user_data = true
-      labels = {
-        "gitpod.io/workload_workspace_headless" = true
-      }
-
-      tags = {
-        "k8s.io/cluster-autoscaler/enabled" = true
-        "k8s.io/cluster-autoscaler/gitpod"  = "owned"
-      }
-
-      pre_bootstrap_user_data = <<-EOT
+  pre_bootstrap_user_data = <<-EOT
         #!/bin/bash
         set -ex
         cat <<-EOF > /etc/profile.d/bootstrap.sh
@@ -241,8 +181,6 @@ module "eks" {
         # Source extra environment variables in bootstrap script
         sed -i '/^set -o errexit/a\\nsource /etc/profile.d/bootstrap.sh' /etc/eks/bootstrap.sh
         EOT
-    }
-  }
 }
 
 module "vpc_cni_irsa" {
@@ -261,17 +199,6 @@ module "vpc_cni_irsa" {
   }
 }
 
-resource "null_resource" "kubeconfig" {
-  depends_on = [module.eks]
-  provisioner "local-exec" {
-    command = "aws eks update-kubeconfig --region ${var.region} --name ${var.cluster_name} --kubeconfig ${var.kubeconfig}"
-  }
-
-  lifecycle {
-    create_before_destroy = true
-  }
-}
-
 data "aws_iam_policy_document" "eks_policy" {
   statement {
     actions = [

install/infra/modules/eks/local.tf

@@ -1,6 +1,6 @@
- locals {
-   aws_cert_manager_enabled = local.domain_name_enabled && var.use_aws_cert_manager == true
-   aws_cert_manager_count   = local.aws_cert_manager_enabled ? 1 : 0
-   domain_name_enabled      = var.domain_name != ""
-   domain_name_count        = local.domain_name_enabled ? 1 : 0
- }
+locals {
+  aws_cert_manager_enabled = local.domain_name_enabled && var.use_aws_cert_manager == true
+  aws_cert_manager_count   = local.aws_cert_manager_enabled ? 1 : 0
+  domain_name_enabled      = var.domain_name != ""
+  domain_name_count        = local.domain_name_enabled ? 1 : 0
+}

install/infra/modules/eks/providers.tf

@@ -1,8 +1,8 @@
 terraform {
   required_providers {
     aws = {
-        version = " ~> 3.0"
-        source = "registry.terraform.io/hashicorp/aws"
+      version = " ~> 3.0"
+      source  = "registry.terraform.io/hashicorp/aws"
     }
     helm = {
       source  = "hashicorp/helm"
@@ -12,5 +12,5 @@ terraform {
 }
 
 provider "aws" {
-  region  = var.region
+  region = var.region
 }

install/infra/modules/eks/variables.tf

@@ -43,7 +43,7 @@ variable "vpc_availability_zones" {
 }
 
 variable "domain_name" {
-  default = ""
+  default     = ""
   description = "Domain name to associate with the route53 zone"
 }
 

install/infra/modules/tools/aws-calico/main.tf

@@ -0,0 +1,19 @@
+variable "kubeconfig" {
+  description = "Path to the KUBECONFIG file to connect to the cluster"
+  default     = "./kubeconfig"
+}
+
+provider "helm" {
+  kubernetes {
+    config_path = var.kubeconfig
+  }
+}
+
+resource "helm_release" "calico" {
+  name             = "tigera-operator"
+  repository       = "https://projectcalico.docs.tigera.io/charts"
+  chart            = "tigera-operator"
+  namespace        = "tigera-operator"
+  version          = "v3.24.1"
+  create_namespace = true
+}

install/infra/modules/tools/azure-external-dns/main.tf

@@ -1,7 +1,7 @@
-variable settings {}
-variable domain_name { default = "test"}
-variable kubeconfig { default = "conf"}
-variable txt_owner_id { default = "nightly-test"}
+variable "settings" {}
+variable "domain_name" { default = "test" }
+variable "kubeconfig" { default = "conf" }
+variable "txt_owner_id" { default = "nightly-test" }
 
 provider "helm" {
   kubernetes {

install/infra/single-cluster/aws/Makefile

@@ -26,12 +26,16 @@ plan-cluster:
 	@terraform plan -target=module.eks
 
 .PHONY: plan-tools
-plan-tools: plan-cm-edns plan-cluster-issuer plan-cluster-autoscaler
+plan-tools: plan-calico plan-cm-edns plan-cluster-issuer plan-cluster-autoscaler
 
 .PHONY: plan-cluster-autoscaler
 plan-cluster-autoscaler:
 	@terraform plan -target=module.cluster-autoscaler
 
+.PHONY: plan-calico
+plan-calico:
+	@terraform plan -target=module.calico
+
 .PHONY: plan-cm-edns
 plan-cm-edns:
 	@terraform plan -target=module.certmanager -target=module.externaldns
@@ -45,7 +49,11 @@ apply-cluster:
 	@terraform apply -target=module.eks --auto-approve
 
 .PHONY: apply-tools
-apply-tools: install-cm-edns install-cluster-issuer install-cluster-autoscaler
+apply-tools: install-calico install-cm-edns install-cluster-issuer install-cluster-autoscaler
+
+.PHONY: install-calico
+install-calico:
+	@terraform apply -target=module.calico --auto-approve
 
 .PHONY: install-cluster-autoscaler
 install-cluster-autoscaler:
@@ -64,7 +72,11 @@ destroy-cluster:
 	@terraform destroy -target=module.eks --auto-approve
 
 .PHONY: destroy-tools
-destroy-tools: destroy-cluster-issuer destroy-cm-edns destroy-cluster-autoscaler
+destroy-tools: destroy-calico destroy-cluster-issuer destroy-cm-edns destroy-cluster-autoscaler
+
+.PHONY: destroy-calico
+destroy-calico:
+	@terraform destroy -target=module.calico --auto-approve
 
 .PHONY: destroy-cluster-autoscaler
 destroy-cluster-autoscaler:

install/infra/single-cluster/aws/output.tf

@@ -3,7 +3,7 @@ output "url" {
 }
 
 output "cluster_name" {
-  value     = var.cluster_name
+  value = var.cluster_name
 }
 
 output "registry_backend" {

install/infra/single-cluster/aws/tools.tf

@@ -29,3 +29,8 @@ module "cluster-autoscaler" {
   cluster_id        = module.eks.cluster_id
   oidc_provider_arn = module.eks.oidc_provider_arn
 }
+
+module "calico" {
+  source     = "../../modules/tools/aws-calico"
+  kubeconfig = var.kubeconfig
+}

install/tests/Makefile

@@ -180,9 +180,15 @@ k3s-standard-cluster: check-env-cluster-version
 	@echo "Done creating k3s cluster"
 
 .PHONY:
-## cert-manager: Installs cert-manager, optionally create secret for cloud-dns access
-cert-manager:
+## calico: Installs calico
+calico:
 	$(MAKE) select-workspace && \
+	terraform apply -target=module.aws-calico -var kubeconfig=${KUBECONFIG} --auto-approve
+	@echo "Done installing Calico"
+
+.PHONY:
+## cert-manager: Installs cert-manager, optionally create secret for cloud-dns access
+cert-manager: check-env-cloud select-workspace
 	terraform apply -target=module.certmanager -var kubeconfig=${KUBECONFIG} --auto-approve
 	@echo "Done installing cert-manager"