[single-cluster/aws] Install Calico as the CNI

Fixes #12953

This PR updates the EKS single cluster reference guides to install
calico as the CNI. This is important for the network policies
to work.

For GKE and AKS, There are already options to do the same which we
use already instead of doing it manually.

Signed-off-by: Tarun Pothulapati <[email protected]>

Author: Pothulapati

install/infra/modules/eks/kubernetes.tf

@@ -89,10 +89,6 @@ module "eks" {
       resolve_conflicts = "OVERWRITE"
     }
     kube-proxy = {}
-    vpc-cni = {
-      resolve_conflicts        = "OVERWRITE"
-      service_account_role_arn = module.vpc_cni_irsa.iam_role_arn
-    }
   }
 
   eks_managed_node_group_defaults = {
@@ -245,22 +241,6 @@ module "eks" {
   }
 }
 
-module "vpc_cni_irsa" {
-  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
-  version = "~> 4.12"
-
-  role_name_prefix      = "VPC-CNI-IRSA"
-  attach_vpc_cni_policy = true
-  vpc_cni_enable_ipv4   = true
-
-  oidc_providers = {
-    main = {
-      provider_arn               = module.eks.oidc_provider_arn
-      namespace_service_accounts = ["kube-system:aws-node"]
-    }
-  }
-}
-
 resource "null_resource" "kubeconfig" {
   depends_on = [module.eks]
   provisioner "local-exec" {

install/infra/modules/eks/local.tf

@@ -1,6 +1,6 @@
- locals {
-   aws_cert_manager_enabled = local.domain_name_enabled && var.use_aws_cert_manager == true
-   aws_cert_manager_count   = local.aws_cert_manager_enabled ? 1 : 0
-   domain_name_enabled      = var.domain_name != ""
-   domain_name_count        = local.domain_name_enabled ? 1 : 0
- }
+locals {
+  aws_cert_manager_enabled = local.domain_name_enabled && var.use_aws_cert_manager == true
+  aws_cert_manager_count   = local.aws_cert_manager_enabled ? 1 : 0
+  domain_name_enabled      = var.domain_name != ""
+  domain_name_count        = local.domain_name_enabled ? 1 : 0
+}

install/infra/modules/eks/providers.tf

@@ -1,8 +1,8 @@
 terraform {
   required_providers {
     aws = {
-        version = " ~> 3.0"
-        source = "registry.terraform.io/hashicorp/aws"
+      version = " ~> 3.0"
+      source  = "registry.terraform.io/hashicorp/aws"
     }
     helm = {
       source  = "hashicorp/helm"
@@ -12,5 +12,5 @@ terraform {
 }
 
 provider "aws" {
-  region  = var.region
+  region = var.region
 }

install/infra/modules/eks/variables.tf

@@ -43,7 +43,7 @@ variable "vpc_availability_zones" {
 }
 
 variable "domain_name" {
-  default = ""
+  default     = ""
   description = "Domain name to associate with the route53 zone"
 }
 

install/infra/modules/tools/aws-calico/main.tf

@@ -0,0 +1,19 @@
+variable "kubeconfig" {
+  description = "Path to the KUBECONFIG file to connect to the cluster"
+  default     = "./kubeconfig"
+}
+
+provider "helm" {
+  kubernetes {
+    config_path = var.kubeconfig
+  }
+}
+
+resource "helm_release" "calico" {
+  name             = "tigera-operator"
+  repository       = "https://projectcalico.docs.tigera.io/charts"
+  chart            = "tigera-operator"
+  namespace        = "tigera-operator"
+  version          = "v3.24.1"
+  create_namespace = true
+}

install/infra/modules/tools/azure-external-dns/main.tf

@@ -1,7 +1,7 @@
-variable settings {}
-variable domain_name { default = "test"}
-variable kubeconfig { default = "conf"}
-variable txt_owner_id { default = "nightly-test"}
+variable "settings" {}
+variable "domain_name" { default = "test" }
+variable "kubeconfig" { default = "conf" }
+variable "txt_owner_id" { default = "nightly-test" }
 
 provider "helm" {
   kubernetes {

install/infra/single-cluster/aws/Makefile

@@ -26,12 +26,16 @@ plan-cluster:
 	@terraform plan -target=module.eks
 
 .PHONY: plan-tools
-plan-tools: plan-cm-edns plan-cluster-issuer plan-cluster-autoscaler
+plan-tools: plan-calico plan-cm-edns plan-cluster-issuer plan-cluster-autoscaler
 
 .PHONY: plan-cluster-autoscaler
 plan-cluster-autoscaler:
 	@terraform plan -target=module.cluster-autoscaler
 
+.PHONY: plan-calico
+plan-calico:
+	@terraform plan -target=module.calico
+
 .PHONY: plan-cm-edns
 plan-cm-edns:
 	@terraform plan -target=module.certmanager -target=module.externaldns
@@ -45,7 +49,11 @@ apply-cluster:
 	@terraform apply -target=module.eks --auto-approve
 
 .PHONY: apply-tools
-apply-tools: install-cm-edns install-cluster-issuer install-cluster-autoscaler
+apply-tools: install-calico install-cm-edns install-cluster-issuer install-cluster-autoscaler
+
+.PHONY: install-calico
+install-calico:
+	@terraform apply -target=module.calico --auto-approve
 
 .PHONY: install-cluster-autoscaler
 install-cluster-autoscaler:
@@ -64,7 +72,11 @@ destroy-cluster:
 	@terraform destroy -target=module.eks --auto-approve
 
 .PHONY: destroy-tools
-destroy-tools: destroy-cluster-issuer destroy-cm-edns destroy-cluster-autoscaler
+destroy-tools: destroy-calico destroy-cluster-issuer destroy-cm-edns destroy-cluster-autoscaler
+
+.PHONY: destroy-calico
+destroy-calico:
+	@terraform destroy -target=module.calico --auto-approve
 
 .PHONY: destroy-cluster-autoscaler
 destroy-cluster-autoscaler:

install/infra/single-cluster/aws/output.tf

@@ -3,7 +3,7 @@ output "url" {
 }
 
 output "cluster_name" {
-  value     = var.cluster_name
+  value = var.cluster_name
 }
 
 output "registry_backend" {

install/infra/single-cluster/aws/tools.tf

@@ -29,3 +29,8 @@ module "cluster-autoscaler" {
   cluster_id        = module.eks.cluster_id
   oidc_provider_arn = module.eks.oidc_provider_arn
 }
+
+module "calico" {
+  source     = "../../modules/tools/aws-calico"
+  kubeconfig = var.kubeconfig
+}