Enable creation of S3 bucket for Flow Logs

Author: miguelaferreira

README.md

@@ -195,7 +195,8 @@ Sometimes it is handy to have public access to Redshift clusters (for example if
 This module is able to provision the collection of VPC Flow Logs by setting `enable_flow_log = true`.
 The [default behaviour](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/complete-vpc) will push the Flow Logs to a CloudWatch log group that gets created together the required IAM role.
 Through the module's arguments it is possible to pass in an existing [CloudWatch log group](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/vpc-flow-provided-cloudwatch-log-group) and/or an existing [IAM role](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/vpc-flow-provided-cloudwatch-role) to be used instead of creating these resources.
-It is also possible to pass in a [S3 bucket](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/vpc-flow-provided-s3-bucket) as the destination for the logs, instead of a CloudWatch log group.
+It is also possible to push FLow Logs to S3.
+To that end the module allows for a [S3 bucket](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/vpc-flow-provided-s3-bucket) to be proivided as the destination for the logs, or to [have it created by the module](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/vpc-flow-create-s3-bucket).
 
 ## Examples
 
@@ -206,6 +207,7 @@ It is also possible to pass in a [S3 bucket](https://github.com/terraform-aws-mo
 * [Flow Log CloudWatch log group provided](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/vpc-flow-provided-cloudwatch-log-group)
 * [Flow Log IAM role for CloudWatch provided](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/vpc-flow-provided-cloudwatch-role)
 * [Flow Log S3 bucket provided](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/vpc-flow-provided-s3-bucket)
+* [Flow Log create S3 bucket](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/vpc-flow-create-s3-bucket)
 * Few tests and edge cases examples: [#46](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/issue-46-no-private-subnets), [#44](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/issue-44-asymmetric-private-subnets), [#108](https://github.com/terraform-aws-modules/terraform-aws-vpc/tree/master/examples/issue-108-route-already-exists)
 
 <!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/vpc-flow-create-s3-bucket/README.md

@@ -0,0 +1,29 @@
+# VPC Flow Logs pushed to new S3 bucket
+
+Configuration in this directory creates set of VPC resources with FLow Logs enabled and configured to push to a new S3 bucket.
+
+## Usage
+
+To run this example you need to execute:
+
+```bash
+$ terraform init
+$ terraform plan
+$ terraform apply
+```
+
+Note that this example may create resources which can cost money (AWS Elastic IP, for example). Run `terraform destroy` when you don't need these resources.
+
+<!-- BEGINNING OF PRE-COMMIT-TERRAFORM DOCS HOOK -->
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| azs | A list of availability zones spefified as argument to this module |
+| nat\_public\_ips | List of public Elastic IPs created for AWS NAT Gateway |
+| private\_subnets | List of IDs of private subnets |
+| public\_subnets | List of IDs of public subnets |
+| vpc\_cidr\_block | The CIDR block of the VPC |
+| vpc\_id | The ID of the VPC |
+
+<!-- END OF PRE-COMMIT-TERRAFORM DOCS HOOK -->

examples/vpc-flow-create-s3-bucket/main.tf

@@ -0,0 +1,37 @@
+provider "aws" {
+  region = "eu-west-1"
+}
+
+data "aws_security_group" "default" {
+  name   = "default"
+  vpc_id = module.vpc.vpc_id
+}
+
+module "vpc" {
+  source = "../../"
+
+  name = "flow-log-provided-s3-bucket"
+
+  cidr = "10.0.0.0/16"
+
+  azs             = ["eu-west-1a", "eu-west-1b", "eu-west-1c"]
+  private_subnets = ["10.0.1.0/24", "10.0.2.0/24", "10.0.3.0/24"]
+  public_subnets  = ["10.0.101.0/24", "10.0.102.0/24", "10.0.103.0/24"]
+
+  assign_generated_ipv6_cidr_block = true
+
+  enable_nat_gateway = true
+  single_nat_gateway = true
+
+  enable_flow_log                      = true
+  create_flow_log_cloudwatch_log_group = false
+  create_flow_log_cloudwatch_iam_role  = false
+  create_flow_log_s3_bucket            = true
+  push_flow_log_to_s3                  = true
+  flow_log_force_destroy_s3_bucket     = true
+
+  tags = {
+    Owner       = "user"
+    Environment = "dev"
+  }
+}

examples/vpc-flow-create-s3-bucket/outputs.tf

@@ -0,0 +1,40 @@
+# VPC
+output "vpc_id" {
+  description = "The ID of the VPC"
+  value       = module.vpc.vpc_id
+}
+
+# CIDR blocks
+output "vpc_cidr_block" {
+  description = "The CIDR block of the VPC"
+  value       = module.vpc.vpc_cidr_block
+}
+
+//output "vpc_ipv6_cidr_block" {
+//  description = "The IPv6 CIDR block"
+//  value       = ["${module.vpc.vpc_ipv6_cidr_block}"]
+//}
+
+# Subnets
+output "private_subnets" {
+  description = "List of IDs of private subnets"
+  value       = module.vpc.private_subnets
+}
+
+output "public_subnets" {
+  description = "List of IDs of public subnets"
+  value       = module.vpc.public_subnets
+}
+
+# NAT gateways
+output "nat_public_ips" {
+  description = "List of public Elastic IPs created for AWS NAT Gateway"
+  value       = module.vpc.nat_public_ips
+}
+
+# AZs
+output "azs" {
+  description = "A list of availability zones spefified as argument to this module"
+  value       = module.vpc.azs
+}
+

examples/vpc-flow-provided-s3-bucket/main.tf

@@ -23,11 +23,11 @@ module "vpc" {
   enable_nat_gateway = true
   single_nat_gateway = true
 
-  enable_flow_log = true
+  enable_flow_log                      = true
   create_flow_log_cloudwatch_log_group = false
-  create_flow_log_cloudwatch_iam_role = false
-  flow_log_destination_type = "s3"
-  flow_log_destination_arn = aws_s3_bucket.vpf_flow_log.arn
+  create_flow_log_cloudwatch_iam_role  = false
+  push_flow_log_to_s3                  = true
+  flow_log_destination_arn             = aws_s3_bucket.vpf_flow_log.arn
 
   tags = {
     Owner       = "user"
@@ -38,6 +38,8 @@ module "vpc" {
 resource "aws_s3_bucket" "vpf_flow_log" {
   bucket = "aws-vpc-module-test-flow-logs"
   policy = data.aws_iam_policy_document.flow_log_s3.json
+
+  force_destroy = true
 }
 
 data "aws_iam_policy_document" "flow_log_s3" {

flow-logs.tf

@@ -4,9 +4,16 @@ locals {
 
   create_flow_log_cloudwatch_iam_role  = local.enable_flow_log && var.create_flow_log_cloudwatch_iam_role
   create_flow_log_cloudwatch_log_group = local.enable_flow_log && var.create_flow_log_cloudwatch_log_group
+  create_flow_log_s3_bucket            = local.enable_flow_log && var.create_flow_log_s3_bucket
+
+  flow_log_cloudwatch_destination = local.create_flow_log_cloudwatch_log_group ? join("", aws_cloudwatch_log_group.flow_log.*.arn) : var.flow_log_destination_arn
+  flow_log_s3_destination         = local.create_flow_log_s3_bucket ? join("", aws_s3_bucket.flow_log.*.arn) : var.flow_log_destination_arn
 
   flow_log_iam_role_arn = local.create_flow_log_cloudwatch_iam_role ? join("", aws_iam_role.vpc_flow_log_cloudwatch.*.arn) : var.flow_log_cloudwatch_iam_role_arn
-  flow_log_destination  = local.create_flow_log_cloudwatch_log_group ? join("", aws_cloudwatch_log_group.flow_log.*.arn) : var.flow_log_destination_arn
+
+  flow_log_destination = var.push_flow_log_to_s3 ? local.flow_log_s3_destination : local.flow_log_cloudwatch_destination
+
+  flow_log_destination_type = var.push_flow_log_to_s3 ? "s3" : "cloud-watch-logs"
 }
 
 ###################
@@ -15,7 +22,7 @@ locals {
 resource "aws_flow_log" "this" {
   count = local.enable_flow_log ? 1 : 0
 
-  log_destination_type = var.flow_log_destination_type
+  log_destination_type = local.flow_log_destination_type
   log_destination      = local.flow_log_destination
   iam_role_arn         = local.flow_log_iam_role_arn
   vpc_id               = local.vpc_id
@@ -94,3 +101,56 @@ data "aws_iam_policy_document" "flow_log_cloudwatch" {
     resources = ["*"]
   }
 }
+
+#############
+# Flow Log S3
+#############
+resource "aws_s3_bucket" "flow_log" {
+  count = local.create_flow_log_s3_bucket ? 1 : 0
+
+  bucket = "flow-log-${local.vpc_id}"
+  policy = join("", data.aws_iam_policy_document.flow_log_s3.*.json)
+
+  force_destroy = var.flow_log_force_destroy_s3_bucket
+
+  tags = merge(var.tags, var.vpc_flow_log_tags)
+}
+
+data "aws_iam_policy_document" "flow_log_s3" {
+  count = local.create_flow_log_s3_bucket ? 1 : 0
+
+  statement {
+    sid = "AWSLogDeliveryWrite"
+
+    principals {
+      type        = "Service"
+      identifiers = ["delivery.logs.amazonaws.com"]
+    }
+
+    effect = "Allow"
+
+    actions = [
+      "s3:PutObject",
+    ]
+
+    resources = [
+      "arn:aws:s3:::flow-log-${local.vpc_id}/*"]
+  }
+
+  statement {
+    sid = "AWSLogDeliveryAclCheck"
+
+    principals {
+      type        = "Service"
+      identifiers = ["delivery.logs.amazonaws.com"]
+    }
+
+    effect = "Allow"
+
+    actions = [
+      "s3:GetBucketAcl",
+    ]
+
+    resources = ["arn:aws:s3:::flow-log-${local.vpc_id}"]
+  }
+}

variables.tf

@@ -1402,10 +1402,10 @@ variable "flow_log_traffic_type" {
   default     = "ALL"
 }
 
-variable "flow_log_destination_type" {
-  description = "The type of the logging destination. Valid values: cloud-watch-logs, s3."
-  type        = string
-  default     = "cloud-watch-logs"
+variable "push_flow_log_to_s3" {
+  description = "Whether or not to change the default behaviour from pushing logs to CloudWatch to pushing to S3"
+  type        = bool
+  default     = false
 }
 
 variable "flow_log_destination_arn" {
@@ -1432,4 +1432,14 @@ variable "flow_log_cloudwatch_iam_role_arn" {
   default     = ""
 }
 
+variable "create_flow_log_s3_bucket" {
+  description = "Whether or not to create an S3 bucket to push the VPC Flow Logs to"
+  type        = bool
+  default     = false
+}
 
+variable "flow_log_force_destroy_s3_bucket" {
+  description = "Whether or not to force destroy the Flow Logs S3 bucket created by this module"
+  type        = bool
+  default     = false
+}