add s3 option for vpc flow logs

defaults.tf

@@ -12,9 +12,9 @@ locals {
 
     # sensiblie defaults that can all be overridden
     log_destination_type = var.vpc_flow_logs.log_destination_type == null ? "cloud-watch-logs" : var.vpc_flow_logs.log_destination_type
-    retention_in_days    = var.vpc_flow_logs.retention_in_days == null ? 180 : var.vpc_flow_logs.retention_in_days
+    retention_in_days    = try(var.vpc_flow_logs.retention_in_days, null)
     traffic_type         = var.vpc_flow_logs.traffic_type == null ? "ALL" : var.vpc_flow_logs.traffic_type
-    destination_options = var.vpc_flow_logs.destination_options == null ? {
+    destination_options = can(var.vpc_flow_logs.destination_options) ? {
       file_format                = "plain-text"
       hive_compatible_partitions = false
       per_hour_partition         = false

examples/public_private_flow_logs/README.md

@@ -7,19 +7,22 @@ At this point, only cloud-watch logs are support, pending: https://github.com/aw
 
 ## Requirements
 
-No requirements.
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 0.15.0 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 3.73.0 |
 
 ## Providers
 
 | Name | Version |
 |------|---------|
-| <a name="provider_aws"></a> [aws](#provider\_aws) | n/a |
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 3.73.0 |
 
 ## Modules
 
 | Name | Source | Version |
 |------|--------|---------|
-| <a name="module_vpc"></a> [vpc](#module\_vpc) | ../.. | n/a |
+| <a name="module_vpc"></a> [vpc](#module\_vpc) | aws-ia/vpc/aws | >= 2.0.0 |
 
 ## Resources
 
@@ -32,6 +35,7 @@ No requirements.
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_kms_key_id"></a> [kms\_key\_id](#input\_kms\_key\_id) | KMS Key ID | `string` | `null` | no |
+| <a name="input_vpc_flow_logs"></a> [vpc\_flow\_logs](#input\_vpc\_flow\_logs) | Whether or not to create VPC flow logs and which type. Options: "cloudwatch", "s3", "none". | <pre>object({<br>    log_destination = optional(string)<br>    iam_role_arn    = optional(string)<br>    kms_key_id      = optional(string)<br><br>    log_destination_type = string<br>    retention_in_days    = optional(number)<br>    tags                 = optional(map(string))<br>    traffic_type         = optional(string)<br>    destination_options = optional(object({<br>      file_format                = optional(string)<br>      hive_compatible_partitions = optional(bool)<br>      per_hour_partition         = optional(bool)<br>    }))<br>  })</pre> | <pre>{<br>  "kms_key_id": null,<br>  "log_destination_type": "s3",<br>  "retention_in_days": null<br>}</pre> | no |
 
 ## Outputs
 

examples/public_private_flow_logs/main.tf

@@ -4,7 +4,7 @@ module "vpc" {
   source  = "aws-ia/vpc/aws"
   version = ">= 2.0.0"
 
-  name       = "tag-test"
+  name       = "public-private-flowlogs"
   cidr_block = "10.0.0.0/20"
   az_count   = 2
 
@@ -26,11 +26,7 @@ module "vpc" {
     }
   }
 
-  vpc_flow_logs = {
-    log_destination_type = "cloud-watch-logs"
-    retention_in_days    = 180
-    kms_key_id           = var.kms_key_id
-  }
+  vpc_flow_logs = var.vpc_flow_logs
 
   tags = {
     "key" = "value"

examples/public_private_flow_logs/outputs.tf

@@ -3,6 +3,13 @@ output "public_subnets" {
   value       = module.vpc.public_subnet_attributes_by_az
 }
 
+output "private_subnets" {
+  description = "Map of private subnet attributes grouped by az."
+  value       = module.vpc.private_subnet_attributes_by_az
+}
+
+## Used for Testing, do not delete
+
 output "public_subnets_tags_length" {
   description = "Count of public subnet tags for a single az."
   value       = length(module.vpc.public_subnet_attributes_by_az[data.aws_availability_zones.current.names[0]].tags)
@@ -12,8 +19,3 @@ output "private_subnets_tags_length" {
   description = "Count of private subnet tags for a single az."
   value       = length(module.vpc.private_subnet_attributes_by_az["private/${data.aws_availability_zones.current.names[0]}"].tags)
 }
-
-output "private_subnets" {
-  description = "Map of private subnet attributes grouped by az."
-  value       = module.vpc.private_subnet_attributes_by_az
-}

examples/public_private_flow_logs/providers.tf

@@ -0,0 +1,10 @@
+terraform {
+  required_version = ">= 0.15.0"
+  experiments      = [module_variable_optional_attrs]
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 3.73.0"
+    }
+  }
+}

examples/public_private_flow_logs/variables.tf

@@ -3,3 +3,28 @@ variable "kms_key_id" {
   type        = string
   default     = null
 }
+
+variable "vpc_flow_logs" {
+  description = "Whether or not to create VPC flow logs and which type. Options: \"cloudwatch\", \"s3\", \"none\"."
+
+  type = object({
+    log_destination = optional(string)
+    iam_role_arn    = optional(string)
+    kms_key_id      = optional(string)
+
+    log_destination_type = string
+    retention_in_days    = optional(number)
+    tags                 = optional(map(string))
+    traffic_type         = optional(string)
+    destination_options = optional(object({
+      file_format                = optional(string)
+      hive_compatible_partitions = optional(bool)
+      per_hour_partition         = optional(bool)
+    }))
+  })
+  default = {
+    log_destination_type = "s3"
+    retention_in_days    = null
+    kms_key_id           = null
+  }
+}

examples/transit_gateway/outputs.tf

@@ -1,3 +1,5 @@
+## Used for Testing, do not delete
+
 output "tgw_subnets_tags_length" {
   description = "Count of tgw subnet tags for a single az."
   value       = length(module.vpc.tgw_subnet_attributes_by_az[data.aws_availability_zones.current.names[0]].tags)

modules/flow_logs/main.tf

@@ -4,12 +4,12 @@ locals {
 
   # which log destination to use
   log_destination = local.create_flow_log_destination ? (
-    var.flow_log_defintion.log_destination_type == "cloud-watch-logs" ? module.cloudwatch_log_group[0].log_group.arn : null # change to s3 when implemented
+    var.flow_log_defintion.log_destination_type == "cloud-watch-logs" ? module.cloudwatch_log_group[0].log_group.arn : module.s3_log_bucket[0].bucket_flow_logs_attributes.arn # change to s3 when implemented
   ) : var.flow_log_defintion.log_destination
 
   # Use IAM from submodule if if not passed
   iam_role_arn = local.create_flow_log_destination ? (
-    var.flow_log_defintion.log_destination_type == "cloud-watch-logs" ? module.cloudwatch_log_group[0].iam_role.arn : null # change to s3 when implemented
+    var.flow_log_defintion.log_destination_type == "cloud-watch-logs" ? module.cloudwatch_log_group[0].iam_role.arn : null # s3: unnecessary, svc creates its own bucket policy
   ) : var.flow_log_defintion.iam_role_arn
 }
 
@@ -20,12 +20,20 @@ module "cloudwatch_log_group" {
   version = "1.0.0"
 
   name                  = var.name
-  retention_in_days     = var.flow_log_defintion.retention_in_days
+  retention_in_days     = var.flow_log_defintion.retention_in_days == null ? 180 : var.flow_log_defintion.retention_in_days
   kms_key_id            = var.flow_log_defintion.kms_key_id
   aws_service_principal = "vpc-flow-logs.amazonaws.com"
   tags                  = var.tags
 }
 
+module "s3_log_bucket" {
+  # if create destination and type = s3
+  count  = (local.create_flow_log_destination && var.flow_log_defintion.log_destination_type == "s3") ? 1 : 0
+  source = "./modules/s3_log_bucket"
+
+  name = var.name
+}
+
 resource "aws_flow_log" "main" {
   log_destination      = local.log_destination
   iam_role_arn         = local.iam_role_arn
@@ -34,12 +42,12 @@ resource "aws_flow_log" "main" {
   vpc_id               = var.vpc_id
 
   dynamic "destination_options" {
-    for_each = var.flow_log_defintion.log_destination_type == "s3" ? var.flow_log_defintion.destination_options : {}
+    for_each = var.flow_log_defintion.log_destination_type == "s3" ? [true] : []
 
     content {
-      file_format                = each.value.file_format
-      per_hour_partition         = each.value.per_hour_partition
-      hive_compatible_partitions = each.value.hive_compatible_partitions
+      file_format                = var.flow_log_defintion.destination_options.file_format
+      per_hour_partition         = var.flow_log_defintion.destination_options.per_hour_partition
+      hive_compatible_partitions = var.flow_log_defintion.destination_options.hive_compatible_partitions
     }
   }
 
@@ -48,4 +56,3 @@ resource "aws_flow_log" "main" {
     var.tags
   )
 }
-

modules/flow_logs/modules/s3_log_bucket/main.tf

@@ -0,0 +1,44 @@
+#tfsec:ignore:aws-s3-encryption-customer-key tfsec:ignore:aws-s3-enable-bucket-logging tfsec:ignore:aws-s3-enable-versioning
+resource "aws_s3_bucket" "flow_logs" {
+  bucket_prefix = "vpc-flow-logs-${var.name}"
+  force_destroy = true
+}
+
+resource "aws_s3_bucket_public_access_block" "flow_logs" {
+  bucket = aws_s3_bucket.flow_logs.bucket
+
+  block_public_acls       = true
+  block_public_policy     = true
+  ignore_public_acls      = true
+  restrict_public_buckets = true
+}
+
+#tfsec:ignore:aws-s3-encryption-customer-key
+resource "aws_s3_bucket_server_side_encryption_configuration" "flow_logs" {
+  bucket = aws_s3_bucket.flow_logs.bucket
+
+  rule {
+    apply_server_side_encryption_by_default {
+      sse_algorithm = "AES256"
+    }
+  }
+}
+
+resource "aws_s3_bucket_lifecycle_configuration" "flow_logs" {
+  bucket = aws_s3_bucket.flow_logs.bucket
+
+  rule {
+    id = "transition"
+
+    transition {
+      days          = 30
+      storage_class = "STANDARD_IA"
+    }
+
+    expiration {
+      days = 60
+    }
+
+    status = "Enabled"
+  }
+}

modules/flow_logs/modules/s3_log_bucket/outputs.tf

@@ -0,0 +1,3 @@
+output "bucket_flow_logs_attributes" {
+  value = aws_s3_bucket.flow_logs
+}

modules/flow_logs/modules/s3_log_bucket/providers.tf

@@ -0,0 +1,9 @@
+terraform {
+  required_version = ">= 0.15.0"
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = ">= 3.72.0"
+    }
+  }
+}

modules/flow_logs/modules/s3_log_bucket/variables.tf

@@ -0,0 +1,4 @@
+variable "name" {
+  type        = string
+  description = "(optional) describe your variable"
+}

test/examples_public_private_test.go

@@ -22,3 +22,23 @@ func TestExamplesPublicPrivate(t *testing.T) {
 	privateTagsLength := terraform.Output(t, terraformOptions, "private_subnets_tags_length")
 	assert.Equal(t, "2", privateTagsLength)
 }
+
+func TestExamplesPublicPrivateS3FlowLogs(t *testing.T) {
+
+	terraformOptions := &terraform.Options{
+		TerraformDir: "../examples/public_private_flow_logs",
+		Vars: map[string]interface{}{
+			"vpc_flow_logs": map[string]interface{}{
+				"log_destination_type": "s3",
+				"kms_key_id":           nil,
+				"desination_options": map[string]interface{}{
+					"file_format": "parquet",
+				},
+			},
+		},
+	}
+
+	defer terraform.Destroy(t, terraformOptions)
+	terraform.InitAndApply(t, terraformOptions)
+	terraform.ApplyAndIdempotent(t, terraformOptions)
+}