feat: Update S3 Logs SQS permissions to allow external account s3 logs bucket event notifications

CHANGELOG.md

@@ -3,6 +3,18 @@ All notable changes to this project will be documented in this file.
 
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [7.9.9] - 2025-02-25
+### Added
+- Update S3 Logs SQS permissions to allow multiple s3 logs buckets to report event notifications.
+
+## [7.9.8] - 2025-02-25
+### Added
+- Update S3 Logs SQS permissions to allow multiple s3 logs buckets to report event notifications.
+
+## [7.9.7] - 2025-02-26
+### Added
+- Update S3 Logs SQS permissions to allow multiple s3 logs buckets to report event notifications.
+
 ## [7.9.6] - 2025-02-26
 ### Fixed
 - Expiration will be 0 when the S3 lifecycle rule is disabled.

common.tf

@@ -41,6 +41,8 @@ locals {
   enable_apiary_s3_log_management = var.apiary_log_bucket == "" ? true : false
   enable_apiary_s3_log_hive       = var.apiary_log_bucket == "" && var.enable_apiary_s3_log_hive ? true : false
   apiary_s3_logs_bucket           = local.enable_apiary_s3_log_management ? "${local.apiary_bucket_prefix}-s3-logs" : ""
+  s3_logs_bucket                  = var.s3_logs_buckets_csv
+  s3_logs_bucket_list             = concat(["arn:aws:s3:::${local.apiary_s3_logs_bucket}"], [for bucket in split(",", var.s3_logs_buckets_csv):"arn:aws:s3:::${bucket}"])
   apiary_s3_hive_logs_bucket      = local.enable_apiary_s3_log_management ? "${local.apiary_s3_logs_bucket}-hive" : ""
   apiary_system_bucket            = "${local.apiary_bucket_prefix}-${replace(var.system_schema_name, "_", "-")}"
 

lf.tf

@@ -0,0 +1,14 @@
+/**
+ * Copyright (C) 2018-2025 Expedia, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ */
+
+resource "aws_lakeformation_resource" "apiary_data_bucket" {
+  for_each = var.create_lf_resource ? {
+    for schema in local.schemas_info : "${schema["schema_name"]}" => schema
+  } : {}
+  arn = aws_s3_bucket.apiary_data_bucket[each.key].arn
+
+  hybrid_access_enabled = true
+}

sns.tf

@@ -96,7 +96,12 @@ resource "aws_sqs_queue" "apiary_managed_logs_queue" {
       "Action": "sqs:SendMessage",
       "Resource": "arn:aws:sqs:*:*:${local.instance_alias}-s3-logs-queue",
       "Condition":{
-          "ArnEquals":{"aws:SourceArn":"arn:aws:s3:::${local.apiary_s3_logs_bucket}"}
+          "ArnEquals":{
+              "aws:SourceArn": [
+                  "arn:aws:s3:::${local.apiary_s3_logs_bucket}",
+                  "arn:aws:s3:::${local.s3_logs_bucket}"
+              ]
+          }
       }
     }
   ]

variables.tf

@@ -108,6 +108,12 @@ variable "s3_logs_sqs_receive_wait_time_seconds" {
   default     = 10
 }
 
+variable "s3_logs_buckets_csv" {
+  description = "This is the s3 logs buckets separated by comma where the s3 logs sqs should receive event notifications from f.i: 'bucketname1,bucketname2...'"
+  type        = string
+  default     = ""
+}
+
 variable "enable_hive_metastore_metrics" {
   description = "Enable sending Hive Metastore metrics to CloudWatch."
   type        = bool
@@ -585,6 +591,12 @@ variable "rw_ingress_cidr" {
   default     = []
 }
 
+variable "create_lf_resource" {
+  description = "Register data buckets in LakeFormation."
+  type        = bool
+  default     = false
+}
+
 variable "disable_glue_db_init" {
   description = "Glue databases are created programatically by default in hms-readwrite bootstrap init action. Setting this variable to true will disable the hms-readwrite bootstrap init action and create Glue databases via Terraform."
   type        = bool

version.tf

@@ -13,7 +13,7 @@ terraform {
     }
     aws = {
       source  = "hashicorp/aws"
-      version = "~> 4.0"
+      version = ">= 5.17.0"
     }
     external = {
       source  = "hashicorp/external"
@@ -28,7 +28,7 @@ terraform {
       version = "3.25.0"
     }
     template = {
-      source = "hashicorp/template"
+      source  = "hashicorp/template"
       version = "~> 2.2"
     }
   }