fix: Allow KMS keys from other account

Author: gtoonstra

README.md

@@ -269,8 +269,10 @@ https://docs.datafold.com/datafold-deployment/dedicated-cloud/aws
 | <a name="input_backend_app_port"></a> [backend\_app\_port](#input\_backend\_app\_port) | The target port to use for the backend services | `number` | `80` | no |
 | <a name="input_backup_lifecycle_expiration_days"></a> [backup\_lifecycle\_expiration\_days](#input\_backup\_lifecycle\_expiration\_days) | Number of days after which clickhouse backup objects will expire and be deleted. | `number` | `6` | no |
 | <a name="input_ch_data_ebs_iops"></a> [ch\_data\_ebs\_iops](#input\_ch\_data\_ebs\_iops) | IOPS of EBS volume | `number` | `3000` | no |
+| <a name="input_ch_data_ebs_kms_key_arn"></a> [ch\_data\_ebs\_kms\_key\_arn](#input\_ch\_data\_ebs\_kms\_key\_arn) | Full ARN of an existing KMS key to encrypt the clickhouse data EBS volume with. Supports keys held in another account, provided the key policy grants this account use of the key. When null, the account default EBS encryption key is used. Must be a key ARN, not an alias: aliases do not resolve across accounts. Changing this on an existing volume forces replacement. | `string` | `null` | no |
 | <a name="input_ch_data_ebs_throughput"></a> [ch\_data\_ebs\_throughput](#input\_ch\_data\_ebs\_throughput) | Throughput of EBS volume | `number` | `1000` | no |
 | <a name="input_ch_logs_ebs_iops"></a> [ch\_logs\_ebs\_iops](#input\_ch\_logs\_ebs\_iops) | IOPS of EBS volume | `number` | `3000` | no |
+| <a name="input_ch_logs_ebs_kms_key_arn"></a> [ch\_logs\_ebs\_kms\_key\_arn](#input\_ch\_logs\_ebs\_kms\_key\_arn) | Full ARN of an existing KMS key to encrypt the clickhouse logs EBS volume with. Supports keys held in another account, provided the key policy grants this account use of the key. When null, the account default EBS encryption key is used. Must be a key ARN, not an alias: aliases do not resolve across accounts. Changing this on an existing volume forces replacement. | `string` | `null` | no |
 | <a name="input_ch_logs_ebs_throughput"></a> [ch\_logs\_ebs\_throughput](#input\_ch\_logs\_ebs\_throughput) | Throughput of EBS volume | `number` | `250` | no |
 | <a name="input_clickhouse_data_size"></a> [clickhouse\_data\_size](#input\_clickhouse\_data\_size) | EBS volume size for clickhouse data in GB | `number` | `40` | no |
 | <a name="input_clickhouse_logs_size"></a> [clickhouse\_logs\_size](#input\_clickhouse\_logs\_size) | EBS volume size for clickhouse logs in GB | `number` | `40` | no |
@@ -347,6 +349,7 @@ https://docs.datafold.com/datafold-deployment/dedicated-cloud/aws
 | <a name="input_rds_identifier"></a> [rds\_identifier](#input\_rds\_identifier) | Name of the RDS instance | `string` | `""` | no |
 | <a name="input_rds_instance"></a> [rds\_instance](#input\_rds\_instance) | EC2 insance type for PostgreSQL RDS database.<br/>Available instance groups: t3, m4, m5, r6i, m6i<br/>Available instance classes: medium and higher. | `string` | `"db.t3.medium"` | no |
 | <a name="input_rds_kms_key_alias"></a> [rds\_kms\_key\_alias](#input\_rds\_kms\_key\_alias) | RDS KMS key alias. | `string` | `"datafold-rds"` | no |
+| <a name="input_rds_kms_key_arn"></a> [rds\_kms\_key\_arn](#input\_rds\_kms\_key\_arn) | Full ARN of an existing KMS key to encrypt the RDS instance with. Supports keys held in another account, provided the key policy grants this account use of the key. Takes precedence over create\_rds\_kms\_key / rds\_kms\_key\_alias. Must be a key ARN, not an alias: aliases do not resolve across accounts. Changing this on an existing instance forces replacement. | `string` | `null` | no |
 | <a name="input_rds_maintenance_window"></a> [rds\_maintenance\_window](#input\_rds\_maintenance\_window) | RDS maintenance window | `string` | `"Mon:00:00-Mon:03:00"` | no |
 | <a name="input_rds_max_allocated_storage"></a> [rds\_max\_allocated\_storage](#input\_rds\_max\_allocated\_storage) | The upper limit the database can grow in GB | `number` | `100` | no |
 | <a name="input_rds_monitoring_interval"></a> [rds\_monitoring\_interval](#input\_rds\_monitoring\_interval) | RDS monitoring interval | `number` | `0` | no |
@@ -362,6 +365,7 @@ https://docs.datafold.com/datafold-deployment/dedicated-cloud/aws
 | <a name="input_rds_version"></a> [rds\_version](#input\_rds\_version) | Postgres RDS version to use. | `string` | `"15.5"` | no |
 | <a name="input_redis_data_size"></a> [redis\_data\_size](#input\_redis\_data\_size) | Redis EBS volume size in GB | `number` | `50` | no |
 | <a name="input_redis_ebs_iops"></a> [redis\_ebs\_iops](#input\_redis\_ebs\_iops) | IOPS of EBS redis volume | `number` | `3000` | no |
+| <a name="input_redis_ebs_kms_key_arn"></a> [redis\_ebs\_kms\_key\_arn](#input\_redis\_ebs\_kms\_key\_arn) | Full ARN of an existing KMS key to encrypt the redis data EBS volume with. Supports keys held in another account, provided the key policy grants this account use of the key. When null, the account default EBS encryption key is used. Must be a key ARN, not an alias: aliases do not resolve across accounts. Changing this on an existing volume forces replacement. | `string` | `null` | no |
 | <a name="input_redis_ebs_throughput"></a> [redis\_ebs\_throughput](#input\_redis\_ebs\_throughput) | Throughput of EBS redis volume | `number` | `125` | no |
 | <a name="input_s3_backup_bucket_name_override"></a> [s3\_backup\_bucket\_name\_override](#input\_s3\_backup\_bucket\_name\_override) | Bucket name override. | `string` | `""` | no |
 | <a name="input_s3_clickhouse_backup_tags"></a> [s3\_clickhouse\_backup\_tags](#input\_s3\_clickhouse\_backup\_tags) | The extra tags to be applied to the S3 clickhouse backup bucket | `map(any)` | `{}` | no |

main.tf

@@ -415,6 +415,16 @@ module "eks" {
   clickhouse_backup_bucket_arn = local.clickhouse_backup_bucket_arn
   service_account_prefix       = var.service_account_prefix
 
+  # The CSI driver needs explicit KMS permissions to attach volumes that are
+  # encrypted with a non-default CMK (e.g. customer-held cross-account keys).
+  ebs_csi_kms_cmk_arns = distinct([
+    for arn in [
+      var.ch_data_ebs_kms_key_arn,
+      var.ch_logs_ebs_kms_key_arn,
+      var.redis_ebs_kms_key_arn,
+    ] : arn if arn != null
+  ])
+
   deploy_temporal             = var.deploy_temporal
   temporal_backup_bucket_arn  = local.temporal_backup_bucket_arn
   temporal_postgres_namespace = var.temporal_postgres_namespace
@@ -445,6 +455,7 @@ module "database" {
   create_rds_kms_key                        = var.create_rds_kms_key
   rds_kms_key_alias                         = var.rds_kms_key_alias
   use_default_rds_kms_key                   = var.use_default_rds_kms_key
+  rds_kms_key_arn                           = var.rds_kms_key_arn
   database_name                             = var.database_name
   db_subnet_group_name                      = var.db_subnet_group_name
   db_parameter_group_name                   = var.db_parameter_group_name
@@ -486,6 +497,7 @@ resource "aws_ebs_volume" "clickhouse_data" {
   availability_zone = local.azs[var.az_index]
   size              = var.clickhouse_data_size
   encrypted         = true
+  kms_key_id        = var.ch_data_ebs_kms_key_arn
   type              = var.ebs_type
   iops              = var.ebs_type != "gp2" ? var.ch_data_ebs_iops : null
   throughput        = var.ebs_type != "gp2" ? var.ch_data_ebs_throughput : null
@@ -499,6 +511,7 @@ resource "aws_ebs_volume" "clickhouse_logs" {
   availability_zone = local.azs[var.az_index]
   size              = var.clickhouse_logs_size
   encrypted         = true
+  kms_key_id        = var.ch_logs_ebs_kms_key_arn
   type              = var.ebs_type
   iops              = var.ebs_type != "gp2" ? var.ch_logs_ebs_iops : null
   throughput        = var.ebs_type != "gp2" ? var.ch_logs_ebs_throughput : null
@@ -512,6 +525,7 @@ resource "aws_ebs_volume" "redis_data" {
   availability_zone = local.azs[var.az_index]
   size              = var.redis_data_size
   encrypted         = true
+  kms_key_id        = var.redis_ebs_kms_key_arn
   type              = var.ebs_type
   iops              = var.ebs_type != "gp2" ? var.redis_ebs_iops : null
   throughput        = var.ebs_type != "gp2" ? var.redis_ebs_throughput : null

modules/database/main.tf

@@ -29,7 +29,7 @@ module "db" {
   allocated_storage     = var.rds_allocated_storage
   max_allocated_storage = var.rds_max_allocated_storage
   storage_encrypted     = true
-  kms_key_id            = var.use_default_rds_kms_key ? null : data.aws_kms_key.rds.arn
+  kms_key_id            = var.use_default_rds_kms_key ? null : local.rds_kms_key_arn
 
   # NOTE: Do NOT use 'user' as the value for 'username' as it throws:
   # "Error creating DB Instance: InvalidParameterValue: MasterUsername
@@ -103,6 +103,8 @@ module "db" {
 locals {
   log_rds_automated_backups_replication_path = "${path.module}/../../logs/rds_automated_backups_replication.log"
   rds_password                               = var.rds_password_override != null ? var.rds_password_override : random_password.rds_master_password.result
+  rds_kms_key_arn                            = var.rds_kms_key_arn != null ? var.rds_kms_key_arn : one(data.aws_kms_key.rds[*].arn)
+  rds_kms_key_id                             = var.rds_kms_key_arn != null ? var.rds_kms_key_arn : one(data.aws_kms_key.rds[*].id)
 }
 
 # https://docs.aws.amazon.com/cli/latest/reference/rds/start-db-instance-automated-backups-replication.html
@@ -113,7 +115,7 @@ resource "null_resource" "rds-automated-backups-replication" {
     command = <<-EOT
       aws rds start-db-instance-automated-backups-replication \
         --source-db-instance-arn ${one(module.db[*].db_instance_arn)} \
-        --kms-key-id ${data.aws_kms_key.rds.id} \
+        --kms-key-id ${local.rds_kms_key_id} \
         --source-region ${var.provider_region} \
         --region ${var.rds_backups_replication_target_region} \
         --backup-retention-period ${var.rds_backups_replication_retention_period} \

modules/database/secrets.tf

@@ -1,18 +1,22 @@
-# KMS key for encrypting RDS instance
+# KMS key for encrypting RDS instance.
+# Not created (and the alias not looked up) when an external key ARN is
+# passed in via rds_kms_key_arn — aliases only resolve in the caller's
+# account, so cross-account keys must be referenced by key ARN.
 resource "aws_kms_key" "rds" {
-  count               = var.create_rds_kms_key ? 1 : 0
+  count               = var.create_rds_kms_key && var.rds_kms_key_arn == null ? 1 : 0
   multi_region        = true
   description         = "KMS key for RDS"
   enable_key_rotation = true
 }
 
 resource "aws_kms_alias" "rds" {
-  count         = var.create_rds_kms_key ? 1 : 0
+  count         = var.create_rds_kms_key && var.rds_kms_key_arn == null ? 1 : 0
   name          = "alias/${var.rds_kms_key_alias}"
   target_key_id = one(resource.aws_kms_key.rds[*].key_id)
 }
 
 data "aws_kms_key" "rds" {
+  count  = var.rds_kms_key_arn == null ? 1 : 0
   key_id = "alias/${var.rds_kms_key_alias}"
 
   depends_on = [

modules/database/variables.tf

@@ -120,6 +120,12 @@ variable "use_default_rds_kms_key" {
   description = "Flag weither or not to use the default RDS KMS encryption key. Not recommended to be used."
 }
 
+variable "rds_kms_key_arn" {
+  type        = string
+  default     = null
+  description = "Full ARN of an existing KMS key (possibly in another account) to encrypt the RDS instance with. Takes precedence over create_rds_kms_key / rds_kms_key_alias."
+}
+
 variable "db_subnet_group_name" {
   type        = string
   default     = ""

modules/eks/main.tf

@@ -6,6 +6,7 @@ module "ebs_csi_irsa_role" {
 
   name                  = "${var.deployment_name}-ebs-csi-controller"
   attach_ebs_csi_policy = true
+  ebs_csi_kms_cmk_arns  = var.ebs_csi_kms_cmk_arns
   use_name_prefix       = false
   policy_name           = "${var.deployment_name}-ebs-csi-controller"
 

modules/eks/variables.tf

@@ -226,3 +226,9 @@ variable "temporal_postgres_namespace" {
   default     = "temporal"
   description = "Kubernetes namespace where the Temporal PostgreSQL CRD (and postgres-pod service account) is deployed."
 }
+
+variable "ebs_csi_kms_cmk_arns" {
+  type        = list(string)
+  default     = []
+  description = "KMS CMK ARNs the EBS CSI driver may use for encrypted volumes (grants + encrypt/decrypt). Needed when volumes are encrypted with a non-default CMK, including keys held in another account."
+}

variables.tf

@@ -531,6 +531,17 @@ variable "use_default_rds_kms_key" {
   description = "Flag weither or not to use the default RDS KMS encryption key. Not recommended."
 }
 
+variable "rds_kms_key_arn" {
+  type        = string
+  default     = null
+  description = "Full ARN of an existing KMS key to encrypt the RDS instance with. Supports keys held in another account, provided the key policy grants this account use of the key. Takes precedence over create_rds_kms_key / rds_kms_key_alias. Must be a key ARN, not an alias: aliases do not resolve across accounts. Changing this on an existing instance forces replacement."
+
+  validation {
+    condition     = var.rds_kms_key_arn == null || can(regex("^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:key/", var.rds_kms_key_arn))
+    error_message = "rds_kms_key_arn must be a full KMS key ARN (arn:aws:kms:<region>:<account-id>:key/<key-id>), not an alias."
+  }
+}
+
 variable "database_name" {
   type        = string
   default     = "datafold"
@@ -661,6 +672,17 @@ variable "ch_data_ebs_throughput" {
   description = "Throughput of EBS volume"
 }
 
+variable "ch_data_ebs_kms_key_arn" {
+  type        = string
+  default     = null
+  description = "Full ARN of an existing KMS key to encrypt the clickhouse data EBS volume with. Supports keys held in another account, provided the key policy grants this account use of the key. When null, the account default EBS encryption key is used. Must be a key ARN, not an alias: aliases do not resolve across accounts. Changing this on an existing volume forces replacement."
+
+  validation {
+    condition     = var.ch_data_ebs_kms_key_arn == null || can(regex("^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:key/", var.ch_data_ebs_kms_key_arn))
+    error_message = "ch_data_ebs_kms_key_arn must be a full KMS key ARN (arn:aws:kms:<region>:<account-id>:key/<key-id>), not an alias."
+  }
+}
+
 variable "ch_logs_ebs_iops" {
   type        = number
   default     = 3000
@@ -673,6 +695,17 @@ variable "ch_logs_ebs_throughput" {
   description = "Throughput of EBS volume"
 }
 
+variable "ch_logs_ebs_kms_key_arn" {
+  type        = string
+  default     = null
+  description = "Full ARN of an existing KMS key to encrypt the clickhouse logs EBS volume with. Supports keys held in another account, provided the key policy grants this account use of the key. When null, the account default EBS encryption key is used. Must be a key ARN, not an alias: aliases do not resolve across accounts. Changing this on an existing volume forces replacement."
+
+  validation {
+    condition     = var.ch_logs_ebs_kms_key_arn == null || can(regex("^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:key/", var.ch_logs_ebs_kms_key_arn))
+    error_message = "ch_logs_ebs_kms_key_arn must be a full KMS key ARN (arn:aws:kms:<region>:<account-id>:key/<key-id>), not an alias."
+  }
+}
+
 variable "s3_backup_bucket_name_override" {
   type        = string
   default     = ""
@@ -701,6 +734,17 @@ variable "redis_ebs_throughput" {
   description = "Throughput of EBS redis volume"
 }
 
+variable "redis_ebs_kms_key_arn" {
+  type        = string
+  default     = null
+  description = "Full ARN of an existing KMS key to encrypt the redis data EBS volume with. Supports keys held in another account, provided the key policy grants this account use of the key. When null, the account default EBS encryption key is used. Must be a key ARN, not an alias: aliases do not resolve across accounts. Changing this on an existing volume forces replacement."
+
+  validation {
+    condition     = var.redis_ebs_kms_key_arn == null || can(regex("^arn:aws[a-z-]*:kms:[a-z0-9-]+:[0-9]{12}:key/", var.redis_ebs_kms_key_arn))
+    error_message = "redis_ebs_kms_key_arn must be a full KMS key ARN (arn:aws:kms:<region>:<account-id>:key/<key-id>), not an alias."
+  }
+}
+
 # ┏━╸╻┏ ┏━┓
 # ┣╸ ┣┻┓┗━┓
 # ┗━╸╹ ╹┗━┛