feat(ecs-cluster): Add task role output to airflow

openclimatefix · Mar 6, 2025 · 8f299b2 · 8f299b2
1 parent 6788b47
commit 8f299b2
Show file tree

Hide file tree

Showing 7 changed files with 28 additions and 8 deletions.
diff --git a/terraform/india/development/main.tf b/terraform/india/development/main.tf
@@ -450,7 +450,7 @@ module "airflow" {
   aws-vpc_id                = module.network.vpc_id
   aws-subnet_id             = module.network.public_subnet_ids[0]
   airflow-db-connection-url = "${module.postgres-rds.instance_connection_url}/airflow"
-  docker-compose-version    = "0.0.9"
+  docker-compose-version    = "0.0.10"
   ecs-subnet_id             = module.network.public_subnet_ids[0]
   ecs-security_group        = module.network.default_security_group_id
   ecs-execution_role_arn    = module.ecs-cluster.ecs_task_execution_role_arn

diff --git a/terraform/modules/ecs_cluster/iam.tf b/terraform/modules/ecs_cluster/iam.tf
@@ -149,19 +149,19 @@ resource "aws_iam_role_policy_attachment" "ecs-task-execution-role-policy-attach
 
 # Create role for ECS task running
 # * This needs S3 access but not secrets access
-resource "aws_iam_role" "ecs_task_role" {
-  name = "ecs-cluster_${var.name}_task-role"
+resource "aws_iam_role" "ecs_task_run_role" {
+  name = "ecs-cluster_${var.name}_task-run-role"
   path = "/ecs-cluster/${var.name}/"
   assume_role_policy = data.aws_iam_policy_document.ecs_assume_role_policy_document.json
 }
 
 # Attach policies to role
-resource "aws_iam_role_policy_attachment" "esc-task-role-policy-attachment-cloudwatch" {
-  role       = aws_iam_role.ecs_task_role.name
+resource "aws_iam_role_policy_attachment" "esc-task-run-role-policy-attachment-cloudwatch" {
+  role       = aws_iam_role.ecs_task_run_role.name
   policy_arn = aws_iam_policy.write_cloudwatch_policy.arn
 }
-resource "Aws_iam_role_policy_attachment" "ecs-task-role-policy-attachment-s3" {
-  role       = aws_iam_role.ecs_task_role.name
+resource "Aws_iam_role_policy_attachment" "ecs-task-run-role-policy-attachment-s3" {
+  role       = aws_iam_role.ecs_task_run_role.name
   policy_arn = aws_iam_policy.s3_policy.arn
 }
 
diff --git a/terraform/modules/ecs_cluster/output.tf b/terraform/modules/ecs_cluster/output.tf
@@ -5,3 +5,8 @@ output "ecs_cluster_arn" {
 output "ecs_task_execution_role_arn" {
   value = aws_iam_role.ecs_task_execution_role.arn
 }
+
+output "ecs_task_run_role_arn" {
+  value = aws_iam_role.ecs_task_run_role.arn
+}
+
diff --git a/terraform/modules/services/airflow/docker-compose.yml b/terraform/modules/services/airflow/docker-compose.yml
@@ -62,6 +62,7 @@ services:
       ECS_SUBNET: $ECS_SUBNET
       ECS_SECURITY_GROUP: $ECS_SECURITY_GROUP
       ECS_EXECUTION_ROLE_ARN: $ECS_EXECUTION_ROLE_ARN
+      ECS_TASK_ROLE_ARN: $ECS_TASK_ROLE_ARN
       AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
       AWS_OWNER_ID: $AWS_OWNER_ID
       ENVIRONMENT: $ENVIRONMENT
@@ -97,6 +98,7 @@ services:
       ECS_SUBNET: $ECS_SUBNET
       ECS_SECURITY_GROUP: $ECS_SECURITY_GROUP
       ECS_EXECUTION_ROLE_ARN: $ECS_EXECUTION_ROLE_ARN
+      ECS_TASK_ROLE_ARN: $ECS_TASK_ROLE_ARN
       AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
       AWS_OWNER_ID: $AWS_OWNER_ID
       ENVIRONMENT: $ENVIRONMENT
@@ -135,6 +137,7 @@ services:
       ECS_SUBNET: $ECS_SUBNET
       ECS_SECURITY_GROUP: $ECS_SECURITY_GROUP
       ECS_EXECUTION_ROLE_ARN: $ECS_EXECUTION_ROLE_ARN
+      ECS_TASK_ROLE_ARN: $ECS_TASK_ROLE_ARN
       AWS_DEFAULT_REGION: $AWS_DEFAULT_REGION
       AWS_OWNER_ID: $AWS_OWNER_ID
       ENVIRONMENT: $ENVIRONMENT

diff --git a/terraform/modules/services/airflow/eb.tf b/terraform/modules/services/airflow/eb.tf
@@ -127,6 +127,13 @@ resource "aws_elastic_beanstalk_environment" "eb-api-env" {
     resource  = ""
   }
 
+  setting {
+    namespace = "aws:elasticbeanstalk:application:environment"
+    name      = "ECS_TASK_ROLE_ARN"
+    value     = var.ecs-task_role_arn
+    resource  = ""
+  }
+
   setting {
     namespace = "aws:elasticbeanstalk:application:environment"
     name      = "AWS_OWNER_ID"

diff --git a/terraform/modules/services/airflow/variables.tf b/terraform/modules/services/airflow/variables.tf
@@ -52,6 +52,11 @@ variable "ecs-execution_role_arn" {
   type = string
 }
 
+variable "ecs-task_role_arn" {
+  description = "The role with which to run ecs tasks"
+  type = string
+}
+
 variable "docker-compose-version" {
   description = "The version of this for ocf. This helps bump the docker compose file"
   type = string

diff --git a/terraform/nowcasting/development/main.tf b/terraform/nowcasting/development/main.tf
@@ -905,7 +905,7 @@ module "airflow" {
   aws-vpc_id                 = module.networking.vpc_id
   aws-subnet_id              = module.networking.public_subnet_ids[0]
   airflow-db-connection-url  = module.database.forecast-database-secret-airflow-url
-  docker-compose-version     = "0.0.11"
+  docker-compose-version     = "0.0.12"
   ecs-subnet_id              = module.networking.public_subnet_ids[0]
   ecs-security_group         = module.networking.default_security_group_id
   ecs-execution_role_arn     = module.ecs.ecs_task_execution_role_arn