From 361edea263674b4d75788faaefb7536c1b68993f Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 28 Jan 2025 08:45:24 +0000 Subject: [PATCH 1/3] save batches for india foreacst --- terraform/india/production/main.tf | 94 +++++++++++++++++++----------- 1 file changed, 61 insertions(+), 33 deletions(-) diff --git a/terraform/india/production/main.tf b/terraform/india/production/main.tf index 65caa82d..28ea8d20 100644 --- a/terraform/india/production/main.tf +++ b/terraform/india/production/main.tf @@ -6,6 +6,7 @@ # 1.3 - ECS Cluster # 2.0 - S3 bucket for NWP data # 2.1 - S3 bucket for Satellite data +# 2.2 - S3 bucket for Forecast data # 3.0 - Secret containing environment variables for the NWP consumer # 3.1 - Secret containing environment variables for the Satellite consumer # 3.2 - Secret containing HF read access @@ -86,6 +87,16 @@ module "s3-satellite-bucket" { lifecycled_prefixes = ["data"] } +# 2.2 +module "s3-forecast-bucket" { + source = "../../modules/storage/s3-private" + environment = local.environment + region = var.region + domain = local.domain + service_name = "forecast" + lifecycled_prefixes = [""] +} + # 3.0 resource "aws_secretsmanager_secret" "nwp_consumer_secret" { name = "${local.environment}/data/nwp-consumer" @@ -320,42 +331,54 @@ module "satellite_consumer_ecs" { } -# 4.5 - Forecast - Client RU +# 4.5 - Forecast - Client RUVNL module "forecast" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=f0ecf51" - - region = var.region - environment = local.environment - app-name = "forecast" - ecs_config = { - docker_image = "openclimatefix/india_forecast_app" - docker_version = var.version-forecast - memory_mb = 4096 - cpu = 1024 - } - rds_config = { - database_secret_arn = module.postgres-rds.secret.arn - database_secret_read_policy_arn = module.postgres-rds.secret-policy.arn - } - s3_nwp_bucket = { - bucket_id = module.s3-nwp-bucket.bucket_id - bucket_read_policy_arn = module.s3-nwp-bucket.read_policy_arn - datadir = "ecmwf/data" - } - s3_satellite_bucket = { - bucket_id = module.s3-satellite-bucket.bucket_id - bucket_read_policy_arn = module.s3-satellite-bucket.read_policy_arn - datadir = "data" - } + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=73255a4" - // this isnt really needed - s3_ml_bucket = { - bucket_id = module.s3-nwp-bucket.bucket_id - bucket_read_policy_arn = module.s3-nwp-bucket.read_policy_arn - } - loglevel = "INFO" + aws-region = var.region + aws-environment = local.environment + + s3-buckets = [ + { + id : module.s3-nwp-bucket.bucket_id, + access_policy_arn : module.s3-nwp-bucket.read_policy_arn + }, + { + id : module.s3-forecast-bucket.bucket_id, + access_policy_arn : module.s3-forecast-bucket.write_policy_arn + } + ] + + ecs-task_name = "forecast" + ecs-task_type = "forecast" ecs-task_execution_role_arn = module.ecs-cluster.ecs_task_execution_role_arn - sentry_dsn = var.sentry_dsn + ecs-task_size = { + memory = 3072 + cpu = 1024 + } + + container-env_vars = [ + { "name" : "AWS_REGION", "value" : var.region }, + { "name" : "ENVIRONMENT", "value" : local.environment }, + { "name" : "LOGLEVEL", "value" : "INFO" }, + { "name" : "NWP_ECMWF_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/ecmwf/data/latest.zarr" }, + { "name" : "NWP_GFS_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/gfs/data/latest.zarr" }, + { "name" : "NWP_MO_GLOBAL_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/metoffice/data/latest.zarr" }, + { "name" : "SENTRY_DSN", "value": var.sentry_dsn}, + { "name" : "USE_SATELLITE", "value": "False"}, + { "name" : "SAVE_BATCHES_DIR", "value": "s3://${module.s3-forecast-bucket.bucket_id}/RUVNL"} + ] + + container-secret_vars = [ + {secret_policy_arn: module.postgres-rds.secret.arn, + values: ["DB_URL"] + } + ] + + container-tag = var.version-forecast + container-name = "india_forecast_app" + container-registry = "openclimatefix" + container-command = [] } # 4.6 - Forecast - Client AD @@ -373,6 +396,10 @@ module "forecast-ad" { { id : module.s3-nwp-bucket.bucket_id, access_policy_arn : module.s3-nwp-bucket.write_policy_arn + }, + { + id : module.s3-forecast-bucket.bucket_id, + access_policy_arn : module.s3-forecast-bucket.write_policy_arn } ] @@ -395,6 +422,7 @@ module "forecast-ad" { { "name" : "SENTRY_DSN", "value": var.sentry_dsn}, { "name" : "USE_SATELLITE", "value": "True"}, { "name" : "CLIENT_NAME", "value": "ad"} + { "name" : "SAVE_BATCHES_DIR", "value": "s3://${module.s3-forecast-bucket.bucket_id}/ad"}, ] container-secret_vars = [ From 333c5b19024cfe832905224883125b3396b4902d Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 28 Jan 2025 08:48:34 +0000 Subject: [PATCH 2/3] fix --- terraform/india/production/main.tf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/terraform/india/production/main.tf b/terraform/india/production/main.tf index 28ea8d20..1ecb1e07 100644 --- a/terraform/india/production/main.tf +++ b/terraform/india/production/main.tf @@ -421,7 +421,7 @@ module "forecast-ad" { { "name" : "SATELLITE_ZARR_PATH", "value": "s3://${module.s3-satellite-bucket.bucket_id}/data/latest/iodc_latest.zarr.zip" }, { "name" : "SENTRY_DSN", "value": var.sentry_dsn}, { "name" : "USE_SATELLITE", "value": "True"}, - { "name" : "CLIENT_NAME", "value": "ad"} + { "name" : "CLIENT_NAME", "value": "ad"}, { "name" : "SAVE_BATCHES_DIR", "value": "s3://${module.s3-forecast-bucket.bucket_id}/ad"}, ] From ee6e2157a6467644f740b0131afe2a711ca76069 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Tue, 28 Jan 2025 08:51:14 +0000 Subject: [PATCH 3/3] upgrade version --- terraform/india/production/main.tf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/terraform/india/production/main.tf b/terraform/india/production/main.tf index 1ecb1e07..d768d3f9 100644 --- a/terraform/india/production/main.tf +++ b/terraform/india/production/main.tf @@ -333,7 +333,7 @@ module "satellite_consumer_ecs" { # 4.5 - Forecast - Client RUVNL module "forecast" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=73255a4" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=7045011" aws-region = var.region aws-environment = local.environment @@ -383,7 +383,7 @@ module "forecast" { # 4.6 - Forecast - Client AD module "forecast-ad" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=73255a4" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=7045011" aws-region = var.region aws-environment = local.environment