From 242276c61f450ed4d24465eeed92eb5959640753 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 13 Jan 2025 13:14:24 +0000 Subject: [PATCH 1/3] update ecmwf to new nwp-consumer --- terraform/nowcasting/production/main.tf | 33 +++++++++++--------- terraform/nowcasting/production/variables.tf | 5 +++ 2 files changed, 24 insertions(+), 14 deletions(-) diff --git a/terraform/nowcasting/production/main.tf b/terraform/nowcasting/production/main.tf index bdd51165..4f4a9545 100644 --- a/terraform/nowcasting/production/main.tf +++ b/terraform/nowcasting/production/main.tf @@ -191,7 +191,7 @@ module "nwp-metoffice" { # 3.3 module "nwp-ecmwf" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=26e3b29" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=2b68542" ecs-task_name = "nwp-consumer-ecmwf-uk" ecs-task_type = "consumer" @@ -209,26 +209,31 @@ module "nwp-ecmwf" { ] container-env_vars = [ + { "name" : "MODEL_REPOSITORY", "value" : "ecmwf-realtime" }, { "name" : "AWS_REGION", "value" : "eu-west-1" }, - { "name" : "AWS_S3_BUCKET", "value" : module.s3.s3-nwp-bucket.id }, + { "name" : "ECMWF_REALTIME_S3_REGION", "value": "eu-west-1" }, + { "name" : "ECMWF_REALTIME_S3_BUCKET", "value" : "ocf-ecmwf-production" }, + { "name" : "ZARRDIR", "value" : "s3://${module.s3.s3-nwp-bucket.id}/ecmwf/data" }, { "name" : "LOGLEVEL", "value" : "DEBUG" }, + { "name" : "SENTRY_DSN", "value" : var.sentry_dsn }, + { "name" : "CONCURRENCY", "value" : "false" }, + # legacy ones + { "name" : "AWS_S3_BUCKET", "value" : module.s3.s3-nwp-bucket.id }, { "name" : "ECMWF_AWS_REGION", "value": "eu-west-1" }, { "name" : "ECMWF_AWS_S3_BUCKET", "value" : "ocf-ecmwf-production" }, { "name" : "ECMWF_AREA", "value" : "uk" }, - { "name" : "SENTRY_DSN", "value" : var.sentry_dsn }, { "name" : "ENVIRONMENT", "value" : local.environment }, + { "name" : "SENTRY_DSN", "value" : var.sentry_dsn }, + { "name" : "LOGLEVEL", "value" : "DEBUG" } ] - container-secret_vars = ["ECMWF_AWS_ACCESS_KEY", "ECMWF_AWS_ACCESS_SECRET"] - container-tag = var.nwp_version - container-name = "openclimatefix/nwp-consumer" - container-command = [ - "download", - "--source=ecmwf-s3", - "--sink=s3", - "--rdir=ecmwf/raw", - "--zdir=ecmwf/data", - "--create-latest" + + container-secret_vars = [ + {secret_policy_arn: aws_secretsmanager_secret.nwp_consumer_secret.arn, + values: ["ECMWF_REALTIME_S3_ACCESS_KEY", "ECMWF_REALTIME_S3_ACCESS_SECRET"]} ] + container-tag = var.nwp_ecmwf_version + container-name = "openclimatefix/nwp-consumer" + container-command = ["consume"] } @@ -681,7 +686,7 @@ module "forecast_blend" { # 5.2 module "airflow" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/airflow?ref=4d6eadd" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/airflow?ref=2b68542" aws-environment = local.environment aws-domain = local.domain diff --git a/terraform/nowcasting/production/variables.tf b/terraform/nowcasting/production/variables.tf index 75678987..4246c8cb 100644 --- a/terraform/nowcasting/production/variables.tf +++ b/terraform/nowcasting/production/variables.tf @@ -18,6 +18,11 @@ variable "nwp_version" { description = "The NWP version" } +variable "nwp_ecmwf_version" { + description = "The ECMWF NWP version" + default="1.0.9" +} + variable "sat_version" { description = "The Satellite version" } From dd698ce9bdfb385856335bac30c53bf28eaa1f67 Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 13 Jan 2025 13:17:30 +0000 Subject: [PATCH 2/3] update --- terraform/nowcasting/production/main.tf | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/terraform/nowcasting/production/main.tf b/terraform/nowcasting/production/main.tf index 4f4a9545..e8c7dd7b 100644 --- a/terraform/nowcasting/production/main.tf +++ b/terraform/nowcasting/production/main.tf @@ -193,20 +193,22 @@ module "nwp-metoffice" { module "nwp-ecmwf" { source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=2b68542" - ecs-task_name = "nwp-consumer-ecmwf-uk" - ecs-task_type = "consumer" + ecs-task_name = "nwp-consumer-ecmwf-uk" + ecs-task_type = "consumer" ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn - aws-region = var.region - aws-environment = local.environment - aws-secretsmanager_secret_arn = aws_secretsmanager_secret.nwp_consumer_secret.arn + ecs-task_size = { + cpu = 512 + memory = 1024 + } - s3-buckets = [ - { - id : module.s3.s3-nwp-bucket.id - access_policy_arn : module.s3.iam-policy-s3-nwp-write.arn - } - ] + aws-region = var.region + aws-environment = local.environment + + s3-buckets = [{ + id : module.s3.s3-nwp-bucket.id + access_policy_arn : module.s3.iam-policy-s3-nwp-write.arn + }] container-env_vars = [ { "name" : "MODEL_REPOSITORY", "value" : "ecmwf-realtime" }, From b35ab665bd75e804672713c333b4a5ab2750d66b Mon Sep 17 00:00:00 2001 From: peterdudfield Date: Mon, 13 Jan 2025 16:17:35 +0000 Subject: [PATCH 3/3] update pvnet DA to use ecs task --- terraform/nowcasting/production/main.tf | 78 +++++++++++++++---------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/terraform/nowcasting/production/main.tf b/terraform/nowcasting/production/main.tf index e8c7dd7b..bd5e5b03 100644 --- a/terraform/nowcasting/production/main.tf +++ b/terraform/nowcasting/production/main.tf @@ -598,39 +598,55 @@ module "forecast_pvnet_ecwmf" { # 4.6 module "forecast_pvnet_day_ahead" { - source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=4d421e0" + source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=2b68542" - region = var.region - environment = local.environment - app-name = "forecast_pvnet_day_ahead" - ecs_config = { - docker_image = "openclimatefix/pvnet_app" - docker_version = var.forecast_pvnet_day_ahead_docker_version - memory_mb = 8192 - cpu = 2048 - } - rds_config = { - database_secret_arn = module.database.forecast-database-secret.arn - database_secret_read_policy_arn = module.database.iam-policy-forecast-db-read.arn - } - s3_ml_bucket = { - bucket_id = module.forecasting_models_bucket.bucket_id - bucket_read_policy_arn = module.forecasting_models_bucket.read_policy_arn - } - s3_nwp_bucket = { - bucket_id = module.s3.s3-nwp-bucket.id - bucket_read_policy_arn = module.s3.iam-policy-s3-nwp-read.arn - datadir = "data-metoffice" - } - s3_satellite_bucket = { - bucket_id = module.s3.s3-sat-bucket.id - bucket_read_policy_arn = module.s3.iam-policy-s3-sat-read.arn - datadir = "data/latest" - } - loglevel = "INFO" + aws-region = var.region + aws-environment = local.environment + + s3-buckets = [ + { + id : module.s3.s3-nwp-bucket.id, + access_policy_arn : module.s3.iam-policy-s3-nwp-read.arn + }, + { + id : module.s3.s3-sat-bucket.id, + access_policy_arn : module.s3.iam-policy-s3-sat-read.arn + } + + ] + + ecs-task_name = "forecast_pvnet_day_ahead" + ecs-task_type = "forecast" ecs-task_execution_role_arn = module.ecs.ecs_task_execution_role_arn - day_ahead_model = "true" - sentry_dsn = var.sentry_dsn + ecs-task_size = { + memory = 8192 + cpu = 2048 + } + + container-env_vars = [ + { "name" : "AWS_REGION", "value" : var.region }, + { "name" : "ENVIRONMENT", "value" : local.environment }, + { "name" : "LOGLEVEL", "value" : "INFO" }, + { "name" : "NWP_ECMWF_ZARR_PATH", "value": "s3://${module.s3.s3-nwp-bucket.id}/ecmwf/data/latest.zarr" }, + { "name" : "NWP_UKV_ZARR_PATH", "value":"s3://${module.s3.s3-nwp-bucket.id}/data-metoffice/latest.zarr"}, + { "name" : "SATELLITE_ZARR_PATH", "value":"s3://${module.s3.s3-sat-bucket.id}/data/latest/latest.zarr.zip"}, + { "name" : "SENTRY_DSN", "value": var.sentry_dsn}, + {"name": "USE_ADJUSTER", "value": "true"}, + {"name": "RUN_EXTRA_MODELS", "value": "false"}, + {"name": "DAY_AHEAD_MODEL", "value": "true"}, + {"name": "USE_OCF_DATA_SAMPLER", "value": "false"} + ] + + container-secret_vars = [ + {secret_policy_arn: module.database.forecast-database-secret.arn, + values: ["DB_URL"] + } + ] + + container-tag = var.forecast_pvnet_day_ahead_docker_version + container-name = "openclimatefix/pvnet_app" + container-registry = "docker.io" + container-command = [] } # 5.1