Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

save batches for india foreacst #758

Merged
merged 3 commits into from
Jan 28, 2025
Merged

save batches for india foreacst #758

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
361edea
save batches for india foreacst
peterdudfield Jan 28, 2025
333c5b1
fix
peterdudfield Jan 28, 2025
ee6e215
upgrade version
peterdudfield Jan 28, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
98 changes: 63 additions & 35 deletions terraform/india/production/main.tf
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
# 1.3 - ECS Cluster
# 2.0 - S3 bucket for NWP data
# 2.1 - S3 bucket for Satellite data
# 2.2 - S3 bucket for Forecast data
# 3.0 - Secret containing environment variables for the NWP consumer
# 3.1 - Secret containing environment variables for the Satellite consumer
# 3.2 - Secret containing HF read access
Expand Down Expand Up @@ -86,6 +87,16 @@ module "s3-satellite-bucket" {
lifecycled_prefixes = ["data"]
}

# 2.2
module "s3-forecast-bucket" {
source = "../../modules/storage/s3-private"
environment = local.environment
region = var.region
domain = local.domain
service_name = "forecast"
lifecycled_prefixes = [""]
}

# 3.0
resource "aws_secretsmanager_secret" "nwp_consumer_secret" {
name = "${local.environment}/data/nwp-consumer"
Expand Down Expand Up @@ -320,47 +331,59 @@ module "satellite_consumer_ecs" {
}


# 4.5 - Forecast - Client RU
# 4.5 - Forecast - Client RUVNL
module "forecast" {
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/forecast_generic?ref=f0ecf51"

region = var.region
environment = local.environment
app-name = "forecast"
ecs_config = {
docker_image = "openclimatefix/india_forecast_app"
docker_version = var.version-forecast
memory_mb = 4096
cpu = 1024
}
rds_config = {
database_secret_arn = module.postgres-rds.secret.arn
database_secret_read_policy_arn = module.postgres-rds.secret-policy.arn
}
s3_nwp_bucket = {
bucket_id = module.s3-nwp-bucket.bucket_id
bucket_read_policy_arn = module.s3-nwp-bucket.read_policy_arn
datadir = "ecmwf/data"
}
s3_satellite_bucket = {
bucket_id = module.s3-satellite-bucket.bucket_id
bucket_read_policy_arn = module.s3-satellite-bucket.read_policy_arn
datadir = "data"
}
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=7045011"

// this isnt really needed
s3_ml_bucket = {
bucket_id = module.s3-nwp-bucket.bucket_id
bucket_read_policy_arn = module.s3-nwp-bucket.read_policy_arn
}
loglevel = "INFO"
aws-region = var.region
aws-environment = local.environment

s3-buckets = [
{
id : module.s3-nwp-bucket.bucket_id,
access_policy_arn : module.s3-nwp-bucket.read_policy_arn
},
{
id : module.s3-forecast-bucket.bucket_id,
access_policy_arn : module.s3-forecast-bucket.write_policy_arn
}
]

ecs-task_name = "forecast"
ecs-task_type = "forecast"
ecs-task_execution_role_arn = module.ecs-cluster.ecs_task_execution_role_arn
sentry_dsn = var.sentry_dsn
ecs-task_size = {
memory = 3072
cpu = 1024
}

container-env_vars = [
{ "name" : "AWS_REGION", "value" : var.region },
{ "name" : "ENVIRONMENT", "value" : local.environment },
{ "name" : "LOGLEVEL", "value" : "INFO" },
{ "name" : "NWP_ECMWF_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/ecmwf/data/latest.zarr" },
{ "name" : "NWP_GFS_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/gfs/data/latest.zarr" },
{ "name" : "NWP_MO_GLOBAL_ZARR_PATH", "value": "s3://${module.s3-nwp-bucket.bucket_id}/metoffice/data/latest.zarr" },
{ "name" : "SENTRY_DSN", "value": var.sentry_dsn},
{ "name" : "USE_SATELLITE", "value": "False"},
{ "name" : "SAVE_BATCHES_DIR", "value": "s3://${module.s3-forecast-bucket.bucket_id}/RUVNL"}
]

container-secret_vars = [
{secret_policy_arn: module.postgres-rds.secret.arn,
values: ["DB_URL"]
}
]

container-tag = var.version-forecast
container-name = "india_forecast_app"
container-registry = "openclimatefix"
container-command = []
}

# 4.6 - Forecast - Client AD
module "forecast-ad" {
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=73255a4"
source = "github.com/openclimatefix/ocf-infrastructure//terraform/modules/services/ecs_task?ref=7045011"

aws-region = var.region
aws-environment = local.environment
Expand All @@ -373,6 +396,10 @@ module "forecast-ad" {
{
id : module.s3-nwp-bucket.bucket_id,
access_policy_arn : module.s3-nwp-bucket.write_policy_arn
},
{
id : module.s3-forecast-bucket.bucket_id,
access_policy_arn : module.s3-forecast-bucket.write_policy_arn
}
]

Expand All @@ -394,7 +421,8 @@ module "forecast-ad" {
{ "name" : "SATELLITE_ZARR_PATH", "value": "s3://${module.s3-satellite-bucket.bucket_id}/data/latest/iodc_latest.zarr.zip" },
{ "name" : "SENTRY_DSN", "value": var.sentry_dsn},
{ "name" : "USE_SATELLITE", "value": "True"},
{ "name" : "CLIENT_NAME", "value": "ad"}
{ "name" : "CLIENT_NAME", "value": "ad"},
{ "name" : "SAVE_BATCHES_DIR", "value": "s3://${module.s3-forecast-bucket.bucket_id}/ad"},
]

container-secret_vars = [
Expand Down