Fix scaleUpChron check for queue time using max_queue_time_minutes (#6618)

The parameter returned by the hud query `queued_jobs_aggregate`
`min_queue_time_minutes` means for the returned number of queued jobs,
what is the minimum of the jobs queue time. The parameter
`min_queue_time_minutes` in contrast the the one with the maximum queued
job waiting for a particular instance type.

Currently we've been filtering for `min_queue_time_minutes`, what
doesn't make a lot of sense. It does not add any additional
checks/protections and can introduce fatal failures. In case the query
have a divergent configuration from the lambda, say 10 minutes over 30
minutes used currently, and new jobs are always coming, the scaleUpChron
will never run.

So, as this is a fine check (it stills don't do exactly what we wanted
it to do, but, better than nothing): I kept the check but validate if at
least the longest queued job for an instance type is at least
`SCALE_UP_MAX_QUEUE_TIME_MINUTES` (default to 30). This still would make
it possible to overprovision in case of hud fails, but it is better than
nothing.

Author: jeanschmidt

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/config.ts

@@ -39,7 +39,7 @@ export class Config {
   readonly scaleConfigOrg: string;
   readonly scaleConfigRepo: string;
   readonly scaleConfigRepoPath: string;
-  readonly scaleUpMinQueueTimeMinutes: number;
+  readonly scaleUpMaxQueueTimeMinutes: number;
   readonly scaleUpChronRecordQueueUrl: string | undefined;
   readonly secretsManagerSecretsId: string | undefined;
   readonly sSMParamCleanupAgeDays: number;
@@ -98,8 +98,8 @@ export class Config {
     this.retryScaleUpRecordJitterPct = Number(process.env.RETRY_SCALE_UP_RECORD_JITTER_PCT || '0');
     this.retryScaleUpRecordQueueUrl = process.env.RETRY_SCALE_UP_CHRON_RECORD_QUEUE_URL;
     this.scaleUpChronRecordQueueUrl = process.env.SCALE_UP_CHRON_HUD_QUERY_URL;
-    this.scaleUpMinQueueTimeMinutes = process.env.SCALE_UP_MIN_QUEUE_TIME_MINUTES
-      ? Number(process.env.SCALE_UP_MIN_QUEUE_TIME_MINUTES)
+    this.scaleUpMaxQueueTimeMinutes = process.env.SCALE_UP_MAX_QUEUE_TIME_MINUTES
+      ? Number(process.env.SCALE_UP_MAX_QUEUE_TIME_MINUTES)
       : 30;
     this.runnerGroupName = process.env.RUNNER_GROUP_NAME;
     this.runnersExtraLabels = process.env.RUNNER_EXTRA_LABELS;

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-up-chron.test.ts

@@ -79,7 +79,7 @@ const runnerTypeInvalid = 'runner_type_invalid';
 
 const baseCfg = {
   scaleConfigOrg: 'test_org1',
-  scaleUpMinQueueTimeMinutes: 30,
+  scaleUpMaxQueueTimeMinutes: 30,
   scaleUpChronRecordQueueUrl: 'url',
 } as unknown as Config;
 

terraform-aws-github-runner/modules/runners/lambdas/runners/src/scale-runners/scale-up-chron.ts

@@ -34,7 +34,7 @@ export async function scaleUpChron(metrics: ScaleUpChronMetrics): Promise<void>
           runner.max_queue_time_minutes,
         );
         return (
-          runner.min_queue_time_minutes >= Config.Instance.scaleUpMinQueueTimeMinutes &&
+          runner.max_queue_time_minutes >= Config.Instance.scaleUpMaxQueueTimeMinutes &&
           runner.org === Config.Instance.scaleConfigOrg
         );
       })

terraform-aws-github-runner/modules/runners/scale-up-chron.tf

@@ -62,7 +62,7 @@ resource "aws_lambda_function" "scale_up_chron" {
       SCALE_CONFIG_REPO_PATH               = var.scale_config_repo_path
       SECRETSMANAGER_SECRETS_ID            = var.secretsmanager_secrets_id
       SCALE_UP_CHRON_HUD_QUERY_URL         = var.retry_scale_up_chron_hud_query_url
-      SCALE_UP_MIN_QUEUE_TIME_MINUTES      = 30
+      SCALE_UP_MAX_QUEUE_TIME_MINUTES      = 30
 
       AWS_REGIONS_TO_VPC_IDS = join(
         ",",