fix(ami-housekeeper): don't delete referenced AMIs in default config (#4623)

In 472cc5f the default config was
migrated to use SSM for AMI lookup. A parameter is created which stores
a reference to the AMI. By default, this parameter is called
`${var.ssm_paths.root}/${var.ssm_paths.config}/ami_id`.

The housekeeper is a process that looks for AMIs which can be deleted
because they're no longer used. It does this in a couple of ways:

1. Check the launch template for the AMI ID.
2. Check the SSM parameter.
3. Apply a threshold to not delete AMIs that are too new, according to
the config.

The problem is that we were looking for SSM parameters like this:

```typescript
const ssmParams = await ssmClient.send(
  new DescribeParametersCommand({
    ParameterFilters: [
      {
        Key: "Name",
        Values: ["ami-id"],
        Option: "Contains",
      },
    ],
  }),
);
```

i.e. we were looking for parameters which contain the hardcoded string
`ami-id`. This is different to the new default of `ami_id`. So we
weren't considering the right AMIs to be in use.

What would be a better approach would be to reference the values
dynamically. This means resolving from the template, and handling the
passed-in options, if there are any. We're documenting that we support
wildcards, so also support that here too.

The default value in the launch template became `resolve:ssm:<id or
AMI>`, so we need to make sure to ask EC2 to resolve for us when looking
up the template. In that way we get the actual AMI ID rather than the
alias.

This can be a bit challenging to understand, so the comments are
improved.

Comprehensive tests are added to try to ensure this all works as
expected.

Closes: #4571

---------

Co-authored-by: Niek Palm <[email protected]>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
Co-authored-by: Niek Palm <[email protected]>

Author: 3 people

examples/prebuilt/README.md

@@ -7,6 +7,18 @@ This module shows how to create GitHub action runners using a prebuilt AMI for t
 
 @@ Usages
 
+
+Steps for the full setup, such as creating a GitHub app can be found in the root module's [README](https://github.com/github-aws-runners/terraform-aws-github-runner). First download the Lambda releases from GitHub. Alternatively you can build the lambdas locally with Node or Docker, there is a simple build script in `<root>/.ci/build.sh`. In the `main.tf` you can simply remove the location of the lambda zip files, the default location will work in this case.
+
+> This example assumes local built lambda's available. Ensure you have built the lambda's. Alternatively you can download the lambda's. The version needs to be set to a GitHub release version, see https://github.com/github-aws-runners/terraform-aws-github-runner/releases
+
+```bash
+cd ../lambdas-download
+terraform init
+terraform apply -var=module_version=<VERSION>
+cd -
+```
+
 ### Packer Image
 
 You will need to build your image. This example deployment uses the image example in `/images/linux-amz2`. You must build this image with packer in your AWS account first. Once you have built this you need to provider your owner ID as a variable
@@ -92,6 +104,8 @@ terraform output webhook_secret
 | Name | Description | Type | Default | Required |
 |------|-------------|------|---------|:--------:|
 | <a name="input_ami_name_filter"></a> [ami\_name\_filter](#input\_ami\_name\_filter) | AMI name filter for the action runner AMI. By default amazon linux 2 is used. | `string` | `"github-runner-al2023-x86_64-*"` | no |
+| <a name="input_aws_region"></a> [aws\_region](#input\_aws\_region) | AWS region. | `string` | `"eu-west-1"` | no |
+| <a name="input_environment"></a> [environment](#input\_environment) | Environment name, used as prefix. | `string` | `null` | no |
 | <a name="input_github_app"></a> [github\_app](#input\_github\_app) | GitHub for API usages. | <pre>object({<br/>    id         = string<br/>    key_base64 = string<br/>  })</pre> | n/a | yes |
 | <a name="input_runner_os"></a> [runner\_os](#input\_runner\_os) | The EC2 Operating System type to use for action runner instances (linux,windows). | `string` | `"linux"` | no |
 

examples/prebuilt/main.tf

@@ -1,6 +1,6 @@
 locals {
-  environment = "prebuilt"
-  aws_region  = "eu-west-1"
+  environment = var.environment != null ? var.environment : "default"
+  aws_region  = var.aws_region
 }
 
 resource "random_id" "random" {
@@ -32,9 +32,12 @@ module "runners" {
     webhook_secret = random_id.random.hex
   }
 
-  webhook_lambda_zip                = "../lambdas-download/webhook.zip"
-  runner_binaries_syncer_lambda_zip = "../lambdas-download/runner-binaries-syncer.zip"
-  runners_lambda_zip                = "../lambdas-download/runners.zip"
+  # link to downloaded lambda zip files.
+  # When not explicitly set lambda zip files are grabbed from the module requiring lambda build.
+  #
+  # webhook_lambda_zip                = "../lambdas-download/webhook.zip"
+  # runner_binaries_syncer_lambda_zip = "../lambdas-download/runner-binaries-syncer.zip"
+  # runners_lambda_zip                = "../lambdas-download/runners.zip"
 
   runner_extra_labels = ["default", "example"]
 
@@ -56,6 +59,44 @@ module "runners" {
 
   # override scaling down
   scale_down_schedule_expression = "cron(* * * * ? *)"
+
+  enable_ami_housekeeper = true
+  ami_housekeeper_cleanup_config = {
+    ssmParameterNames = ["*/ami_id"]
+    minimumDaysOld    = 1
+    dryRun            = true
+    amiFilters = [
+      {
+        Name   = "name"
+        Values = ["*al2023*"]
+      }
+    ]
+  }
+
+  # variable "runners_ssm_housekeeper" {
+  #   description = <<EOF
+  #   Configuration for the SSM housekeeper lambda. This lambda deletes token / JIT config from SSM.
+
+  #   `schedule_expression`: is used to configure the schedule for the lambda.
+  #   `enabled`: enable or disable the lambda trigger via the EventBridge.
+  #   `lambda_memory_size`: lambda memery size limit.
+  #   `lambda_timeout`: timeout for the lambda in seconds.
+  #   `config`: configuration for the lambda function. Token path will be read by default from the module.
+  #   EOF
+  #   type = object({
+  #     schedule_expression = optional(string, "rate(1 day)")
+  #     enabled             = optional(bool, true)
+  #     lambda_memory_size  = optional(number, 512)
+  #     lambda_timeout      = optional(number, 60)
+  #     config = object({
+  #       tokenPath      = optional(string)
+  #       minimumDaysOld = optional(number, 1)
+  #       dryRun         = optional(bool, false)
+  #     })
+  #   })
+  #   default = { config = {} }
+
+  # log_level = "debug"
 }
 
 module "webhook_github_app" {

examples/prebuilt/variables.tf

@@ -7,6 +7,20 @@ variable "github_app" {
   })
 }
 
+variable "environment" {
+  description = "Environment name, used as prefix."
+
+  type    = string
+  default = null
+}
+
+variable "aws_region" {
+  description = "AWS region."
+
+  type    = string
+  default = "eu-west-1"
+}
+
 variable "runner_os" {
   description = "The EC2 Operating System type to use for action runner instances (linux,windows)."
 

images/.gitignore

@@ -0,0 +1,28 @@
+# Created by https://www.toptal.com/developers/gitignore/api/packer
+# Edit at https://www.toptal.com/developers/gitignore?templates=packer
+
+### Packer ###
+# Cache objects
+packer_cache/
+
+# Crash log
+crash.log
+
+# https://www.packer.io/guides/hcl/variables
+# Exclude all .pkrvars.hcl files, which are likely to contain sensitive data,
+# such as password, private keys, and other secrets. These should not be part of
+# version control as they are data points which are potentially sensitive and
+# subject to change depending on the environment.
+#
+*.pkrvars.hcl
+
+# For built boxes
+*.box
+
+### Packer Patch ###
+# ignore temporary output files
+output-*/
+
+# End of https://www.toptal.com/developers/gitignore/api/packer
+
+**/manifest.json