Terragrunt config export

.github/workflows/terragrunt-config-export.yml

@@ -0,0 +1,38 @@
+name: Terragrunt Config Export
+
+on:
+  push:
+    branches:
+      - master
+    paths:
+      - 'terragrunt-config-export/**'
+  pull_request:
+    branches:
+      - '*'
+    paths:
+      - 'terragrunt-config-export/**'
+
+jobs:
+  build-and-push-docker:
+    runs-on: ubuntu-latest
+    if: github.ref == 'refs/heads/master'
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Build and push Docker image
+        uses: docker/build-push-action@v5
+        with:
+          context: terragrunt-config-export
+          file: terragrunt-config-export/Dockerfile
+          push: true
+          tags: sykescottages/bitbucket-pipes:terragrunt-config-export

terragrunt-config-export/.gitignore

@@ -0,0 +1 @@
+.idea

terragrunt-config-export/Dockerfile

@@ -0,0 +1,12 @@
+FROM python:3.9-slim
+
+WORKDIR /app
+
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
+
+COPY main.py .
+
+RUN chmod +x main.py
+
+ENTRYPOINT ["python", "/app/main.py"]

terragrunt-config-export/README.md

@@ -0,0 +1,82 @@
+# Bitbucket Pipe: Terragrunt config export
+
+This pipe retrieves all the config needed to pass into a helm chart to deploy the sandbox environments.
+## YAML Definition
+
+Add the following to your `bitbucket-pipelines.yml` file:
+
+```yaml
+- pipe: 
+  variables:
+    ECS_CLUSTER: 'my-ecs-cluster'
+    ECS_SERVICE: 'my-ecs-service'
+    AWS_OIDC_ROLE_ARN: 'arn:aws:iam::account-id:role/role-name'
+    # Optional variables
+    EXTRA_ENV: #Extra env vars to include in the config
+       BASE_URL: example.com
+    ENDPOINTS: ['authenticate']
+    IAM_ROLE: 'arn:aws:iam::account-id:role/role-name'
+    AWS_PROFILE: 'staging' 
+    OUTPUT_FILE: 'values.yml' 
+    AWS_REGION: 'eu-west-1'
+```
+
+## Variables
+
+| Variable | Usage                                 | Required |
+| -------- |---------------------------------------| -------- |
+| ECS_CLUSTER | Name of the ECS cluster               | Yes |
+| ECS_SERVICE | Name of the ECS service               | Yes |
+| AWS_OIDC_ROLE_ARN | OIDC Role to assume                   | Yes |
+| EXTRA_ENV | Extra environment vars for the sevice | No |
+| ENDPOINTS | Endpoints for the target groups       | No |
+| IAM_ROLE | IAM role for the service to use       | No |
+| AWS_PROFILE | Profile to use to get the config      | No |
+| OUTPUT_FILE | File to write the config to           | No |
+| AWS_REGION | AWS region                            | No |
+
+## Examples
+
+### Basic Usage
+
+```yaml
+- pipe: your-docker-registry/ecs-container-definitions-pipe:latest
+  variables:
+    ECS_CLUSTER: 'my-ecs-cluster'
+    ECS_SERVICE: 'my-ecs-service'
+    AWS_OIDC_ROLE_ARN: 'arn:aws:iam::account-id:role/role-name'
+```
+
+### Write output to a file and use in subsequent steps
+
+```yaml
+- step:
+    name: Extract ECS Configuration
+    script:
+      - pipe: your-docker-registry/ecs-container-definitions-pipe:latest
+        variables:
+          ECS_CLUSTER: 'my-ecs-cluster'
+          ECS_SERVICE: 'my-ecs-service'
+          AWS_OIDC_ROLE_ARN: 'arn:aws:iam::account-id:role/role-name'
+          OUTPUT_FILE: 'values.yml'
+      - cat values.yml.json
+```
+
+## Development
+
+To build and test this pipe locally:
+
+1. Build the Docker image:
+   ```bash
+   docker build -t terragrunt-config-export .
+   ```
+
+2. Run the pipe locally:
+   ```bash
+   docker run \
+     -v ~/.aws:/root/.aws \
+     -e ECS_CLUSTER="ew1-s-hyperion" \
+     -e ECS_SERVICE="ew1-s-hyperion-webapp" \
+     -e AWS_PROFILE="staging" \
+     terragrunt-config-export
+   ```

terragrunt-config-export/main.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+
+import os
+import sys
+import boto3
+import yaml
+import time
+import stat
+import configparser
+
+
+def auth_oidc():
+    random_number = str(time.time_ns())
+    aws_config_directory = os.path.join(os.environ["HOME"], '.aws')
+    oidc_token_directory = os.path.join(aws_config_directory, '.aws-oidc')
+
+    os.makedirs(aws_config_directory, exist_ok=True)
+    os.makedirs(oidc_token_directory, exist_ok=True)
+
+    web_identity_token_path = os.path.join(oidc_token_directory, f'oidc_token_{random_number}')
+    with open(web_identity_token_path, 'w') as f:
+        f.write(os.getenv('BITBUCKET_STEP_OIDC_TOKEN'))
+
+    os.chmod(web_identity_token_path, mode=stat.S_IRUSR)
+    print('Web identity token file is created')
+
+    aws_configfile_path = os.path.join(aws_config_directory, 'config')
+    with open(aws_configfile_path, 'w') as configfile:
+        config = configparser.ConfigParser()
+        config['default'] = {
+            'role_arn': os.getenv('\fi'),
+            'web_identity_token_file': web_identity_token_path
+        }
+        config.write(configfile)
+    print('Configured settings for authentication with assume web identity role')
+
+
+def get_boto3_client(service_name='ecs'):
+    """
+    Create and return a boto3 client with appropriate authentication.
+    """
+    region = os.getenv('AWS_REGION', os.getenv('AWS_DEFAULT_REGION', 'eu-west-1'))
+    profile_name = os.getenv('AWS_PROFILE')
+    oidc = os.getenv('AWS_OIDC_ROLE_ARN')
+
+    if oidc:
+        auth_oidc()
+        return boto3.client(service_name,region_name=region)
+    elif profile_name:
+        session = boto3.Session(profile_name=profile_name)
+        return session.client(service_name, region_name=region)
+
+
+def convert_to_terragrunt_format(task_definition, service_name):
+    """
+    Convert ECS task definition to the required Terragrunt format.
+    """
+    # Initialize the terragrunt config structure
+    config = {
+        "deployment": {
+            "extraEnv": os.getenv('EXTRA_ENV', [])
+        },
+        "terragruntConfig": {
+            "name": f"({service_name})",
+            "secrets": [],
+            "containers": [],
+            "mainContainerName": "",
+            "resources": [],
+            "endpoints": os.getenv('ENDPOINTS', []),
+            "iamRole": os.getenv('IAM_ROLE', "")
+        }
+    }
+
+    config["terragruntConfig"]['resources'].append({'cpu': task_definition.get("cpu", 0)})
+    config["terragruntConfig"]['resources'].append({'memory': task_definition.get("memory", 0)})
+
+    # Process each container definition
+    for container in task_definition.get("containerDefinitions", []):
+        terragrunt_container = {
+            "name": container.get("name", ""),
+            "image": container.get("image", ""),
+            "environment": [],
+            "ports": [],
+            "dependencies": []
+        }
+
+        if "secrets" in container:
+            for secret in container["secrets"]:
+                secret_name = secret.get("valueFrom", "").split(":")[-1].rsplit("-", 1)[0]
+                if secret_name not in config["terragruntConfig"]["secrets"]:
+                    config["terragruntConfig"]["secrets"].append(
+                        secret_name
+                    )
+
+        # Process environment variables
+        if "environment" in container:
+            for env in container["environment"]:
+                terragrunt_container["environment"].append({
+                    "name": env.get("name", ""),
+                    "value": env.get("value", "")
+                })
+
+        # Process port mappings
+        if "portMappings" in container:
+            for port in container["portMappings"]:
+                terragrunt_container["ports"].append({
+                    "name": port.get("name", ""),
+                    "hostPort": port.get("hostPort", 0),
+                    "containerPort": port.get("containerPort", 0),
+                    "protocol": port.get("protocol", "")
+                })
+
+        # Process dependencies (if any)
+        if "dependsOn" in container:
+            for dep in container["dependsOn"]:
+                terragrunt_container["dependencies"].append({
+                    "condition": dep.get("condition", ""),
+                    "containerName": dep.get("containerName", "")
+                })
+
+        config["terragruntConfig"]["containers"].append(terragrunt_container)
+
+    return config
+
+
+def get_container_definitions():
+    """
+    Get container definitions from ECS service
+    """
+    # Required parameters
+    cluster = os.getenv('ECS_CLUSTER')
+    service = os.getenv('ECS_SERVICE')
+
+    # Validate required parameters
+    if not cluster:
+        print("Error: ECS_CLUSTER is required")
+        sys.exit(1)
+
+    if not service:
+        print("Error: ECS_SERVICE is required")
+        sys.exit(1)
+
+    try:
+        # Get the ECS client
+        ecs_client = get_boto3_client('ecs')
+
+        # Get the service details
+        print(f"Fetching service details for {service} in cluster {cluster}...")
+        service_response = ecs_client.describe_services(
+            cluster=cluster,
+            services=[service]
+        )
+
+        if not service_response['services']:
+            print(f"Error: Service {service} not found in cluster {cluster}")
+            sys.exit(1)
+
+        # Get the task definition ARN
+        task_definition_arn = service_response['services'][0]['taskDefinition']
+        print(f"Found task definition: {task_definition_arn}")
+
+        # Get the task definition details
+        task_def_response = ecs_client.describe_task_definition(
+            taskDefinition=task_definition_arn
+        )
+
+        print("Converting to Terragrunt format...")
+        response = convert_to_terragrunt_format(
+            task_def_response['taskDefinition'],
+            service
+        )
+
+        output_content = yaml.dump(response, default_flow_style=False, sort_keys=False)
+
+        # Write to file if specified
+        if os.getenv('OUTPUT_FILE'):
+            with open(os.getenv('OUTPUT_FILE'), 'w') as f:
+                f.write(output_content)
+            print(f"Output written to {os.getenv('OUTPUT_FILE')}")
+        else:
+            print(output_content)
+
+        print("✅ Successfully retrieved container definitions")
+        return 0
+
+    except Exception as e:
+        print(f"❌ Error: {str(e)}")
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(get_container_definitions())

terragrunt-config-export/pipe.yml

@@ -0,0 +1,39 @@
+name: Terragrunt config export
+description: Retrieve the config needed for the helm chart from the ECS Service
+image:
+category: Utilities
+repository: https://bitbucket.org/sykescottagesltd/terragrunt-config-export
+vendor:
+  name: Forge Holiday Group
+  website: https://www.forgeholidays.com/
+variables:
+  ECS_CLUSTER:
+    description: Name of the ECS cluster
+    required: true
+  ECS_SERVICE:
+    description: Name of the ECS service
+    required: true
+  AWS_OIDC_ROLE_ARN:
+    description: ARN of IAM OIDC Role to assume
+    required: true
+  EXTRA_ENV:
+    type: Map
+    required: false
+    default: {}
+  ENDPOINTS:
+    type: Array
+    required: false
+    default: []
+  IAM_ROLE:
+    description: The IAM Role for the service
+    required: false
+  OUTPUT_FILE:
+    description: File to write the output to (if not specified, outputs to console)
+    required: false
+  AWS_PROFILE:
+    description: Profile to assume when running locally
+    required: false
+  AWS_REGION:
+    description: AWS region
+    default: "eu-west-1"
+    required: false

terragrunt-config-export/requirements.txt

@@ -0,0 +1,2 @@
+boto3==1.28.50
+pyyaml==6.0.1