From 005d86904ad5608bed3cbf7b7331989ceeb03c53 Mon Sep 17 00:00:00 2001 From: Matteo Figus Date: Wed, 3 Mar 2021 17:45:36 +0000 Subject: [PATCH] Improvements on Fargate service (#240) * Test for new Fargate Platform version * Improvements on Fargate Instrumentation code + add ECR API VPC endpoint * Update changelog * Pin to 1.4.0 * Review guide --- CHANGELOG.md | 6 +++ .../enable_container_insights.py | 29 -------------- docs/USER_GUIDE.md | 15 ++++--- templates/deletion_flow.yaml | 27 ++++--------- templates/template.yaml | 4 +- templates/vpc.yaml | 13 ++++++ .../crs/test_cr_enable_container_insights.py | 40 ------------------- 7 files changed, 38 insertions(+), 96 deletions(-) delete mode 100644 backend/lambdas/custom_resources/enable_container_insights.py delete mode 100644 tests/unit/crs/test_cr_enable_container_insights.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cdce6c6..25acbd53 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Change Log +## v0.24 + +- [#240](https://github.com/awslabs/amazon-s3-find-and-forget/pull/240): Add ECR + API Endpoint and migrate to + [Fargate Platform version 1.4.0](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/platform_versions.html#platform-version-migration) + ## v0.23 - [#238](https://github.com/awslabs/amazon-s3-find-and-forget/pull/238): Upgrade diff --git a/backend/lambdas/custom_resources/enable_container_insights.py b/backend/lambdas/custom_resources/enable_container_insights.py deleted file mode 100644 index d21f7d9e..00000000 --- a/backend/lambdas/custom_resources/enable_container_insights.py +++ /dev/null @@ -1,29 +0,0 @@ -import os -import boto3 -from crhelper import CfnResource -from decorators import with_logging - -helper = CfnResource(json_logging=False, log_level="DEBUG", boto_level="CRITICAL") - -ecs_client = boto3.client("ecs") - - -@with_logging -@helper.create -@helper.update -def create(event, context): - ecs_client.update_cluster_settings( - cluster=os.getenv("Cluster"), - settings=[{"name": "containerInsights", "value": "enabled"}], - ) - return None - - -@with_logging -@helper.delete -def delete(event, context): - return None - - -def handler(event, context): - helper(event, context) diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md index af1d0eab..fdc1af98 100644 --- a/docs/USER_GUIDE.md +++ b/docs/USER_GUIDE.md @@ -42,12 +42,15 @@ The Fargate tasks used by this solution to perform deletions must be able to access the following AWS services, either via an Internet Gateway or via [VPC Endpoints]: -- Amazon S3 -- Amazon DynamoDB -- Amazon CloudWatch (monitoring and logs) -- AWS ECR -- Amazon SQS -- AWS STS +- Amazon S3 (gateway endpoint _com.amazonaws.**region**.s3_) +- Amazon DynamoDB (gateway endpoint _com.amazonaws.**region**.dynamodb_) +- Amazon CloudWatch Monitoring (interface endpoint + _com.amazonaws.**region**.monitoring_) and Logs (interface endpoint + _com.amazonaws.**region**.logs_) +- AWS ECR API (interface endpoint _com.amazonaws.**region**.ecr.api_) and Docker + (interface endpoint _com.amazonaws.**region**.ecr.dkr_) +- Amazon SQS (interface endpoint _com.amazonaws.**region**.sqs_) +- AWS STS (interface endpoint _com.amazonaws.**region**.sts_) #### Creating a New VPC diff --git a/templates/deletion_flow.yaml b/templates/deletion_flow.yaml index 4b6417fe..350d3eb6 100644 --- a/templates/deletion_flow.yaml +++ b/templates/deletion_flow.yaml @@ -52,6 +52,13 @@ Resources: ECSCluster: Type: AWS::ECS::Cluster + Properties: + ClusterSettings: + - Name: containerInsights + Value: !If + - WithContainerInsights + - enabled + - disabled ECRRepository: Type: AWS::ECR::Repository @@ -139,6 +146,7 @@ Resources: Cluster: !GetAtt ECSCluster.Arn DesiredCount: 0 LaunchType: FARGATE + PlatformVersion: 1.4.0 NetworkConfiguration: AwsvpcConfiguration: SecurityGroups: !Ref VpcSecurityGroups @@ -189,25 +197,6 @@ Resources: KmsMasterKeyId: alias/aws/sqs ReceiveMessageWaitTimeSeconds: 0 - UpdateClusterConfig: - Type: Custom::Setup - Condition: WithContainerInsights - Properties: - ServiceToken: !GetAtt UpdateClusterEnableContainerInsights.Arn - - UpdateClusterEnableContainerInsights: - Type: AWS::Serverless::Function - Condition: WithContainerInsights - Properties: - Handler: enable_container_insights.handler - CodeUri: ../backend/lambdas/custom_resources/ - Description: Custom Lambda resource for the Amazon S3 Find and Forget Cloudformation Stack - Policies: - - Statement: - - Effect: Allow - Action: ecs:UpdateClusterSettings - Resource: !Sub arn:aws:ecs:${AWS::Region}:${AWS::AccountId}:cluster/${ECSCluster} - Outputs: DeleteObjectsQueueUrl: Value: !Ref DelObjQ diff --git a/templates/template.yaml b/templates/template.yaml index c8663070..06aba93d 100644 --- a/templates/template.yaml +++ b/templates/template.yaml @@ -1,6 +1,6 @@ AWSTemplateFormatVersion: "2010-09-09" Transform: AWS::Serverless-2016-10-31 -Description: Amazon S3 Find and Forget (uksb-1q2j8beb0) (version:v0.23) +Description: Amazon S3 Find and Forget (uksb-1q2j8beb0) (version:v0.24) Parameters: AccessControlAllowOriginOverride: @@ -135,7 +135,7 @@ Conditions: Mappings: Solution: Constants: - Version: 'v0.23' + Version: 'v0.24' Resources: TempBucket: diff --git a/templates/vpc.yaml b/templates/vpc.yaml index 10a9564a..8cc63ef1 100644 --- a/templates/vpc.yaml +++ b/templates/vpc.yaml @@ -249,6 +249,19 @@ Resources: VpcEndpointType: Interface VpcId: !Ref VPC + ECRAPIEndpoint: + Type: AWS::EC2::VPCEndpoint + Properties: + PrivateDnsEnabled: true + SecurityGroupIds: [!Ref SecurityGroup] + ServiceName: !Sub 'com.amazonaws.${AWS::Region}.ecr.api' + SubnetIds: + - !Ref PrivateSubnet1 + - !Ref PrivateSubnet2 + - !If [HasThreeAZs, !Ref PrivateSubnet3, !Ref 'AWS::NoValue'] + VpcEndpointType: Interface + VpcId: !Ref VPC + S3Endpoint: Type: AWS::EC2::VPCEndpoint Properties: diff --git a/tests/unit/crs/test_cr_enable_container_insights.py b/tests/unit/crs/test_cr_enable_container_insights.py deleted file mode 100644 index 703a2e35..00000000 --- a/tests/unit/crs/test_cr_enable_container_insights.py +++ /dev/null @@ -1,40 +0,0 @@ -from types import SimpleNamespace - -import json -import pytest -from mock import patch, MagicMock - -from backend.lambdas.custom_resources.enable_container_insights import ( - create, - delete, - handler, -) - -pytestmark = [pytest.mark.unit, pytest.mark.task] - - -@patch("backend.lambdas.custom_resources.enable_container_insights.ecs_client") -@patch("os.getenv") -def test_it_updates_cluster_setting(getenv_mock, mock_client): - getenv_mock.return_value = "cluster-name" - resp = create({}, MagicMock()) - mock_client.update_cluster_settings.assert_called_with( - cluster="cluster-name", - settings=[{"name": "containerInsights", "value": "enabled"}], - ) - - assert not resp - - -@patch("backend.lambdas.custom_resources.enable_container_insights.ecs_client") -def test_it_does_nothing_on_delete(mock_client): - mock_client.return_value = MagicMock() - resp = delete({}, MagicMock()) - mock_client.assert_not_called() - assert resp == None - - -@patch("backend.lambdas.custom_resources.enable_container_insights.helper") -def test_it_delegates_to_cr_helper(cr_helper): - handler(1, 2) - cr_helper.assert_called_with(1, 2)