From 7826a4fdfeca94179d6d211f7b669e1592a81ed2 Mon Sep 17 00:00:00 2001 From: philmcmahon Date: Mon, 29 Jan 2024 13:37:38 +0000 Subject: [PATCH] Temporarily remove all steps from Dockerfile to speed up ECR debugging --- .github/workflows/build-whisper-docker.yml | 36 ++++++++++++---------- packages/cdk/bin/cdk.ts | 2 +- packages/cdk/lib/repository.ts | 32 ++++++++++++++++--- whisper_container/Dockerfile | 8 ++--- 4 files changed, 51 insertions(+), 27 deletions(-) diff --git a/.github/workflows/build-whisper-docker.yml b/.github/workflows/build-whisper-docker.yml index 98517a0a..5d6bd366 100644 --- a/.github/workflows/build-whisper-docker.yml +++ b/.github/workflows/build-whisper-docker.yml @@ -5,7 +5,7 @@ name: Build and publish whisper docker image on: workflow_dispatch: push: - branches: ['main'] + branches: ['main', 'whisper-docker'] paths: - "whisper_container/Dockerfile" - ".github/workflows/build-whisper-docker.yml" @@ -23,6 +23,7 @@ jobs: permissions: contents: read packages: write + id-token: write steps: - name: Checkout repository @@ -30,7 +31,7 @@ jobs: - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - # Uses the `docker/login-action` action to log in to the Container registry registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. + # Uses the `docker/login-action` action to log in to the Container registry using the account and password that will publish the packages. Once published, the packages are scoped to the account defined here. - name: Log in to the Container registry uses: docker/login-action@v3 with: @@ -42,7 +43,20 @@ jobs: id: meta uses: docker/metadata-action@v5 with: - images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + images: | + ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + ${{ secrets.TRANSCRIPTION_SERVICE_ECR_URI }} + tags: | + type=sha + type=raw,value=latest + - uses: aws-actions/configure-aws-credentials@v4 + name: Configure AWS credentials for pushing to ECR + with: + role-to-assume: ${{ secrets.INVESTIGATIONS_GITHUB_ACTIONS_ROLE_ARN }} + aws-region: eu-west-1 + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@v2 # This step uses the `docker/build-push-action` action to build the image, based on your repository's `Dockerfile`. If the build succeeds, it pushes the image to GitHub Packages. # It uses the `context` parameter to define the build's context as the set of files located in the specified path. For more information, see "[Usage](https://github.com/docker/build-push-action#usage)" in the README of the `docker/build-push-action` repository. # It uses the `tags` and `labels` parameters to tag and label the image with the output from the "meta" step. @@ -55,17 +69,5 @@ jobs: push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} - - uses: aws-actions/configure-aws-credentials@v4 - name: Configure AWS credentials for pushing to ECR - with: - role-to-assume: ${{ secrets.INVESTIGATIONS_GITHUB_ACTIONS_ROLE_ARN }} - aws-region: eu-west-1 - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@v2 - - name: Push image to ECR - run: | - docker buildx imagetools create \ - --tag ${{ secrets.TRANSCRIPTION_SERVICE_ECR_URI }}:latest \ - --tag ${{ secrets.TRANSCRIPTION_SERVICE_ECR_URI }}:${{ env.GITHUB_RUN_NUMBER }} \ - transcription-service:latest + + diff --git a/packages/cdk/bin/cdk.ts b/packages/cdk/bin/cdk.ts index b3dab4b2..9e9f456a 100644 --- a/packages/cdk/bin/cdk.ts +++ b/packages/cdk/bin/cdk.ts @@ -1,7 +1,7 @@ import 'source-map-support/register'; import { GuRoot } from '@guardian/cdk/lib/constructs/root'; -import { TranscriptionService } from '../lib/transcription-service'; import {TranscriptionServiceRepository} from "../lib/repository"; +import { TranscriptionService } from '../lib/transcription-service'; const app = new GuRoot(); new TranscriptionService(app, 'TranscriptionService-CODE', { diff --git a/packages/cdk/lib/repository.ts b/packages/cdk/lib/repository.ts index e82bc004..8d6c7450 100644 --- a/packages/cdk/lib/repository.ts +++ b/packages/cdk/lib/repository.ts @@ -1,20 +1,42 @@ -import type { GuStackProps} from "@guardian/cdk/lib/constructs/core"; -import {GuStack} from "@guardian/cdk/lib/constructs/core"; +import type {GuStackProps} from "@guardian/cdk/lib/constructs/core"; +import {GuStack, GuStringParameter} from "@guardian/cdk/lib/constructs/core"; import type {App} from "aws-cdk-lib"; -import { RemovalPolicy} from "aws-cdk-lib"; +import {RemovalPolicy} from "aws-cdk-lib"; import {Repository, TagMutability} from "aws-cdk-lib/aws-ecr"; +import {ArnPrincipal, Effect, PolicyStatement} from "aws-cdk-lib/aws-iam"; export class TranscriptionServiceRepository extends GuStack { constructor(scope: App, id: string, props: GuStackProps) { super(scope, id, props); - new Repository(this, "TranscriptionServiceRepository", { + const githubActionsIAMRoleArn = new GuStringParameter(this, "GithubActionsIAMRoleArn", { + description: "IAM role for role used by github actions workflows" + }) + const repository = new Repository(this, "TranscriptionServiceRepository", { repositoryName: `transcription-service`, lifecycleRules: [{ maxImageCount: 5 }], imageTagMutability: TagMutability.IMMUTABLE, removalPolicy: RemovalPolicy.DESTROY, - imageScanOnPush: true + imageScanOnPush: true, }) + repository.addToResourcePolicy(new PolicyStatement({ + principals: [new ArnPrincipal(githubActionsIAMRoleArn.valueAsString)], + actions: [ + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:GetDownloadUrlForLayer", + "ecr:GetRepositoryPolicy", + "ecr:DescribeRepositories", + "ecr:ListImages", + "ecr:DescribeImages", + "ecr:BatchGetImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:PutImage" + ], + effect: Effect.ALLOW + })) } } \ No newline at end of file diff --git a/whisper_container/Dockerfile b/whisper_container/Dockerfile index 27d421f8..6ef6803b 100644 --- a/whisper_container/Dockerfile +++ b/whisper_container/Dockerfile @@ -3,10 +3,10 @@ WORKDIR /opt LABEL com.theguardian.transcription-service.whisper-container="Whisper.cpp container with models downloaded, including ffmpeg" RUN apt-get update -RUN apt-get install -y ffmpeg curl wget git build-essential -RUN git clone https://github.com/ggerganov/whisper.cpp -RUN cd whisper.cpp && make -RUN bash /opt/whisper.cpp/models/download-ggml-model.sh medium +#RUN apt-get install -y ffmpeg curl wget git build-essential +#RUN git clone https://github.com/ggerganov/whisper.cpp +#RUN cd whisper.cpp && make +#RUN bash /opt/whisper.cpp/models/download-ggml-model.sh medium # Large model not currently in use - but we might want to add it as an option at some point #RUN bash /opt/whisper.cpp/models/download-ggml-model.sh large-v2 \ No newline at end of file