From 098350a0e2efa0ee1db712a4d6e260115656f294 Mon Sep 17 00:00:00 2001 From: flakey5 <73616808+flakey5@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:01:34 -0700 Subject: [PATCH 1/4] actions: promote release action Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com> --- .github/workflows/promote.yml | 22 +++++++ scripts/promote-release.js | 115 ++++++++++++++++++++++++++++++++++ 2 files changed, 137 insertions(+) create mode 100644 .github/workflows/promote.yml create mode 100755 scripts/promote-release.js diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml new file mode 100644 index 0000000..20859d2 --- /dev/null +++ b/.github/workflows/promote.yml @@ -0,0 +1,22 @@ +name: Promote Release + +on: + # Triggered by the dist server + workflow_dispatch: + inputs: + path: + description: 'path to promote' + required: true + +jobs: + promote-release: + name: Promote Release + runs-on: ubuntu-latest + + steps: + - name: Promote Files + env: + CF_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }} + CF_SECRET_ACCESS_KEY: ${{ secrets.CF_SECRET_ACCESS_KEY }} + runs: | + node scripts/promote-release.js ${{ inputs.path }} diff --git a/scripts/promote-release.js b/scripts/promote-release.js new file mode 100755 index 0000000..e3d3ed5 --- /dev/null +++ b/scripts/promote-release.js @@ -0,0 +1,115 @@ +#!/usr/bin/env node + +import { S3Client, ListObjectsV2Command, CopyObjectCommand } from '@aws-sdk/client-s3'; + +if (process.argv.length !== 3) { + console.error(`usage: promote-release `); + process.exit(1); +} + +if (!process.env.CF_ACCESS_KEY_ID) { + console.error('CF_ACCESS_KEY_ID missing'); + process.exit(1); +} + +if (!process.env.CF_SECRET_ACCESS_KEY) { + console.error('CF_SECRET_ACCESS_KEY missing'); + process.exit(1); +} + +const ENDPOINT = + 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com'; +const PROD_BUCKET = 'dist-prod' +const STAGING_BUCKET = 'dist-staging' +const RETRY_LIMIT = 3; + +const client = new S3Client({ + endpoint: ENDPOINT, + region: 'auto', + credentials: { + accessKeyId: process.env.CF_ACCESS_KEY_ID, + secretAccessKey: process.env.CF_SECRET_ACCESS_KEY + } +}) + +const path = process.argv[2] +const files = await getFilesToPromote(path); + +for (const file of files) { + promoteFile(file) +} + +/** + * @param {string} path + * @returns {string[]} + */ +async function getFilesToPromote(path) { + let paths = []; + + let truncated = true; + let continuationToken; + while (truncated) { + const data = await retryWrapper( + async () => { + return await client.send( + new ListObjectsV2Command({ + Bucket: STAGING_BUCKET, + Delimiter: '/', + Prefix: path, + ContinuationToken: continuationToken + }) + ) + } + ) + + if (data.CommonPrefixes) { + for (const directory of data.CommonPrefixes) { + paths.push(...await getFilesToPromote(directory.Prefix)) + } + } + + if (data.Contents) { + for (const object of data.Contents) { + paths.push(object.Key) + } + } + + truncated = data.IsTruncated ?? false + continuationToken = data.NextContinuationToken + } + + return paths +} + +/** + * @param {string} file + */ +async function promoteFile(file) { + console.log(`Promoting ${file}`) + + await client.send(new CopyObjectCommand({ + Bucket: PROD_BUCKET, + CopySource: `${STAGING_BUCKET}/${file}`, + Key: file + })) +} + +/** + * @param {() => Promise} request + * @returns {Promise} + */ +async function retryWrapper (request, retryLimit) { + let r2Error; + + for (let i = 0; i < RETRY_LIMIT; i++) { + try { + const result = await request(); + return result; + } catch (err) { + r2Error = err; + process.emitWarning(`error when contacting r2: ${err}`) + } + } + + throw r2Error +} From 5d94b1c79211d713e9a6a928e899d4ddb1d0d219 Mon Sep 17 00:00:00 2001 From: flakey5 <73616808+flakey5@users.noreply.github.com> Date: Fri, 4 Oct 2024 19:18:26 -0700 Subject: [PATCH 2/4] setup node Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com> --- .github/workflows/promote.yml | 6 +++ scripts/promote-release.js | 80 ++++++++++++++++++----------------- 2 files changed, 48 insertions(+), 38 deletions(-) diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 20859d2..30a206c 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -14,6 +14,12 @@ jobs: runs-on: ubuntu-latest steps: + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: lts/* + cache: 'npm' + - name: Promote Files env: CF_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }} diff --git a/scripts/promote-release.js b/scripts/promote-release.js index e3d3ed5..54ab062 100755 --- a/scripts/promote-release.js +++ b/scripts/promote-release.js @@ -1,6 +1,10 @@ #!/usr/bin/env node -import { S3Client, ListObjectsV2Command, CopyObjectCommand } from '@aws-sdk/client-s3'; +import { + S3Client, + ListObjectsV2Command, + CopyObjectCommand, +} from '@aws-sdk/client-s3'; if (process.argv.length !== 3) { console.error(`usage: promote-release `); @@ -19,8 +23,8 @@ if (!process.env.CF_SECRET_ACCESS_KEY) { const ENDPOINT = 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com'; -const PROD_BUCKET = 'dist-prod' -const STAGING_BUCKET = 'dist-staging' +const PROD_BUCKET = 'dist-prod'; +const STAGING_BUCKET = 'dist-staging'; const RETRY_LIMIT = 3; const client = new S3Client({ @@ -28,20 +32,20 @@ const client = new S3Client({ region: 'auto', credentials: { accessKeyId: process.env.CF_ACCESS_KEY_ID, - secretAccessKey: process.env.CF_SECRET_ACCESS_KEY - } -}) + secretAccessKey: process.env.CF_SECRET_ACCESS_KEY, + }, +}); -const path = process.argv[2] +const path = process.argv[2]; const files = await getFilesToPromote(path); for (const file of files) { - promoteFile(file) + promoteFile(file); } /** * @param {string} path - * @returns {string[]} + * @returns {string[]} */ async function getFilesToPromote(path) { let paths = []; @@ -49,56 +53,56 @@ async function getFilesToPromote(path) { let truncated = true; let continuationToken; while (truncated) { - const data = await retryWrapper( - async () => { - return await client.send( - new ListObjectsV2Command({ - Bucket: STAGING_BUCKET, - Delimiter: '/', - Prefix: path, - ContinuationToken: continuationToken - }) - ) - } - ) + const data = await retryWrapper(async () => { + return await client.send( + new ListObjectsV2Command({ + Bucket: STAGING_BUCKET, + Delimiter: '/', + Prefix: path, + ContinuationToken: continuationToken, + }) + ); + }); if (data.CommonPrefixes) { for (const directory of data.CommonPrefixes) { - paths.push(...await getFilesToPromote(directory.Prefix)) + paths.push(...(await getFilesToPromote(directory.Prefix))); } } if (data.Contents) { for (const object of data.Contents) { - paths.push(object.Key) + paths.push(object.Key); } } - truncated = data.IsTruncated ?? false - continuationToken = data.NextContinuationToken + truncated = data.IsTruncated ?? false; + continuationToken = data.NextContinuationToken; } - return paths + return paths; } /** - * @param {string} file + * @param {string} file */ async function promoteFile(file) { - console.log(`Promoting ${file}`) - - await client.send(new CopyObjectCommand({ - Bucket: PROD_BUCKET, - CopySource: `${STAGING_BUCKET}/${file}`, - Key: file - })) + console.log(`Promoting ${file}`); + + await client.send( + new CopyObjectCommand({ + Bucket: PROD_BUCKET, + CopySource: `${STAGING_BUCKET}/${file}`, + Key: file, + }) + ); } /** - * @param {() => Promise} request + * @param {() => Promise} request * @returns {Promise} */ -async function retryWrapper (request, retryLimit) { +async function retryWrapper(request, retryLimit) { let r2Error; for (let i = 0; i < RETRY_LIMIT; i++) { @@ -107,9 +111,9 @@ async function retryWrapper (request, retryLimit) { return result; } catch (err) { r2Error = err; - process.emitWarning(`error when contacting r2: ${err}`) + process.emitWarning(`error when contacting r2: ${err}`); } } - throw r2Error + throw r2Error; } From 443e2e8a0c19ce561e3814bf2ad819436f64ee0e Mon Sep 17 00:00:00 2001 From: flakey5 <73616808+flakey5@users.noreply.github.com> Date: Thu, 10 Oct 2024 12:14:53 -0700 Subject: [PATCH 3/4] code review Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com> --- package.json | 4 +- scripts/constants.mjs | 11 ++++++ ...promote-release.js => promote-release.mjs} | 37 +++++++++++-------- 3 files changed, 35 insertions(+), 17 deletions(-) create mode 100644 scripts/constants.mjs rename scripts/{promote-release.js => promote-release.mjs} (80%) diff --git a/package.json b/package.json index 858a441..b1c92d6 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,8 @@ "type": "module", "scripts": { "start": "wrangler dev --remote", - "format": "prettier --check --write \"**/*.{ts,js,json,md}\"", - "prettier": "prettier --check \"**/*.{ts,js,json,md}\"", + "format": "prettier --check --write \"**/*.{ts,js,mjs,json,md}\"", + "prettier": "prettier --check \"**/*.{ts,js,mjs,json,md}\"", "lint": "eslint ./src", "test": "npm run test:unit && npm run test:e2e", "test:unit": "node --test --test-reporter=@reporters/github --test-reporter-destination=stdout --test-reporter=spec --test-reporter-destination=stdout --import=tsx ./tests/unit/index.test.ts", diff --git a/scripts/constants.mjs b/scripts/constants.mjs new file mode 100644 index 0000000..f1a7be9 --- /dev/null +++ b/scripts/constants.mjs @@ -0,0 +1,11 @@ +'use strict'; + +export const ENDPOINT = + process.env.ENDPOINT ?? + 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com'; + +export const PROD_BUCKET = process.env.PROD_BUCKET ?? 'dist-prod'; + +export const STAGING_BUCKET = process.env.STAGING_BUCKET ?? 'dist-staging'; + +export const R2_RETRY_COUNT = 3; diff --git a/scripts/promote-release.js b/scripts/promote-release.mjs similarity index 80% rename from scripts/promote-release.js rename to scripts/promote-release.mjs index 54ab062..f65213e 100755 --- a/scripts/promote-release.js +++ b/scripts/promote-release.mjs @@ -1,10 +1,21 @@ #!/usr/bin/env node +/** + * Usage: `promote-release ` + * ex/ `promote-release nodejs/release/v20.0.0` + */ + import { S3Client, ListObjectsV2Command, CopyObjectCommand, } from '@aws-sdk/client-s3'; +import { + ENDPOINT, + PROD_BUCKET, + STAGING_BUCKET, + R2_RETRY_COUNT, +} from './constants.mjs'; if (process.argv.length !== 3) { console.error(`usage: promote-release `); @@ -21,12 +32,6 @@ if (!process.env.CF_SECRET_ACCESS_KEY) { process.exit(1); } -const ENDPOINT = - 'https://07be8d2fbc940503ca1be344714cb0d1.r2.cloudflarestorage.com'; -const PROD_BUCKET = 'dist-prod'; -const STAGING_BUCKET = 'dist-staging'; -const RETRY_LIMIT = 3; - const client = new S3Client({ endpoint: ENDPOINT, region: 'auto', @@ -40,7 +45,7 @@ const path = process.argv[2]; const files = await getFilesToPromote(path); for (const file of files) { - promoteFile(file); + await promoteFile(file); } /** @@ -89,13 +94,15 @@ async function getFilesToPromote(path) { async function promoteFile(file) { console.log(`Promoting ${file}`); - await client.send( - new CopyObjectCommand({ - Bucket: PROD_BUCKET, - CopySource: `${STAGING_BUCKET}/${file}`, - Key: file, - }) - ); + await retryWrapper(async () => { + return await client.send( + new CopyObjectCommand({ + Bucket: PROD_BUCKET, + CopySource: `${STAGING_BUCKET}/${file}`, + Key: file, + }) + ); + }, R2_RETRY_COUNT); } /** @@ -105,7 +112,7 @@ async function promoteFile(file) { async function retryWrapper(request, retryLimit) { let r2Error; - for (let i = 0; i < RETRY_LIMIT; i++) { + for (let i = 0; i < R2_RETRY_COUNT; i++) { try { const result = await request(); return result; From fcd95a93833299dc515b766cb608426f74f5b8a2 Mon Sep 17 00:00:00 2001 From: flakey5 <73616808+flakey5@users.noreply.github.com> Date: Thu, 10 Oct 2024 12:29:30 -0700 Subject: [PATCH 4/4] --recursive Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com> Signed-off-by: flakey5 <73616808+flakey5@users.noreply.github.com> --- .github/workflows/promote.yml | 11 +++++++++-- scripts/promote-release.mjs | 21 +++++++++++++++------ 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/promote.yml b/.github/workflows/promote.yml index 30a206c..f8a596f 100644 --- a/.github/workflows/promote.yml +++ b/.github/workflows/promote.yml @@ -7,6 +7,10 @@ on: path: description: 'path to promote' required: true + recursive: + description: 'is the path a directory' + type: boolean + required: true jobs: promote-release: @@ -14,6 +18,9 @@ jobs: runs-on: ubuntu-latest steps: + - name: Git Checkout + uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 + - name: Setup Node uses: actions/setup-node@v4 with: @@ -24,5 +31,5 @@ jobs: env: CF_ACCESS_KEY_ID: ${{ secrets.CF_ACCESS_KEY_ID }} CF_SECRET_ACCESS_KEY: ${{ secrets.CF_SECRET_ACCESS_KEY }} - runs: | - node scripts/promote-release.js ${{ inputs.path }} + run: | + node scripts/promote-release.mjs ${{ inputs.path }} ${{ inputs.recursive == true && '--recursive' || '' }} diff --git a/scripts/promote-release.mjs b/scripts/promote-release.mjs index f65213e..2bcbc66 100755 --- a/scripts/promote-release.mjs +++ b/scripts/promote-release.mjs @@ -1,8 +1,8 @@ #!/usr/bin/env node /** - * Usage: `promote-release ` - * ex/ `promote-release nodejs/release/v20.0.0` + * Usage: `promote-release [--recursive]` + * ex/ `promote-release nodejs/release/v20.0.0/ --recursive` */ import { @@ -17,8 +17,10 @@ import { R2_RETRY_COUNT, } from './constants.mjs'; -if (process.argv.length !== 3) { - console.error(`usage: promote-release `); +if (process.argv.length !== 3 && process.argv.length !== 4) { + console.error( + `usage: promote-release [--recursive]` + ); process.exit(1); } @@ -42,9 +44,16 @@ const client = new S3Client({ }); const path = process.argv[2]; -const files = await getFilesToPromote(path); +const recursive = + process.argv.length === 4 && process.argv[3] === '--recursive'; -for (const file of files) { +if (recursive) { + const files = await getFilesToPromote(path); + + for (const file of files) { + await promoteFile(file); + } +} else { await promoteFile(file); }