diff --git a/.serverless_plugins/remove-storage/index.js b/.serverless_plugins/remove-storage/index.js index 1af7841..ea58fbf 100644 --- a/.serverless_plugins/remove-storage/index.js +++ b/.serverless_plugins/remove-storage/index.js @@ -28,15 +28,21 @@ class RemoveStorageBucket { .Properties .BucketName; + this.bucketCache = this.serverless.service.resources + .Resources + .PackageCacheStorage + .Properties + .BucketName; + this.hooks = { 'before:remove:remove': this.beforeRemove.bind(this), }; } - listAllKeys(token) { + listAllKeys(bucket, token) { const allKeys = []; return this.s3.listObjectsV2({ - Bucket: this.bucket, + Bucket: bucket, ContinuationToken: token, }) .promise() @@ -44,7 +50,7 @@ class RemoveStorageBucket { allKeys.push(data.Contents); if (data.IsTruncated) { - return this.listAllKeys(data.NextContinuationToken); + return this.listAllKeys(bucket, data.NextContinuationToken); } return [].concat(...allKeys).map(({ Key }) => ({ Key })); @@ -53,19 +59,40 @@ class RemoveStorageBucket { beforeRemove() { return new Promise((resolve, reject) => { - return this.listAllKeys() + this.listAllKeys(this.bucketCache) .then((keys) => { if (keys.length > 0) { return this.s3 .deleteObjects({ - Bucket: this.bucket, + Bucket: this.bucketCache, Delete: { Objects: keys, }, }).promise(); } - - return true; + }) + .then(() => { + return this.s3 + .deleteBucket({ + Bucket: this.bucketCache, + }).promise() + .then(() => { + this.serverless.cli.log('AWS Package Storage Cache Removed'); + }); + }) + .then(() => { + return this.listAllKeys(this.bucket) + .then((keys) => { + if (keys.length > 0) { + return this.s3 + .deleteObjects({ + Bucket: this.bucket, + Delete: { + Objects: keys, + }, + }).promise(); + } + }) }) .then(() => { return this.s3 diff --git a/.serverless_plugins/update-cache-env/index.js b/.serverless_plugins/update-cache-env/index.js new file mode 100644 index 0000000..922cd06 --- /dev/null +++ b/.serverless_plugins/update-cache-env/index.js @@ -0,0 +1,78 @@ +const util = require('util'); + +class UpdateCacheEnv { + constructor(serverless, options) { + this.options = options; + this.serverless = serverless; + this.provider = this.serverless.getProvider('aws'); + + this.awsInfo = this.serverless + .pluginManager + .plugins + .find(p => p.constructor.name === 'AwsInfo'); + + this.bucketCache = this.serverless + .service + .provider + .environment + .bucketCache; + + this.registry = this.serverless + .service + .provider + .environment + .registry; + + this.cacheEnabled = this.serverless + .service + .provider + .environment + .cacheEnabled; + + this.hooks = { + 'after:deploy:deploy': this.afterDeploy.bind(this), + }; + } + + afterDeploy() { + const lambda = new this.provider.sdk.Lambda({ + signatureVersion: 'v4', + region: this.options.region, + }); + + const cacheFunction = this + .awsInfo + .gatheredData + .info + .functions + .find(f => f.name === 'cache'); + + if (!cacheFunction) { + throw new Error('Cache function has not been deployed correctly to AWS.'); + } + + const params = { + FunctionName: cacheFunction.deployedName, + Environment: { + Variables: { + apiEndpoint: `${this.awsInfo.gatheredData.info.endpoint}/registry`, + region: this.options.region, + bucketCache: this.bucketCache, + registry: this.registry, + cacheEnabled: this.cacheEnabled, + } + }, + }; + + lambda + .updateFunctionConfiguration(params) + .promise() + .then(() => { + this.serverless.cli.log('AWS Cache Environment Ready.'); + }).catch(err => { + this.serverless.cli.log(err.message); + }); + } +} + +module.exports = UpdateCacheEnv; diff --git a/README.md b/README.md index eb8808a..1e499fe 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ export CODEBOX_GITHUB_URL="https://api.github.com/" # The GitHub / GitHub Enterp export CODEBOX_GITHUB_CLIENT_ID="client_id" # The client id for your GitHub application export CODEBOX_GITHUB_SECRET="secret" # The secret for your GitHub application export CODEBOX_RESTRICTED_ORGS="" # OPTIONAL: Comma seperated list of github organisations to only allow access to users in that org (e.g. "craftship,myorg"). Useful if using public GitHub for authentication, as by default all authenticated users would have access. +export CODEBOX_CACHE="false" # OPTIONAL: Any npm install will cache dependencies from public registry in a separate S3 bucket ``` * `serverless deploy --stage prod` (pick which ever stage you wish) * `npm set registry ` - `` being the base url shown in the terminal after deployment completes, such as: @@ -64,6 +65,11 @@ Once done ensure you have a project based `.npmrc` config setup a per the "Using Yarn does not require an explicit `yarn login` as in this scenario it uses your `.npmrc` config instead. +## Caching +If you enable `CODEBOX_CACHE="true"` when using the registry all requests to packages that hit the public registry will then be cached. This allows you to have a cache / mirror of all dependencies in your project. Helps with robust deployments and better response times when hosting your CI in the same region as your Codebox npm registry. + +**NOTE: Your AWS bill will rise due to this scheduled task ensuring that cached dependnecies are up to date.** + ## Admins / Publishing Packages `npm publish` works as it normally does via the npm CLI. By default all users that authenticate have read only access. If you wish to allow publish rights then you need to set the `CODEBOX_ADMINS` environment variable to a comma separated list of GitHub usernames such as `jonsharratt,kadikraman` and re-deploy. diff --git a/serverless.yml b/serverless.yml index e032a0c..75d6c13 100644 --- a/serverless.yml +++ b/serverless.yml @@ -4,6 +4,7 @@ plugins: - environment-variables - remove-storage - serverless-webpack + - update-cache-env - content-handling - codebox-tools @@ -22,7 +23,9 @@ provider: githubClientId: ${env:CODEBOX_GITHUB_CLIENT_ID} githubSecret: ${env:CODEBOX_GITHUB_SECRET} bucket: ${env:CODEBOX_BUCKET}-${self:provider.stage} + bucketCache: ${env:CODEBOX_BUCKET}-cache-${self:provider.stage} region: ${self:provider.region} + cacheEnabled: ${env:CODEBOX_CACHE} clientId: ${env:CODEBOX_INSIGHTS_CLIENT_ID} secret: ${env:CODEBOX_INSIGHTS_SECRET} @@ -38,6 +41,7 @@ provider: - "sns:Publish" Resource: - "arn:aws:s3:::${self:provider.environment.bucket}*" + - "arn:aws:s3:::${self:provider.environment.bucketCache}*" - "Fn::Join": - "" - - "arn:aws:sns:" @@ -50,6 +54,12 @@ functions: authorizerGithub: handler: authorizerGithub.default + cache: + handler: cache.default + timeout: 300 + events: + - schedule: rate(1 hour) + put: handler: put.default events: @@ -134,6 +144,11 @@ resources: Properties: AccessControl: Private BucketName: ${self:provider.environment.bucket} + PackageCacheStorage: + Type: AWS::S3::Bucket + Properties: + AccessControl: Private + BucketName: ${self:provider.environment.bucketCache} custom: webpackIncludeModules: true diff --git a/src/adapters/s3.js b/src/adapters/s3.js index 1f1bbc1..54f913c 100644 --- a/src/adapters/s3.js +++ b/src/adapters/s3.js @@ -25,4 +25,20 @@ export default class Storage { return meta.Body; } + + async listAllKeys(token = null, keys = []) { + const data = await this.S3.listObjectsV2({ + ContinuationToken: token, + }) + .promise(); + + keys.push(data.Contents); + + if (data.IsTruncated) { + return this.listAllKeys(data.NextContinuationToken, keys); + } + + return [].concat(...keys).map(({ Key }) => Key); + } } + diff --git a/src/contextFactory.js b/src/contextFactory.js index de2600e..cafcc25 100644 --- a/src/contextFactory.js +++ b/src/contextFactory.js @@ -53,7 +53,9 @@ const log = (cmd, namespace, region, topic) => { export default (namespace, { headers, requestContext }) => { const { registry, + cacheEnabled, bucket, + bucketCache, region, logTopic, } = process.env; @@ -62,9 +64,11 @@ export default (namespace, { headers, requestContext }) => { return { command: cmd, + cacheEnabled: (cacheEnabled === 'true'), registry, user: user(requestContext.authorizer), storage: storage(region, bucket), + cache: storage(region, bucketCache), log: log(cmd, namespace, region, logTopic), npm, }; diff --git a/src/get/lib.js b/src/get/lib.js index a26a85f..45ede8f 100644 --- a/src/get/lib.js +++ b/src/get/lib.js @@ -1,64 +1,107 @@ -export default async ({ pathParameters }, { +export default async ({ pathParameters, body }, { registry, + cacheEnabled, user, storage, + cache, npm, log, }, callback) => { const name = `${decodeURIComponent(pathParameters.name)}`; try { - const pkgBuffer = await storage.get(`${name}/index.json`); - const json = JSON.parse(pkgBuffer.toString()); - json._attachments = {}; // eslint-disable-line no-underscore-dangle + if (!cacheEnabled) { + const cacheDisabledError = new Error('Cache currently disabled, set CODEBOX_CACHE=true to enable it.'); + cacheDisabledError.code = 'CacheDisabled'; + throw cacheDisabledError; + } - const version = json['dist-tags'].latest; + const cachedBuffer = await cache.get(`${name}/index.json`); + const cachedJson = JSON.parse(cachedBuffer.toString()); - await log.info(user, { - name: json.name, - version, - }); + if (cachedJson._codebox.cached) { // eslint-disable-line no-underscore-dangle + return callback(null, { + statusCode: 200, + body: JSON.stringify(cachedJson), + }); + } - return callback(null, { - statusCode: 200, - body: JSON.stringify(json), - }); - } catch (storageError) { - if (storageError.code === 'NoSuchKey') { + const notCachedError = new Error('Not yet cached within npm cache storage.'); + notCachedError.code = 'NotCached'; + throw notCachedError; + } catch (cachedStorageErr) { + if (cachedStorageErr.code === 'NoSuchKey' || + cachedStorageErr.code === 'CacheDisabled' || + cachedStorageErr.code === 'NotCached') { try { - const json = await npm.package(registry, pathParameters.name); - - const version = json['dist-tags'].latest; + // Could be a private package that has been published + const pkgBuffer = await storage.get(`${name}/index.json`); + const json = JSON.parse(pkgBuffer.toString()); + json._attachments = {}; // eslint-disable-line no-underscore-dangle await log.info(user, { name: json.name, - version, }); return callback(null, { statusCode: 200, body: JSON.stringify(json), }); - } catch (npmError) { - if (npmError.status === 500) { - await log.error(user, npmError); + } catch (storageError) { + if (storageError.code === 'NoSuchKey') { + try { + const json = await npm.package(registry, pathParameters.name); + + // Store json ready for scheduled cache task + // to fetch the package. + if (cacheEnabled) { + json._codebox = { // eslint-disable-line no-underscore-dangle + cached: false, + }; + + await cache.put( + `${name}/index.json`, + JSON.stringify(json), + ); + } + + await log.info(user, { + name: json.name, + }); + + return callback(null, { + statusCode: 200, + body: JSON.stringify(json), + }); + } catch (npmError) { + if (npmError.status === 500) { + await log.error(user, npmError); + } + + return callback(null, { + statusCode: npmError.status, + body: JSON.stringify({ + error: npmError.message, + }), + }); + } } + await log.error(user, storageError); + return callback(null, { - statusCode: npmError.status, + statusCode: 500, body: JSON.stringify({ - error: npmError.message, + error: storageError.message, }), }); } } - await log.error(user, storageError); - return callback(null, { statusCode: 500, body: JSON.stringify({ - error: storageError.message, + error: cachedStorageErr.message, }), }); } diff --git a/src/scheduled/cache/index.js b/src/scheduled/cache/index.js new file mode 100644 index 0000000..5966c81 --- /dev/null +++ b/src/scheduled/cache/index.js @@ -0,0 +1,29 @@ +import npm from '../../adapters/npm'; +import S3 from '../../adapters/s3'; +import lib from './lib'; + +export default async (event, _, callback) => { + if (process.env.cacheEnabled !== 'true') { + return callback(null, { + status: 'CACHE_DISABLED', + }); + } + + try { + const storage = new S3({ + region: process.env.region, + bucket: process.env.bucketCache, + }); + + return lib( + event, { + registry: process.env.registry, + storage, + npm, + }, + callback, + ); + } catch (err) { + return callback(err); + } +}; diff --git a/src/scheduled/cache/lib.js b/src/scheduled/cache/lib.js new file mode 100644 index 0000000..ac5f632 --- /dev/null +++ b/src/scheduled/cache/lib.js @@ -0,0 +1,77 @@ +export default async (event, { + registry, + storage, + npm, + cacheEnabled, +}, callback) => { + const keys = await storage.listAllKeys(); + + keys.filter((key) => { + const parts = key.split('/'); + return parts[parts.length - 1] === 'index.json'; + }) + .forEach(async (k) => { + const pkgBuffer = await storage.get(k); + const cacheJson = JSON.parse(pkgBuffer.toString()); + const name = cacheJson._id; // eslint-disable-line no-underscore-dangle + const npmJson = await npm.package( + registry, + name, + ); + + if (npmJson._rev !== cacheJson._rev || // eslint-disable-line no-underscore-dangle + !cacheJson._codebox.cached) { // eslint-disable-line no-underscore-dangle + try { + Object.keys(npmJson.versions).forEach(async (v) => { + const versionData = npmJson.versions[v]; + const tarballUrl = versionData.dist.tarball; + const tarball = tarballUrl.split('/').slice(-1); + const tar = await npm.tar(registry, `${name}/-/${tarball}`); + + await storage.put( + `${name}/${v}.tgz`, + tar, + 'base64', + ); + }); + + Object.keys(npmJson.versions).forEach((v) => { + if (process.env.apiEndpoint) { + const version = npmJson.versions[v]; + + if (version.dist && version.dist.tarball) { + const tarballParts = version.dist.tarball.split('/'); + const currentHost = tarballParts[2]; + const currentProtocol = tarballParts[0]; + + version.dist.tarball = version.dist.tarball + .replace(currentHost, `${process.env.apiEndpoint}`) + .replace(currentProtocol, 'https:'); + + npmJson.versions[v] = version; + } + } + }); + + npmJson._codebox = { // eslint-disable-line no-underscore-dangle + cached: true, + }; + + await storage.put( + `${name}/index.json`, + JSON.stringify(npmJson), + ); + } catch (err) { + return callback(err); + } + } + + return callback(null, { + status: 'OK', + }); + }); + + return callback(null, { + status: 'OK', + }); +}; diff --git a/test/adapters/s3.test.js b/test/adapters/s3.test.js index d7a7db7..133852e 100644 --- a/test/adapters/s3.test.js +++ b/test/adapters/s3.test.js @@ -6,16 +6,19 @@ describe('S3', () => { let awsSpy; let putObjectStub; let getObjectStub; + let listObjectsV2Stub; beforeEach(() => { awsSpy = { S3: spy(() => { getObjectStub = stub().returns({ promise: () => Promise.resolve() }); putObjectStub = stub().returns({ promise: () => Promise.resolve() }); + listObjectsV2Stub = stub().returns({ promise: () => Promise.resolve({ Contents: ['foo-key-1', 'foo-key-2'] }) }); const awsS3Instance = createStubInstance(AWS.S3); awsS3Instance.putObject = putObjectStub; awsS3Instance.getObject = getObjectStub; + awsS3Instance.listObjectsV2 = listObjectsV2Stub; return awsS3Instance; }), @@ -24,6 +27,32 @@ describe('S3', () => { Subject.__Rewire__('AWS', awsSpy); }); + describe('#listAllKeys()', () => { + it('should call AWS with correct parameters', async () => { + const subject = new Subject({ + region: 'foo-region', + bucket: 'bar-bucket', + }); + + await subject.listAllKeys(); + + assert(listObjectsV2Stub.calledWithExactly({ + ContinuationToken: null, + })); + }); + + it('should return an array of keys', async () => { + const subject = new Subject({ + region: 'foo-region', + bucket: 'bar-bucket', + }); + + const result = await subject.listAllKeys(); + + assert(result, ['foo-key-1', 'foo-key-2']); + }); + }); + describe('#put()', () => { context('base64', () => { it('should call AWS with correct parameters', async () => { diff --git a/test/fixtures/package.js b/test/fixtures/package.js index ba7b444..d1cd64c 100644 --- a/test/fixtures/package.js +++ b/test/fixtures/package.js @@ -3,6 +3,7 @@ export default { major, minor, patch, + cached = false, }) => new Buffer( JSON.stringify({ _id: 'foo-bar-package', @@ -25,12 +26,16 @@ export default { data: 'foo-package-data', }, }, + _codebox: { + cached, + }, }), ), withAttachments: ({ major, minor, patch, + cached = false, }) => new Buffer( JSON.stringify({ _id: 'foo-bar-package', @@ -52,12 +57,16 @@ export default { data: 'foo-package-data', }, }, + _codebox: { + cached, + }, }), ), withoutAttachments: ({ major, minor, patch, + cached = false, }) => new Buffer( JSON.stringify({ _id: 'foo-bar-package', @@ -75,6 +84,9 @@ export default { }, }, _attachments: {}, + _codebox: { + cached, + }, }), ), }; diff --git a/test/get/lib.test.js b/test/get/lib.test.js index 3225895..6a2d3cf 100644 --- a/test/get/lib.test.js +++ b/test/get/lib.test.js @@ -180,6 +180,132 @@ describe('GET /registry/{name}', () => { }); }); + context('cache enabled', () => { + let storageStub; + let cacheStub; + + beforeEach(() => { + const getPackageStub = stub().returns( + pkg.withoutAttachments({ + major: 1, + minor: 0, + patch: 0, + })); + + cacheStub = { + get: getPackageStub, + }; + }); + + it('should attempt to get package from cache', async () => { + await subject(event, { + registry: 'https://example.com', + user: stub(), + cacheEnabled: true, + cache: cacheStub, + log: { + error: stub(), + }, + npm: stub(), + storage: stub(), + }, callback); + + assert(cacheStub.get.calledWithExactly( + 'foo-bar-package/index.json', + )); + }); + + context('package has not been cached and not a private package', () => { + beforeEach(() => { + const notInStorageErr = new Error('Not Found'); + notInStorageErr.code = 'NoSuchKey'; + + const notInStorageStub = stub().throws(notInStorageErr); + + cacheStub = { + get: notInStorageStub, + put: stub(), + }; + + storageStub = { + get: notInStorageStub, + }; + }); + + it('should add package ready to be cached by schedule', async () => { + await subject(event, { + registry: 'https://example.com', + user: stub(), + cacheEnabled: true, + cache: cacheStub, + log: { + info: stub(), + }, + npm: { + package: stub().returns( + JSON.parse( + pkg.withoutAttachments({ + major: 1, + minor: 0, + patch: 0, + }).toString())), + }, + storage: storageStub, + }, callback); + + assert(cacheStub.put.calledWithExactly( + 'foo-bar-package/index.json', + pkg.withoutAttachments({ + major: 1, + minor: 0, + patch: 0, + cached: false, + }).toString(), + )); + }); + }); + + context('package has been cached', () => { + beforeEach(() => { + const getPackageStub = stub().returns( + pkg.withoutAttachments({ + major: 1, + minor: 0, + patch: 0, + cached: true, + })); + + cacheStub = { + get: getPackageStub, + }; + }); + + it('should return package from cache', async () => { + await subject(event, { + registry: 'https://example.com', + user: stub(), + cacheEnabled: true, + cache: cacheStub, + log: { + error: stub(), + }, + npm: stub(), + storage: stub(), + }, callback); + + assert(callback.calledWithExactly(null, { + statusCode: 200, + body: pkg.withoutAttachments({ + major: 1, + minor: 0, + patch: 0, + cached: true, + }).toString(), + })); + }); + }); + }); + context('storage get errors', () => { let storageStub; diff --git a/test/serverless_plugins/remove-storage/index.test.js b/test/serverless_plugins/remove-storage/index.test.js index 4000656..53e71d3 100644 --- a/test/serverless_plugins/remove-storage/index.test.js +++ b/test/serverless_plugins/remove-storage/index.test.js @@ -35,6 +35,11 @@ describe('Plugin: RemoveStorageBucket', () => { BucketName: 'foo-bucket', }, }, + PackageCacheStorage: { + Properties: { + BucketName: 'foo-bucket-cache', + }, + }, }, }, }, diff --git a/webpack.config.js b/webpack.config.js index 265f253..b567eaf 100644 --- a/webpack.config.js +++ b/webpack.config.js @@ -6,6 +6,7 @@ module.exports = { externals: [nodeExternals()], entry: { authorizerGithub: ['./bootstrap', './src/authorizers/github.js'], + cache: ['./bootstrap', './src/scheduled/cache/index.js'], put: ['./bootstrap', './src/put/index.js'], get: ['./bootstrap', './src/get/index.js'], distTagsGet: ['./bootstrap', './src/dist-tags/get.js'],