Caching

.serverless_plugins/remove-storage/index.js

@@ -28,23 +28,29 @@ class RemoveStorageBucket {
       .Properties
       .BucketName;
 
+    this.bucketCache = this.serverless.service.resources
+      .Resources
+      .PackageCacheStorage
+      .Properties
+      .BucketName;
+
     this.hooks = {
       'before:remove:remove': this.beforeRemove.bind(this),
     };
   }
 
-  listAllKeys(token) {
+  listAllKeys(bucket, token) {
     const allKeys = [];
     return this.s3.listObjectsV2({
-      Bucket: this.bucket,
+      Bucket: bucket,
       ContinuationToken: token,
     })
     .promise()
     .then((data) => {
       allKeys.push(data.Contents);
 
       if (data.IsTruncated) {
-        return this.listAllKeys(data.NextContinuationToken);
+        return this.listAllKeys(bucket, data.NextContinuationToken);
       }
 
       return [].concat(...allKeys).map(({ Key }) => ({ Key }));
@@ -53,19 +59,40 @@ class RemoveStorageBucket {
 
   beforeRemove() {
     return new Promise((resolve, reject) => {
-      return this.listAllKeys()
+      this.listAllKeys(this.bucketCache)
       .then((keys) => {
         if (keys.length > 0) {
           return this.s3
           .deleteObjects({
-            Bucket: this.bucket,
+            Bucket: this.bucketCache,
             Delete: {
               Objects: keys,
             },
           }).promise();
         }
-
-        return true;
+      })
+      .then(() => {
+        return this.s3
+          .deleteBucket({
+            Bucket: this.bucketCache,
+          }).promise()
+          .then(() => {
+            this.serverless.cli.log('AWS Package Storage Cache Removed');
+          });
+      })
+      .then(() => {
+        return this.listAllKeys(this.bucket)
+        .then((keys) => {
+          if (keys.length > 0) {
+            return this.s3
+            .deleteObjects({
+              Bucket: this.bucket,
+              Delete: {
+                Objects: keys,
+              },
+            }).promise();
+          }
+        })
       })
       .then(() => {
         return this.s3

.serverless_plugins/update-cache-env/index.js

@@ -0,0 +1,78 @@
+const util = require('util');
+
+class UpdateCacheEnv {
+  constructor(serverless, options) {
+    this.options = options;
+    this.serverless = serverless;
+    this.provider = this.serverless.getProvider('aws');
+
+    this.awsInfo = this.serverless
+      .pluginManager
+      .plugins
+      .find(p => p.constructor.name === 'AwsInfo');
+
+    this.bucketCache = this.serverless
+    .service
+    .provider
+    .environment
+    .bucketCache;
+
+    this.registry = this.serverless
+    .service
+    .provider
+    .environment
+    .registry;
+
+    this.cacheEnabled = this.serverless
+    .service
+    .provider
+    .environment
+    .cacheEnabled;
+
+    this.hooks = {
+      'after:deploy:deploy': this.afterDeploy.bind(this),
+    };
+  }
+
+  afterDeploy() {
+    const lambda = new this.provider.sdk.Lambda({
+      signatureVersion: 'v4',
+      region: this.options.region,
+    });
+
+    const cacheFunction = this
+    .awsInfo
+    .gatheredData
+    .info
+    .functions
+    .find(f => f.name === 'cache');
+
+    if (!cacheFunction) {
+      throw new Error('Cache function has not been deployed correctly to AWS.');
+    }
+
+    const params = {
+      FunctionName: cacheFunction.deployedName,
+      Environment: {
+        Variables: {
+          apiEndpoint: `${this.awsInfo.gatheredData.info.endpoint}/registry`,
+          region: this.options.region,
+          bucketCache: this.bucketCache,
+          registry: this.registry,
+          cacheEnabled: this.cacheEnabled,
+        }
+      },
+    };
+
+    lambda
+    .updateFunctionConfiguration(params)
+    .promise()
+    .then(() => {
+      this.serverless.cli.log('AWS Cache Environment Ready.');
+    }).catch(err => {
+      this.serverless.cli.log(err.message);
+    });
+  }
+}
+
+module.exports = UpdateCacheEnv;

README.md

@@ -32,6 +32,7 @@ export CODEBOX_GITHUB_URL="https://api.github.com/" # The GitHub / GitHub Enterp
 export CODEBOX_GITHUB_CLIENT_ID="client_id" # The client id for your GitHub application
 export CODEBOX_GITHUB_SECRET="secret" # The secret for your GitHub application
 export CODEBOX_RESTRICTED_ORGS="" # OPTIONAL: Comma seperated list of github organisations to only allow access to users in that org (e.g. "craftship,myorg").  Useful if using public GitHub for authentication, as by default all authenticated users would have access.
+export CODEBOX_CACHE="false" # OPTIONAL: Any npm install will cache dependencies from public registry in a separate S3 bucket
 ```
 * `serverless deploy --stage prod` (pick which ever stage you wish)
 * `npm set registry <url>` - `<url>` being the base url shown in the terminal after deployment completes, such as:
@@ -64,6 +65,11 @@ Once done ensure you have a project based `.npmrc` config setup a per the "Using
 
 Yarn does not require an explicit `yarn login` as in this scenario it uses your `.npmrc` config instead.
 
+## Caching
+If you enable `CODEBOX_CACHE="true"` when using the registry all requests to packages that hit the public registry will then be cached.  This allows you to have a cache / mirror of all dependencies in your project.  Helps with robust deployments and better response times when hosting your CI in the same region as your Codebox npm registry.
+
+**NOTE: Your AWS bill will rise due to this scheduled task ensuring that cached dependnecies are up to date.**
+
 ## Admins / Publishing Packages
 `npm publish` works as it normally does via the npm CLI.  By default all users that authenticate have read only access.  If you wish to allow publish rights then you need to set the `CODEBOX_ADMINS` environment variable to a comma separated list of GitHub usernames such as `jonsharratt,kadikraman` and re-deploy.
 

serverless.yml

@@ -4,6 +4,7 @@ plugins:
   - environment-variables
   - remove-storage
   - serverless-webpack
+  - update-cache-env
   - content-handling
   - codebox-tools
 
@@ -22,7 +23,9 @@ provider:
     githubClientId:  ${env:CODEBOX_GITHUB_CLIENT_ID}
     githubSecret:  ${env:CODEBOX_GITHUB_SECRET}
     bucket: ${env:CODEBOX_BUCKET}-${self:provider.stage}
+    bucketCache: ${env:CODEBOX_BUCKET}-cache-${self:provider.stage}
     region: ${self:provider.region}
+    cacheEnabled: ${env:CODEBOX_CACHE}
 
     clientId: ${env:CODEBOX_INSIGHTS_CLIENT_ID}
     secret: ${env:CODEBOX_INSIGHTS_SECRET}
@@ -38,6 +41,7 @@ provider:
         - "sns:Publish"
       Resource:
         - "arn:aws:s3:::${self:provider.environment.bucket}*"
+        - "arn:aws:s3:::${self:provider.environment.bucketCache}*"
         - "Fn::Join":
           - ""
           - - "arn:aws:sns:"
@@ -50,6 +54,12 @@ functions:
   authorizerGithub:
     handler: authorizerGithub.default
 
+  cache:
+    handler: cache.default
+    timeout: 300
+    events:
+      - schedule: rate(1 hour)
+
   put:
     handler: put.default
     events:
@@ -134,6 +144,11 @@ resources:
       Properties:
         AccessControl: Private
         BucketName: ${self:provider.environment.bucket}
+    PackageCacheStorage:
+      Type: AWS::S3::Bucket
+      Properties:
+        AccessControl: Private
+        BucketName: ${self:provider.environment.bucketCache}
 
 custom:
   webpackIncludeModules: true

src/adapters/s3.js

@@ -25,4 +25,20 @@ export default class Storage {
 
     return meta.Body;
   }
+
+  async listAllKeys(token = null, keys = []) {
+    const data = await this.S3.listObjectsV2({
+      ContinuationToken: token,
+    })
+    .promise();
+
+    keys.push(data.Contents);
+
+    if (data.IsTruncated) {
+      return this.listAllKeys(data.NextContinuationToken, keys);
+    }
+
+    return [].concat(...keys).map(({ Key }) => Key);
+  }
 }
+

src/contextFactory.js

@@ -53,7 +53,9 @@ const log = (cmd, namespace, region, topic) => {
 export default (namespace, { headers, requestContext }) => {
   const {
     registry,
+    cacheEnabled,
     bucket,
+    bucketCache,
     region,
     logTopic,
   } = process.env;
@@ -62,9 +64,11 @@ export default (namespace, { headers, requestContext }) => {
 
   return {
     command: cmd,
+    cacheEnabled: (cacheEnabled === 'true'),
     registry,
     user: user(requestContext.authorizer),
     storage: storage(region, bucket),
+    cache: storage(region, bucketCache),
     log: log(cmd, namespace, region, logTopic),
     npm,
   };