Merge pull request #6196 from serverless/s3-package-artifacts

Add support for S3 hosted package artifacts

Author: pmuens

docs/providers/aws/guide/packaging.md

@@ -90,17 +90,18 @@ Serverless won't zip your service if this is configured and therefore `exclude`
 
 The artifact option is especially useful in case your development environment allows you to generate a deployable artifact like Maven does for Java.
 
-### Example
+#### Service package
 
 ```yml
 service: my-service
 package:
   artifact: path/to/my-artifact.zip
 ```
 
-You can also use this to package functions individually.
+#### Individual function packages
+
+You can also use this to package functions individually:
 
-### Example
 ```yml
 service: my-service
 
@@ -118,6 +119,34 @@ functions:
         method: get
 ```
 
+#### Artifacst hosted on S3
+
+Artifacts can also be fetched from a remote S3 bucket. In this case you just need to provide the S3 object URL as the artifact value. This applies to both, service-wide and function-level artifact setups.
+
+##### Service package
+
+```yml
+service: my-service
+
+package:
+  artifact: https://s3.amazonaws.com/some-bucket/service-artifact.zip
+```
+
+##### Individual function packages
+
+```yml
+service: my-service
+
+package:
+  individually: true
+
+functions:
+  hello:
+    handler: com.serverless.Handler
+  package:
+    artifact: https://s3.amazonaws.com/some-bucket/function-artifact.zip
+```
+
 ### Packaging functions separately
 
 If you want even more controls over your functions for deployment you can configure them to be packaged independently. This allows you more control for optimizing your deployment. To enable individual packaging set `individually` to true in the service or function wide packaging settings.

lib/classes/Utils.js

@@ -1,5 +1,7 @@
 'use strict';
 
+const os = require('os');
+const crypto = require('crypto');
 const fs = require('fs');
 const path = require('path');
 const ci = require('ci-info');
@@ -31,6 +33,13 @@ class Utils {
     return dirExistsSync(dirPath);
   }
 
+  getTmpDirPath() {
+    const dirPath = path.join(os.tmpdir(),
+      'tmpdirs-serverless', crypto.randomBytes(8).toString('hex'));
+    fse.ensureDirSync(dirPath);
+    return dirPath;
+  }
+
   fileExistsSync(filePath) {
     return fileExistsSync(filePath);
   }

lib/classes/Utils.test.js

@@ -25,6 +25,15 @@ describe('Utils', () => {
     utils = new Utils(serverless);
   });
 
+  describe('#getTmpDirPath()', () => {
+    it('should create a scoped tmp directory', () => {
+      const dirPath = serverless.utils.getTmpDirPath();
+      const stats = fse.statSync(dirPath);
+      expect(dirPath).to.include('tmpdirs-serverless');
+      expect(stats.isDirectory()).to.equal(true);
+    });
+  });
+
   describe('#dirExistsSync()', () => {
     describe('When reading a directory', () => {
       it('should detect if a directory exists', () => {

lib/plugins/aws/package/compile/functions/index.js

@@ -1,5 +1,6 @@
 'use strict';
 
+const AWS = require('aws-sdk');
 const BbPromise = require('bluebird');
 const crypto = require('crypto');
 const fs = require('fs');
@@ -23,6 +24,7 @@ class AwsCompileFunctions {
 
     this.hooks = {
       'package:compileFunctions': () => BbPromise.bind(this)
+        .then(this.downloadPackageArtifacts)
         .then(this.compileFunctions),
     };
   }
@@ -66,6 +68,46 @@ class AwsCompileFunctions {
     }
   }
 
+  downloadPackageArtifact(functionName) {
+    const { region } = this.options;
+    const S3 = new AWS.S3({ region });
+
+    const functionObject = this.serverless.service.getFunction(functionName);
+    const artifactFilePath = _.get(functionObject, 'package.artifact') ||
+      _.get(this, 'serverless.service.package.artifact');
+
+    const regex = new RegExp('.*s3.amazonaws.com/(.+)/(.+)');
+    const match = artifactFilePath.match(regex);
+
+    if (match) {
+      return new BbPromise((resolve, reject) => {
+        const tmpDir = this.serverless.utils.getTmpDirPath();
+        const filePath = path.join(tmpDir, match[2]);
+
+        const readStream = S3.getObject({
+          Bucket: match[1],
+          Key: match[2],
+        }).createReadStream();
+
+        const writeStream = fs.createWriteStream(filePath);
+
+        readStream.on('error', (error) => reject(error));
+        readStream.pipe(writeStream)
+          .on('error', reject)
+          .on('close', () => {
+            if (functionObject.package.artifact) {
+              functionObject.package.artifact = filePath;
+            } else {
+              this.serverless.service.package.artifact = filePath;
+            }
+            return resolve(filePath);
+          });
+      });
+    }
+
+    return BbPromise.resolve();
+  }
+
   compileFunction(functionName) {
     const newFunction = this.cfLambdaFunctionTemplate();
     const functionObject = this.serverless.service.getFunction(functionName);
@@ -76,6 +118,7 @@ class AwsCompileFunctions {
 
     let artifactFilePath = functionObject.package.artifact ||
       this.serverless.service.package.artifact;
+
     if (!artifactFilePath ||
       (this.serverless.service.artifact && !functionObject.package.artifact)) {
       let artifactFileName = serviceArtifactFileName;
@@ -452,12 +495,18 @@ class AwsCompileFunctions {
     });
   }
 
+  downloadPackageArtifacts() {
+    const allFunctions = this.serverless.service.getAllFunctions();
+    return BbPromise.each(
+      allFunctions,
+      functionName => this.downloadPackageArtifact(functionName));
+  }
+
   compileFunctions() {
     const allFunctions = this.serverless.service.getAllFunctions();
     return BbPromise.each(
       allFunctions,
-      functionName => this.compileFunction(functionName)
-    );
+      functionName => this.compileFunction(functionName));
   }
 
   // helper functions

lib/plugins/aws/package/compile/functions/index.test.js

@@ -1,19 +1,24 @@
 'use strict';
 
+const AWS = require('aws-sdk');
+const fs = require('fs');
 const _ = require('lodash');
 const path = require('path');
 const chai = require('chai');
+const sinon = require('sinon');
 const AwsProvider = require('../../../provider/awsProvider');
 const AwsCompileFunctions = require('./index');
 const Serverless = require('../../../../../Serverless');
-const { getTmpDirPath } = require('../../../../../../tests/utils/fs');
+const { getTmpDirPath, createTmpFile } = require('../../../../../../tests/utils/fs');
 
 chai.use(require('chai-as-promised'));
+chai.use(require('sinon-chai'));
 
 const expect = chai.expect;
 
 describe('AwsCompileFunctions', () => {
   let serverless;
+  let awsProvider;
   let awsCompileFunctions;
   const functionName = 'test';
   const compiledFunctionName = 'TestLambdaFunction';
@@ -24,7 +29,8 @@ describe('AwsCompileFunctions', () => {
       region: 'us-east-1',
     };
     serverless = new Serverless(options);
-    serverless.setProvider('aws', new AwsProvider(serverless, options));
+    awsProvider = new AwsProvider(serverless, options);
+    serverless.setProvider('aws', awsProvider);
     serverless.cli = new serverless.classes.CLI();
     awsCompileFunctions = new AwsCompileFunctions(serverless, options);
     awsCompileFunctions.serverless.service.provider.compiledCloudFormationTemplate = {
@@ -74,6 +80,60 @@ describe('AwsCompileFunctions', () => {
       expect(awsCompileFunctions.isArnRefGetAttOrImportValue({ Blah: 'vtha' })).to.equal(false));
   });
 
+  describe('#downloadPackageArtifacts()', () => {
+    let requestStub;
+    let testFilePath;
+    const s3BucketName = 'test-bucket';
+    const s3ArtifactName = 's3-hosted-artifact.zip';
+
+    beforeEach(() => {
+      testFilePath = createTmpFile('dummy-artifact');
+      requestStub = sinon.stub(AWS, 'S3').returns({
+        getObject: () => ({
+          createReadStream() {
+            return fs.createReadStream(testFilePath);
+          },
+        }),
+      });
+    });
+
+    afterEach(() => {
+      AWS.S3.restore();
+    });
+
+    it('should download the file and replace the artifact path for function packages', () => {
+      awsCompileFunctions.serverless.service.package.individually = true;
+      awsCompileFunctions.serverless.service.functions[functionName]
+        .package.artifact = `https://s3.amazonaws.com/${s3BucketName}/${s3ArtifactName}`;
+
+      return expect(awsCompileFunctions.downloadPackageArtifacts()).to.be.fulfilled
+        .then(() => {
+          const artifactFileName = awsCompileFunctions.serverless.service
+            .functions[functionName].package.artifact.split(path.sep).pop();
+
+          expect(requestStub.callCount).to.equal(1);
+          expect(artifactFileName).to.equal(s3ArtifactName);
+        });
+    });
+
+    it('should download the file and replace the artifact path for service-wide packages', () => {
+      awsCompileFunctions.serverless.service.package.individually = false;
+      awsCompileFunctions.serverless.service.functions[functionName]
+        .package.artifact = false;
+      awsCompileFunctions.serverless.service.package
+        .artifact = `https://s3.amazonaws.com/${s3BucketName}/${s3ArtifactName}`;
+
+      return expect(awsCompileFunctions.downloadPackageArtifacts()).to.be.fulfilled
+        .then(() => {
+          const artifactFileName = awsCompileFunctions.serverless.service.package.artifact
+            .split(path.sep).pop();
+
+          expect(requestStub.callCount).to.equal(1);
+          expect(artifactFileName).to.equal(s3ArtifactName);
+        });
+    });
+  });
+
   describe('#compileFunctions()', () => {
     it('should use service artifact if not individually', () => {
       awsCompileFunctions.serverless.service.package.individually = false;

tests/utils/fs/index.js

@@ -3,6 +3,7 @@
 const os = require('os');
 const path = require('path');
 const fs = require('fs');
+const fse = require('fs-extra');
 const crypto = require('crypto');
 const YAML = require('js-yaml');
 const JSZip = require('jszip');
@@ -18,6 +19,12 @@ function getTmpFilePath(fileName) {
   return path.join(getTmpDirPath(), fileName);
 }
 
+function createTmpFile(name) {
+  const filePath = getTmpFilePath(name);
+  fse.ensureFileSync(filePath);
+  return filePath;
+}
+
 function replaceTextInFile(filePath, subString, newSubString) {
   const fileContent = fs.readFileSync(filePath).toString();
   fs.writeFileSync(filePath, fileContent.replace(subString, newSubString));
@@ -43,6 +50,7 @@ module.exports = {
   tmpDirCommonPath,
   getTmpDirPath,
   getTmpFilePath,
+  createTmpFile,
   replaceTextInFile,
   readYamlFile,
   writeYamlFile,