Skip to content

Commit 27b70f4

Browse files
stijzermansStijn IJzermans
stijzermans
and
Stijn IJzermans
authored
fix: Ensure proper support for mixed runtimes and architectures (#815)
* feat: Use function runtime & arch for docker * docs: Update readme for python3.9 * feat: Do not zip req for non-py functions * ci: Bump internal package version / python version * fix: Rename mixed test name to be more descriptive --------- Co-authored-by: Stijn IJzermans <[email protected]>
1 parent 787b479 commit 27b70f4

File tree

20 files changed

+187
-25
lines changed

20 files changed

+187
-25
lines changed

‎.python-version

+1-1
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
3.7
1+
3.9

‎CONTRIBUTING.md

+6-4
Original file line numberDiff line numberDiff line change
@@ -13,12 +13,14 @@ Welcome, and thanks in advance for your help!
1313
## Setup
1414

1515
Pre-Reqs:
16-
* Python 3.7
17-
* [poetry](https://python-poetry.org/docs/) (if you use multiple versions of Python be sure to install it with python 3.7)
18-
* Perl (used in the tests)
19-
* Node v14 or v16
16+
17+
- Python 3.9
18+
- [poetry](https://python-poetry.org/docs/) (if you use multiple versions of Python be sure to install it with python 3.9)
19+
- Perl (used in the tests)
20+
- Node v14 or v16
2021

2122
Then, to begin development:
23+
2224
1. fork the repository
2325
2. `npm install -g serverless@<VERSION>` (check the peer dependencies in the root `package.json` file for the version)
2426
3. run `npm install` in its root folder

‎index.js

+13-6
Original file line numberDiff line numberDiff line change
@@ -106,13 +106,8 @@ class ServerlessPythonRequirements {
106106
throw new Error(
107107
'Python Requirements: you can provide a dockerImage or a dockerFile option, not both.'
108108
);
109-
} else if (!options.dockerFile) {
110-
// If no dockerFile is provided, use default image
111-
const architecture =
112-
this.serverless.service.provider.architecture || 'x86_64';
113-
const defaultImage = `public.ecr.aws/sam/build-${this.serverless.service.provider.runtime}:latest-${architecture}`;
114-
options.dockerImage = options.dockerImage || defaultImage;
115109
}
110+
116111
if (options.layer) {
117112
// If layer was set as a boolean, set it to an empty object to use the layer defaults.
118113
if (options.layer === true) {
@@ -188,6 +183,18 @@ class ServerlessPythonRequirements {
188183
this.commands.requirements.type = 'container';
189184
}
190185

186+
this.dockerImageForFunction = (funcOptions) => {
187+
const runtime =
188+
funcOptions.runtime || this.serverless.service.provider.runtime;
189+
190+
const architecture =
191+
funcOptions.architecture ||
192+
this.serverless.service.provider.architecture ||
193+
'x86_64';
194+
const defaultImage = `public.ecr.aws/sam/build-${runtime}:latest-${architecture}`;
195+
return this.options.dockerImage || defaultImage;
196+
};
197+
191198
const isFunctionRuntimePython = (args) => {
192199
// If functionObj.runtime is undefined, python.
193200
if (!args[1].functionObj || !args[1].functionObj.runtime) {

‎lib/pip.js

+7-6
Original file line numberDiff line numberDiff line change
@@ -125,12 +125,13 @@ async function pipAcceptsSystem(pythonBin, pluginInstance) {
125125
/**
126126
* Install requirements described from requirements in the targetFolder into that same targetFolder
127127
* @param {string} targetFolder
128-
* @param {Object} serverless
129-
* @param {Object} options
128+
* @param {Object} pluginInstance
129+
* @param {Object} funcOptions
130130
* @return {undefined}
131131
*/
132-
async function installRequirements(targetFolder, pluginInstance) {
133-
const { options, serverless, log, progress } = pluginInstance;
132+
async function installRequirements(targetFolder, pluginInstance, funcOptions) {
133+
const { options, serverless, log, progress, dockerImageForFunction } =
134+
pluginInstance;
134135
const targetRequirementsTxt = path.join(targetFolder, 'requirements.txt');
135136

136137
let installProgress;
@@ -253,7 +254,7 @@ async function installRequirements(targetFolder, pluginInstance) {
253254
buildDockerImageProgress && buildDockerImageProgress.remove();
254255
}
255256
} else {
256-
dockerImage = options.dockerImage;
257+
dockerImage = dockerImageForFunction(funcOptions);
257258
}
258259
if (log) {
259260
log.info(`Docker Image: ${dockerImage}`);
@@ -691,7 +692,7 @@ async function installRequirementsIfNeeded(
691692
fse.copySync(slsReqsTxt, path.join(workingReqsFolder, 'requirements.txt'));
692693

693694
// Then install our requirements from this folder
694-
await installRequirements(workingReqsFolder, pluginInstance);
695+
await installRequirements(workingReqsFolder, pluginInstance, funcOptions);
695696

696697
// Copy vendor libraries to requirements folder
697698
if (options.vendor) {

‎lib/zip.js

+5
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,11 @@ function packRequirements() {
114114
if (this.options.zip) {
115115
if (this.serverless.service.package.individually) {
116116
return BbPromise.resolve(this.targetFuncs)
117+
.filter((func) => {
118+
return (
119+
func.runtime || this.serverless.service.provider.runtime
120+
).match(/^python.*/);
121+
})
117122
.map((f) => {
118123
if (!get(f, 'module')) {
119124
set(f, ['module'], '.');

‎test.js

+83
Original file line numberDiff line numberDiff line change
@@ -1373,6 +1373,89 @@ test(
13731373
{ skip: !canUseDocker() || process.platform === 'win32' }
13741374
);
13751375

1376+
test(
1377+
'py3.9 can package flask running in docker with module runtime & architecture of function',
1378+
async (t) => {
1379+
process.chdir('tests/individually_mixed_runtime');
1380+
const path = npm(['pack', '../..']);
1381+
npm(['i', path]);
1382+
1383+
sls(['package'], {
1384+
env: { dockerizePip: 'true' },
1385+
});
1386+
1387+
const zipfiles_hello2 = await listZipFiles(
1388+
'.serverless/module2-sls-py-req-test-indiv-mixed-runtime-dev-hello2.zip'
1389+
);
1390+
t.true(
1391+
zipfiles_hello2.includes('handler2.py'),
1392+
'handler2.py is packaged at root level in function hello2'
1393+
);
1394+
t.true(
1395+
zipfiles_hello2.includes(`flask${sep}__init__.py`),
1396+
'flask is packaged in function hello2'
1397+
);
1398+
},
1399+
{
1400+
skip: !canUseDocker() || process.platform === 'win32',
1401+
}
1402+
);
1403+
1404+
test(
1405+
'py3.9 can package flask succesfully when using mixed architecture, docker and zipping',
1406+
async (t) => {
1407+
process.chdir('tests/individually_mixed_runtime');
1408+
const path = npm(['pack', '../..']);
1409+
1410+
npm(['i', path]);
1411+
sls(['package'], { env: { dockerizePip: 'true', zip: 'true' } });
1412+
1413+
const zipfiles_hello = await listZipFiles('.serverless/hello1.zip');
1414+
t.true(
1415+
zipfiles_hello.includes(`module1${sep}handler1.ts`),
1416+
'handler1.ts is packaged in module dir for hello1'
1417+
);
1418+
t.false(
1419+
zipfiles_hello.includes('handler2.py'),
1420+
'handler2.py is NOT packaged at root level in function hello1'
1421+
);
1422+
t.false(
1423+
zipfiles_hello.includes(`flask${sep}__init__.py`),
1424+
'flask is NOT packaged in function hello1'
1425+
);
1426+
1427+
const zipfiles_hello2 = await listZipFiles(
1428+
'.serverless/module2-sls-py-req-test-indiv-mixed-runtime-dev-hello2.zip'
1429+
);
1430+
const zippedReqs = await listRequirementsZipFiles(
1431+
'.serverless/module2-sls-py-req-test-indiv-mixed-runtime-dev-hello2.zip'
1432+
);
1433+
t.true(
1434+
zipfiles_hello2.includes('handler2.py'),
1435+
'handler2.py is packaged at root level in function hello2'
1436+
);
1437+
t.false(
1438+
zipfiles_hello2.includes(`module1${sep}handler1.ts`),
1439+
'handler1.ts is NOT included at module1 level in hello2'
1440+
);
1441+
t.false(
1442+
zipfiles_hello2.includes(`pyaml${sep}__init__.py`),
1443+
'pyaml is NOT packaged in function hello2'
1444+
);
1445+
t.false(
1446+
zipfiles_hello2.includes(`boto3${sep}__init__.py`),
1447+
'boto3 is NOT included in zipfile'
1448+
);
1449+
t.true(
1450+
zippedReqs.includes(`flask${sep}__init__.py`),
1451+
'flask is packaged in function hello2 in requirements.zip'
1452+
);
1453+
1454+
t.end();
1455+
},
1456+
{ skip: !canUseDocker() || process.platform === 'win32' }
1457+
);
1458+
13761459
test(
13771460
'py3.9 uses download cache by default option',
13781461
async (t) => {

‎tests/base/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
"author": "",
1010
"license": "ISC",
1111
"dependencies": {
12-
"serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz"
12+
"serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz"
1313
}
1414
}

‎tests/individually/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
"author": "",
1010
"license": "ISC",
1111
"dependencies": {
12-
"serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz"
12+
"serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz"
1313
}
1414
}

‎tests/individually_mixed_runtime/module1/handler1.ts

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
function hello() {
2+
return "hello"
3+
}

‎tests/individually_mixed_runtime/module2/handler2.py

+6
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
import flask
2+
3+
def hello(event, context):
4+
return {
5+
'status': 200,
6+
}

‎tests/individually_mixed_runtime/module2/requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
flask==2.0.3

‎tests/individually_mixed_runtime/package.json

+14
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
{
2+
"name": "example",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "index.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"author": "",
10+
"license": "ISC",
11+
"dependencies": {
12+
"serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz"
13+
}
14+
}

‎tests/individually_mixed_runtime/requirements-common.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
boto3

‎tests/individually_mixed_runtime/serverless.yml

+39
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
service: sls-py-req-test-indiv-mixed-runtime
2+
3+
provider:
4+
name: aws
5+
runtime: nodejs18.x
6+
architecture: arm64
7+
8+
package:
9+
individually: true
10+
11+
custom:
12+
pythonRequirements:
13+
dockerizePip: ${env:dockerizePip, self:custom.defaults.dockerizePip}
14+
zip: ${env:zip, self:custom.defaults.zip}
15+
defaults:
16+
dockerizePip: false
17+
zip: false
18+
19+
functions:
20+
hello1:
21+
handler: handler1.hello
22+
architecture: x86_64
23+
package:
24+
patterns:
25+
- '!**'
26+
- 'module1/**'
27+
28+
hello2:
29+
handler: handler2.hello
30+
module: module2
31+
runtime: python3.9
32+
architecture: x86_64
33+
package:
34+
patterns:
35+
- '!**'
36+
- 'module2/**'
37+
38+
plugins:
39+
- serverless-python-requirements

‎tests/non_build_pyproject/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
"author": "",
1010
"license": "ISC",
1111
"dependencies": {
12-
"serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz"
12+
"serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz"
1313
}
1414
}

‎tests/non_poetry_pyproject/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
"author": "",
1010
"license": "ISC",
1111
"dependencies": {
12-
"serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz"
12+
"serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz"
1313
}
1414
}

‎tests/pipenv/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
"author": "",
1010
"license": "ISC",
1111
"dependencies": {
12-
"serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz"
12+
"serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz"
1313
}
1414
}

‎tests/poetry/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
"author": "",
1010
"license": "ISC",
1111
"dependencies": {
12-
"serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz"
12+
"serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz"
1313
}
1414
}

‎tests/poetry_individually/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
"author": "",
1010
"license": "ISC",
1111
"dependencies": {
12-
"serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz"
12+
"serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz"
1313
}
1414
}

‎tests/poetry_packages/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,6 @@
99
"author": "",
1010
"license": "ISC",
1111
"dependencies": {
12-
"serverless-python-requirements": "file:serverless-python-requirements-6.0.0.tgz"
12+
"serverless-python-requirements": "file:serverless-python-requirements-6.0.1.tgz"
1313
}
1414
}

0 commit comments

Comments
 (0)