Skip to content

Commit 28b1121

Browse files
authored
PYTHON-3461 Test FaaS (AWS Lambda) Behavior Per Driver (#1310)
1 parent 0d44783 commit 28b1121

13 files changed

+387
-0
lines changed

.evergreen/config.yml

+50
Original file line numberDiff line numberDiff line change
@@ -1303,6 +1303,33 @@ task_groups:
13031303
tasks:
13041304
- testazurekms-task
13051305

1306+
- name: test_aws_lambda_task_group
1307+
setup_group:
1308+
- func: fetch source
1309+
- func: prepare resources
1310+
- command: subprocess.exec
1311+
params:
1312+
working_dir: src
1313+
binary: bash
1314+
add_expansions_to_env: true
1315+
args:
1316+
- ${DRIVERS_TOOLS}/.evergreen/atlas/setup-atlas-cluster.sh
1317+
- command: expansions.update
1318+
params:
1319+
file: src/atlas-expansion.yml
1320+
teardown_group:
1321+
- command: subprocess.exec
1322+
params:
1323+
working_dir: src
1324+
binary: bash
1325+
add_expansions_to_env: true
1326+
args:
1327+
- ${DRIVERS_TOOLS}/.evergreen/atlas/teardown-atlas-cluster.sh
1328+
setup_group_can_fail_task: true
1329+
setup_group_timeout_secs: 1800
1330+
tasks:
1331+
- test-aws-lambda-deployed
1332+
13061333
- name: test_atlas_task_group_search_indexes
13071334
setup_group:
13081335
- func: fetch source
@@ -1785,6 +1812,23 @@ tasks:
17851812
vars:
17861813
TEST_DATA_LAKE: "true"
17871814

1815+
- name: "test-aws-lambda-deployed"
1816+
commands:
1817+
- func: "install dependencies"
1818+
- command: ec2.assume_role
1819+
params:
1820+
role_arn: ${LAMBDA_AWS_ROLE_ARN}
1821+
duration_seconds: 3600
1822+
- command: subprocess.exec
1823+
params:
1824+
working_dir: src
1825+
binary: bash
1826+
add_expansions_to_env: true
1827+
args:
1828+
- .evergreen/run-deployed-lambda-aws-tests.sh
1829+
env:
1830+
TEST_LAMBDA_DIRECTORY: ${PROJECT_DIRECTORY}/test/lambda
1831+
17881832
- name: test-ocsp-rsa-valid-cert-server-staples
17891833
tags: ["ocsp", "ocsp-rsa", "ocsp-staple"]
17901834
commands:
@@ -3358,6 +3402,12 @@ buildvariants:
33583402
batchtime: 20160 # Use a batchtime of 14 days as suggested by the CSFLE test README
33593403
- testazurekms-fail-task
33603404

3405+
- name: rhel8-test-lambda
3406+
display_name: AWS Lambda handler tests
3407+
run_on: rhel87-small
3408+
tasks:
3409+
- name: test_aws_lambda_task_group
3410+
33613411
- name: Release
33623412
display_name: Release
33633413
batchtime: 20160 # 14 days
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#!/bin/bash
2+
set -o errexit # Exit the script with error if any of the commands fail
3+
4+
export PATH="/opt/python/3.9/bin:${PATH}"
5+
python --version
6+
pushd ./test/lambda
7+
8+
. build.sh
9+
popd
10+
. ${DRIVERS_TOOLS}/.evergreen/aws_lambda/run-deployed-lambda-aws-tests.sh

.gitignore

+7
Original file line numberDiff line numberDiff line change
@@ -17,3 +17,10 @@ mongocryptd.pid
1717
.idea/
1818
.nova/
1919
venv/
20+
21+
# Lambda temp files
22+
test/lambda/.aws-sam
23+
test/lambda/env.json
24+
test/lambda/mongodb/pymongo/*
25+
test/lambda/mongodb/gridfs/*
26+
test/lambda/mongodb/bson/*

.pre-commit-config.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ repos:
77
- id: check-case-conflict
88
- id: check-toml
99
- id: check-yaml
10+
exclude: template.yaml
1011
- id: debug-statements
1112
- id: end-of-file-fixer
1213
exclude: WHEEL

test/lambda/README.md

+17
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,17 @@
1+
AWS Lambda Testing
2+
------------------
3+
4+
Running locally
5+
===============
6+
7+
Prerequisites:
8+
9+
- AWS SAM CLI
10+
- Docker daemon running
11+
12+
Usage
13+
=====
14+
15+
- Start a local mongodb instance on port 27017
16+
- Run ``build.sh``
17+
- Run ``test.sh``

test/lambda/build.sh

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
#!/bin/bash
2+
set -o errexit # Exit the script with error if any of the commands fail
3+
set -o xtrace
4+
5+
rm -rf mongodb/pymongo
6+
rm -rf mongodb/gridfs
7+
rm -rf mongodb/bson
8+
9+
pushd ../..
10+
rm -f pymongo/*.so
11+
rm -f bson/*.so
12+
image="quay.io/pypa/manylinux2014_x86_64:latest"
13+
14+
DOCKER=$(command -v docker) || true
15+
if [ -z "$DOCKER" ]; then
16+
PODMAN=$(command -v podman) || true
17+
if [ -z "$PODMAN" ]; then
18+
echo "docker or podman are required!"
19+
exit 1
20+
fi
21+
DOCKER=podman
22+
fi
23+
24+
$DOCKER run --rm -v "`pwd`:/src" $image /src/test/lambda/build_internal.sh
25+
cp -r pymongo ./test/lambda/mongodb/pymongo
26+
cp -r bson ./test/lambda/mongodb/bson
27+
cp -r gridfs ./test/lambda/mongodb/gridfs
28+
popd

test/lambda/build_internal.sh

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash -ex
2+
3+
cd /src
4+
PYTHON=/opt/python/cp39-cp39/bin/python
5+
$PYTHON -m pip install -v -e .

test/lambda/events/event.json

+62
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,62 @@
1+
{
2+
"body": "{\"message\": \"hello world\"}",
3+
"resource": "/hello",
4+
"path": "/hello",
5+
"httpMethod": "GET",
6+
"isBase64Encoded": false,
7+
"queryStringParameters": {
8+
"foo": "bar"
9+
},
10+
"pathParameters": {
11+
"proxy": "/path/to/resource"
12+
},
13+
"stageVariables": {
14+
"baz": "qux"
15+
},
16+
"headers": {
17+
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
18+
"Accept-Encoding": "gzip, deflate, sdch",
19+
"Accept-Language": "en-US,en;q=0.8",
20+
"Cache-Control": "max-age=0",
21+
"CloudFront-Forwarded-Proto": "https",
22+
"CloudFront-Is-Desktop-Viewer": "true",
23+
"CloudFront-Is-Mobile-Viewer": "false",
24+
"CloudFront-Is-SmartTV-Viewer": "false",
25+
"CloudFront-Is-Tablet-Viewer": "false",
26+
"CloudFront-Viewer-Country": "US",
27+
"Host": "1234567890.execute-api.us-east-1.amazonaws.com",
28+
"Upgrade-Insecure-Requests": "1",
29+
"User-Agent": "Custom User Agent String",
30+
"Via": "1.1 08f323deadbeefa7af34d5feb414ce27.cloudfront.net (CloudFront)",
31+
"X-Amz-Cf-Id": "cDehVQoZnx43VYQb9j2-nvCh-9z396Uhbp027Y2JvkCPNLmGJHqlaA==",
32+
"X-Forwarded-For": "127.0.0.1, 127.0.0.2",
33+
"X-Forwarded-Port": "443",
34+
"X-Forwarded-Proto": "https"
35+
},
36+
"requestContext": {
37+
"accountId": "123456789012",
38+
"resourceId": "123456",
39+
"stage": "prod",
40+
"requestId": "c6af9ac6-7b61-11e6-9a41-93e8deadbeef",
41+
"requestTime": "09/Apr/2015:12:34:56 +0000",
42+
"requestTimeEpoch": 1428582896000,
43+
"identity": {
44+
"cognitoIdentityPoolId": null,
45+
"accountId": null,
46+
"cognitoIdentityId": null,
47+
"caller": null,
48+
"accessKey": null,
49+
"sourceIp": "127.0.0.1",
50+
"cognitoAuthenticationType": null,
51+
"cognitoAuthenticationProvider": null,
52+
"userArn": null,
53+
"userAgent": "Custom User Agent String",
54+
"user": null
55+
},
56+
"path": "/prod/hello",
57+
"resourcePath": "/hello",
58+
"httpMethod": "POST",
59+
"apiId": "1234567890",
60+
"protocol": "HTTP/1.1"
61+
}
62+
}

test/lambda/mongodb/Makefile

+4
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
2+
build-MongoDBFunction:
3+
cp -r . $(ARTIFACTS_DIR)
4+
python -m pip install -t $(ARTIFACTS_DIR) dnspython

test/lambda/mongodb/__init__.py

Whitespace-only changes.

test/lambda/mongodb/app.py

+149
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,149 @@
1+
"""
2+
Lambda function for Python Driver testing
3+
4+
Creates the client that is cached for all requests, subscribes to
5+
relevant events, and forces the connection pool to get populated.
6+
"""
7+
import json
8+
import os
9+
10+
from bson import has_c as has_bson_c
11+
from pymongo import MongoClient
12+
from pymongo import has_c as has_pymongo_c
13+
from pymongo.monitoring import (
14+
CommandListener,
15+
ConnectionPoolListener,
16+
ServerHeartbeatListener,
17+
)
18+
19+
open_connections = 0
20+
heartbeat_count = 0
21+
total_heartbeat_duration = 0
22+
total_commands = 0
23+
total_command_duration = 0
24+
25+
# Ensure we are using C extensions
26+
assert has_bson_c()
27+
assert has_pymongo_c()
28+
29+
30+
class CommandHandler(CommandListener):
31+
def started(self, event):
32+
print("command started", event)
33+
34+
def succeeded(self, event):
35+
global total_commands, total_command_duration
36+
total_commands += 1
37+
total_command_duration += event.duration_micros / 1e6
38+
print("command succeeded", event)
39+
40+
def failed(self, event):
41+
global total_commands, total_command_duration
42+
total_commands += 1
43+
total_command_duration += event.duration_micros / 1e6
44+
print("command failed", event)
45+
46+
47+
class ServerHeartbeatHandler(ServerHeartbeatListener):
48+
def started(self, event):
49+
print("server heartbeat started", event)
50+
51+
def succeeded(self, event):
52+
global heartbeat_count, total_heartbeat_duration
53+
heartbeat_count += 1
54+
total_heartbeat_duration += event.duration
55+
print("server heartbeat succeeded", event)
56+
57+
def failed(self, event):
58+
global heartbeat_count, total_heartbeat_duration
59+
heartbeat_count += 1
60+
total_heartbeat_duration += event.duration
61+
print("server heartbeat failed", event)
62+
63+
64+
class ConnectionHandler(ConnectionPoolListener):
65+
def connection_created(self, event):
66+
global open_connections
67+
open_connections += 1
68+
print("connection created")
69+
70+
def connection_ready(self, event):
71+
pass
72+
73+
def connection_closed(self, event):
74+
global open_connections
75+
open_connections -= 1
76+
print("connection closed")
77+
78+
def connection_check_out_started(self, event):
79+
pass
80+
81+
def connection_check_out_failed(self, event):
82+
pass
83+
84+
def connection_checked_out(self, event):
85+
pass
86+
87+
def connection_checked_in(self, event):
88+
pass
89+
90+
def pool_created(self, event):
91+
pass
92+
93+
def pool_ready(self, event):
94+
pass
95+
96+
def pool_cleared(self, event):
97+
pass
98+
99+
def pool_closed(self, event):
100+
pass
101+
102+
103+
listeners = [CommandHandler(), ServerHeartbeatHandler(), ConnectionHandler()]
104+
print("Creating client")
105+
client = MongoClient(os.environ["MONGODB_URI"], event_listeners=listeners)
106+
107+
108+
# Populate the connection pool.
109+
print("Connecting")
110+
client.lambdaTest.list_collections()
111+
print("Connected")
112+
113+
114+
# Create the response to send back.
115+
def create_response():
116+
return dict(
117+
averageCommandDuration=total_command_duration / total_commands,
118+
averageHeartbeatDuration=total_heartbeat_duration / heartbeat_count,
119+
openConnections=open_connections,
120+
heartbeatCount=heartbeat_count,
121+
)
122+
123+
124+
# Reset the numbers.
125+
def reset():
126+
global open_connections, heartbeat_count, total_heartbeat_duration, total_commands, total_command_duration
127+
open_connections = 0
128+
heartbeat_count = 0
129+
total_heartbeat_duration = 0
130+
total_commands = 0
131+
total_command_duration = 0
132+
133+
134+
def lambda_handler(event, context):
135+
"""
136+
The handler function itself performs an insert/delete and returns the
137+
id of the document in play.
138+
"""
139+
print("initializing")
140+
db = client.lambdaTest
141+
collection = db.test
142+
result = collection.insert_one({"n": 1})
143+
collection.delete_one({"_id": result.inserted_id})
144+
# Create the response and then reset the numbers.
145+
response = json.dumps(create_response())
146+
reset()
147+
print("finished!")
148+
149+
return dict(statusCode=200, body=response)

test/lambda/run.sh

+5
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/bin/bash
2+
set -o errexit # Exit the script with error if any of the commands fail
3+
4+
sam build
5+
sam local invoke --docker-network host --parameter-overrides "MongoDbUri=mongodb://host.docker.internal:27017"

0 commit comments

Comments
 (0)