Skip to content

Commit e4166fb

Browse files
xiazhveraxiazhvera
authored
Device Advisor Automation Scripts (#230)
Description of changes: Add the device advisor scripts to enable GitHub Actions to automatically run device advisor test on push GitHub Setting Changes: Added Repository secrets: AWS_DATEST_ACCESS_KEY_ID, AWS_DATEST_SECRET_ACCESS_KEY The secrets are set to aws-sdk-common-runtime user: IotSDKDeviceAdvisorCIAutomation By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
1 parent 4a0fdbe commit e4166fb

File tree

12 files changed

+319
-44
lines changed

12 files changed

+319
-44
lines changed

‎.github/workflows/ci.yml

Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,11 @@ env:
1313
PACKAGE_NAME: aws-iot-device-sdk-js-v2
1414
LINUX_BASE_IMAGE: ubuntu-16-x64
1515
RUN: ${{ github.run_id }}-${{ github.run_number }}
16-
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
17-
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
18-
AWS_DEFAULT_REGION: ${{ secrets.AWS_DEFAULT_REGION }}
19-
AWS_REGION: us-east-1
16+
# AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
17+
# AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
18+
AWS_ACCESS_KEY_ID: ${{ secrets.AWS_DATEST_ACCESS_KEY_ID }}
19+
AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_DATEST_SECRET_ACCESS_KEY }}
20+
AWS_DEFAULT_REGION: us-east-1
2021

2122
jobs:
2223

@@ -28,7 +29,6 @@ jobs:
2829
run: |
2930
aws s3 cp s3://aws-crt-test-stuff/ci/${{ env.BUILDER_VERSION }}/linux-container-ci.sh ./linux-container-ci.sh && chmod a+x ./linux-container-ci.sh
3031
./linux-container-ci.sh ${{ env.BUILDER_VERSION }} aws-crt-al2-x64 build -p ${{ env.PACKAGE_NAME }}
31-
3232
windows:
3333
runs-on: windows-latest
3434
steps:
@@ -46,4 +46,3 @@ jobs:
4646
chmod a+x builder
4747
./builder build -p ${{ env.PACKAGE_NAME }}
4848
49-

‎builder.json

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,5 +7,14 @@
77

88
"build_steps": [
99
"npm install"
10-
]
10+
],
11+
"test_steps": [
12+
"python3 -m pip install boto3",
13+
"python3 deviceadvisor/script/DATestRun.py"],
14+
"env": {
15+
"DA_TOPIC": "test/da",
16+
"DA_SHADOW_PROPERTY": "datest",
17+
"DA_SHADOW_VALUE_SET": "ON",
18+
"DA_SHADOW_VALUE_DEFAULT": "OFF"
19+
}
1120
}

‎deviceadvisor/script/DATestConfig.json

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"tests" :["MQTT Connect", "MQTT Publish", "MQTT Subscribe", "Shadow Publish", "Shadow Update"],
3+
"test_suite_ids" :
4+
{
5+
"MQTT Connect" : "ejbdzmo3hf3v",
6+
"MQTT Publish" : "euw7favf6an4",
7+
"MQTT Subscribe" : "01o8vo6no7sd",
8+
"Shadow Publish" : "elztm2jebc1q",
9+
"Shadow Update" : "vuydgrbbbfce"
10+
},
11+
"test_exe_path" :
12+
{
13+
"MQTT Connect" : "mqtt_connect",
14+
"MQTT Publish" : "mqtt_publish",
15+
"MQTT Subscribe" : "mqtt_subscribe",
16+
"Shadow Publish" : "shadow_update",
17+
"Shadow Update" : "shadow_update"
18+
}
19+
}

‎deviceadvisor/script/DATestRun.py

Lines changed: 225 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,225 @@
1+
import boto3
2+
import uuid
3+
import json
4+
import os
5+
import subprocess
6+
from time import sleep
7+
8+
##############################################
9+
# Cleanup Certificates and Things and created certificate and private key file
10+
def delete_thing_with_certi(thingName, certiId, certiArn):
11+
client.detach_thing_principal(
12+
thingName = thingName,
13+
principal = certiArn)
14+
client.update_certificate(
15+
certificateId =certiId,
16+
newStatus ='INACTIVE')
17+
client.delete_certificate(certificateId = certiId, forceDelete = True)
18+
client.delete_thing(thingName = thingName)
19+
os.remove(os.environ["DA_CERTI"])
20+
os.remove(os.environ["DA_KEY"])
21+
22+
23+
##############################################
24+
# Initialize variables
25+
# create aws clients
26+
client = boto3.client('iot')
27+
dataClient = boto3.client('iot-data')
28+
deviceAdvisor = boto3.client('iotdeviceadvisor')
29+
30+
# load test config
31+
f = open('deviceadvisor/script/DATestConfig.json')
32+
DATestConfig = json.load(f)
33+
f.close()
34+
35+
# create an temporary certificate/key file path
36+
certificate_path = os.path.join(os.getcwd(), 'certificate.pem.crt')
37+
key_path = os.path.join(os.getcwd(), 'private.pem.key')
38+
39+
# load environment variables requried for testing
40+
shadowProperty = os.environ['DA_SHADOW_PROPERTY']
41+
shadowDefault = os.environ['DA_SHADOW_VALUE_DEFAULT']
42+
43+
##############################################
44+
# install iot-device-sdk-v2
45+
subprocess.run("npm install --unsafe-perm", shell = True)
46+
47+
# test result
48+
test_result = {}
49+
50+
##############################################
51+
# Run device advisor
52+
for test_name in DATestConfig['tests']:
53+
##############################################
54+
# create a test thing
55+
thing_name = "DATest_" + str(uuid.uuid4())
56+
try:
57+
# create_thing_response:
58+
# {
59+
# 'thingName': 'string',
60+
# 'thingArn': 'string',
61+
# 'thingId': 'string'
62+
# }
63+
print("[Device Advisor]Info: Started to create thing...")
64+
create_thing_response = client.create_thing(
65+
thingName=thing_name
66+
)
67+
os.environ["DA_THING_NAME"] = thing_name
68+
69+
except Exception as e:
70+
print("[Device Advisor]Error: Failed to create thing: " + thing_name)
71+
exit(-1)
72+
73+
74+
##############################################
75+
# create certificate and keys used for testing
76+
try:
77+
print("[Device Advisor]Info: Started to create certificate...")
78+
# create_cert_response:
79+
# {
80+
# 'certificateArn': 'string',
81+
# 'certificateId': 'string',
82+
# 'certificatePem': 'string',
83+
# 'keyPair':
84+
# {
85+
# 'PublicKey': 'string',
86+
# 'PrivateKey': 'string'
87+
# }
88+
# }
89+
create_cert_response = client.create_keys_and_certificate(
90+
setAsActive=True
91+
)
92+
# write certificate to file
93+
f = open(certificate_path, "w")
94+
f.write(create_cert_response['certificatePem'])
95+
f.close()
96+
97+
# write private key to file
98+
f = open(key_path, "w")
99+
f.write(create_cert_response['keyPair']['PrivateKey'])
100+
f.close()
101+
102+
# setup environment variable
103+
os.environ["DA_CERTI"] = certificate_path
104+
os.environ["DA_KEY"] = key_path
105+
106+
except:
107+
client.delete_thing(thingName = thing_name)
108+
print("[Device Advisor]Error: Failed to create certificate.")
109+
exit(-1)
110+
111+
##############################################
112+
# attach certification to thing
113+
try:
114+
print("[Device Advisor]Info: Attach certificate to test thing...")
115+
# attache the certificate to thing
116+
client.attach_thing_principal(
117+
thingName = thing_name,
118+
principal = create_cert_response['certificateArn']
119+
)
120+
121+
certificate_arn = create_cert_response['certificateArn']
122+
certificate_id = create_cert_response['certificateId']
123+
124+
except:
125+
delete_thing_with_certi(thing_name, certificate_id ,certificate_arn )
126+
print("[Device Advisor]Error: Failed to attach certificate.")
127+
exit(-1)
128+
129+
try:
130+
######################################
131+
# set default shadow, for shadow update, if the
132+
# shadow does not exists, update will fail
133+
payload_shadow = json.dumps(
134+
{
135+
"state": {
136+
"desired": {
137+
shadowProperty: shadowDefault
138+
},
139+
"reported": {
140+
shadowProperty: shadowDefault
141+
}
142+
}
143+
})
144+
shadow_response = dataClient.update_thing_shadow(
145+
thingName = thing_name,
146+
payload = payload_shadow)
147+
get_shadow_response = dataClient.get_thing_shadow(thingName = thing_name)
148+
# make sure shadow is created before we go to next step
149+
while(get_shadow_response is None):
150+
get_shadow_response = dataClient.get_thing_shadow(thingName = thing_name)
151+
152+
# start device advisor test
153+
# test_start_response
154+
# {
155+
# 'suiteRunId': 'string',
156+
# 'suiteRunArn': 'string',
157+
# 'createdAt': datetime(2015, 1, 1)
158+
# }
159+
print("[Device Advisor]Info: Start device advisor test: " + test_name)
160+
test_start_response = deviceAdvisor.start_suite_run(
161+
suiteDefinitionId=DATestConfig['test_suite_ids'][test_name],
162+
suiteRunConfiguration={
163+
'primaryDevice': {
164+
'thingArn': create_thing_response['thingArn'],
165+
},
166+
'parallelRun': True
167+
})
168+
169+
# get DA endpoint
170+
endpoint_response = deviceAdvisor.get_endpoint(
171+
thingArn = create_thing_response['thingArn']
172+
)
173+
os.environ['DA_ENDPOINT'] = endpoint_response['endpoint']
174+
175+
working_dir = os.getcwd()
176+
exe_path = os.path.join("deviceadvisor/tests",DATestConfig['test_exe_path'][test_name])
177+
os.chdir(exe_path)
178+
subprocess.run("npm install --unsafe-perm", shell = True)
179+
180+
while True:
181+
# sleep for 1s every loop to avoid TooManyRequestsException
182+
sleep(1)
183+
test_result_responds = deviceAdvisor.get_suite_run(
184+
suiteDefinitionId=DATestConfig['test_suite_ids'][test_name],
185+
suiteRunId=test_start_response['suiteRunId']
186+
)
187+
# If the status is PENDING or the responds does not loaded, the test suite is still loading
188+
if (test_result_responds['status'] == 'PENDING' or
189+
len(test_result_responds['testResult']['groups']) == 0 or # test group has not been loaded
190+
len(test_result_responds['testResult']['groups'][0]['tests']) == 0 or #test case has not been loaded
191+
test_result_responds['testResult']['groups'][0]['tests'][0]['status'] == 'PENDING'):
192+
continue
193+
194+
# Start to run the test sample after the status turns into RUNNING
195+
elif (test_result_responds['status'] == 'RUNNING' and
196+
test_result_responds['testResult']['groups'][0]['tests'][0]['status'] == 'RUNNING'):
197+
file_path = 'dist/'+ DATestConfig['test_exe_path'][test_name] + '/index.js'
198+
subprocess.run('node ' + file_path, shell = True)
199+
# If the test finalizing then store the test result
200+
elif (test_result_responds['status'] != 'RUNNING'):
201+
test_result[test_name] = test_result_responds['status']
202+
# Delete the thing if the test passed
203+
if(test_result[test_name] == "PASS"):
204+
delete_thing_with_certi(thing_name, certificate_id ,certificate_arn )
205+
break
206+
207+
os.chdir(working_dir)
208+
except Exception as e:
209+
print("[Device Advisor]Error: Failed to test: "+ test_name + e)
210+
exit(-1)
211+
212+
##############################################
213+
# print result and cleanup things
214+
print(test_result)
215+
failed = False
216+
for test in test_result:
217+
if(test_result[test] != "PASS" and
218+
test_result[test] != "PASS_WITH_WARNINGS"):
219+
print("[Device Advisor]Error: Test \"" + test + "\" Failed with status:" + test_result[test])
220+
failed = True
221+
if failed:
222+
# if the test failed, we dont clean the Thing so that we can track the error
223+
exit(-1)
224+
225+
exit(0)

‎deviceadvisor/tests/mqtt_connect/index.ts

Lines changed: 8 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -19,9 +19,14 @@ async function main() {
1919
// ToDo: we can get rid of this but it requires a refactor of the native connection binding that includes
2020
// pinning the libuv event loop while the connection is active or potentially active.
2121
const timer = setInterval(() => { }, 60 * 1000);
22-
23-
await connection.connect();
24-
await connection.disconnect();
22+
try
23+
{
24+
await connection.connect();
25+
await connection.disconnect();
26+
} catch
27+
{
28+
process.exit(-1);
29+
}
2530

2631
// Allow node to die if the promise above resolved
2732
clearTimeout(timer);

‎deviceadvisor/tests/mqtt_connect/package.json

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"AWS SDK Common Runtime Team <[email protected]>"
1212
],
1313
"license": "Apache-2.0",
14-
"main": "./dist/index.ts",
14+
"main": "./dist/index.js",
1515
"scripts": {
1616
"tsc": "tsc",
1717
"prepare": "npm run tsc"
@@ -21,6 +21,7 @@
2121
"typescript": "^3.9.7"
2222
},
2323
"dependencies": {
24-
"aws-iot-device-sdk-v2": "../../../"
24+
"aws-iot-device-sdk-v2": "../../../",
25+
"clean": "^4.0.2"
2526
}
2627
}

‎deviceadvisor/tests/mqtt_publish/index.ts

Lines changed: 18 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,19 +20,24 @@ async function main() {
2020
// pinning the libuv event loop while the connection is active or potentially active.
2121
const timer = setInterval(() => { }, 60 * 1000);
2222

23-
// connect to mqtt
24-
await connection.connect();
25-
26-
// publish message to topic
27-
const msg = {
28-
message: "Device Advisor Test"
29-
};
30-
const json_msg = JSON.stringify(msg);
31-
await connection.publish(datest_utils.topic, json_msg, mqtt.QoS.AtLeastOnce);
32-
33-
// disconnect
34-
await connection.disconnect();
35-
23+
try
24+
{
25+
// connect to mqtt
26+
await connection.connect();
27+
28+
// publish message to topic
29+
const msg = {
30+
message: "Device Advisor Test"
31+
};
32+
const json_msg = JSON.stringify(msg);
33+
await connection.publish(datest_utils.topic, json_msg, mqtt.QoS.AtMostOnce);
34+
35+
// disconnect
36+
await connection.disconnect();
37+
} catch
38+
{
39+
process.exit(-1)
40+
}
3641
// Allow node to die if the promise above resolved
3742
clearTimeout(timer);
3843
process.exit(0);

‎deviceadvisor/tests/mqtt_publish/package.json

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111
"AWS SDK Common Runtime Team <[email protected]>"
1212
],
1313
"license": "Apache-2.0",
14-
"main": "./dist/index.ts",
14+
"main": "./dist/index.js",
1515
"scripts": {
1616
"tsc": "tsc",
1717
"prepare": "npm run tsc"

0 commit comments

Comments
 (0)