ExistsSync fixes

.github/workflows/ci.yml

@@ -8,10 +8,13 @@ on:
 
 env:
   RUN: ${{ github.run_id }}-${{ github.run_number }}
-  AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-  AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
   PACKAGE_NAME: aws-iot-device-sdk-js
   AWS_EC2_METADATA_DISABLED: true
+  CRT_CI_ROLE: ${{ secrets.CRT_CI_ROLE_ARN }}
+  AWS_DEFAULT_REGION: us-east-1
+
+permissions:
+  id-token: write # This is required for requesting the JWT
 
 jobs:
   unit-tests:
@@ -44,4 +47,8 @@ jobs:
         uses: actions/setup-node@v2
         with:
           node-version: ${{ matrix.node-version }}
+      - uses: aws-actions/configure-aws-credentials@v4
+        with:
+          role-to-assume: ${{ env.CRT_CI_ROLE }}
+          aws-region: ${{ env.AWS_DEFAULT_REGION }}
       - run: cd integration-testing && ./run-tests.sh ${{ matrix.test-type }}

common/lib/tls-reader.js

@@ -86,20 +86,26 @@ module.exports = function(options) {
    // Parse PEM files.  Options ending in 'Path' must be files
    // and will override options which do not end in 'Path'.
 
-   if (filesys.existsSync(options.keyPath)) {
-      options.key = filesys.readFileSync(options.keyPath);
-   } else if (!isUndefined(options.keyPath)) {
-      throw new Error(exceptions.INVALID_KEY_PATH_OPTION);
+   if (!isUndefined(options.keyPath)) {
+      if (filesys.existsSync(options.keyPath)) {
+         options.key = filesys.readFileSync(options.keyPath);
+      } else {
+         throw new Error(exceptions.INVALID_KEY_PATH_OPTION);
+      }
    }
-   if (filesys.existsSync(options.certPath)) {
-      options.cert = filesys.readFileSync(options.certPath);
-   } else if (!isUndefined(options.certPath)) {
-      throw new Error(exceptions.INVALID_CERT_PATH_OPTION);
+   if (!isUndefined(options.certPath)) {
+      if (filesys.existsSync(options.certPath)) {
+         options.cert = filesys.readFileSync(options.certPath);
+      } else {
+         throw new Error(exceptions.INVALID_CERT_PATH_OPTION);
+      }
    }
-   if (filesys.existsSync(options.caPath)) {
-      options.ca = filesys.readFileSync(options.caPath);
-   } else if (!isUndefined(options.caPath)) {
-      throw new Error(exceptions.INVALID_CA_PATH_OPTION);
+   if (!isUndefined(options.caPath)) {
+      if (filesys.existsSync(options.caPath)) {
+         options.ca = filesys.readFileSync(options.caPath);
+      } else {
+         throw new Error(exceptions.INVALID_CA_PATH_OPTION);
+      }
    }
 
    // request certificate from partner

device/index.js

@@ -113,10 +113,6 @@ function signUrl(method, scheme, hostname, path, queryParams, accessId, secretKe
 
    var signingKey = getSignatureKey(secretKey, today, region, serviceName);
 
-   if (debug === true) {
-      console.log('signing key: ' + signingKey + '\n');
-   }
-
    var signature = hmacSHA256(stringToSign, signingKey, {
       asBytes: true
    });
@@ -183,6 +179,28 @@ function arrayEach(array, iterFunction) {
     }
 }
 
+function isSensitiveProperty(property) {
+    return property === "username" ||
+        property === "password" ||
+        property === "key" ||
+        property === "customAuthHeaders" ||
+        property === "cert";
+}
+
+function logSensitiveObject(options) {
+    var optionsLog = ["{"];
+    Object.keys(options).forEach(function(property) {
+        if (!isSensitiveProperty(property)) {
+            optionsLog.push("  " + property + " : " + options[property]);
+        } else {
+            optionsLog.push("  " + property + " : [redacted]");
+        }
+    });
+    optionsLog.push("}");
+
+    console.log(optionsLog.join("\n"));
+}
+
 function getCredentials(ini) {
     //Get shared credential function from AWS SDK.
     var map = {};
@@ -541,7 +559,7 @@ function DeviceClient(options) {
    }
 
    if ((!isUndefined(options)) && (options.debug === true)) {
-      console.log(options);
+       logSensitiveObject(options);
       console.log('attempting new mqtt connection...');
    }
    //connect and return the client instance to map all mqttjs apis

integration-testing/integration-tests/jobs-integration-test.js

@@ -22,7 +22,7 @@ const jobsModule   = require('..').jobs;
 const cmdLineProcess = require('../examples/lib/cmdline');
 const isUndefined    = require('../common/lib/is-undefined.js');
 const awsSDK = require('aws-sdk');
-awsSDK.config.update({ "accessKeyId": process.env.JOBS_AWS_ACCESS_KEY_ID, "secretAccessKey": process.env.JOBS_AWS_SECRET_ACCESS_KEY, "region": "us-east-1" });
+awsSDK.config.update({ "region": "us-east-1" });
 
 var iot = new awsSDK.Iot();
 
@@ -139,7 +139,7 @@ jobs
         var jobIdPrefix = 'test-job-id-' + (Math.floor(Math.random() * 99999999)).toString();
 
         for (var i = 0; i < jobCount; i++) {
-          iot.createJob({ jobId: jobIdPrefix + '-' + i.toString(), targets: [ 'arn:aws:iot:us-east-1:809478692717:thing/' + preRegisteredThingName ], document: '{ "operation":"test' + i.toString() + '", "jobNum": ' + i.toString() + ' }' }, function(err, data) {
+          iot.createJob({ jobId: jobIdPrefix + '-' + i.toString(), targets: [ 'arn:aws:iot:us-east-1:123124136734:thing/' + preRegisteredThingName ], document: '{ "operation":"test' + i.toString() + '", "jobNum": ' + i.toString() + ' }' }, function(err, data) {
             console.log('createJob:');
             if (isUndefined(err)) {
               console.log(data);

integration-testing/integration-tests/run-device-integration-test.sh

@@ -32,8 +32,7 @@ set -o pipefail
 #
 RECEIVES_REQUIRED=46
 TRANSMITS_TOTAL=48
-export HOSTNAME="ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com"
-export CUSTOM_AUTH_HOST=$(aws --region us-east-1 secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+export HOSTNAME=$(aws --region us-east-1 secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
 #
 # Process output will be captured in these files.
 #
@@ -64,9 +63,9 @@ then
     echo "###################################################################"
     echo ${0##*/}": running device integration test (websocket/custom auth)"
     echo "###################################################################"
-    $NODE $INT_TEST_DIR/device-integration-test.js -H $CUSTOM_AUTH_HOST -P=wss-custom-auth -t2 --debug=true -T $TEST_TAG | tee $PROC2_OUTFILE &
+    $NODE $INT_TEST_DIR/device-integration-test.js -H $HOSTNAME -P=wss-custom-auth -t2 --debug=true -T $TEST_TAG | tee $PROC2_OUTFILE &
     PROC2_PID=$!
-    $NODE $INT_TEST_DIR/device-integration-test.js -H $CUSTOM_AUTH_HOST -P=wss-custom-auth -t1 --debug=true -T $TEST_TAG | tee $PROC1_OUTFILE &
+    $NODE $INT_TEST_DIR/device-integration-test.js -H $HOSTNAME -P=wss-custom-auth -t1 --debug=true -T $TEST_TAG | tee $PROC1_OUTFILE &
     PROC1_PID=$!
 else
     echo "###################################################################"

integration-testing/integration-tests/run-jobs-integration-test.sh

@@ -34,7 +34,7 @@ RECEIVES_REQUIRED=46
 TRANSMITS_TOTAL=48
 COMPLETED_JOBS_REQUIRED=4
 CREATED_JOBS_TOTAL=5
-export HOSTNAME="ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com"
+export HOSTNAME=$(aws --region us-east-1 secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
 #
 # Process output will be captured in these files.
 #

integration-testing/integration-tests/run-offline-publishing-test.sh

@@ -20,8 +20,7 @@ NODE=node
 # that all topic names used are unique to this test run.
 #
 TEST_TAG="test-"$RANDOM
-export HOSTNAME="ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com"
-export CUSTOM_AUTH_HOST=$(aws --region us-east-1 secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+export HOSTNAME=$(aws --region us-east-1 secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
 #
 # Capture the exit code of the first command which fails in a pipeline.
 #
@@ -80,13 +79,13 @@ then
     echo "###################################################################"
     echo ${0##*/}": running device integration test (websocket/custom auth)"
     echo "###################################################################"
-    $NODE $INT_TEST_DIR/offline-publishing-test.js -H $CUSTOM_AUTH_HOST -P=wss-custom-auth -t1 --debug=true -T $TEST_TAG | tee $PROC1_OUTFILE &
+    $NODE $INT_TEST_DIR/offline-publishing-test.js -H $HOSTNAME -P=wss-custom-auth -t1 --debug=true -T $TEST_TAG | tee $PROC1_OUTFILE &
     PROC1_PID=$!
     sleep 3       # wait 3 seconds prior to starting transmitting process
     #
     # transmit 4x/second
     #
-    $NODE $INT_TEST_DIR/offline-publishing-test.js -H $CUSTOM_AUTH_HOST -P=wss-custom-auth -t2 --debug=true --delay-ms=250 -T $TEST_TAG | tee $PROC2_OUTFILE &
+    $NODE $INT_TEST_DIR/offline-publishing-test.js -H $HOSTNAME -P=wss-custom-auth -t2 --debug=true --delay-ms=250 -T $TEST_TAG | tee $PROC2_OUTFILE &
     PROC2_PID=$!
 else
     echo "###################################################################"

integration-testing/integration-tests/run-thing-integration-test.sh

@@ -20,8 +20,7 @@ NODE=node
 # that all topic names used are unique to this test run.
 #
 TEST_TAG="test-"$RANDOM
-export HOSTNAME="ajje7lpljulm4-ats.iot.us-east-1.amazonaws.com"
-export CUSTOM_AUTH_HOST=$(aws --region us-east-1 secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
+export HOSTNAME=$(aws --region us-east-1 secretsmanager get-secret-value --secret-id "unit-test/endpoint" --query "SecretString" | cut -f2 -d":" | sed -e 's/[\\\"\}]//g')
 #
 # Capture the exit code of the first command which fails in a pipeline.
 #
@@ -64,9 +63,9 @@ then
     echo "###################################################################"
     echo ${0##*/}": running device integration test (websocket/custom auth)"
     echo "###################################################################"
-    $NODE $INT_TEST_DIR/thing-integration-test.js -H $CUSTOM_AUTH_HOST -P=wss-custom-auth -t1 --debug=true -T $TEST_TAG | tee $PROC1_OUTFILE &
+    $NODE $INT_TEST_DIR/thing-integration-test.js -H $HOSTNAME -P=wss-custom-auth -t1 --debug=true -T $TEST_TAG | tee $PROC1_OUTFILE &
     PROC1_PID=$!
-    $NODE $INT_TEST_DIR/thing-integration-test.js -H $CUSTOM_AUTH_HOST -P=wss-custom-auth -t2 --debug=true -T $TEST_TAG | tee $PROC2_OUTFILE &
+    $NODE $INT_TEST_DIR/thing-integration-test.js -H $HOSTNAME -P=wss-custom-auth -t2 --debug=true -T $TEST_TAG | tee $PROC2_OUTFILE &
     PROC2_PID=$!
 else
     echo "###################################################################"

integration-testing/run-tests.sh

@@ -131,38 +131,9 @@ then
    exit 4
 fi
 
-#
-# The SDK installed without errors; now, retrieve credentials
-#
-echo "###################################################################"
-echo ${0##*/}": retrieving AWS credentials from AWS SecretsManager"
-echo "###################################################################"
-# fetch secret value and strip quotes with sed
-principal=$(aws --region us-east-1 secretsmanager get-secret-value --secret-id V1IotSdkIntegrationTestWebsocketAccessKeyId --query SecretString | sed -n 's/^"\(.*\)"/\1/p')
-if [ $? == "0" ]
-then
-    echo ${0##*/}": retrieved ws testing access key id"
-else
-    echo ${0##*/}": couldn't retrieve ws testing access key id!"
-    exit 5
-fi
-
-# fetch secret value and strip quotes with sed
-credential=$(aws --region us-east-1 secretsmanager get-secret-value --secret-id V1IotSdkIntegrationTestWebsocketSecretAccessKey --query SecretString | sed -n 's/^"\(.*\)"/\1/p')
-if [ $? == "0" ]
-then
-    echo ${0##*/}": retrieved ws testing secret access key"
-else
-    echo ${0##*/}": couldn't retrieve ws testing secret access key!"
-    exit 6
-fi
-
 case $AUTHENTICATION_TYPE"" in
 
    websocket)
-       export AWS_ACCESS_KEY_ID=$principal
-       export AWS_SECRET_ACCESS_KEY=$credential
-
        $RUN_INTEGRATION_TESTS
        exit $?
        ;;
@@ -182,17 +153,14 @@ case $AUTHENTICATION_TYPE"" in
        ;;
 
    certificate)
-       export JOBS_AWS_ACCESS_KEY_ID=$principal
-       export JOBS_AWS_SECRET_ACCESS_KEY=$credential
-
        export CERT_DIR=$NPMTEST_DIR/certs
        mkdir -p $CERT_DIR
        echo "###################################################################"
        echo ${0##*/}": retrieving certificate credentials from AWS Secrets Manager"
        echo "###################################################################"
 
        # fetch secret value, strip quotes and replace "\n" with an actual newline
-       aws --region us-east-1 secretsmanager get-secret-value --secret-id V1IotSdkIntegrationTestCertificate --query SecretString | sed -n 's/^"\(.*\)"/\1/p' | sed 's/\\n/\
+       aws --region us-east-1 secretsmanager get-secret-value --secret-id ci/mqtt5/us/Mqtt5Prod/cert --query SecretString | sed -n 's/^"\(.*\)"/\1/p' | sed 's/\\n/\
 /g' > $CERT_DIR/certificate.pem.crt
        if [ $? == "0" ]
        then
@@ -203,7 +171,7 @@ case $AUTHENTICATION_TYPE"" in
        fi
 
        # fetch secret value, strip quotes and replace "\n" with an actual newline
-       aws --region us-east-1 secretsmanager get-secret-value --secret-id V1IotSdkIntegrationTestPrivateKey --query SecretString | sed -n 's/^"\(.*\)"/\1/p' | sed 's/\\n/\
+       aws --region us-east-1 secretsmanager get-secret-value --secret-id ci/mqtt5/us/Mqtt5Prod/key --query SecretString | sed -n 's/^"\(.*\)"/\1/p' | sed 's/\\n/\
 /g' > $CERT_DIR/private.pem.key
        if [ $? == "0" ]
        then