From 694f609c92826a9c1dca45d1907712dcf1d174a1 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Wed, 29 Jan 2025 16:34:54 -0800 Subject: [PATCH 01/27] chore(GHA): Run Java CI testing for MPL Latest Release --- .../ci_test_latest_released_mpl_java.yml | 32 ++++++++++++++++++ .github/workflows/dafny-interop.yml | 6 ++-- .github/workflows/mpl-head.yml | 10 +++--- .github/workflows/mpl_head_version.yml | 33 +++++++++++++++++-- .../workflows/mpl_latest_release_version.yml | 25 ++++++++++++++ codebuild/staging/release-staging.yml | 2 -- project.properties | 5 +++ 7 files changed, 101 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/ci_test_latest_released_mpl_java.yml create mode 100644 .github/workflows/mpl_latest_release_version.yml diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml new file mode 100644 index 000000000..d24daae42 --- /dev/null +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -0,0 +1,32 @@ +# This workflow is for testing that the latest released version +# of the MPL is compatible with the current DB-ESDK Head +name: Test Latest Released MPL Java with DB-ESDK HEAD + +on: + schedule: + - cron: "00 16 * * 1-5" + +jobs: + getDBESDKDafnyVersion: + # Don't run the cron builds on forks + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/dafny_version.yml + getVerifyDBESDKDafnyVersion: + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/dafny_verify_version.yml + getLatestReleaseMplVersion: + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/mpl_latest_release_version.yml + getMplDafnyVersion: + needs: getLatestReleaseMplVersion + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/mpl_head_version.yml + with: + mpl-head: ${{needs.getLatestReleaseMplVersion.outputs.version}} + test-latest-released-mpl-java: + needs: [getDafnyVersion, getLatestReleaseMplVersion, getMplDafnyVersion] + uses: ./.github/workflows/dafny-interop.yml + with: + mpl-dafny: ${{needs.getLatestReleaseMplVersion.outputs.version}} + mpl-commit: ${{needs.getMplDafnyVersion.outputs.dafnyVersion}} + dbesdk-dafny: ${{needs.getDBESDKDafnyVersion.outputs.version}} diff --git a/.github/workflows/dafny-interop.yml b/.github/workflows/dafny-interop.yml index da395dbfa..9ffc00e0a 100644 --- a/.github/workflows/dafny-interop.yml +++ b/.github/workflows/dafny-interop.yml @@ -31,7 +31,7 @@ jobs: with: mpl-dafny: ${{inputs.mpl-dafny}} mpl-commit: ${{inputs.mpl-commit}} - mpl-version: ${{needs.getMplHeadVersion.outputs.version}} + mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} dbesdk-dafny: ${{inputs.dbesdk-dafny}} dafny-interop-java-test-vectors: needs: getMplHeadVersion @@ -39,7 +39,7 @@ jobs: with: mpl-dafny: ${{inputs.mpl-dafny}} mpl-commit: ${{inputs.mpl-commit}} - mpl-version: ${{needs.getMplHeadVersion.outputs.version}} + mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} dbesdk-dafny: ${{inputs.dbesdk-dafny}} dafny-interop-java-examples: needs: getMplHeadVersion @@ -47,7 +47,7 @@ jobs: with: mpl-dafny: ${{inputs.mpl-dafny}} mpl-commit: ${{inputs.mpl-commit}} - mpl-version: ${{needs.getMplHeadVersion.outputs.version}} + mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} dbesdk-dafny: ${{inputs.dbesdk-dafny}} dafny-interop-net: uses: ./.github/workflows/dafny_interop_test_net.yml diff --git a/.github/workflows/mpl-head.yml b/.github/workflows/mpl-head.yml index 6e2e06234..56a89bb40 100644 --- a/.github/workflows/mpl-head.yml +++ b/.github/workflows/mpl-head.yml @@ -31,35 +31,35 @@ jobs: uses: ./.github/workflows/library_dafny_verification.yml with: dafny: ${{needs.getVerifyVersion.outputs.version}} - mpl-version: ${{needs.getMplHeadVersion.outputs.version}} + mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} mpl-head: true mpl-head-ci-test-vector-verification: needs: [getVerifyVersion, getMplHeadVersion] uses: ./.github/workflows/test_vector_verification.yml with: dafny: ${{needs.getVerifyVersion.outputs.version}} - mpl-version: ${{needs.getMplHeadVersion.outputs.version}} + mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} mpl-head: true mpl-head-ci-java: needs: [getVersion, getMplHeadVersion] uses: ./.github/workflows/ci_test_java.yml with: dafny: ${{needs.getVersion.outputs.version}} - mpl-version: ${{needs.getMplHeadVersion.outputs.version}} + mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} mpl-head: true mpl-head-ci-java-test-vectors: needs: [getVersion, getMplHeadVersion] uses: ./.github/workflows/ci_test_vector_java.yml with: dafny: ${{needs.getVersion.outputs.version}} - mpl-version: ${{needs.getMplHeadVersion.outputs.version}} + mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} mpl-head: true mpl-head-ci-java-examples: needs: [getVersion, getMplHeadVersion] uses: ./.github/workflows/ci_examples_java.yml with: dafny: ${{needs.getVersion.outputs.version}} - mpl-version: ${{needs.getMplHeadVersion.outputs.version}} + mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} mpl-head: true mpl-head-ci-net: needs: getVersion diff --git a/.github/workflows/mpl_head_version.yml b/.github/workflows/mpl_head_version.yml index 994cdaa96..5475b8911 100644 --- a/.github/workflows/mpl_head_version.yml +++ b/.github/workflows/mpl_head_version.yml @@ -12,9 +12,12 @@ on: default: main type: string outputs: - version: - description: "The dafny version for verify" + mplVersion: + description: "The MPL version" value: ${{ jobs.getMplHeadVersion.outputs.version }} + dafnyVersion: + description: "The dafny version for the MPL" + value: ${{ jobs.getMplHeadDafnyVersion.outputs.version }} jobs: getMplHeadVersion: @@ -40,3 +43,29 @@ jobs: with: path: "submodules/MaterialProviders/project.properties" properties: "mplVersion" + + getMplHeadDafnyVersion: + runs-on: ubuntu-22.04 + outputs: + version: ${{ steps.read_property.outputs.dafnyVersion }} + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + fetch-depth: 0 + - name: Update MPL submodule locally if requested + working-directory: submodules/MaterialProviders + shell: bash + run: | + git checkout ${{inputs.mpl-head}} + git submodule update --init --recursive + git rev-parse HEAD + + - name: Get the MPL version from the MPL submodule + id: read_property + uses: christian-draeger/read-properties@1.1.1 + with: + path: "submodules/MaterialProviders/project.properties" + properties: "dafnyVersion" + + diff --git a/.github/workflows/mpl_latest_release_version.yml b/.github/workflows/mpl_latest_release_version.yml new file mode 100644 index 000000000..77b6f6b2d --- /dev/null +++ b/.github/workflows/mpl_latest_release_version.yml @@ -0,0 +1,25 @@ +# This workflow reads the project.properties +# into the environment variables +# and then creates an output variable for `mplDependencyJavaLatestReleaseVersionTag` +name: Latest Released MPL Version + +on: + workflow_call: + outputs: + version: + description: "The latest released MPL version" + value: ${{ jobs.getMPLVersion.outputs.version }} + +jobs: + getMPLVersion: + runs-on: ubuntu-22.04 + outputs: + version: ${{ steps.read_property.outputs.mplDependencyJavaLatestReleaseVersionTag }} + steps: + - uses: actions/checkout@v4 + - name: Read version from Properties-file + id: read_property + uses: christian-draeger/read-properties@1.1.1 + with: + path: "./project.properties" + properties: "mplDependencyJavaLatestReleaseVersionTag" diff --git a/codebuild/staging/release-staging.yml b/codebuild/staging/release-staging.yml index 541dbe66d..5fddc3446 100644 --- a/codebuild/staging/release-staging.yml +++ b/codebuild/staging/release-staging.yml @@ -47,8 +47,6 @@ phases: - aws sts get-caller-identity build: commands: - # Validate the MPL submodule points to the correct release - - scripts/validate-mpl-submodule.sh # Build and deploy to maven local - cd DynamoDbEncryption/ - make transpile_implementation_java diff --git a/project.properties b/project.properties index 584f9f1ec..1915209b4 100644 --- a/project.properties +++ b/project.properties @@ -4,3 +4,8 @@ dafnyVersion=4.9.0 dafnyVerifyVersion=4.9.0 dafnyRuntimeJavaVersion=4.9.0 smithyDafnyJavaConversionVersion=0.1.1 +# mplDependencyJavaLatestReleaseVersionTag is used in ci_test_latest_released_mpl_java.yml +# to test whether the HEAD of DB_ESDK is compatible with the latest released version of +# the MPL or not. +# Add here the tag of the latest released MPL version, for instance, v1.8.0 +mplDependencyJavaLatestReleaseVersionTag="v1.8.0" From 9d1012045cf503b3e5acdbf14375a92e687651b8 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Wed, 29 Jan 2025 16:48:29 -0800 Subject: [PATCH 02/27] m --- .github/workflows/mpl_head_version.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/mpl_head_version.yml b/.github/workflows/mpl_head_version.yml index 5475b8911..8bac70bd3 100644 --- a/.github/workflows/mpl_head_version.yml +++ b/.github/workflows/mpl_head_version.yml @@ -43,7 +43,7 @@ jobs: with: path: "submodules/MaterialProviders/project.properties" properties: "mplVersion" - + getMplHeadDafnyVersion: runs-on: ubuntu-22.04 outputs: @@ -67,5 +67,3 @@ jobs: with: path: "submodules/MaterialProviders/project.properties" properties: "dafnyVersion" - - From b048d81ff95798a5eb0bc64a7e688acb4a3b1a83 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 09:56:43 -0800 Subject: [PATCH 03/27] m --- .github/workflows/ci_test_latest_released_mpl_java.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index d24daae42..babfa0f0e 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -5,6 +5,7 @@ name: Test Latest Released MPL Java with DB-ESDK HEAD on: schedule: - cron: "00 16 * * 1-5" + workflow_dispatch: # allows triggering this manually through the Actions UI jobs: getDBESDKDafnyVersion: From b5686442d4b06adab1c77e9af8e17be0411b27d9 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 10:26:54 -0800 Subject: [PATCH 04/27] m --- .github/workflows/ci_test_latest_released_mpl_java.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index babfa0f0e..da22d11a3 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -5,7 +5,7 @@ name: Test Latest Released MPL Java with DB-ESDK HEAD on: schedule: - cron: "00 16 * * 1-5" - workflow_dispatch: # allows triggering this manually through the Actions UI + workflow_dispatch: # allows triggering this manually through the Actions UI jobs: getDBESDKDafnyVersion: From be6cb892075ba996ca5651aee4e776dae3e8470b Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 12:45:23 -0800 Subject: [PATCH 05/27] m --- .../ci_test_latest_released_mpl_java.yml | 113 ++++++++++++++---- .github/workflows/dafny-interop.yml | 6 +- .github/workflows/mpl-head.yml | 10 +- .github/workflows/mpl_head_version.yml | 29 +---- .../workflows/mpl_latest_release_version.yml | 25 ---- project.properties | 5 - 6 files changed, 99 insertions(+), 89 deletions(-) delete mode 100644 .github/workflows/mpl_latest_release_version.yml diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index da22d11a3..1e965e22f 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -8,26 +8,93 @@ on: workflow_dispatch: # allows triggering this manually through the Actions UI jobs: - getDBESDKDafnyVersion: - # Don't run the cron builds on forks - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/dafny_version.yml - getVerifyDBESDKDafnyVersion: - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/dafny_verify_version.yml - getLatestReleaseMplVersion: - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/mpl_latest_release_version.yml - getMplDafnyVersion: - needs: getLatestReleaseMplVersion - if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/mpl_head_version.yml - with: - mpl-head: ${{needs.getLatestReleaseMplVersion.outputs.version}} - test-latest-released-mpl-java: - needs: [getDafnyVersion, getLatestReleaseMplVersion, getMplDafnyVersion] - uses: ./.github/workflows/dafny-interop.yml - with: - mpl-dafny: ${{needs.getLatestReleaseMplVersion.outputs.version}} - mpl-commit: ${{needs.getMplDafnyVersion.outputs.dafnyVersion}} - dbesdk-dafny: ${{needs.getDBESDKDafnyVersion.outputs.version}} + testJava: + strategy: + max-parallel: 1 + matrix: + java-version: [8, 11, 16, 17] + os: [macos-13] + runs-on: ${{ matrix.os }} + permissions: + id-token: write + contents: read + steps: + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + aws-region: us-west-2 + role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-DDBEC-Dafny-Role-us-west-2 + role-session-name: DDBEC-Dafny-Java-Tests + + - uses: actions/checkout@v3 + with: + submodules: recursive + + - name: Setup Java ${{ matrix.java-version }} + uses: actions/setup-java@v4 + with: + distribution: "corretto" + java-version: ${{ matrix.java-version }} + + - name: Setup Dafny + uses: dafny-lang/setup-dafny-action@v1.8.0 + with: + dafny-version: ${{ inputs.dafny }} + + - name: Regenerate code using smithy-dafny if necessary + if: ${{ inputs.regenerate-code }} + uses: ./.github/actions/polymorph_codegen + with: + dafny: ${{ env.DAFNY_VERSION }} + library: DynamoDbEncryption + diff-generated-code: false + update-and-regenerate-mpl: true + + # The following two steps: "Build and deploy to maven local" and "Run Extensive Tests" + # mimic the tests in ./codebuild/staging/release-staging.yml + - name: Build and deploy to maven local + shell: bash + working-directory: ./DynamoDbEncryption + run: | + # Run transpile by itself. We don't want to locally build the MPL because + # we want to verify that the version pulled down from maven works correctly + make transpile_implementation_java + make transpile_test_java + make mvn_local_deploy + make test_java + + - name: Run Extensive Tests + working-directory: ./DynamoDbEncryption + run: | + gradle -p runtimes/java clean + gradle -p runtimes/java test + + # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", + # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/release-staging.yml + - name: Transpile MPL Test Vectors without recursively building the MPL + working-directory: ./submodules/MaterialProviders/TestVectorsAwsCryptographicMaterialProviders + run: | + # Run transpile by itself. We don't want to locally build the MPL because + # we want to verify that the version pulled down from maven works correctly + make transpile_implementation_java + make transpile_test_java + make mvn_local_deploy + + - name: Run Test Vectors + working-directory: ./TestVectors + run: | + # Spin up ddb local + docker run --name dynamodb -d -p 8000:8000 amazon/dynamodb-local -jar DynamoDBLocal.jar -port 8000 -inMemory -cors * + # Run transpile by itself so we don't locally build the MPL. + make transpile_implementation_java + make transpile_test_java + gradle -p runtimes/java runTests + + - name: Test Examples + working-directory: ./Examples + run: | + # Run Simple Examples + gradle -p runtimes/java/DynamoDbEncryption test + # Run Migration Examples + gradle -p runtimes/java/Migration/PlaintextToAWSDBE test + gradle -p runtimes/java/Migration/DDBECToAWSDBE test diff --git a/.github/workflows/dafny-interop.yml b/.github/workflows/dafny-interop.yml index 9ffc00e0a..da395dbfa 100644 --- a/.github/workflows/dafny-interop.yml +++ b/.github/workflows/dafny-interop.yml @@ -31,7 +31,7 @@ jobs: with: mpl-dafny: ${{inputs.mpl-dafny}} mpl-commit: ${{inputs.mpl-commit}} - mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} + mpl-version: ${{needs.getMplHeadVersion.outputs.version}} dbesdk-dafny: ${{inputs.dbesdk-dafny}} dafny-interop-java-test-vectors: needs: getMplHeadVersion @@ -39,7 +39,7 @@ jobs: with: mpl-dafny: ${{inputs.mpl-dafny}} mpl-commit: ${{inputs.mpl-commit}} - mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} + mpl-version: ${{needs.getMplHeadVersion.outputs.version}} dbesdk-dafny: ${{inputs.dbesdk-dafny}} dafny-interop-java-examples: needs: getMplHeadVersion @@ -47,7 +47,7 @@ jobs: with: mpl-dafny: ${{inputs.mpl-dafny}} mpl-commit: ${{inputs.mpl-commit}} - mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} + mpl-version: ${{needs.getMplHeadVersion.outputs.version}} dbesdk-dafny: ${{inputs.dbesdk-dafny}} dafny-interop-net: uses: ./.github/workflows/dafny_interop_test_net.yml diff --git a/.github/workflows/mpl-head.yml b/.github/workflows/mpl-head.yml index 56a89bb40..6e2e06234 100644 --- a/.github/workflows/mpl-head.yml +++ b/.github/workflows/mpl-head.yml @@ -31,35 +31,35 @@ jobs: uses: ./.github/workflows/library_dafny_verification.yml with: dafny: ${{needs.getVerifyVersion.outputs.version}} - mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} + mpl-version: ${{needs.getMplHeadVersion.outputs.version}} mpl-head: true mpl-head-ci-test-vector-verification: needs: [getVerifyVersion, getMplHeadVersion] uses: ./.github/workflows/test_vector_verification.yml with: dafny: ${{needs.getVerifyVersion.outputs.version}} - mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} + mpl-version: ${{needs.getMplHeadVersion.outputs.version}} mpl-head: true mpl-head-ci-java: needs: [getVersion, getMplHeadVersion] uses: ./.github/workflows/ci_test_java.yml with: dafny: ${{needs.getVersion.outputs.version}} - mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} + mpl-version: ${{needs.getMplHeadVersion.outputs.version}} mpl-head: true mpl-head-ci-java-test-vectors: needs: [getVersion, getMplHeadVersion] uses: ./.github/workflows/ci_test_vector_java.yml with: dafny: ${{needs.getVersion.outputs.version}} - mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} + mpl-version: ${{needs.getMplHeadVersion.outputs.version}} mpl-head: true mpl-head-ci-java-examples: needs: [getVersion, getMplHeadVersion] uses: ./.github/workflows/ci_examples_java.yml with: dafny: ${{needs.getVersion.outputs.version}} - mpl-version: ${{needs.getMplHeadVersion.outputs.mplVersion}} + mpl-version: ${{needs.getMplHeadVersion.outputs.version}} mpl-head: true mpl-head-ci-net: needs: getVersion diff --git a/.github/workflows/mpl_head_version.yml b/.github/workflows/mpl_head_version.yml index 8bac70bd3..e270095f4 100644 --- a/.github/workflows/mpl_head_version.yml +++ b/.github/workflows/mpl_head_version.yml @@ -12,12 +12,9 @@ on: default: main type: string outputs: - mplVersion: + version: description: "The MPL version" value: ${{ jobs.getMplHeadVersion.outputs.version }} - dafnyVersion: - description: "The dafny version for the MPL" - value: ${{ jobs.getMplHeadDafnyVersion.outputs.version }} jobs: getMplHeadVersion: @@ -43,27 +40,3 @@ jobs: with: path: "submodules/MaterialProviders/project.properties" properties: "mplVersion" - - getMplHeadDafnyVersion: - runs-on: ubuntu-22.04 - outputs: - version: ${{ steps.read_property.outputs.dafnyVersion }} - steps: - - uses: actions/checkout@v4 - with: - submodules: recursive - fetch-depth: 0 - - name: Update MPL submodule locally if requested - working-directory: submodules/MaterialProviders - shell: bash - run: | - git checkout ${{inputs.mpl-head}} - git submodule update --init --recursive - git rev-parse HEAD - - - name: Get the MPL version from the MPL submodule - id: read_property - uses: christian-draeger/read-properties@1.1.1 - with: - path: "submodules/MaterialProviders/project.properties" - properties: "dafnyVersion" diff --git a/.github/workflows/mpl_latest_release_version.yml b/.github/workflows/mpl_latest_release_version.yml deleted file mode 100644 index 77b6f6b2d..000000000 --- a/.github/workflows/mpl_latest_release_version.yml +++ /dev/null @@ -1,25 +0,0 @@ -# This workflow reads the project.properties -# into the environment variables -# and then creates an output variable for `mplDependencyJavaLatestReleaseVersionTag` -name: Latest Released MPL Version - -on: - workflow_call: - outputs: - version: - description: "The latest released MPL version" - value: ${{ jobs.getMPLVersion.outputs.version }} - -jobs: - getMPLVersion: - runs-on: ubuntu-22.04 - outputs: - version: ${{ steps.read_property.outputs.mplDependencyJavaLatestReleaseVersionTag }} - steps: - - uses: actions/checkout@v4 - - name: Read version from Properties-file - id: read_property - uses: christian-draeger/read-properties@1.1.1 - with: - path: "./project.properties" - properties: "mplDependencyJavaLatestReleaseVersionTag" diff --git a/project.properties b/project.properties index 1915209b4..584f9f1ec 100644 --- a/project.properties +++ b/project.properties @@ -4,8 +4,3 @@ dafnyVersion=4.9.0 dafnyVerifyVersion=4.9.0 dafnyRuntimeJavaVersion=4.9.0 smithyDafnyJavaConversionVersion=0.1.1 -# mplDependencyJavaLatestReleaseVersionTag is used in ci_test_latest_released_mpl_java.yml -# to test whether the HEAD of DB_ESDK is compatible with the latest released version of -# the MPL or not. -# Add here the tag of the latest released MPL version, for instance, v1.8.0 -mplDependencyJavaLatestReleaseVersionTag="v1.8.0" From 6a46b6079898da11d51c1cb012b4760eef408dc6 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 12:50:59 -0800 Subject: [PATCH 06/27] m --- .github/workflows/pull.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 3f237ee0d..560d1b49c 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -64,6 +64,8 @@ jobs: uses: ./.github/workflows/ci_examples_net.yml with: dafny: ${{needs.getVersion.outputs.version}} + pr-ci-net-examples: + uses: ./.github/workflows/ci_test_latest_released_mpl_java.yml pr-ci-all-required: if: always() needs: From 702ac5632c1e7600dfabe52d1e51c917b251fec8 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 12:51:23 -0800 Subject: [PATCH 07/27] m --- .github/workflows/pull.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 560d1b49c..bb1e5731e 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -64,7 +64,7 @@ jobs: uses: ./.github/workflows/ci_examples_net.yml with: dafny: ${{needs.getVersion.outputs.version}} - pr-ci-net-examples: + pr-ci-latest-released-mpl-remove: uses: ./.github/workflows/ci_test_latest_released_mpl_java.yml pr-ci-all-required: if: always() From ed6ecf1994ef1a0e1e737d036bb0c1ce8442c9d3 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 12:54:50 -0800 Subject: [PATCH 08/27] m --- .github/workflows/pull.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index bb1e5731e..139ce9c09 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -82,6 +82,7 @@ jobs: - pr-ci-rust - pr-ci-net-test-vectors - pr-ci-net-examples + - pr-ci-latest-released-mpl-remove runs-on: ubuntu-22.04 steps: - name: Verify all required jobs passed From eb6f8b96c8efe1e0a72b96ea431452899e0c5eb1 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 13:11:33 -0800 Subject: [PATCH 09/27] m --- .github/workflows/ci_test_latest_released_mpl_java.yml | 1 + .github/workflows/pull.yml | 3 --- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 1e965e22f..5f75ed4cd 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -6,6 +6,7 @@ on: schedule: - cron: "00 16 * * 1-5" workflow_dispatch: # allows triggering this manually through the Actions UI + pull_request: jobs: testJava: diff --git a/.github/workflows/pull.yml b/.github/workflows/pull.yml index 139ce9c09..3f237ee0d 100644 --- a/.github/workflows/pull.yml +++ b/.github/workflows/pull.yml @@ -64,8 +64,6 @@ jobs: uses: ./.github/workflows/ci_examples_net.yml with: dafny: ${{needs.getVersion.outputs.version}} - pr-ci-latest-released-mpl-remove: - uses: ./.github/workflows/ci_test_latest_released_mpl_java.yml pr-ci-all-required: if: always() needs: @@ -82,7 +80,6 @@ jobs: - pr-ci-rust - pr-ci-net-test-vectors - pr-ci-net-examples - - pr-ci-latest-released-mpl-remove runs-on: ubuntu-22.04 steps: - name: Verify all required jobs passed From 736e38715ee7c0a7f808d131c8ce9c3d0588a06d Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 13:15:20 -0800 Subject: [PATCH 10/27] m --- .github/workflows/ci_test_latest_released_mpl_java.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 5f75ed4cd..2aecde27e 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -9,7 +9,15 @@ on: pull_request: jobs: + getVersion: + # Don't run the cron builds on forks + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/dafny_version.yml + getVerifyVersion: + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/dafny_verify_version.ym testJava: + needs: getVersion strategy: max-parallel: 1 matrix: @@ -40,7 +48,7 @@ jobs: - name: Setup Dafny uses: dafny-lang/setup-dafny-action@v1.8.0 with: - dafny-version: ${{ inputs.dafny }} + dafny-version: ${{ needs.getVersion.outputs.version }} - name: Regenerate code using smithy-dafny if necessary if: ${{ inputs.regenerate-code }} From 8598979660dacfc7778eae39c3af6c39c3b2512d Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 13:17:28 -0800 Subject: [PATCH 11/27] update --- .github/workflows/ci_test_latest_released_mpl_java.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 2aecde27e..849b40706 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -15,7 +15,7 @@ jobs: uses: ./.github/workflows/dafny_version.yml getVerifyVersion: if: github.event_name != 'schedule' || github.repository_owner == 'aws' - uses: ./.github/workflows/dafny_verify_version.ym + uses: ./.github/workflows/dafny_verify_version.yml testJava: needs: getVersion strategy: From 00a9456efcdbb01f38b3c623e0523988d69d4933 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 13:56:05 -0800 Subject: [PATCH 12/27] m --- TestVectors/runtimes/java/build.gradle.kts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/TestVectors/runtimes/java/build.gradle.kts b/TestVectors/runtimes/java/build.gradle.kts index d5839b95c..62b9cf443 100644 --- a/TestVectors/runtimes/java/build.gradle.kts +++ b/TestVectors/runtimes/java/build.gradle.kts @@ -90,7 +90,7 @@ dependencies { implementation("software.amazon.smithy.dafny:conversion:${smithyDafnyJavaConversionVersion}") implementation("software.amazon.cryptography:aws-cryptographic-material-providers:${mplVersion}") implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:${ddbecVersion}") - implementation("software.amazon.cryptography:TestAwsCryptographicMaterialProviders:1.8.0-SNAPSHOT") + implementation("software.amazon.cryptography:TestAwsCryptographicMaterialProviders:${mplVersion}") implementation(platform("software.amazon.awssdk:bom:2.26.25")) implementation("software.amazon.awssdk:dynamodb") From f71f7408f2a197c823ddd29b7f4796d0f468d736 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Thu, 30 Jan 2025 15:05:13 -0800 Subject: [PATCH 13/27] m --- .github/workflows/ci_test_latest_released_mpl_java.yml | 6 ++++-- TestVectors/runtimes/java/build.gradle.kts | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 849b40706..0165ec918 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -6,6 +6,8 @@ on: schedule: - cron: "00 16 * * 1-5" workflow_dispatch: # allows triggering this manually through the Actions UI + # TODO: Remove this before merging to main. We don't want this workflow running in CI + # on PRs because we expect it to fail when the MPL is actually updated since the last version. pull_request: jobs: @@ -21,7 +23,7 @@ jobs: strategy: max-parallel: 1 matrix: - java-version: [8, 11, 16, 17] + java-version: [17] os: [macos-13] runs-on: ${{ matrix.os }} permissions: @@ -79,7 +81,7 @@ jobs: gradle -p runtimes/java test # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", - # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/release-staging.yml + # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/validate-staging.yml - name: Transpile MPL Test Vectors without recursively building the MPL working-directory: ./submodules/MaterialProviders/TestVectorsAwsCryptographicMaterialProviders run: | diff --git a/TestVectors/runtimes/java/build.gradle.kts b/TestVectors/runtimes/java/build.gradle.kts index 62b9cf443..d5839b95c 100644 --- a/TestVectors/runtimes/java/build.gradle.kts +++ b/TestVectors/runtimes/java/build.gradle.kts @@ -90,7 +90,7 @@ dependencies { implementation("software.amazon.smithy.dafny:conversion:${smithyDafnyJavaConversionVersion}") implementation("software.amazon.cryptography:aws-cryptographic-material-providers:${mplVersion}") implementation("software.amazon.cryptography:aws-database-encryption-sdk-dynamodb:${ddbecVersion}") - implementation("software.amazon.cryptography:TestAwsCryptographicMaterialProviders:${mplVersion}") + implementation("software.amazon.cryptography:TestAwsCryptographicMaterialProviders:1.8.0-SNAPSHOT") implementation(platform("software.amazon.awssdk:bom:2.26.25")) implementation("software.amazon.awssdk:dynamodb") From a9627c0e1cf319bd9608aeedace3324700edacd3 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Fri, 31 Jan 2025 16:22:55 -0800 Subject: [PATCH 14/27] bump mpl; update ci_test_latest_released_mpl_java.yml --- .../ci_test_latest_released_mpl_java.yml | 12 ++++++++- .../workflows/mpl_dependency_java_version.yml | 25 +++++++++++++++++++ .github/workflows/mpl_head_version.yml | 4 +-- submodules/MaterialProviders | 2 +- 4 files changed, 39 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/mpl_dependency_java_version.yml diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 0165ec918..d5fbbb29b 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -18,8 +18,11 @@ jobs: getVerifyVersion: if: github.event_name != 'schedule' || github.repository_owner == 'aws' uses: ./.github/workflows/dafny_verify_version.yml + getMplDependencyJavaVersion: + if: github.event_name != 'schedule' || github.repository_owner == 'aws' + uses: ./.github/workflows/mpl_dependency_java_version.yml testJava: - needs: getVersion + needs: [getVersion, getMplDependencyJavaVersion] strategy: max-parallel: 1 matrix: @@ -80,6 +83,13 @@ jobs: gradle -p runtimes/java clean gradle -p runtimes/java test + # This makes sure that we are using the correct MPL version to test the DB-ESDK. + # If this contains a SNAPSHOT version, this will fail because' + # we are NOT building the MPL recursively but pulling from Maven. + - name: Update project.properties if using MPL HEAD + run: | + sed "s/mplDependencyJavaVersion=.*/mplDependencyJavaVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties + # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/validate-staging.yml - name: Transpile MPL Test Vectors without recursively building the MPL diff --git a/.github/workflows/mpl_dependency_java_version.yml b/.github/workflows/mpl_dependency_java_version.yml new file mode 100644 index 000000000..1d360ef85 --- /dev/null +++ b/.github/workflows/mpl_dependency_java_version.yml @@ -0,0 +1,25 @@ +# This workflow reads the project.properties +# into the environment variables +# and then creates an output variable for `mplDependencyJavaVersion` +name: MPL Dependency Java Version + +on: + workflow_call: + outputs: + version: + description: "The MPL Dependency Java version from project.properties" + value: ${{ jobs.getMplDependencyJavaVersion.outputs.version }} + +jobs: + getMplDependencyJavaVersion: + runs-on: ubuntu-22.04 + outputs: + version: ${{ steps.read_property.outputs.mplVersion }} + steps: + - uses: actions/checkout@v4 + - name: Read version from Properties-file + id: read_property + uses: christian-draeger/read-properties@1.1.1 + with: + path: "./project.properties" + properties: "mplDependencyJavaVersion" diff --git a/.github/workflows/mpl_head_version.yml b/.github/workflows/mpl_head_version.yml index e270095f4..0d32688fb 100644 --- a/.github/workflows/mpl_head_version.yml +++ b/.github/workflows/mpl_head_version.yml @@ -1,6 +1,6 @@ -# This workflow reads the project.properties +# This workflow reads the project.properties in the MPL submodule # into the environment variables -# and then creates an output variable for `dafnyVerifyVersion ` +# and then creates an output variable for `mplVersion` name: MPL HEAD Version on: diff --git a/submodules/MaterialProviders b/submodules/MaterialProviders index 9c80544bb..7383f0573 160000 --- a/submodules/MaterialProviders +++ b/submodules/MaterialProviders @@ -1 +1 @@ -Subproject commit 9c80544bbfeac5547915aa93795f9b79eac7db6b +Subproject commit 7383f05733841a8a2059531e80dd36f0a27e2548 From 4ba5139426a22b1fe964e79ea037369dd38665ce Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Fri, 31 Jan 2025 16:47:06 -0800 Subject: [PATCH 15/27] m --- .github/workflows/ci_test_latest_released_mpl_java.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index d5fbbb29b..a39022aa0 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -86,9 +86,10 @@ jobs: # This makes sure that we are using the correct MPL version to test the DB-ESDK. # If this contains a SNAPSHOT version, this will fail because' # we are NOT building the MPL recursively but pulling from Maven. - - name: Update project.properties if using MPL HEAD + - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) + working-directory: ./submodules/MaterialProviders/ run: | - sed "s/mplDependencyJavaVersion=.*/mplDependencyJavaVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties + sed "s/mplVersion=.*/mplVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/validate-staging.yml From 0a0a3f735bd59d168b97c3abb91a028b86702e5c Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Fri, 31 Jan 2025 17:08:31 -0800 Subject: [PATCH 16/27] m --- .../ci_test_latest_released_mpl_java.yml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index a39022aa0..5ca2bbd2c 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -74,14 +74,14 @@ jobs: # we want to verify that the version pulled down from maven works correctly make transpile_implementation_java make transpile_test_java - make mvn_local_deploy - make test_java + # make mvn_local_deploy + # make test_java - - name: Run Extensive Tests - working-directory: ./DynamoDbEncryption - run: | - gradle -p runtimes/java clean - gradle -p runtimes/java test + # - name: Run Extensive Tests + # working-directory: ./DynamoDbEncryption + # run: | + # gradle -p runtimes/java clean + # gradle -p runtimes/java test # This makes sure that we are using the correct MPL version to test the DB-ESDK. # If this contains a SNAPSHOT version, this will fail because' @@ -89,7 +89,10 @@ jobs: - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) working-directory: ./submodules/MaterialProviders/ run: | + echo ${{needs.getMplDependencyJavaVersion.outputs.version}} + cat project.properties sed "s/mplVersion=.*/mplVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties + cat project.properties # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/validate-staging.yml @@ -98,6 +101,7 @@ jobs: run: | # Run transpile by itself. We don't want to locally build the MPL because # we want to verify that the version pulled down from maven works correctly + cat ../project.properties make transpile_implementation_java make transpile_test_java make mvn_local_deploy From bf2e657124112707e43440f19306a12231cd80a2 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Fri, 31 Jan 2025 17:17:05 -0800 Subject: [PATCH 17/27] m --- submodules/MaterialProviders | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/submodules/MaterialProviders b/submodules/MaterialProviders index 7383f0573..8c4b7f123 160000 --- a/submodules/MaterialProviders +++ b/submodules/MaterialProviders @@ -1 +1 @@ -Subproject commit 7383f05733841a8a2059531e80dd36f0a27e2548 +Subproject commit 8c4b7f123c407fac7f7501f54b41b2c43dc1cadb From 9af7f137501e8d07da281342eda22c5a9fe20d39 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Fri, 31 Jan 2025 17:26:37 -0800 Subject: [PATCH 18/27] m --- .../ci_test_latest_released_mpl_java.yml | 44 +++++++++---------- submodules/MaterialProviders | 2 +- 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 5ca2bbd2c..87a40164b 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -50,30 +50,30 @@ jobs: distribution: "corretto" java-version: ${{ matrix.java-version }} - - name: Setup Dafny - uses: dafny-lang/setup-dafny-action@v1.8.0 - with: - dafny-version: ${{ needs.getVersion.outputs.version }} + # - name: Setup Dafny + # uses: dafny-lang/setup-dafny-action@v1.8.0 + # with: + # dafny-version: ${{ needs.getVersion.outputs.version }} - - name: Regenerate code using smithy-dafny if necessary - if: ${{ inputs.regenerate-code }} - uses: ./.github/actions/polymorph_codegen - with: - dafny: ${{ env.DAFNY_VERSION }} - library: DynamoDbEncryption - diff-generated-code: false - update-and-regenerate-mpl: true + # - name: Regenerate code using smithy-dafny if necessary + # if: ${{ inputs.regenerate-code }} + # uses: ./.github/actions/polymorph_codegen + # with: + # dafny: ${{ env.DAFNY_VERSION }} + # library: DynamoDbEncryption + # diff-generated-code: false + # update-and-regenerate-mpl: true # The following two steps: "Build and deploy to maven local" and "Run Extensive Tests" # mimic the tests in ./codebuild/staging/release-staging.yml - - name: Build and deploy to maven local - shell: bash - working-directory: ./DynamoDbEncryption - run: | - # Run transpile by itself. We don't want to locally build the MPL because - # we want to verify that the version pulled down from maven works correctly - make transpile_implementation_java - make transpile_test_java + # - name: Build and deploy to maven local + # shell: bash + # working-directory: ./DynamoDbEncryption + # run: | + # # Run transpile by itself. We don't want to locally build the MPL because + # # we want to verify that the version pulled down from maven works correctly + # make transpile_implementation_java + # make transpile_test_java # make mvn_local_deploy # make test_java @@ -89,9 +89,9 @@ jobs: - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) working-directory: ./submodules/MaterialProviders/ run: | - echo ${{needs.getMplDependencyJavaVersion.outputs.version}} + echo ${{ needs.getMplDependencyJavaVersion.outputs.version }} cat project.properties - sed "s/mplVersion=.*/mplVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties + sed "s/mplVersion=.*/mplVersion=${{ needs.getMplDependencyJavaVersion.outputs.version }}/g" project.properties > project.properties2; mv project.properties2 project.properties cat project.properties # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", diff --git a/submodules/MaterialProviders b/submodules/MaterialProviders index 8c4b7f123..7383f0573 160000 --- a/submodules/MaterialProviders +++ b/submodules/MaterialProviders @@ -1 +1 @@ -Subproject commit 8c4b7f123c407fac7f7501f54b41b2c43dc1cadb +Subproject commit 7383f05733841a8a2059531e80dd36f0a27e2548 From 364d8d39fe31cee32c5850bb39ecd7dd08d256d9 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Fri, 31 Jan 2025 17:29:35 -0800 Subject: [PATCH 19/27] m --- .github/workflows/ci_test_latest_released_mpl_java.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 87a40164b..548a3db12 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -89,6 +89,7 @@ jobs: - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) working-directory: ./submodules/MaterialProviders/ run: | + echo ${{ needs.getVersion.outputs.version }} echo ${{ needs.getMplDependencyJavaVersion.outputs.version }} cat project.properties sed "s/mplVersion=.*/mplVersion=${{ needs.getMplDependencyJavaVersion.outputs.version }}/g" project.properties > project.properties2; mv project.properties2 project.properties From 4c0d7bb6a90cdbf7d819559c5c7fe9a157eedfb7 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Fri, 31 Jan 2025 17:31:42 -0800 Subject: [PATCH 20/27] m1 --- .github/workflows/ci_test_latest_released_mpl_java.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 548a3db12..ac82d6a39 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -89,7 +89,7 @@ jobs: - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) working-directory: ./submodules/MaterialProviders/ run: | - echo ${{ needs.getVersion.outputs.version }} + echo ${{ needs.getVersion.outputs.version }} echo ${{ needs.getMplDependencyJavaVersion.outputs.version }} cat project.properties sed "s/mplVersion=.*/mplVersion=${{ needs.getMplDependencyJavaVersion.outputs.version }}/g" project.properties > project.properties2; mv project.properties2 project.properties From a990c61312fa0180d965afbb48d2f9c13d94ebba Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Fri, 31 Jan 2025 17:34:58 -0800 Subject: [PATCH 21/27] m2 --- .github/workflows/ci_test_latest_released_mpl_java.yml | 4 +--- .github/workflows/mpl_dependency_java_version.yml | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index ac82d6a39..7d12795c2 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -89,10 +89,8 @@ jobs: - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) working-directory: ./submodules/MaterialProviders/ run: | - echo ${{ needs.getVersion.outputs.version }} - echo ${{ needs.getMplDependencyJavaVersion.outputs.version }} cat project.properties - sed "s/mplVersion=.*/mplVersion=${{ needs.getMplDependencyJavaVersion.outputs.version }}/g" project.properties > project.properties2; mv project.properties2 project.properties + sed "s/mplVersion=.*/mplVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties cat project.properties # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", diff --git a/.github/workflows/mpl_dependency_java_version.yml b/.github/workflows/mpl_dependency_java_version.yml index 1d360ef85..640b82e33 100644 --- a/.github/workflows/mpl_dependency_java_version.yml +++ b/.github/workflows/mpl_dependency_java_version.yml @@ -14,7 +14,7 @@ jobs: getMplDependencyJavaVersion: runs-on: ubuntu-22.04 outputs: - version: ${{ steps.read_property.outputs.mplVersion }} + version: ${{ steps.read_property.outputs.mplDependencyJavaVersion }} steps: - uses: actions/checkout@v4 - name: Read version from Properties-file From 5d4f30c9f83b2e669e4293a622674f8c94478bb8 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Fri, 31 Jan 2025 17:38:35 -0800 Subject: [PATCH 22/27] finally --- .../ci_test_latest_released_mpl_java.yml | 57 +++++++++---------- 1 file changed, 27 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 7d12795c2..a39022aa0 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -50,38 +50,38 @@ jobs: distribution: "corretto" java-version: ${{ matrix.java-version }} - # - name: Setup Dafny - # uses: dafny-lang/setup-dafny-action@v1.8.0 - # with: - # dafny-version: ${{ needs.getVersion.outputs.version }} + - name: Setup Dafny + uses: dafny-lang/setup-dafny-action@v1.8.0 + with: + dafny-version: ${{ needs.getVersion.outputs.version }} - # - name: Regenerate code using smithy-dafny if necessary - # if: ${{ inputs.regenerate-code }} - # uses: ./.github/actions/polymorph_codegen - # with: - # dafny: ${{ env.DAFNY_VERSION }} - # library: DynamoDbEncryption - # diff-generated-code: false - # update-and-regenerate-mpl: true + - name: Regenerate code using smithy-dafny if necessary + if: ${{ inputs.regenerate-code }} + uses: ./.github/actions/polymorph_codegen + with: + dafny: ${{ env.DAFNY_VERSION }} + library: DynamoDbEncryption + diff-generated-code: false + update-and-regenerate-mpl: true # The following two steps: "Build and deploy to maven local" and "Run Extensive Tests" # mimic the tests in ./codebuild/staging/release-staging.yml - # - name: Build and deploy to maven local - # shell: bash - # working-directory: ./DynamoDbEncryption - # run: | - # # Run transpile by itself. We don't want to locally build the MPL because - # # we want to verify that the version pulled down from maven works correctly - # make transpile_implementation_java - # make transpile_test_java - # make mvn_local_deploy - # make test_java + - name: Build and deploy to maven local + shell: bash + working-directory: ./DynamoDbEncryption + run: | + # Run transpile by itself. We don't want to locally build the MPL because + # we want to verify that the version pulled down from maven works correctly + make transpile_implementation_java + make transpile_test_java + make mvn_local_deploy + make test_java - # - name: Run Extensive Tests - # working-directory: ./DynamoDbEncryption - # run: | - # gradle -p runtimes/java clean - # gradle -p runtimes/java test + - name: Run Extensive Tests + working-directory: ./DynamoDbEncryption + run: | + gradle -p runtimes/java clean + gradle -p runtimes/java test # This makes sure that we are using the correct MPL version to test the DB-ESDK. # If this contains a SNAPSHOT version, this will fail because' @@ -89,9 +89,7 @@ jobs: - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) working-directory: ./submodules/MaterialProviders/ run: | - cat project.properties sed "s/mplVersion=.*/mplVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties - cat project.properties # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/validate-staging.yml @@ -100,7 +98,6 @@ jobs: run: | # Run transpile by itself. We don't want to locally build the MPL because # we want to verify that the version pulled down from maven works correctly - cat ../project.properties make transpile_implementation_java make transpile_test_java make mvn_local_deploy From c04fa3fa99c4416ddce5f0b92b69853f1d4a2535 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 4 Feb 2025 16:01:56 -0800 Subject: [PATCH 23/27] add flag to run / not run tvs --- .github/workflows/ci_test_latest_released_mpl_java.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index a39022aa0..3af69a003 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -6,6 +6,12 @@ on: schedule: - cron: "00 16 * * 1-5" workflow_dispatch: # allows triggering this manually through the Actions UI + inputs: + run_test_vectors: + description: 'Run Test Vectors?' + required: false + default: true + type: boolean # TODO: Remove this before merging to main. We don't want this workflow running in CI # on PRs because we expect it to fail when the MPL is actually updated since the last version. pull_request: @@ -103,6 +109,8 @@ jobs: make mvn_local_deploy - name: Run Test Vectors + # TODO: Remove pull_request from here + if: github.event_name == 'pull_request' || github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && ${{inputs.run_test_vectors}}) working-directory: ./TestVectors run: | # Spin up ddb local From 576e13ec169f47a6e4035a39fbfc41e75f5d4ac6 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 4 Feb 2025 16:05:56 -0800 Subject: [PATCH 24/27] m --- .github/workflows/ci_test_latest_released_mpl_java.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 3af69a003..23760dd2d 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -8,7 +8,7 @@ on: workflow_dispatch: # allows triggering this manually through the Actions UI inputs: run_test_vectors: - description: 'Run Test Vectors?' + description: "Run Test Vectors?" required: false default: true type: boolean From 438730f949075381108878eff11ec06f7e1ef89c Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 4 Feb 2025 16:47:27 -0800 Subject: [PATCH 25/27] try --- .../ci_test_latest_released_mpl_java.yml | 70 +++++++++---------- 1 file changed, 35 insertions(+), 35 deletions(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 23760dd2d..150b9b4d6 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -33,7 +33,7 @@ jobs: max-parallel: 1 matrix: java-version: [17] - os: [macos-13] + os: [ubuntu-22.04] runs-on: ${{ matrix.os }} permissions: id-token: write @@ -70,43 +70,43 @@ jobs: diff-generated-code: false update-and-regenerate-mpl: true - # The following two steps: "Build and deploy to maven local" and "Run Extensive Tests" - # mimic the tests in ./codebuild/staging/release-staging.yml - - name: Build and deploy to maven local - shell: bash - working-directory: ./DynamoDbEncryption - run: | - # Run transpile by itself. We don't want to locally build the MPL because - # we want to verify that the version pulled down from maven works correctly - make transpile_implementation_java - make transpile_test_java - make mvn_local_deploy - make test_java + # # The following two steps: "Build and deploy to maven local" and "Run Extensive Tests" + # # mimic the tests in ./codebuild/staging/release-staging.yml + # - name: Build and deploy to maven local + # shell: bash + # working-directory: ./DynamoDbEncryption + # run: | + # # Run transpile by itself. We don't want to locally build the MPL because + # # we want to verify that the version pulled down from maven works correctly + # make transpile_implementation_java + # make transpile_test_java + # make mvn_local_deploy + # make test_java - - name: Run Extensive Tests - working-directory: ./DynamoDbEncryption - run: | - gradle -p runtimes/java clean - gradle -p runtimes/java test + # - name: Run Extensive Tests + # working-directory: ./DynamoDbEncryption + # run: | + # gradle -p runtimes/java clean + # gradle -p runtimes/java test - # This makes sure that we are using the correct MPL version to test the DB-ESDK. - # If this contains a SNAPSHOT version, this will fail because' - # we are NOT building the MPL recursively but pulling from Maven. - - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) - working-directory: ./submodules/MaterialProviders/ - run: | - sed "s/mplVersion=.*/mplVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties + # # This makes sure that we are using the correct MPL version to test the DB-ESDK. + # # If this contains a SNAPSHOT version, this will fail because' + # # we are NOT building the MPL recursively but pulling from Maven. + # - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) + # working-directory: ./submodules/MaterialProviders/ + # run: | + # sed "s/mplVersion=.*/mplVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties - # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", - # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/validate-staging.yml - - name: Transpile MPL Test Vectors without recursively building the MPL - working-directory: ./submodules/MaterialProviders/TestVectorsAwsCryptographicMaterialProviders - run: | - # Run transpile by itself. We don't want to locally build the MPL because - # we want to verify that the version pulled down from maven works correctly - make transpile_implementation_java - make transpile_test_java - make mvn_local_deploy + # # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", + # # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/validate-staging.yml + # - name: Transpile MPL Test Vectors without recursively building the MPL + # working-directory: ./submodules/MaterialProviders/TestVectorsAwsCryptographicMaterialProviders + # run: | + # # Run transpile by itself. We don't want to locally build the MPL because + # # we want to verify that the version pulled down from maven works correctly + # make transpile_implementation_java + # make transpile_test_java + # make mvn_local_deploy - name: Run Test Vectors # TODO: Remove pull_request from here From 0241874c8596e31dd0283d627d9c0647d3f20f74 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Tue, 4 Feb 2025 16:54:16 -0800 Subject: [PATCH 26/27] m --- .../ci_test_latest_released_mpl_java.yml | 68 +++++++++---------- 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index 150b9b4d6..b16d08d42 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -70,43 +70,43 @@ jobs: diff-generated-code: false update-and-regenerate-mpl: true - # # The following two steps: "Build and deploy to maven local" and "Run Extensive Tests" - # # mimic the tests in ./codebuild/staging/release-staging.yml - # - name: Build and deploy to maven local - # shell: bash - # working-directory: ./DynamoDbEncryption - # run: | - # # Run transpile by itself. We don't want to locally build the MPL because - # # we want to verify that the version pulled down from maven works correctly - # make transpile_implementation_java - # make transpile_test_java - # make mvn_local_deploy - # make test_java + # The following two steps: "Build and deploy to maven local" and "Run Extensive Tests" + # mimic the tests in ./codebuild/staging/release-staging.yml + - name: Build and deploy to maven local + shell: bash + working-directory: ./DynamoDbEncryption + run: | + # Run transpile by itself. We don't want to locally build the MPL because + # we want to verify that the version pulled down from maven works correctly + make transpile_implementation_java + make transpile_test_java + make mvn_local_deploy + make test_java - # - name: Run Extensive Tests - # working-directory: ./DynamoDbEncryption - # run: | - # gradle -p runtimes/java clean - # gradle -p runtimes/java test + - name: Run Extensive Tests + working-directory: ./DynamoDbEncryption + run: | + gradle -p runtimes/java clean + gradle -p runtimes/java test - # # This makes sure that we are using the correct MPL version to test the DB-ESDK. - # # If this contains a SNAPSHOT version, this will fail because' - # # we are NOT building the MPL recursively but pulling from Maven. - # - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) - # working-directory: ./submodules/MaterialProviders/ - # run: | - # sed "s/mplVersion=.*/mplVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties + # This makes sure that we are using the correct MPL version to test the DB-ESDK. + # If this contains a SNAPSHOT version, this will fail because' + # we are NOT building the MPL recursively but pulling from Maven. + - name: Update project.properties to use the correct MPL version (from project.properties in DB-ESDK) + working-directory: ./submodules/MaterialProviders/ + run: | + sed "s/mplVersion=.*/mplVersion=${{needs.getMplDependencyJavaVersion.outputs.version}}/g" project.properties > project.properties2; mv project.properties2 project.properties - # # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", - # # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/validate-staging.yml - # - name: Transpile MPL Test Vectors without recursively building the MPL - # working-directory: ./submodules/MaterialProviders/TestVectorsAwsCryptographicMaterialProviders - # run: | - # # Run transpile by itself. We don't want to locally build the MPL because - # # we want to verify that the version pulled down from maven works correctly - # make transpile_implementation_java - # make transpile_test_java - # make mvn_local_deploy + # The following three steps: "Transpile MPL Test Vectors without recursively building the MPL", + # "Run Test Vectors", and "Test Examples" mimic the tests in ./codebuild/staging/validate-staging.yml + - name: Transpile MPL Test Vectors without recursively building the MPL + working-directory: ./submodules/MaterialProviders/TestVectorsAwsCryptographicMaterialProviders + run: | + # Run transpile by itself. We don't want to locally build the MPL because + # we want to verify that the version pulled down from maven works correctly + make transpile_implementation_java + make transpile_test_java + make mvn_local_deploy - name: Run Test Vectors # TODO: Remove pull_request from here From b024b4e920e7a0d3da6b94768ca876a79d8da7e7 Mon Sep 17 00:00:00 2001 From: Ritvik Kapila Date: Wed, 5 Feb 2025 12:58:30 -0800 Subject: [PATCH 27/27] remove pull_request from workflow --- .github/workflows/ci_test_latest_released_mpl_java.yml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/workflows/ci_test_latest_released_mpl_java.yml b/.github/workflows/ci_test_latest_released_mpl_java.yml index b16d08d42..30a4fd030 100644 --- a/.github/workflows/ci_test_latest_released_mpl_java.yml +++ b/.github/workflows/ci_test_latest_released_mpl_java.yml @@ -12,9 +12,6 @@ on: required: false default: true type: boolean - # TODO: Remove this before merging to main. We don't want this workflow running in CI - # on PRs because we expect it to fail when the MPL is actually updated since the last version. - pull_request: jobs: getVersion: @@ -109,8 +106,7 @@ jobs: make mvn_local_deploy - name: Run Test Vectors - # TODO: Remove pull_request from here - if: github.event_name == 'pull_request' || github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && ${{inputs.run_test_vectors}}) + if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && ${{inputs.run_test_vectors}}) working-directory: ./TestVectors run: | # Spin up ddb local