[WIP] Python DBESDK

.github/workflows/ci_examples_python.yml

@@ -0,0 +1,94 @@
+# This workflow performs tests in Python.
+name: Python Examples
+
+on:
+  workflow_call:
+    inputs:
+      dafny:
+        description: "The Dafny version to run"
+        required: true
+        type: string
+      regenerate-code:
+        description: "Regenerate code using smithy-dafny"
+        required: false
+        default: false
+        type: boolean
+      mpl-version:
+        description: "MPL version to use"
+        required: false
+        type: string
+      mpl-head:
+        description: "Running on MPL HEAD"
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  testPython:
+    strategy:
+      matrix:
+        python-version: [3.11]
+        os: [macos-13]
+    runs-on: ${{ matrix.os }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: us-west-2
+          role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-DDBEC-Dafny-Role-us-west-2
+          role-session-name: DDBEC-Dafny-Python-Tests
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Python ${{ matrix.python-version }} for running tests
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
+      - name: Setup Dafny
+        uses: ./submodules/MaterialProviders/.github/actions/setup_dafny/
+        with:
+          dafny-version: ${{ inputs.dafny }}
+
+      - name: Update MPL submodule if using MPL HEAD
+        if: ${{ inputs.mpl-head == true }}
+        working-directory: submodules/MaterialProviders
+        run: |
+          git checkout main
+          git pull
+          git submodule update --init --recursive
+          git rev-parse HEAD
+
+      - name: Regenerate code using smithy-dafny if necessary
+        if: ${{ inputs.regenerate-code }}
+        uses: ./.github/actions/polymorph_codegen
+        with:
+          dafny: ${{ env.DAFNY_VERSION }}
+          library: DynamoDbEncryption
+          diff-generated-code: false
+          update-and-regenerate-mpl: true
+
+      - name: Build and locally deploy dependencies for examples
+        shell: bash
+        working-directory: ./DynamoDbEncryption
+        run: |
+          make transpile_python
+
+      - name: Test DynamoDbEncryption Examples
+        working-directory: ./Examples/runtimes/python
+        run: |
+          # Run simple examples
+          tox -e dynamodbencryption
+          # Run migration examples
+          # tox -e migration

.github/workflows/ci_test_python.yml

@@ -0,0 +1,130 @@
+# This workflow runs only Dafny-transpiled Python tests.
+name: test python
+
+on:
+  workflow_call:
+    inputs:
+      dafny:
+        description: "The Dafny version to run"
+        required: true
+        type: string
+      regenerate-code:
+        description: "Regenerate code using smithy-dafny"
+        required: false
+        default: false
+        type: boolean
+      mpl-head:
+        description: "Running on MPL HEAD"
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  testPython:
+    strategy:
+      fail-fast: false
+      matrix:
+        library: [DynamoDbEncryption]
+        python-version: ["3.11"]
+        os: [
+          # macos-13,
+          ubuntu-22.04,
+          # Dafny-transpiled Python tests use a PYTHONPATH hack that doesn't work on Windows.
+          # Windows is tested with non-Dafny-transpiled Python tests.
+          # windows-latest
+        ]
+    runs-on: ${{ matrix.os }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+
+      - name: Support longpaths on Git checkout
+        run: |
+          git config --global core.longpaths true
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Python ${{ matrix.python-version }} for running tests
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
+      - name: Setup Dafny
+        uses: ./submodules/MaterialProviders/.github/actions/setup_dafny/
+        with:
+          dafny-version: ${{ inputs.dafny }}
+
+      - name: Update MPL submodule if using MPL HEAD
+        if: ${{ inputs.mpl-head == true }}
+        working-directory: submodules/MaterialProviders
+        run: |
+          git checkout main
+          git pull
+          git submodule update --init --recursive
+          git rev-parse HEAD
+
+      - name: Regenerate code using smithy-dafny if necessary
+        if: ${{ inputs.regenerate-code }}
+        uses: ./.github/actions/polymorph_codegen
+        with:
+          dafny: ${{ env.DAFNY_VERSION }}
+          library: ${{ matrix.library }}
+          diff-generated-code: false
+          update-and-regenerate-mpl: true
+
+      - name: Download Dependencies
+        working-directory: ./${{ matrix.library }}
+        run: make setup_python
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: us-west-2
+          role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-DDBEC-Dafny-Role-us-west-2
+          role-session-name: DDBEC-Dafny-Net-Tests
+
+      - name: Compile ${{ matrix.library }} implementation
+        shell: bash
+        working-directory: ./${{ matrix.library }}
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_python CORES=$CORES
+
+      - name: Test ${{ matrix.library }} Dafny-transpiled Python tests
+        # Dafny-transpiled Python tests use a PYTHONPATH hack that doesn't work on Windows.
+        # Windows is tested with non-Dafny-transpiled Python tests.
+        if: ${{ matrix.os != 'windows-latest' }}
+        working-directory: ./${{ matrix.library }}/runtimes/python
+        shell: bash
+        run: |
+          tox -e dafnytests
+
+      - name: Test ${{ matrix.library }} Python unit tests
+        working-directory: ./${{ matrix.library }}/runtimes/python
+        shell: bash
+        run: |
+          tox -e unit
+
+      - name: Test ${{ matrix.library }} Python integration tests
+        working-directory: ./${{ matrix.library }}/runtimes/python
+        shell: bash
+        run: |
+          tox -e integ
+
+      - name: Test ${{ matrix.library }} Python coverage
+        working-directory: ./${{ matrix.library }}/runtimes/python
+        shell: bash
+        run: |
+          tox -e encrypted-interface-coverage
+          tox -e client-to-resource-conversions-coverage
+          tox -e resource-to-client-conversions-coverage
+

.github/workflows/ci_test_vector_python.yml

@@ -0,0 +1,102 @@
+# This workflow performs test vectors in Python.
+name: Library Python Test Vectors
+
+on:
+  workflow_call:
+    inputs:
+      dafny:
+        description: "The Dafny version to run"
+        required: true
+        type: string
+      regenerate-code:
+        description: "Regenerate code using smithy-dafny"
+        required: false
+        default: false
+        type: boolean
+      mpl-version:
+        description: "MPL version to use"
+        required: false
+        type: string
+      mpl-head:
+        description: "Running on MPL HEAD"
+        required: false
+        default: false
+        type: boolean
+
+jobs:
+  testPython:
+    strategy:
+      fail-fast: false
+      matrix:
+        library: [TestVectors]
+        python-version: [3.11, 3.12, 3.13]
+        os: [ubuntu-22.04]
+        interface: [client, resource, table]
+    runs-on: ${{ matrix.os }}
+    permissions:
+      id-token: write
+      contents: read
+    steps:
+      - name: Setup DynamoDB Local
+        uses: rrainn/[email protected]
+        with:
+          port: 8000
+          cors: "*"
+
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v4
+        with:
+          aws-region: us-west-2
+          role-to-assume: arn:aws:iam::370957321024:role/GitHub-CI-DDBEC-Dafny-Role-us-west-2
+          role-session-name: DDBEC-Dafny-Python-Tests
+
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+
+      - name: Setup Dafny
+        uses: ./submodules/MaterialProviders/.github/actions/setup_dafny/
+        with:
+          dafny-version: ${{ inputs.dafny }}
+
+      - name: Update MPL submodule if using MPL HEAD
+        if: ${{ inputs.mpl-head == true }}
+        working-directory: submodules/MaterialProviders
+        run: |
+          git checkout main
+          git pull
+          git submodule update --init --recursive
+          git rev-parse HEAD
+
+      - name: Regenerate code using smithy-dafny if necessary
+        if: ${{ inputs.regenerate-code }}
+        uses: ./.github/actions/polymorph_codegen
+        with:
+          dafny: ${{ env.DAFNY_VERSION }}
+          library: ${{ matrix.library }}
+          diff-generated-code: false
+          update-and-regenerate-mpl: true
+
+      - name: Setup Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Setup Python ${{ matrix.python-version }} for running tests
+        run: |
+          python -m pip install --upgrade pip
+          pip install --upgrade tox
+          pip install poetry
+
+      - name: Build Python TestVectors implementation
+        shell: bash
+        working-directory: ${{matrix.library}}
+        run: |
+          # This works because `node` is installed by default on GHA runners
+          CORES=$(node -e 'console.log(os.cpus().length)')
+          make transpile_python CORES=$CORES
+
+      - name: Test Python TestVectors with ${{matrix.interface}} interface
+        working-directory: ${{matrix.library}}
+        run: |
+          make test_python_${{matrix.interface}}_interface

.github/workflows/daily_ci.yml

@@ -61,6 +61,16 @@ jobs:
     uses: ./.github/workflows/library_rust_tests.yml
     with:
       dafny: ${{needs.getVersion.outputs.version}}
+  daily-ci-python:
+    needs: getVersion
+    uses: ./.github/workflows/ci_test_python.yml
+    with:
+      dafny: ${{needs.getVersion.outputs.version}}
+  daily-ci-python-examples:
+    needs: getVersion
+    uses: ./.github/workflows/ci_examples_python.yml
+    with:
+      dafny: ${{needs.getVersion.outputs.version}}
   daily-ci-net-test-vectors:
     needs: getVersion
     uses: ./.github/workflows/ci_test_vector_net.yml

.github/workflows/manual.yml

@@ -57,6 +57,16 @@ jobs:
     with:
       dafny: ${{ inputs.dafny }}
       regenerate-code: ${{ inputs.regenerate-code }}
+  manual-ci-python:
+    uses: ./.github/workflows/ci_test_python.yml
+    with:
+      dafny: ${{ inputs.dafny }}
+      regenerate-code: ${{ inputs.regenerate-code }}
+  manual-ci-python-examples:
+    uses: ./.github/workflows/ci_examples_python.yml
+    with:
+      dafny: ${{ inputs.dafny }}
+      regenerate-code: ${{ inputs.regenerate-code }}
   manual-ci-net-test-vectors:
     uses: ./.github/workflows/ci_test_vector_net.yml
     with:

.github/workflows/mpl-head.yml

@@ -73,6 +73,18 @@ jobs:
     with:
       dafny: ${{needs.getVersion.outputs.version}}
       mpl-head: true
+  mpl-head-ci-python:
+    needs: getVersion
+    uses: ./.github/workflows/ci_test_python.yml
+    with:
+      dafny: ${{needs.getVersion.outputs.version}}
+      mpl-head: true
+  mpl-head-ci-python-examples:
+    needs: getVersion
+    uses: ./.github/workflows/ci_examples_python.yml
+    with:
+      dafny: ${{needs.getVersion.outputs.version}}
+      mpl-head: true
   mpl-head-ci-net-test-vectors:
     needs: getVersion
     uses: ./.github/workflows/ci_test_vector_net.yml

.github/workflows/nightly.yml

@@ -58,6 +58,12 @@ jobs:
     with:
       dafny: "nightly-latest"
       regenerate-code: true
+  dafny-nightly-python:
+    if: github.event_name != 'schedule' || github.repository_owner == 'aws'
+    uses: ./.github/workflows/ci_test_python.yml
+    with:
+      dafny: "nightly-latest"
+      regenerate-code: true
   dafny-nightly-test-vectors-net:
     if: github.event_name != 'schedule' || github.repository_owner == 'aws'
     uses: ./.github/workflows/ci_test_vector_net.yml