chore(ci): Adds platform integration test (#282)

* chore(ci): Adds platform integration test We have migrated to a new set of backend services and are deprecating the existing python based services. This replaces the recommended sample code with platform compatible code * disable nano roundtrip test
opentdf · May 22, 2024 · f4e2f49 · f4e2f49
1 parent 9d344cb
commit f4e2f49
Show file tree

Hide file tree

Showing 13 changed files with 494 additions and 26 deletions.
diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml
@@ -207,18 +207,54 @@ jobs:
           kubernetes-version: 1.30.0
       - name: Run tilt ci
         env:
-          TEST_SCRIPT: ./wait-and-test.sh
-          OPENTDF_INGRESS_HOST_PORT: "5432"
+          TEST_SCRIPT: ./wait-and-test.sh backend
+          OPENTDF_INGRESS_HOST_PORT: '5432'
           PLAYWRIGHT_TESTS_TO_RUN: roundtrip
         run: tilt ci
 
+  platform-roundtrip:
+    needs:
+      - cli
+      - lib
+      - web-app
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: .github/workflows/roundtrip
+    timeout-minutes: 45
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20'
+          cache: 'npm'
+          cache-dependency-path: './web-app/package-lock.json'
+      - uses: actions/download-artifact@v4
+        with:
+          name: opentdf-client-lib
+          path: lib/
+      - uses: actions/download-artifact@v4
+        with:
+          name: opentdf-cli
+          path: cli/
+      - uses: actions/setup-go@0c52d547c9bc32b1aa3301fd7a9cb496313a4491
+        with:
+          go-version: '1.22'
+      - run: docker compose up -d --wait --wait-timeout 240
+      - env:
+          OPENTDF_SERVICES_AUTHORIZATION_URL: 'http://localhost:65432/'
+          PLAYWRIGHT_TESTS_TO_RUN: roundtrip
+        run: |-
+          ./wait-and-test.sh platform
+
   deliver-ghp:
     needs:
       - lib
       - cli
       - web-app
       - scripts
       - backend-roundtrip
+      - platform-roundtrip
     runs-on: ubuntu-latest
     timeout-minutes: 5
     # To publish from a release or feature branch, remove the ref == condition below
@@ -260,8 +296,8 @@ jobs:
           with version=[${{ steps.guess-build-metadata.outputs.FULL_VERSION }}]"
       - run: >-
           .github/workflows/publish-to.sh
-          ${{ steps.guess-build-metadata.outputs.FULL_VERSION }}
-          ${{ steps.guess-build-metadata.outputs.DIST_TAG }}
+            ${{ steps.guess-build-metadata.outputs.FULL_VERSION }}
+            ${{ steps.guess-build-metadata.outputs.DIST_TAG }}
         env:
           NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
       - run: echo "- [Client Library](https://github.com/opentdf/client-web/pkgs/npm/client)">>$GITHUB_STEP_SUMMARY

diff --git a/.github/workflows/roundtrip/config-demo-idp.sh b/.github/workflows/roundtrip/config-demo-idp.sh
@@ -0,0 +1,46 @@
+#!/usr/bin/env bash
+
+set -x
+
+: "${KC_VERSION:=24.0.3}"
+
+if ! which kcadm.sh; then
+  KCADM_URL=https://github.com/keycloak/keycloak/releases/download/${KC_VERSION}/keycloak-${KC_VERSION}.zip
+  echo "DOWNLOADING ${KCADM_URL}"
+  if ! curl --output kc.zip --fail --location "${KCADM_URL}"; then
+    echo "[ERROR] Failed to download ${KCADM_URL}"
+    exit 3
+  fi
+  ls -l
+  if ! unzip ./kc.zip; then
+    echo "[ERROR] Failed to unzip file from ${KCADM_URL}"
+    exit 3
+  fi
+  ls -l
+  ls -l "$(pwd)/keycloak-${KC_VERSION}/bin"
+  PATH=$PATH:"$(pwd)/keycloak-${KC_VERSION}/bin"
+  export PATH
+  if ! which kcadm.sh; then
+    echo "[ERROR] Failed to find kcadm.sh"
+    exit 3
+  fi
+fi
+
+kcadm.sh config credentials --server http://localhost:65432/auth \
+  --realm master --user admin --password changeme
+
+kcadm.sh create clients -r opentdf \
+  -s clientId=browsertest \
+  -s enabled=true \
+  -s 'redirectUris=["http://localhost:65432/"]' \
+  -s consentRequired=false \
+  -s standardFlowEnabled=true \
+  -s directAccessGrantsEnabled=true \
+  -s serviceAccountsEnabled=false \
+  -s publicClient=true \
+  -s protocol=openid-connect \
+  -s 'protocolMappers=[{"name":"aud","protocol":"openid-connect","protocolMapper":"oidc-audience-mapper","consentRequired":false,"config":{"access.token.claim":"true","included.custom.audience":"http://localhost:65432"}}]' \
+  -s 'attributes={"dpop.bound.access.tokens":"true"}'
+
+kcadm.sh create users -r opentdf -s username=user1 -s enabled=true -s firstName=Alice -s lastName=User
+kcadm.sh set-password -r opentdf --username user1 --new-password testuser123
diff --git a/.github/workflows/roundtrip/docker-compose.yaml b/.github/workflows/roundtrip/docker-compose.yaml
@@ -0,0 +1,80 @@
+services:
+  keycloak:
+    # This is kc 24.0.1 with opentdf protocol mapper on board
+    image: quay.io/keycloak/keycloak:24.0
+    restart:
+      always
+      # To enable debugging, use this CMD and also set and expose the DEBUG_PORT
+    # command: ["--debug", "start-dev", "--log-level=DEBUG"]
+    command: ['start-dev']
+    environment:
+      # DEBUG_PORT: "*:30012"
+      KC_DB_VENDOR: postgres
+      KC_DB_URL_HOST: keycloakdb
+      KC_DB_URL_PORT: 5432
+      KC_DB_URL_DATABASE: keycloak
+      KC_DB_USERNAME: keycloak
+      KC_DB_PASSWORD: changeme
+      KC_FEATURES: 'preview,token-exchange'
+      KC_HEALTH_ENABLED: 'true'
+      KC_HOSTNAME_ADMIN_URL: 'http://localhost:65432/auth'
+      KC_HOSTNAME_PORT: '65432'
+      KC_HOSTNAME_STRICT: 'false'
+      KC_HOSTNAME_STRICT_BACKCHANNEL: 'false'
+      KC_HOSTNAME_STRICT_HTTPS: 'false'
+      KC_HOSTNAME_URL: 'http://localhost:65432/auth'
+      KC_HTTP_ENABLED: 'true'
+      KC_HTTP_PORT: '8888'
+      KC_HTTP_RELATIVE_PATH: '/auth'
+      KC_PROXY_HEADERS: 'xforwarded'
+      KEYCLOAK_ADMIN: admin
+      KEYCLOAK_ADMIN_PASSWORD: changeme
+    ports:
+      - '8888:8888'
+      # - "30012:30012"
+    healthcheck:
+      test:
+        - CMD-SHELL
+        - >-
+          [ -f /tmp/HealthCheck.java ] 
+          || echo "public class HealthCheck {
+            public static void main(String[] args) throws java.lang.Throwable {
+              System.exit(
+                java.net.HttpURLConnection.HTTP_OK ==
+                  ((java.net.HttpURLConnection) new java.net.URL(args[0]).openConnection())
+                    .getResponseCode() ? 0 : 1);
+            }
+          }" >/tmp/HealthCheck.java
+          && java /tmp/HealthCheck.java http://localhost:8888/auth/health/live
+      interval: 5s
+      timeout: 10s
+      retries: 3
+      start_period: 2m
+  keycloakdb:
+    image: postgres
+    restart: always
+    user: postgres
+    environment:
+      POSTGRES_PASSWORD: changeme
+      POSTGRES_USER: postgres
+      POSTGRES_DB: keycloak
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+  opentdfdb:
+    image: public.ecr.aws/docker/library/postgres:15-alpine
+    restart: always
+    user: postgres
+    environment:
+      POSTGRES_USER: postgres
+      POSTGRES_PASSWORD: changeme
+      POSTGRES_DB: opentdf
+    healthcheck:
+      test: ['CMD-SHELL', 'pg_isready']
+      interval: 5s
+      timeout: 5s
+      retries: 10
+    ports:
+      - '5432:5432'
diff --git a/.github/workflows/roundtrip/init-temp-keys.sh b/.github/workflows/roundtrip/init-temp-keys.sh
@@ -0,0 +1,69 @@
+#!/bin/sh
+# init-temporary-keys.sh
+# Initialize temporary keys for use with a KAS
+
+USAGE="Usage:  ${CMD:=${0##*/}} [(-v|--verbose)] [-H|--hsm]"
+
+# helper functions
+exit2() {
+  printf >&2 "%s:  %s: '%s'\n%s\n" "$CMD" "$1" "$2" "$USAGE"
+  exit 2
+}
+check() { { [ "$1" != "$EOL" ] && [ "$1" != '--' ]; } || exit2 "missing argument" "$2"; }
+
+# parse command-line options
+set -- "$@" "${EOL:=$(printf '\1\3\3\7')}" # end-of-list marker
+while [ "$1" != "$EOL" ]; do
+  opt="$1"
+  shift
+  case "$opt" in
+    -H | --hsm) opt_hsm='true' ;;
+    -v | --verbose) opt_verbose='true' ;;
+    -h | --help)
+      printf "%s\n" "$USAGE"
+      exit 0
+      ;;
+
+    # process special cases
+    -[A-Za-z0-9] | -*[!A-Za-z0-9]*) exit2 "invalid option" "$opt" ;;
+  esac
+done
+shift
+
+if [ "$opt_verbose" = true ]; then
+  set -x
+fi
+
+if [ "$opt_hsm" = true ]; then
+  : "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN:=12345}"
+  : "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_EC_LABEL:=development-ec-kas}"
+  : "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_RSA_LABEL:=development-rsa-kas}"
+
+  if [ -z "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" ]; then
+    if which brew; then
+      OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH=$(brew --prefix)/lib/softhsm/libsofthsm2.so
+    else
+      OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH=/lib/softhsm/libsofthsm2.so
+    fi
+  fi
+
+  if softhsm2-util --show-slots | grep dev-token; then
+    echo "[INFO] dev-token slot is already configured"
+    exit 0
+  fi
+
+  softhsm2-util --init-token --free --label "dev-token" --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" --so-pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}"
+  pkcs11-tool --module "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" --login --show-info --list-objects --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}"
+fi
+
+openssl req -x509 -nodes -newkey RSA:2048 -subj "/CN=kas" -keyout kas-private.pem -out kas-cert.pem -days 365
+openssl ecparam -name prime256v1 >ecparams.tmp
+openssl req -x509 -nodes -newkey ec:ecparams.tmp -subj "/CN=kas" -keyout kas-ec-private.pem -out kas-ec-cert.pem -days 365
+
+if [ "$opt_hsm" = true ]; then
+  pkcs11-tool --module "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" --login --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" --write-object kas-private.pem --type privkey --label "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_RSA_LABEL}"
+  pkcs11-tool --module "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" --login --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" --write-object kas-cert.pem --type cert --label "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_RSA_LABEL}"
+  # https://manpages.ubuntu.com/manpages/jammy/man1/pkcs11-tool.1.html --usage-derive
+  pkcs11-tool --module "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" --login --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" --write-object kas-ec-private.pem --type privkey --label "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_EC_LABEL}" --usage-derive
+  pkcs11-tool --module "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_MODULEPATH}" --login --pin "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_PIN}" --write-object kas-ec-cert.pem --type cert --label "${OPENTDF_SERVER_CRYPTOPROVIDER_HSM_KEYS_EC_LABEL}"
+fi