Update k8s deployment files

.editorconfig

@@ -0,0 +1,30 @@
+root = true
+
+[*.java]
+indent_style = tab
+indent_size = 4
+
+[*.adoc]
+indent_style = tab
+indent_size = 4
+
+[*.groovy]
+indent_style = tab
+indent_size = 4
+
+[*.xml]
+indent_style = tab
+indent_size = 4
+
+[*.yml]
+indent_style = space
+indent_size = 2
+
+[*.yaml]
+indent_style = space
+indent_size = 2
+
+[*.sh]
+indent_style = space
+indent_size = 4
+end_of_line = lf

.github/actions/build-images/action.yml

@@ -10,72 +10,34 @@ inputs:
   dockerhub-password:
     description: 'dockerhub password'
     required: true
+  GCR_JSON_KEY:
+    description: 'GCR_JSON_KEY'
+    required: true
 runs:
   using: "composite"
   steps:
   - name: Install pack
     uses: jvalkeal/[email protected]
     with:
-      pack-version: 0.18.0
-  - name: Build Images
-    env:
-      TAG: ${{ inputs.version }}
-    shell: bash
-    run: |
-      for v in 8 11 17
-      do
-        pack build \
-          --path spring-cloud-dataflow-server/target/spring-cloud-dataflow-server-$TAG.jar \
-          --builder gcr.io/paketo-buildpacks/builder:base \
-          --env BP_JVM_VERSION=$v springcloud/spring-cloud-dataflow-server:$TAG-jdk$v
-        pack build \
-          --path spring-cloud-dataflow-composed-task-runner/target/spring-cloud-dataflow-composed-task-runner-$TAG.jar \
-          --builder gcr.io/paketo-buildpacks/builder:base \
-          --env BP_JVM_VERSION=$v springcloud/spring-cloud-dataflow-composed-task-runner:$TAG-jdk$v
-        pack build \
-          --path spring-cloud-dataflow-tasklauncher/spring-cloud-dataflow-tasklauncher-sink-kafka/target/spring-cloud-dataflow-tasklauncher-sink-kafka-$TAG.jar \
-          --builder gcr.io/paketo-buildpacks/builder:base \
-          --env BP_JVM_VERSION=$v springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:$TAG-jdk$v
-        pack build \
-          --path spring-cloud-dataflow-tasklauncher/spring-cloud-dataflow-tasklauncher-sink-rabbit/target/spring-cloud-dataflow-tasklauncher-sink-rabbit-$TAG.jar \
-          --builder gcr.io/paketo-buildpacks/builder:base \
-          --env BP_JVM_VERSION=$v springcloud/spring-cloud-dataflow-tasklauncher-sink-rabbit:$TAG-jdk$v
-        pack build \
-          --path spring-cloud-dataflow-single-step-batch-job/target/spring-cloud-dataflow-single-step-batch-job-$TAG.jar \
-          --builder gcr.io/paketo-buildpacks/builder:base \
-          --env BP_JVM_VERSION=$v springcloud/spring-cloud-dataflow-single-step-batch-job:$TAG-jdk$v
-      done
+      pack-version: 0.30.0
 
-  # docker hub login
-  - uses: docker/login-action@v1
+    # docker hub login
+  - name: Login to docker.io
+    uses: docker/login-action@v3
     with:
       username: ${{ inputs.dockerhub-username }}
       password: ${{ inputs.dockerhub-password }}
-
-  # push images
-  - name: Push images
+  - name: Login to GCR
+    uses: docker/login-action@v3
+    with:
+      registry: gcr.io
+      username: _json_key
+      password: ${{ inputs.GCR_JSON_KEY }}
+  - name: Build Images
     env:
       TAG: ${{ inputs.version }}
     shell: bash
-    run: |
-      docker tag springcloud/spring-cloud-dataflow-server:$TAG-jdk11 springcloud/spring-cloud-dataflow-server:$TAG
-      docker tag springcloud/spring-cloud-dataflow-composed-task-runner:$TAG-jdk11 springcloud/spring-cloud-dataflow-composed-task-runner:$TAG
-      docker tag springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:$TAG-jdk11 springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:$TAG
-      docker tag springcloud/spring-cloud-dataflow-tasklauncher-sink-rabbit:$TAG-jdk11 springcloud/spring-cloud-dataflow-tasklauncher-sink-rabbit:$TAG
-      docker tag springcloud/spring-cloud-dataflow-single-step-batch-job:$TAG-jdk11 springcloud/spring-cloud-dataflow-single-step-batch-job:$TAG
-      for v in 8 11 17
-      do
-        docker push springcloud/spring-cloud-dataflow-server:$TAG-jdk$v
-        docker push springcloud/spring-cloud-dataflow-composed-task-runner:$TAG-jdk$v
-        docker push springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:$TAG-jdk$v
-        docker push springcloud/spring-cloud-dataflow-tasklauncher-sink-rabbit:$TAG-jdk$v
-        docker push springcloud/spring-cloud-dataflow-single-step-batch-job:$TAG-jdk$v
-      done
-      docker push springcloud/spring-cloud-dataflow-server:$TAG
-      docker push springcloud/spring-cloud-dataflow-composed-task-runner:$TAG
-      docker push springcloud/spring-cloud-dataflow-tasklauncher-sink-kafka:$TAG
-      docker push springcloud/spring-cloud-dataflow-tasklauncher-sink-rabbit:$TAG
-      docker push springcloud/spring-cloud-dataflow-single-step-batch-job:$TAG
+    run: ${{ github.action_path }}/build-images.sh
 
   # build/publish support images
   - name: Publish Grafana Prometheus

.github/actions/build-images/build-images.sh

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+if [ "$TAG" == "" ]; then
+    echo "TAG not found"
+    exit 1
+fi
+if [ "$DEFAULT_JDK" = "" ]; then
+    echo "DEFAULT_JDK not found using 11"
+    DEFAULT_JDK=11
+else
+    echo "DEFAULT_JDK=$DEFAULT_JDK"
+fi
+
+function pack_image {
+    JAR="$1-$TAG.jar"
+    REPO="$2"
+    v="$3"
+    if [ ! -f "$JAR" ]; then
+        echo "File not found $JAR"
+        exit 2
+    fi
+    echo "Creating: $REPO:$TAG-jdk$v"
+    # --buildpack "paketo-buildpacks/[email protected]" --buildpack "paketo-buildpacks/[email protected]"
+    pack build --builder gcr.io/paketo-buildpacks/builder:base \
+            --path "$JAR" \
+            --trust-builder --verbose \
+            --env BP_JVM_VERSION=$v "$REPO:$TAG-jdk$v"
+    RC=$?
+    if ((RC!=0)); then
+        echo "Error $RC packaging $JAR"
+        exit $RC
+    fi
+    echo "Created: $REPO:$TAG-jdk$v"
+}
+LEN=$(jq '.include | length' .github/workflows/images.json)
+for ((i = 0; i < LEN; i++)); do
+    TARGET="$(jq -r --argjson index $i '.include[$index] | .path' .github/workflows/images.json)"
+    IMAGE="$(jq -r --argjson index $i '.include[$index] | .image' .github/workflows/images.json)"
+    ARTIFACT_ID="$(jq -r --argjson index $i '.include[$index] | .name' .github/workflows/images.json)"
+    # 8 11 17 21
+    for v in 8 11 17; do
+        pack_image "$TARGET/$ARTIFACT_ID"  $IMAGE $v $ARTIFACT_ID
+        RC=$?
+        if [ $RC -ne 0 ]; then
+            exit $RC
+        fi
+        docker push "$IMAGE:$TAG-jdk$v"
+        echo "Pushed $IMAGE:$TAG-jdk$v"
+        if [ "$DEFAULT_JDK" == "$v" ]; then
+            docker tag "$IMAGE:$TAG-jdk$DEFAULT_JDK" "$IMAGE:$TAG"
+            docker push "$IMAGE:$TAG"
+            echo "Pushed $IMAGE:$TAG"
+        fi
+    done
+done
+echo "Pruning Docker"
+docker system prune -f
+docker system prune --volumes -f

.github/actions/build-package-bundle/action.yml

@@ -7,6 +7,21 @@ inputs:
   skipper-version:
     description: skipper version
     required: true
+  server-version:
+    description: server version
+    required: true
+  server-repository:
+    description: server repository
+    required: true
+  skipper-repository:
+    description: skipper repository
+    required: true
+  ctr-version:
+    description: ctr version
+    required: true
+  package-name:
+    description: package name
+    required: true
   package-bundle-template:
     description: path to package bundle template
     required: true
@@ -16,6 +31,15 @@ inputs:
   config:
     description: path to ytt config files dir
     required: true
+  project-directory:
+    description: The working directory
+    required: true
+  SRP_CLIENT_ID:
+    description: secrets.SRP_CLIENT_ID
+    required: false
+  SRP_CLIENT_SECRET:
+    description: secrets.SRP_CLIENT_SECRET
+    required: false
 outputs:
   bundle-path:
     description: Location path where bundle was build
@@ -24,43 +48,37 @@ outputs:
 runs:
   using: "composite"
   steps:
-  - uses: vmware-tanzu/carvel-setup-action@v1
   - name: Tanzu Dance
     id: tanzu-dance
     env:
       DATAFLOW_VERSION: ${{ inputs.dataflow-version }}
       SKIPPER_VERSION: ${{ inputs.skipper-version }}
+      SKIPPER_REPOSITORY: ${{ inputs.skipper-repository }}
+      SERVER_VERSION: ${{ inputs.server-version }}
+      SERVER_REPOSITORY: ${{ inputs.server-repository }}
+      CTR_VERSION: ${{ inputs.ctr-version }}
+      PACKAGE_NAME: ${{ inputs.package-name }}
       PACKAGE_BUNDLE_TEMPLATE: ${{ inputs.package-bundle-template }}
       PACKAGE_BUNDLE_GENERATED: ${{ runner.temp }}/generated/packagebundle
       VENDIR_SRC_IN: ${{ inputs.config }}
       IMGPKG_LOCK_TEMPLATE: ${{ inputs.imgpkg-lock-template }}
       IMGPKG_LOCK_GENERATED_IN: ${{ runner.temp }}/generated/imgpkgin
       IMGPKG_LOCK_GENERATED_OUT: ${{ runner.temp }}/generated/imgpkgout
+      SRP_CLIENT_ID: ${{ inputs.SRP_CLIENT_ID }}
+      SRP_CLIENT_SECRET: ${{ inputs.SRP_CLIENT_SECRET }}
     shell: bash
+    working-directory: ${{ inputs.project-directory }}
     run: |
-      echo "::set-output name=bundle-path::$PACKAGE_BUNDLE_GENERATED"
-      ytt \
-        -f $PACKAGE_BUNDLE_TEMPLATE \
-        --output-files $PACKAGE_BUNDLE_GENERATED \
-        --data-value-yaml project.version=$DATAFLOW_VERSION \
-        --data-value-yaml spring.cloud.skipper.version=$SKIPPER_VERSION \
-        --file-mark 'config/values.yml:type=text-template'
-      ytt \
-        -f $IMGPKG_LOCK_TEMPLATE \
-        --output-files $IMGPKG_LOCK_GENERATED_IN \
-        --data-value-yaml project.version=$DATAFLOW_VERSION \
-        --data-value-yaml spring.cloud.skipper.version=$SKIPPER_VERSION \
-        --file-mark '**/*.yml:type=text-template'
-      cp -R $VENDIR_SRC_IN $PACKAGE_BUNDLE_GENERATED/config/upstream
-      vendir sync --chdir $PACKAGE_BUNDLE_GENERATED
-      mkdir -p $IMGPKG_LOCK_GENERATED_OUT
-
-      for DIR in $(ls $IMGPKG_LOCK_GENERATED_IN)
-      do
-        ytt -f $PACKAGE_BUNDLE_GENERATED -f $IMGPKG_LOCK_GENERATED_IN/$DIR > $IMGPKG_LOCK_GENERATED_OUT/$DIR.yml
-      done
-
-      mkdir -p $PACKAGE_BUNDLE_GENERATED/.imgpkg
-      kbld \
-        -f $IMGPKG_LOCK_GENERATED_OUT \
-        --imgpkg-lock-output $PACKAGE_BUNDLE_GENERATED/.imgpkg/images.yml
+      echo "bundle-path=$PACKAGE_BUNDLE_GENERATED" >> $GITHUB_OUTPUT
+      SCDF_DIR="${{ inputs.project-directory || '.' }}"
+      if [ "$USE_SRP" == "true" ]; then
+        if [ "$SRP_SCRIPTS" == "" ]; then
+          echo "SRP_SCRIPTS not defined"
+          exit 2
+        fi
+        export OBSERVATION=package-bundle
+        export OUTPUT_VERSION=$SERVER_VERSION
+        $SRP_SCRIPTS/run-under-observer.sh "${{ github.action_path }}/build-package-bundle.sh"
+      else
+        "${{ github.action_path }}/build-package-bundle.sh"
+      fi

.github/actions/build-package-bundle/build-package-bundle.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+
+function check_env() {
+  eval ev='$'$1
+  if [ "$ev" == "" ]; then
+    echo "env var $1 not defined"
+    if ((sourced != 0)); then
+      return 1
+    else
+      exit 1
+    fi
+  fi
+}
+
+TMP=$(mktemp -d)
+if [ "$PACKAGE_BUNDLE_GENERATED" = "" ]; then
+    export PACKAGE_BUNDLE_GENERATED="$TMP/generated/packagebundle"
+fi
+mkdir -p "$PACKAGE_BUNDLE_GENERATED"
+if [ "$IMGPKG_LOCK_GENERATED_IN" = "" ]; then
+    export IMGPKG_LOCK_GENERATED_IN="$TMP/generated/imgpkgin"
+fi
+mkdir -p "$IMGPKG_LOCK_GENERATED_IN"
+if [ "$IMGPKG_LOCK_GENERATED_OUT" = "" ]; then
+    export IMGPKG_LOCK_GENERATED_OUT="$TMP/generated/imgpkgout"
+fi
+mkdir -p "$IMGPKG_LOCK_GENERATED_OUT"
+
+check_env PACKAGE_BUNDLE_TEMPLATE
+check_env SERVER_VERSION
+check_env SERVER_REPOSITORY
+check_env DATAFLOW_VERSION
+check_env SKIPPER_VERSION
+check_env SKIPPER_REPOSITORY
+check_env PACKAGE_NAME
+check_env IMGPKG_LOCK_TEMPLATE
+check_env VENDIR_SRC_IN
+
+echo "Build Package Bundle: $PACKAGE_BUNDLE_TEMPLATE package.name=$PACKAGE_NAME, server.repository=$SERVER_REPOSITORY, server.version=$SERVER_VERSION,skipper.repository=$SKIPPER_REPOSITORY, skipper.version=$SKIPPER_VERSION, output=$PACKAGE_BUNDLE_GENERATED"
+set +e
+time ls > /dev/null 2>&1
+RC=$?
+if((RC=0)); then
+  MEASURE="time -v -o times.txt -a"
+else
+  MEASURE=""
+fi
+set -e
+echo "ytt -f $PACKAGE_BUNDLE_TEMPLATE" > times.txt
+
+$MEASURE ytt -f "$PACKAGE_BUNDLE_TEMPLATE" \
+    --output-files "$PACKAGE_BUNDLE_GENERATED" \
+    --data-value-yaml server.version="$SERVER_VERSION" \
+    --data-value-yaml server.repository="$SERVER_REPOSITORY" \
+    --data-value-yaml ctr.version="$DATAFLOW_VERSION" \
+    --data-value-yaml dataflow.version="$DATAFLOW_VERSION" \
+    --data-value-yaml skipper.version="$SKIPPER_VERSION" \
+    --data-value-yaml skipper.repository="$SKIPPER_REPOSITORY" \
+    --data-value-yaml grafana.version="$DATAFLOW_VERSION" \
+    --data-value-yaml package.name="$PACKAGE_NAME" \
+    --file-mark 'config/values.yml:type=text-template' \
+    --file-mark '.imgpkg/bundle.yaml:type=text-template'
+echo "ytt -f $IMGPKG_LOCK_TEMPLATE" >> times.txt
+$MEASURE ytt -f "$IMGPKG_LOCK_TEMPLATE" \
+    --output-files "$IMGPKG_LOCK_GENERATED_IN" \
+    --data-value-yaml server.version="$SERVER_VERSION" \
+    --data-value-yaml server.repository="$SERVER_REPOSITORY" \
+    --data-value-yaml ctr.version="$DATAFLOW_VERSION" \
+    --data-value-yaml dataflow.version="$DATAFLOW_VERSION" \
+    --data-value-yaml skipper.version="$SKIPPER_VERSION" \
+    --data-value-yaml skipper.repository="$SKIPPER_REPOSITORY" \
+    --data-value-yaml grafana.version="$DATAFLOW_VERSION" \
+    --file-mark '**/*.yml:type=text-template'
+
+mkdir -p "$PACKAGE_BUNDLE_GENERATED/config/upstream"
+cp -R "$VENDIR_SRC_IN" "$PACKAGE_BUNDLE_GENERATED/config/upstream"
+echo "vendir -f $IMGPKG_LOCK_TEMPLATE" >> times.txt
+$MEASURE vendir sync --chdir "$PACKAGE_BUNDLE_GENERATED"
+mkdir -p "$IMGPKG_LOCK_GENERATED_OUT"
+
+for DIR in $(ls $IMGPKG_LOCK_GENERATED_IN); do
+    echo "ytt for $DIR" >> times.txt
+    $MEASURE ytt -f "$PACKAGE_BUNDLE_GENERATED" -f "$IMGPKG_LOCK_GENERATED_IN/$DIR" > "$IMGPKG_LOCK_GENERATED_OUT/$DIR.yml"
+done
+
+mkdir -p "$PACKAGE_BUNDLE_GENERATED/.imgpkg"
+echo "kbld -f $IMGPKG_LOCK_GENERATED_OUT" >> times.txt
+$MEASURE kbld -f "$IMGPKG_LOCK_GENERATED_OUT" \
+    --imgpkg-lock-output "$PACKAGE_BUNDLE_GENERATED/.imgpkg/images.yml"
+
+cat times.txt