Merge branch 'main' into next

stackabletech · Oct 24, 2024 · 5bc2356 · 5bc2356
2 parents 958f174 + 8841a8d
commit 5bc2356
Show file tree

Hide file tree

Showing 14 changed files with 395 additions and 33 deletions.
diff --git a/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml b/.github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml
@@ -0,0 +1,89 @@
+---
+name: Build and publish jupyter-pyspark-with-alibi-detect
+
+env:
+  IMAGE_NAME: jupyter-pyspark-with-alibi-detect
+  IMAGE_VERSION: python-3.9
+  REGISTRY_PATH: stackable
+  DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-jupyter"
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - demos/signal-processing/Dockerfile-jupyter
+      - demos/signal-processing/requirements.txt
+      - .github/workflows/dev_jupyter-pyspark-with-alibi-detect.yaml
+
+jobs:
+  build:
+    name: Build/Publish ${{ matrix.runner.arch }} Image
+    permissions:
+      id-token: write
+    runs-on: ${{ matrix.runner.name }}
+    strategy:
+      matrix:
+        runner:
+          - {name: "ubuntu-latest", arch: "amd64"}
+          - {name: "ubicloud-standard-8-arm", arch: "arm64"}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Build image
+        id: build
+        uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-name: ${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+          container-file: ${{ env.DOCKERFILE_PATH }}
+
+      - name: Publish Container Image on docker.stackable.tech
+        uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: docker.stackable.tech
+          image-registry-username: github
+          image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
+          image-repository: demos/${{ env.IMAGE_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
+      - name: Publish Container Image on oci.stackable.tech
+        uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: oci.stackable.tech
+          image-registry-username: robot$stackable+github-action-build
+          image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
+  publish_manifests:
+    name: Build/Publish Manifest
+    needs: [build]
+    permissions:
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Publish and Sign Image Index Manifest to docker.stackable.tech
+        uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: docker.stackable.tech
+          image-registry-username: github
+          image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
+          image-repository: demos/${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+
+      - name: Publish and Sign Image Index Manifest to oci.stackable.tech
+        uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: oci.stackable.tech
+          image-registry-username: robot$stackable+github-action-build
+          image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
diff --git a/.github/workflows/dev_nifi.yaml b/.github/workflows/dev_nifi.yaml
@@ -0,0 +1,88 @@
+---
+name: Build and publish NiFi for signal-processing demo
+
+env:
+  IMAGE_NAME: nifi
+  IMAGE_VERSION: 1.27.0-postgresql
+  REGISTRY_PATH: stackable
+  DOCKERFILE_PATH: "demos/signal-processing/Dockerfile-nifi"
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - demos/signal-processing/Dockerfile-nifi
+      - .github/workflows/dev_nifi.yaml
+
+jobs:
+  build:
+    name: Build/Publish ${{ matrix.runner.arch }} Image
+    permissions:
+      id-token: write
+    runs-on: ${{ matrix.runner.name }}
+    strategy:
+      matrix:
+        runner:
+          - {name: "ubuntu-latest", arch: "amd64"}
+          - {name: "ubicloud-standard-8-arm", arch: "arm64"}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Build image
+        id: build
+        uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-name: ${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+          container-file: ${{ env.DOCKERFILE_PATH }}
+
+      - name: Publish Container Image on docker.stackable.tech
+        uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: docker.stackable.tech
+          image-registry-username: github
+          image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
+          image-repository: demos/${{ env.IMAGE_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
+      - name: Publish Container Image on oci.stackable.tech
+        uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: oci.stackable.tech
+          image-registry-username: robot$stackable+github-action-build
+          image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
+  publish_manifests:
+    name: Build/Publish Manifest
+    needs: [build]
+    permissions:
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Publish and Sign Image Index Manifest to docker.stackable.tech
+        uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: docker.stackable.tech
+          image-registry-username: github
+          image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
+          image-repository: demos/${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+
+      - name: Publish and Sign Image Index Manifest to oci.stackable.tech
+        uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: oci.stackable.tech
+          image-registry-username: robot$stackable+github-action-build
+          image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
diff --git a/.github/workflows/dev_spark-k8s-with-scikit-learn.yaml b/.github/workflows/dev_spark-k8s-with-scikit-learn.yaml
@@ -0,0 +1,96 @@
+---
+name: Build and publish spark-k8s-with-scikit-learn
+
+env:
+  IMAGE_NAME: spark-k8s-with-scikit-learn
+  IMAGE_VERSION: 3.5.0-stackable24.3.0
+  REGISTRY_PATH: stackable
+  DOCKERFILE_PATH: "demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile"
+
+on:
+  workflow_dispatch:
+  push:
+    branches:
+      - main
+    paths:
+      - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile
+      - demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt
+      - .github/workflows/dev_spark-k8s-with-scikit-learn.yaml
+
+jobs:
+  build:
+    name: Build/Publish ${{ matrix.runner.arch }} Image
+    permissions:
+      id-token: write
+    runs-on: ${{ matrix.runner.name }}
+    strategy:
+      matrix:
+        runner:
+          - {name: "ubuntu-latest", arch: "amd64"}
+          # TODO: the image 3.5.0-stackable24.3.0 does not have an arm64 build.
+          # Re-activate the arm runner when the image is updated to one that does.
+          # Also adjust publish_manifest step to include arm architecture
+          #- {name: "ubicloud-standard-8-arm", arch: "arm64"}
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      - name: Build image
+        id: build
+        uses: stackabletech/actions/build-container-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-name: ${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+          container-file: ${{ env.DOCKERFILE_PATH }}
+
+      - name: Publish Container Image on docker.stackable.tech
+        uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: docker.stackable.tech
+          image-registry-username: github
+          image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
+          image-repository: demos/${{ env.IMAGE_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
+      - name: Publish Container Image on oci.stackable.tech
+        uses: stackabletech/actions/publish-image@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: oci.stackable.tech
+          image-registry-username: robot$stackable+github-action-build
+          image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
+          image-manifest-tag: ${{ steps.build.outputs.image-manifest-tag }}
+          source-image-uri: ${{ steps.build.outputs.image-manifest-uri }}
+
+  publish_manifests:
+    name: Build/Publish Manifest
+    needs: [build]
+    permissions:
+      id-token: write
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+
+      # TODO: remove image-architecture key once arm image is also built
+      - name: Publish and Sign Image Index Manifest to docker.stackable.tech
+        uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: docker.stackable.tech
+          image-registry-username: github
+          image-registry-password: ${{ secrets.NEXUS_PASSWORD }}
+          image-repository: demos/${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+          image-architectures: '["amd64"]'
+
+      # TODO: remove image-architecture key once arm image is also built
+      - name: Publish and Sign Image Index Manifest to oci.stackable.tech
+        uses: stackabletech/actions/publish-index-manifest@013e6482fbc0edf2d38cf9220fc931f6a81336fb # v0.0.6
+        with:
+          image-registry-uri: oci.stackable.tech
+          image-registry-username: robot$stackable+github-action-build
+          image-registry-password: ${{ secrets.HARBOR_ROBOT_STACKABLE_GITHUB_ACTION_BUILD_SECRET }}
+          image-repository: ${{ env.REGISTRY_PATH }}/${{ env.IMAGE_NAME }}
+          image-index-manifest-tag: ${{ env.IMAGE_VERSION }}
+          image-architectures: '["amd64"]'
diff --git a/demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.xml b/demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.xml
@@ -1385,7 +1385,7 @@
                     </entry>
                     <entry>
                         <key>Remote URL</key>
-                        <value>https://ip-api.mobidata-bw.de/v1/NVBW/gbfs/v2/free_bike_status.json</value>
+                        <value>https://api.mobidata-bw.de/sharing/gbfs/v2/deer/free_bike_status</value>
                     </entry>
                     <entry>
                         <key>disable-http2</key>
@@ -2963,7 +2963,7 @@
                     </entry>
                     <entry>
                         <key>Remote URL</key>
-                        <value>https://ip-api.mobidata-bw.de/v1/NVBW/gbfs/v2/station_information.json</value>
+                        <value>https://api.mobidata-bw.de/sharing/gbfs/v2/deer/station_information</value>
                     </entry>
                     <entry>
                         <key>disable-http2</key>
@@ -5584,7 +5584,7 @@
                     </entry>
                     <entry>
                         <key>Remote URL</key>
-                        <value>https://ip-api.mobidata-bw.de/v1/NVBW/gbfs/v2/station_status.json</value>
+                        <value>https://api.mobidata-bw.de/sharing/gbfs/v2/deer/station_status</value>
                     </entry>
                     <entry>
                         <key>disable-http2</key>

diff --git a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile
@@ -1,8 +1,6 @@
-# docker build -t docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.4.0-stackable0.0.0-dev .
+FROM docker.stackable.tech/stackable/spark-k8s:3.5.0-stackable24.3.0
 
-FROM docker.stackable.tech/stackable/pyspark-k8s:3.4.0-stackable23.7.0
-
-COPY requirements.txt .
+COPY demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt .
 
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir -r ./requirements.txt
diff --git a/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt b/demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/requirements.txt
@@ -1,2 +1,2 @@
-scikit-learn==1.1.3
-pandas==1.5.1
+scikit-learn==1.3.1
+pandas==2.0.3
diff --git a/demos/signal-processing/Dockerfile-jupyter b/demos/signal-processing/Dockerfile-jupyter
@@ -1,8 +1,6 @@
-# docker build -f ./Dockerfile-jupyter -t docker.stackable.tech/demos/jupyter-pyspark-with-alibi-detect:python-3.9 .
-
 FROM jupyter/pyspark-notebook:python-3.9
 
-COPY requirements.txt .
+COPY demos/signal-processing/requirements.txt .
 
 RUN pip install --no-cache-dir --upgrade pip && \
     pip install --no-cache-dir -r ./requirements.txt
diff --git a/demos/signal-processing/requirements.txt b/demos/signal-processing/requirements.txt
@@ -1,3 +1,2 @@
 psycopg2-binary==2.9.9
-tsfresh==0.20.1
-alibi-detect==0.11.4
+alibi-detect==0.11.4
diff --git a/docs/modules/demos/images/data-lakehouse-iceberg-trino-spark/nifi_2.png b/docs/modules/demos/images/data-lakehouse-iceberg-trino-spark/nifi_2.png