mount notebook

Author: adwk67

stacks/jupyterhub-keycloak/jupyterhub.yaml

@@ -146,6 +146,13 @@ options:
       enabled: false
     extraLabels:
       stackable.tech/vendor: Stackable
+    initContainers:
+      - name: download-notebook
+        image: oci.stackable.tech/sdp/tools:1.0.0-stackable0.0.0-dev
+        command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/feat/keycloak-jupyterhub/stacks/jupyterhub-keycloak/process-s3.ipynb -o /notebook/process-s3.ipynb']
+        volumeMounts:
+          - mountPath: /notebook
+            name: notebook
     storage:
       extraVolumes:
         - name: tls-ca-cert
@@ -164,11 +171,16 @@ options:
         - name: minio-s3-credentials
           secret:
             secretName: minio-s3-credentials
+        - name: notebook
+          emptyDir:
+            sizeLimit: 500Mi
       extraVolumeMounts:
         - name: tls-ca-cert
           mountPath: /stackable/secrets/tls-ca-cert
         - name: minio-s3-credentials
           mountPath: /minio-s3-credentials
+        - name: notebook
+          mountPath: /home/jovyan/notebook
     profileList:
       - display_name: "Default"
         description: "Default profile"
@@ -203,38 +215,10 @@ options:
           image:
             display_name: Image
             choices:
-              "jupyter/pyspark-notebook:python-3.8":
-                display_name: "jupyter/pyspark-notebook:python-3.8"
-                kubespawner_override:
-                  image: "jupyter/pyspark-notebook:python-3.8"
-              "jupyter/pyspark-notebook:python-3.9":
-                display_name: "jupyter/pyspark-notebook:python-3.9"
-                kubespawner_override:
-                  image: "jupyter/pyspark-notebook:python-3.9"
-              "jupyter/pyspark-notebook:python-3.11":
-                display_name: "jupyter/pyspark-notebook:python-3.11"
-                kubespawner_override:
-                  image: "jupyter/pyspark-notebook:python-3.11"
-              "quay.io/jupyter/pyspark-notebook:python-3.11.7":
-                display_name: "quay.io/jupyter/pyspark-notebook:python-3.11.7"
-                kubespawner_override:
-                  image: "quay.io/jupyter/pyspark-notebook:python-3.11.7"
-              "quay.io/jupyter/pyspark-notebook:python-3.11.8":
-                display_name: "quay.io/jupyter/pyspark-notebook:python-3.11.8"
-                kubespawner_override:
-                  image: "quay.io/jupyter/pyspark-notebook:python-3.11.8"
               "quay.io/jupyter/pyspark-notebook:python-3.11.9":
                 display_name: "quay.io/jupyter/pyspark-notebook:python-3.11.9"
                 kubespawner_override:
                   image: "quay.io/jupyter/pyspark-notebook:python-3.11.9"
-              "quay.io/jupyter/pyspark-notebook:spark-3.5.0":
-                display_name: "quay.io/jupyter/pyspark-notebook:spark-3.5.0"
-                kubespawner_override:
-                  image: "quay.io/jupyter/pyspark-notebook:spark-3.5.0"
-              "quay.io/jupyter/pyspark-notebook:spark-3.5.1":
-                display_name: "quay.io/jupyter/pyspark-notebook:spark-3.5.1"
-                kubespawner_override:
-                  image: "quay.io/jupyter/pyspark-notebook:spark-3.5.1"
               "quay.io/jupyter/pyspark-notebook:spark-3.5.2":
                 display_name: "quay.io/jupyter/pyspark-notebook:spark-3.5.2"
                 kubespawner_override:

stacks/jupyterhub-keycloak/process-s3.ipynb

@@ -7,7 +7,7 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "! ls -al /usr/local/spark/jars | grep spark-core\n",
+    "# Output notebook versions\n",
     "! python3 -V\n",
     "! java --version\n",
     "! pyspark --version"
@@ -37,13 +37,12 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "import os\n",
     "from pyspark.sql import SparkSession\n",
     "\n",
     "NAMESPACE = os.environ.get(\"NAMESPACE\", \"default\")\n",
     "POD_NAME = os.environ.get(\"HOSTNAME\", f\"jupyter-{os.environ.get('USER', 'default')}-{NAMESPACE}\")\n",
     "\n",
-    "EXECUTOR_IMAGE = \"spark:3.5.2-python311\" \n",
+    "EXECUTOR_IMAGE = \"oci.stackable.tech/sandbox/spark:3.5.2-python311\" \n",
     "\n",
     "spark = (\n",
     "    SparkSession.builder\n",
@@ -89,9 +88,10 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "# Manual check via pyarrow.fs\n",
+    "# Manual S3 file check via pyarrow.fs\n",
     "import pyarrow.fs as fs\n",
-    "s3 = fs.S3FileSystem(endpoint_override=\"http://minio:9000/\", access_key=\"admin\", secret_key=\"adminadmin\", scheme=\"http\")\n",
+    "\n",
+    "s3 = fs.S3FileSystem(endpoint_override=\"http://minio:9000/\", access_key=minio_user, secret_key=minio_pwd, scheme=\"http\")\n",
     "files = s3.get_file_info(fs.FileSelector(\"demo/gas-sensor/raw/\", recursive=True))\n",
     "for f in files:\n",
     "    print(\"Found file:\", f.path)"