Skip to content

Commit 803e520

Browse files
adwk67adwk67
committed
use secret for minio credentials, add demo entry
1 parent 53132a8 commit 803e520

File tree

7 files changed

+108
-10
lines changed

7 files changed

+108
-10
lines changed

‎demos/demos-v2.yaml

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,3 +226,19 @@ demos:
226226
cpu: "3"
227227
memory: 5098Mi
228228
pvc: 16Gi
229+
jupyterhub-keycloak:
230+
description: Demo showing jupyterhub notebooks secured with keycloak
231+
documentation: TODO
232+
stackableStack: jupyterhub-keycloak
233+
labels:
234+
- jupyterhub
235+
- keycloak
236+
- spark
237+
- S3
238+
manifests:
239+
- plainYaml: demos/jupyterhub-keycloak/load-gas-data.yaml
240+
supportedNamespaces: []
241+
resourceRequests:
242+
cpu: 6400m
243+
memory: 12622Mi
244+
pvc: 20Gi

‎stacks/jupyterhub-keycloak/Dockerfile

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# kind load docker-image spark:3.5.2-python311 -n stackable-data-platform
2+
3+
FROM spark:3.5.2-scala2.12-java17-ubuntu
4+
5+
USER root
6+
7+
RUN set -ex; \
8+
apt-get update; \
9+
# Install dependencies for Python 3.11
10+
apt-get install -y \
11+
software-properties-common \
12+
&& add-apt-repository ppa:deadsnakes/ppa \
13+
&& apt-get update && apt-get install -y \
14+
python3.11 \
15+
python3.11-venv \
16+
python3.11-dev \
17+
&& rm -rf /var/lib/apt/lists/*; \
18+
# Install pip manually for Python 3.11
19+
curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py && \
20+
python3.11 get-pip.py && \
21+
rm get-pip.py
22+
23+
# Make Python 3.11 the default Python version
24+
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.11 1 \
25+
&& update-alternatives --install /usr/bin/pip pip /usr/local/bin/pip3 1
26+
27+
USER spark

‎stacks/jupyterhub-keycloak/README.md

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,8 @@ Deploy the scripts:
44

55
```shell
66
stackablectl stack install jupyterhub-keycloak --release dev -s stacks/stacks-v2.yaml
7-
87
kubectl apply -f ./demos/jupyterhub-keycloak/load-gas-data.yaml
8+
9+
# or (with test data)
10+
stackablectl demo install jupyterhub-keycloak --release dev -s stacks/stacks-v2.yaml -d demos/demos-v2.yaml
911
```

‎stacks/jupyterhub-keycloak/jupyterhub.yaml

Lines changed: 9 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,10 +14,10 @@ options:
1414
GenericOAuthenticator:
1515
client_id: jupyterhub
1616
client_secret: jupyterhubjupyterhub
17-
oauth_callback_url: http://172.19.0.3:31095/hub/oauth_callback
18-
authorize_url: https://172.19.0.3:31093/realms/demo/protocol/openid-connect/auth
19-
token_url: https://172.19.0.3:31093/realms/demo/protocol/openid-connect/token
20-
userdata_url: https://172.19.0.3:31093/realms/demo/protocol/openid-connect/userinfo
17+
oauth_callback_url: http://172.19.0.4:31095/hub/oauth_callback
18+
authorize_url: https://172.19.0.4:31093/realms/demo/protocol/openid-connect/auth
19+
token_url: https://172.19.0.4:31093/realms/demo/protocol/openid-connect/token
20+
userdata_url: https://172.19.0.4:31093/realms/demo/protocol/openid-connect/userinfo
2121
username_claim: preferred_username
2222
scope:
2323
- openid
@@ -139,9 +139,14 @@ options:
139139
resources:
140140
requests:
141141
storage: "1"
142+
- name: minio-s3-credentials
143+
secret:
144+
secretName: minio-s3-credentials
142145
extraVolumeMounts:
143146
- name: tls-ca-cert
144147
mountPath: /stackable/secrets/tls-ca-cert
148+
- name: minio-s3-credentials
149+
mountPath: /minio-s3-credentials
145150
profileList:
146151
- display_name: "Default"
147152
description: "Default profile"

‎stacks/jupyterhub-keycloak/process-s3.ipynb

Lines changed: 20 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,23 @@
1313
"! pyspark --version"
1414
]
1515
},
16+
{
17+
"cell_type": "code",
18+
"execution_count": null,
19+
"id": "bd941fee",
20+
"metadata": {},
21+
"outputs": [],
22+
"source": [
23+
"import os\n",
24+
"\n",
25+
"# get minio credentials\n",
26+
"with open(\"/minio-s3-credentials/accessKey\", \"r\") as f:\n",
27+
" minio_user = f.read().strip()\n",
28+
"\n",
29+
"with open(\"/minio-s3-credentials/secretKey\", \"r\") as f:\n",
30+
" minio_pwd = f.read().strip()"
31+
]
32+
},
1633
{
1734
"cell_type": "code",
1835
"execution_count": null,
@@ -67,8 +84,8 @@
6784
" #.config(\"spark.executorEnv.LD_PRELOAD\", \"/opt/bitnami/common/lib/libnss_wrapper.so\")\n",
6885
" .config(\"spark.hadoop.fs.s3a.endpoint\", \"http://minio:9000/\")\n",
6986
" .config(\"spark.hadoop.fs.s3a.path.style.access\", \"true\")\n",
70-
" .config(\"spark.hadoop.fs.s3a.access.key\", \"admin\")\n",
71-
" .config(\"spark.hadoop.fs.s3a.secret.key\", \"adminadmin\")\n",
87+
" .config(\"spark.hadoop.fs.s3a.access.key\", minio_user)\n",
88+
" .config(\"spark.hadoop.fs.s3a.secret.key\", minio_pwd)\n",
7289
" .config(\"spark.hadoop.fs.s3a.aws.credentials.provider\", \"org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider\")\n",
7390
" .config(\"spark.jars.packages\", \"org.apache.hadoop:hadoop-client-api:3.3.4,org.apache.hadoop:hadoop-client-runtime:3.3.4,org.apache.hadoop:hadoop-aws:3.3.4,org.apache.hadoop:hadoop-common:3.3.4,com.amazonaws:aws-java-sdk-bundle:1.12.162\")\n",
7491
" .config(\"spark.submit.deployMode\", \"client\")\n",
@@ -110,7 +127,7 @@
110127
"metadata": {},
111128
"outputs": [],
112129
"source": [
113-
"df = spark.read.csv(\"s3a://demo/gas-sensor/raw/\")\n",
130+
"df = spark.read.csv(\"s3a://demo/gas-sensor/raw/\", header = True)\n",
114131
"df.show()"
115132
]
116133
},

‎stacks/jupyterhub-keycloak/s3-connection.yaml

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
---
2+
apiVersion: s3.stackable.tech/v1alpha1
3+
kind: S3Connection
4+
metadata:
5+
name: minio
6+
spec:
7+
host: minio
8+
port: 9000
9+
accessStyle: Path
10+
credentials:
11+
secretClass: minio-s3-credentials
12+
---
13+
apiVersion: secrets.stackable.tech/v1alpha1
14+
kind: SecretClass
15+
metadata:
16+
name: minio-s3-credentials
17+
spec:
18+
backend:
19+
k8sSearch:
20+
searchNamespace:
21+
pod: {}
22+
---
23+
apiVersion: v1
24+
kind: Secret
25+
metadata:
26+
name: minio-s3-credentials
27+
labels:
28+
secrets.stackable.tech/class: minio-s3-credentials
29+
stringData:
30+
accessKey: admin
31+
secretKey: {{ minioAdminPassword }}

‎stacks/stacks-v2.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -666,13 +666,13 @@ stacks:
666666
memory: 9010Mi
667667
pvc: 24Gi
668668
manifests:
669+
- helmChart: stacks/_templates/minio.yaml
669670
- plainYaml: stacks/jupyterhub-keycloak/keycloak-serviceaccount.yaml
670671
- plainYaml: stacks/jupyterhub-keycloak/keycloak-realm-config.yaml
671672
- plainYaml: stacks/jupyterhub-keycloak/keycloak.yaml
672673
- helmChart: stacks/jupyterhub-keycloak/jupyterhub.yaml
673674
- plainYaml: stacks/jupyterhub-keycloak/serviceaccount.yaml
674-
- helmChart: stacks/_templates/minio.yaml
675-
- plainYaml: stacks/trino-superset-s3/s3-connection.yaml
675+
- plainYaml: stacks/jupyterhub-keycloak/s3-connection.yaml
676676
parameters:
677677
- name: keycloakAdminPassword
678678
description: Password of the Keycloak admin user

0 commit comments

Comments
 (0)