Skip to content

Commit 3289d83

Browse files
shengnuoshengnuo
committed
Support DataStore V2
1 parent 554a4de commit 3289d83

File tree

4 files changed

+144
-12
lines changed

4 files changed

+144
-12
lines changed

‎api/apps/v1alpha1/nemo_datastorev2_types.go

Lines changed: 82 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -196,11 +196,44 @@ func (n *NemoDatastoreV2) GetStandardEnv() []corev1.EnvVar {
196196
Name: "HOME",
197197
Value: "/data/gitea/git",
198198
},
199+
{
200+
Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID",
201+
ValueFrom: &corev1.EnvVarSource{
202+
SecretKeyRef: &corev1.SecretKeySelector{
203+
Key: "objectStoreKey",
204+
LocalObjectReference: corev1.LocalObjectReference{
205+
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
206+
},
207+
},
208+
},
209+
},
210+
{
211+
Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY",
212+
ValueFrom: &corev1.EnvVarSource{
213+
SecretKeyRef: &corev1.SecretKeySelector{
214+
Key: "objectStoreSecret",
215+
LocalObjectReference: corev1.LocalObjectReference{
216+
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
217+
},
218+
},
219+
},
220+
},
221+
{
222+
Name: "GITEA__SERVER__LFS_JWT_SECRET",
223+
ValueFrom: &corev1.EnvVarSource{
224+
SecretKeyRef: &corev1.SecretKeySelector{
225+
Key: "jwtSecret",
226+
LocalObjectReference: corev1.LocalObjectReference{
227+
Name: n.Spec.DataStoreParams.LfsJwtSecret,
228+
},
229+
},
230+
},
231+
},
199232
{
200233
Name: "GITEA__DATABASE__PASSWD",
201234
ValueFrom: &corev1.EnvVarSource{
202235
SecretKeyRef: &corev1.SecretKeySelector{
203-
Key: "password",
236+
Key: "postgresPassword",
204237
LocalObjectReference: corev1.LocalObjectReference{
205238
Name: n.Spec.DataStoreParams.DBSecret,
206239
},
@@ -233,16 +266,48 @@ func (n *NemoDatastoreV2) GetInitContainerEnv() []corev1.EnvVar {
233266
Name: "GITEA_TEMP",
234267
Value: "/tmp/gitea",
235268
},
236-
237269
{
238270
Name: "HOME",
239271
Value: "/data/gitea/git",
240272
},
273+
{
274+
Name: "GITEA__LFS__MINIO_ACCESS_KEY_ID",
275+
ValueFrom: &corev1.EnvVarSource{
276+
SecretKeyRef: &corev1.SecretKeySelector{
277+
Key: "objectStoreKey",
278+
LocalObjectReference: corev1.LocalObjectReference{
279+
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
280+
},
281+
},
282+
},
283+
},
284+
{
285+
Name: "GITEA__LFS__MINIO_SECRET_ACCESS_KEY",
286+
ValueFrom: &corev1.EnvVarSource{
287+
SecretKeyRef: &corev1.SecretKeySelector{
288+
Key: "objectStoreSecret",
289+
LocalObjectReference: corev1.LocalObjectReference{
290+
Name: n.Spec.DataStoreParams.ObjectStoreSecret,
291+
},
292+
},
293+
},
294+
},
295+
{
296+
Name: "GITEA__SERVER__LFS_JWT_SECRET",
297+
ValueFrom: &corev1.EnvVarSource{
298+
SecretKeyRef: &corev1.SecretKeySelector{
299+
Key: "jwtSecret",
300+
LocalObjectReference: corev1.LocalObjectReference{
301+
Name: n.Spec.DataStoreParams.LfsJwtSecret,
302+
},
303+
},
304+
},
305+
},
241306
{
242307
Name: "GITEA__DATABASE__PASSWD",
243308
ValueFrom: &corev1.EnvVarSource{
244309
SecretKeyRef: &corev1.SecretKeySelector{
245-
Key: "password",
310+
Key: "postgresPassword",
246311
LocalObjectReference: corev1.LocalObjectReference{
247312
Name: n.Spec.DataStoreParams.DBSecret,
248313
},
@@ -253,7 +318,7 @@ func (n *NemoDatastoreV2) GetInitContainerEnv() []corev1.EnvVar {
253318
Name: "GITEA_ADMIN_USERNAME",
254319
ValueFrom: &corev1.EnvVarSource{
255320
SecretKeyRef: &corev1.SecretKeySelector{
256-
Key: "username",
321+
Key: "GITEA_ADMIN_USERNAME",
257322
LocalObjectReference: corev1.LocalObjectReference{
258323
Name: n.Spec.DataStoreParams.GiteaAdminSecret,
259324
},
@@ -264,7 +329,7 @@ func (n *NemoDatastoreV2) GetInitContainerEnv() []corev1.EnvVar {
264329
Name: "GITEA_ADMIN_PASSWORD",
265330
ValueFrom: &corev1.EnvVarSource{
266331
SecretKeyRef: &corev1.SecretKeySelector{
267-
Key: "password",
332+
Key: "GITEA_ADMIN_PASSWORD",
268333
LocalObjectReference: corev1.LocalObjectReference{
269334
Name: n.Spec.DataStoreParams.GiteaAdminSecret,
270335
},
@@ -583,14 +648,22 @@ func (n *NemoDatastoreV2) GetInitContainers() []corev1.Container {
583648
user := int64(1000)
584649
return []corev1.Container{
585650
{
586-
Name: "init-datastore",
651+
Name: "init-directories",
587652
Image: n.GetImage(),
588653
ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()),
589654
Command: []string{
590-
"/bin/sh", "-c",
655+
"/usr/sbin/init/init_directory_structure.sh",
591656
},
592-
Args: []string{
593-
"/usr/sbin/init/init_directory_structure.sh && /usr/sbin/config_environment.sh",
657+
VolumeMounts: n.GetVolumeMountsInitContainer(),
658+
Env: n.GetInitContainerEnv(),
659+
EnvFrom: n.GetInitAppIniEnvFrom(),
660+
},
661+
{
662+
Name: "init-app-ini",
663+
Image: n.GetImage(),
664+
ImagePullPolicy: corev1.PullPolicy(n.GetImagePullPolicy()),
665+
Command: []string{
666+
"/usr/sbin/config_environment.sh",
594667
},
595668
VolumeMounts: n.GetVolumeMountsInitContainer(),
596669
Env: n.GetInitContainerEnv(),

‎config/rbac/role.yaml

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,6 @@ rules:
109109
- apiGroups:
110110
- autoscaling
111111
resources:
112-
- horizontalpodautoscalars
113112
- horizontalpodautoscalers
114113
verbs:
115114
- create

‎config/samples/apps_v1alpha1_nimdatastorev2.yaml

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
apiVersion: apps.nvidia.com/v1alpha1
2+
kind: NemoDatastoreV2
3+
metadata:
4+
name: nemodatastore
5+
spec:
6+
authSecret: ngc-image-pull-secret
7+
dataStoreParams:
8+
datastoreConfigSecret: "nemo-ms-nemo-datastore"
9+
datastoreInitSecret: "nemo-ms-nemo-datastore-init"
10+
datastoreInlineConfigSecret: "nemo-ms-nemo-datastore-inline-config"
11+
datastoreSettingsSecret: "nemo-ms-nemo-datastore-setting"
12+
dbSecret: "nds-pg-existing-secret"
13+
giteaAdminSecret: "gitea-admin-credentials"
14+
lfsJwtSecret: "nemo-ms-nemo-datastore--lfs-jwt"
15+
objStoreSecret: "nds-minio-existing-secret"
16+
pvcSharedData: "pvc-shared-data"
17+
size: "10Gi"
18+
sshEnabled: false
19+
storageClass: "local-path"
20+
expose:
21+
service:
22+
port: 3000
23+
type: ClusterIP
24+
image:
25+
repository: "nvcr.io/nvidian/nemo-llm/datastore"
26+
tag: "25.01-rc8"
27+
pullPolicy: IfNotPresent
28+
replicas: 1
29+
resources:
30+
requests:
31+
memory: "256Mi"
32+
cpu: "500m"
33+
limits:
34+
memory: "512Mi"
35+
cpu: "1"
36+
livenessProbe:
37+
enabled: true
38+
probe:
39+
httpGet:
40+
path: /v1/health
41+
port: 3000
42+
scheme: HTTP
43+
initialDelaySeconds: 10
44+
timeoutSeconds: 5
45+
readinessProbe:
46+
enabled: true
47+
probe:
48+
httpGet:
49+
path: /v1/health
50+
port: 3000
51+
scheme: HTTP
52+
initialDelaySeconds: 5
53+
timeoutSeconds: 3
54+
# metrics:
55+
# enabled: true
56+
# serviceMonitor:
57+
# additionalLabels:
58+
# app: sample-nemodatastore
59+
# interval: "30s"
60+
# scrapeTimeout: "10s"

‎internal/controller/nemo_datastorev2_controller.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -95,7 +95,7 @@ func NewNemoDatastoreV2Reconciler(client client.Client, scheme *runtime.Scheme,
9595
// +kubebuilder:rbac:groups=batch,resources=jobs,verbs=get;list;watch;create;update;patch;delete
9696
// +kubebuilder:rbac:groups=route.openshift.io,resources=routes,verbs=get;list;watch;create;update;patch;delete
9797
// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
98-
// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalars,verbs=get;list;watch;create;update;patch;delete
98+
// +kubebuilder:rbac:groups=autoscaling,resources=horizontalpodautoscalers,verbs=get;list;watch;create;update;patch;delete
9999
// +kubebuilder:rbac:groups="",resources=events,verbs=create;update;patch
100100

101101
// Reconcile is part of the main kubernetes reconciliation loop which aims to
@@ -289,7 +289,7 @@ func (r *NemoDatastoreV2Reconciler) reconcileNemoDatastoreV2(ctx context.Context
289289
corev1.ResourceStorage: size,
290290
},
291291
}
292-
pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}
292+
pvc.Spec.AccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce}
293293
pvc.Namespace = nemoDatastore.Namespace
294294

295295
err = r.Create(ctx, &pvc)

0 commit comments

Comments
 (0)