Add nodemanager and resource manager components

miyurud · Feb 27, 2025 · d5a056d · d5a056d
1 parent 9006759
commit d5a056d
Show file tree

Hide file tree

Showing 5 changed files with 107 additions and 0 deletions.
diff --git a/k8s/hdfs/nodemanager-deployment.yaml b/k8s/hdfs/nodemanager-deployment.yaml
@@ -0,0 +1,36 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hdfs-nodemanager
+  labels:
+    app: hdfs
+    role: nodemanager
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hdfs
+      role: nodemanager
+  template:
+    metadata:
+      labels:
+        app: hdfs
+        role: nodemanager
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: nodemanager
+          image: bde2020/hadoop-nodemanager:2.0.0-hadoop3.2.1-java8
+          ports:
+            - containerPort: 8042
+          env:
+            - name: SERVICE_PRECONDITION
+              value: "hdfs-namenode.default.svc.cluster.local:9000 hdfs-namenode.default.svc.cluster.local:9870 hdfs-datanode.default.svc.cluster.local:9864 hdfs-resourcemanager.default.svc.cluster.local:8088"
+          resources:
+            requests:
+              memory: "512Mi"
+              cpu: "500m"
+            limits:
+              memory: "1Gi"
+              cpu: "1000m"
+
diff --git a/k8s/hdfs/nodemanager-service.yaml b/k8s/hdfs/nodemanager-service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hdfs-nodemanager
+  labels:
+    app: hdfs
+    role: nodemanager
+spec:
+  ports:
+    - port: 8042
+      targetPort: 8042
+  selector:
+    app: hdfs
+    role: nodemanager
+  type: ClusterIP
diff --git a/k8s/hdfs/resourcemanager-deployment.yaml b/k8s/hdfs/resourcemanager-deployment.yaml
@@ -0,0 +1,35 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: hdfs-resourcemanager
+  labels:
+    app: hdfs
+    role: resourcemanager
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: hdfs
+      role: resourcemanager
+  template:
+    metadata:
+      labels:
+        app: hdfs
+        role: resourcemanager
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: resourcemanager
+          image: bde2020/hadoop-resourcemanager:2.0.0-hadoop3.2.1-java8
+          ports:
+            - containerPort: 8088
+          env:
+            - name: SERVICE_PRECONDITION
+              value: "hdfs-namenode.default.svc.cluster.local:9000 hdfs-namenode.default.svc.cluster.local:9870 hdfs-datanode.default.svc.cluster.local:9864"
+          resources:
+            requests:
+              memory: "512Mi"
+              cpu: "500m"
+            limits:
+              memory: "1Gi"
+              cpu: "1000m"
diff --git a/k8s/hdfs/resourcemanager-service.yaml b/k8s/hdfs/resourcemanager-service.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: hdfs-resourcemanager
+  labels:
+    app: hdfs
+    role: resourcemanager
+spec:
+  ports:
+    - port: 8088
+      targetPort: 8088
+  selector:
+    app: hdfs
+    role: resourcemanager
+  type: ClusterIP
diff --git a/test-k8s.sh b/test-k8s.sh
@@ -78,6 +78,12 @@ ready_hdfs() {
     kubectl apply -f ./k8s/hdfs/datanode-deployment.yaml
     kubectl apply -f ./k8s/hdfs/datanode-service.yaml
 
+    # Deploy YARN ResourceManager and NodeManager
+    kubectl apply -f ./k8s/hdfs/resourcemanager-deployment.yaml
+    kubectl apply -f ./k8s/hdfs/resourcemanager-service.yaml
+    kubectl apply -f ./k8s/hdfs/nodemanager-deployment.yaml
+    kubectl apply -f ./k8s/hdfs/nodemanager-service.yaml
+
     echo "Fetching JasmineGraph Master pod name..."
     MASTER_POD=$(kubectl get pods | grep jasminegraph-master | awk '{print $1}')