csi: use different ports for controller and node plugins

Usubg host network produces port collisions. So we use different ports for controller plugin deployments and node plugin deamonsets to avoid collisions. We also make sure that rbd and cephfs drivers don't collide Signed-off-by: Michael Adam <[email protected]>
ceph · Feb 10, 2025 · cc75fb1 · cc75fb1
1 parent c265e41
commit cc75fb1
Show file tree

Hide file tree

Showing 2 changed files with 37 additions and 7 deletions.
diff --git a/internal/controller/driver_controller.go b/internal/controller/driver_controller.go
@@ -759,6 +759,7 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
 						}
 						// Addons Sidecar Container
 						if !r.isNfsDriver() && ptr.Deref(r.driver.Spec.DeployCsiAddons, false) {
+							port := r.ControllerPluginCsiAddonsContainerPort()
 							containers = append(containers, corev1.Container{
 								Name:            "csi-addons",
 								Image:           r.images["addons"],
@@ -772,15 +773,15 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
 										utils.PodContainerArg,
 										utils.PodUidContainerArg,
 										utils.CsiAddonsAddressContainerArg,
-										utils.ControllerPortContainerArg,
+										utils.ContainerPortArg(port),
 										utils.NamespaceContainerArg,
 										utils.If(logRotationEnabled, utils.LogToStdErrContainerArg, ""),
 										utils.If(logRotationEnabled, utils.AlsoLogToStdErrContainerArg, ""),
 										utils.If(logRotationEnabled, utils.LogFileContainerArg("csi-addons"), ""),
 									),
 								),
 								Ports: []corev1.ContainerPort{
-									utils.CsiAddonsContainerPort,
+									port,
 								},
 								Env: []corev1.EnvVar{
 									utils.NodeIdEnvVar,
@@ -922,6 +923,20 @@ func (r *driverReconcile) reconcileControllerPluginDeployment() error {
 	return err
 }
 
+func (r *driverReconcile) ControllerPluginCsiAddonsContainerPort() corev1.ContainerPort {
+
+	// the cephFS and rbd drivers need to use different ports
+	// to avoid port collisions with host network.
+	port := utils.ControllerPluginCsiAddonsContainerRbdPort
+	if r.isCephFsDriver() {
+		port = utils.ControllerPluginCsiAddonsContainerCephFsPort
+
+	}
+
+	return port
+
+}
+
 func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
 	daemonSet := &appsv1.DaemonSet{}
 	daemonSet.Name = r.generateName("nodeplugin")
@@ -1107,6 +1122,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
 						}
 						// CSI Addons Sidecar Container
 						if r.isRdbDriver() && ptr.Deref(r.driver.Spec.DeployCsiAddons, false) {
+							port := utils.NodePluginCsiAddonsContainerPort
 							containers = append(containers, corev1.Container{
 								Name:            "csi-addons",
 								Image:           r.images["addons"],
@@ -1122,7 +1138,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
 										utils.CsiAddonsNodeIdContainerArg,
 										utils.LogVerbosityContainerArg(logVerbosity),
 										utils.CsiAddonsAddressContainerArg,
-										utils.ControllerPortContainerArg,
+										utils.ContainerPortArg(port),
 										utils.PodContainerArg,
 										utils.NamespaceContainerArg,
 										utils.PodUidContainerArg,
@@ -1133,7 +1149,7 @@ func (r *driverReconcile) reconcileNodePluginDeamonSet() error {
 									},
 								),
 								Ports: []corev1.ContainerPort{
-									utils.CsiAddonsContainerPort,
+									port,
 								},
 								Env: []corev1.EnvVar{
 									utils.NodeIdEnvVar,

diff --git a/internal/utils/csi.go b/internal/utils/csi.go
@@ -337,11 +337,25 @@ var DriverNamespaceEnvVar = corev1.EnvVar{
 	},
 }
 
-// CSI Addons container port definition
-var CsiAddonsContainerPort = corev1.ContainerPort{
+// CSI Addons container port definitions
+var ControllerPluginCsiAddonsContainerRbdPort = corev1.ContainerPort{
 	ContainerPort: int32(9070),
 }
 
+var ControllerPluginCsiAddonsContainerCephFsPort = corev1.ContainerPort{
+	ContainerPort: int32(9080),
+}
+
+var NodePluginCsiAddonsContainerPort = corev1.ContainerPort{
+	ContainerPort: int32(9071),
+}
+
+func ContainerPortArg(port corev1.ContainerPort) string {
+
+	return fmt.Sprintf("--controller-port=%d", port.ContainerPort)
+
+}
+
 // Ceph CSI common container arguments
 var CsiAddressContainerArg = fmt.Sprintf("--csi-address=%s", csiEndpoint)
 var EndpointContainerArg = fmt.Sprintf("--endpoint=%s", csiEndpoint)
@@ -359,7 +373,7 @@ var HandleVolumeInuseErrorContainerArg = "--handle-volume-inuse-error=false"
 var PodUidContainerArg = fmt.Sprintf("--pod-uid=$(%s)", PodUidEnvVar.Name)
 var PodContainerArg = fmt.Sprintf("--pod=$(%s)", PodNameEnvVar.Name)
 var NamespaceContainerArg = fmt.Sprintf("--namespace=$(%s)", PodNamespaceEnvVar.Name)
-var ControllerPortContainerArg = fmt.Sprintf("--controller-port=%d", CsiAddonsContainerPort.ContainerPort)
+
 var DriverNamespaceContainerArg = fmt.Sprintf("--drivernamespace=$(%s)", DriverNamespaceEnvVar.Name)
 var MetricsPathContainerArg = "--metricspath=/metrics"
 var PoolTimeContainerArg = "--polltime=60s"