Listener support/external access (#450)

* Remove pod svc controller * Add listener volume * Dont deploy rolegroup Service * Deploy rolegroup services again, namenode/journalnode listeners * Advertise as pod address * Run manual tests against 3.3.4, fix børked registration keys * Use listener addresses to generate discovery * Fix datanode port for HTTPS * Use configured listenerclasses * Move listenerclass into role config * Remove useless journalnode listenerclass option * Docs * Changelog * Disabled start of test * Update CRD * Remove reference to custom image * Document namenode_listener_refs * Break out the listener volume name to a constant * Update tests/templates/kuttl/external-access/operate-on-cluster.sh Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Avoid cloning FQDN override path * Use the port name constants we already have * Re-remove redundant role argument from ContainerConfig::volumes * Bounds-check port number * Separate data volumes from listener volume * Add a comment on discovery ordering * Remove stale TODO * bash != fish * Break other volumes out of `container_log_config` * Update docs/modules/hdfs/pages/usage-guide/listenerclass.adoc Co-authored-by: Siegfried Weber <[email protected]> * Drop external access test limit range --------- Co-authored-by: Sebastian Bernauer <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> Co-authored-by: Siegfried Weber <[email protected]>
stackabletech · Jan 26, 2024 · 2d751d7 · 2d751d7
1 parent 4b47b2c
commit 2d751d7
Show file tree

Hide file tree

Showing 24 changed files with 455 additions and 297 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,10 +7,12 @@ All notable changes to this project will be documented in this file.
 ### Added
 
 - More CRD documentation ([#433]).
+- Support for exposing HDFS clusters to clients outside of Kubernetes ([#450]).
 
 ### Changed
 
 - `operator-rs` `0.56.1` -> `0.57.0` ([#433]).
+- [BREAKING] `.spec.clusterConfig.listenerClass` has been renamed to `.spec.nameNodes.config.listenerClass`, migration will be required when using `external-unstable` ([#450]).
 - Change default value of `dfs.ha.nn.not-become-active-in-safemode` from `true` to `false` ([#458]).
 
 ### Fixed
@@ -19,6 +21,7 @@ All notable changes to this project will be documented in this file.
   and `dfs.datanode.kerberos.principal` in the discovery ConfigMap in case Kerberos is enabled ([#451]).
 
 [#433]: https://github.com/stackabletech/hdfs-operator/pull/433
+[#450]: https://github.com/stackabletech/hdfs-operator/pull/450
 [#451]: https://github.com/stackabletech/hdfs-operator/pull/451
 [#458]: https://github.com/stackabletech/hdfs-operator/pull/458
 

diff --git a/Cargo.lock b/Cargo.lock
diff --git a/deploy/helm/hdfs-operator/crds/crds.yaml b/deploy/helm/hdfs-operator/crds/crds.yaml
@@ -57,20 +57,6 @@ spec:
                       format: uint8
                       minimum: 0.0
                       type: integer
-                    listenerClass:
-                      default: cluster-internal
-                      description: |-
-                        This field controls which type of Service the Operator creates for this HdfsCluster:
-
-                        * cluster-internal: Use a ClusterIP service
-
-                        * external-unstable: Use a NodePort service
-
-                        This is a temporary solution with the goal to keep yaml manifests forward compatible. In the future, this setting will control which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) will be used to expose the service, and ListenerClass names will stay the same, allowing for a non-breaking change.
-                      enum:
-                        - cluster-internal
-                        - external-unstable
-                      type: string
                     vectorAggregatorConfigMapName:
                       description: Name of the Vector aggregator [discovery ConfigMap](https://docs.stackable.tech/home/nightly/concepts/service_discovery). It must contain the key `ADDRESS` with the address of the Vector aggregator. Follow the [logging tutorial](https://docs.stackable.tech/home/nightly/tutorials/logging-vector-aggregator) to learn how to configure log aggregation with Vector.
                       nullable: true
@@ -585,6 +571,10 @@ spec:
                           description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
                           nullable: true
                           type: string
+                        listenerClass:
+                          description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. DataNodes should have a direct ListenerClass, such as `cluster-internal` or `external-unstable`.
+                          nullable: true
+                          type: string
                         logging:
                           default:
                             enableVectorAgent: null
@@ -4093,6 +4083,10 @@ spec:
                                 description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
                                 nullable: true
                                 type: string
+                              listenerClass:
+                                description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. DataNodes should have a direct ListenerClass, such as `cluster-internal` or `external-unstable`.
+                                nullable: true
+                                type: string
                               logging:
                                 default:
                                   enableVectorAgent: null
@@ -14669,6 +14663,10 @@ spec:
                           description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
                           nullable: true
                           type: string
+                        listenerClass:
+                          description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. NameNodes should have a stable ListenerClass, such as `cluster-internal` or `external-stable`.
+                          nullable: true
+                          type: string
                         logging:
                           default:
                             enableVectorAgent: null
@@ -18168,6 +18166,10 @@ spec:
                                 description: Time period Pods have to gracefully shut down, e.g. `30m`, `1h` or `2d`. Consult the operator documentation for details.
                                 nullable: true
                                 type: string
+                              listenerClass:
+                                description: This field controls which [ListenerClass](https://docs.stackable.tech/home/nightly/listener-operator/listenerclass.html) is used to expose this rolegroup. NameNodes should have a stable ListenerClass, such as `cluster-internal` or `external-stable`.
+                                nullable: true
+                                type: string
                               logging:
                                 default:
                                   enableVectorAgent: null

diff --git a/deploy/helm/hdfs-operator/templates/roles.yaml b/deploy/helm/hdfs-operator/templates/roles.yaml
@@ -88,6 +88,12 @@ rules:
       - events
     verbs:
       - create
+  - apiGroups:
+      - listeners.stackable.tech
+    resources:
+      - listeners
+    verbs:
+      - get
   - apiGroups:
       - {{ include "operator.name" . }}.stackable.tech
     resources:

diff --git a/docs/modules/hdfs/examples/getting_started/hdfs.yaml b/docs/modules/hdfs/examples/getting_started/hdfs.yaml
@@ -5,16 +5,19 @@ metadata:
   name: simple-hdfs
 spec:
   image:
-    productVersion: 3.3.6
+    productVersion: 3.3.4
   clusterConfig:
     zookeeperConfigMapName: simple-hdfs-znode
-    listenerClass: external-unstable
     dfsReplication: 1
   nameNodes:
+    config:
+      listenerClass: external-stable
     roleGroups:
       default:
         replicas: 2
   dataNodes:
+    config:
+      listenerClass: external-unstable
     roleGroups:
       default:
         replicas: 1

diff --git a/docs/modules/hdfs/pages/usage-guide/listenerclass.adoc b/docs/modules/hdfs/pages/usage-guide/listenerclass.adoc
@@ -1,15 +1,18 @@
 = Service exposition with ListenerClasses
 
-The Operator deploys a service called `<name>-<role>-<rolegroup>` (where `<name>` is the name of the HdfsCluster, `<role>` is the role and `<rolegroup>` the name of the role group) through which the different HDFS processes can be accessed. Unlike many other Stackable Operators, the HDFS Operator does not deploy role-level Services.
+The operator deploys a xref:listener-operator:listener.adoc[Listener] for each DataNode and NameNode pod. They both default to only being accessible from within the Kubernetes cluster, but this can be changed by setting `.spec.{data,name}Nodes.config.listenerClass`.
 
-These services can have either the `cluster-internal` or `external-unstable` type. `external-stable` is not supported for HDFS at the moment. Read more about the types in the xref:concepts:service-exposition.adoc[service exposition] documentation at platform level.
-
-This is how the listener class is configured:
+The cluster can be configured to be accessible from outside of Kubernetes like this:
 
 [source,yaml]
 ----
 spec:
-  clusterConfig:
-    listenerClass: cluster-internal  # <1>
+  dataNodes:
+    config:
+      listenerClass: external-unstable # <1>
+  nameNodes:
+    config:
+      listenerClass: external-stable # <2>
 ----
-<1> The default `cluster-internal` setting. 
+<1> DataNode listeners should prioritize having a direct connection, to minimize network transfer overhead.
+<2> NameNode listeners should prioritize having a stable address, since they will be baked into the client configuration.
diff --git a/rust/crd/Cargo.toml b/rust/crd/Cargo.toml
@@ -17,6 +17,7 @@ stackable-operator.workspace = true
 product-config.workspace = true
 strum.workspace = true
 tracing.workspace = true
+futures.workspace = true
 
 [dev-dependencies]
 serde_yaml.workspace = true

diff --git a/rust/crd/src/constants.rs b/rust/crd/src/constants.rs
@@ -9,9 +9,6 @@ pub const FIELD_MANAGER_SCOPE_POD: &str = "pod-service";
 
 pub const APP_NAME: &str = "hdfs";
 
-pub const LABEL_ENABLE: &str = "hdfs.stackable.tech/pod-service";
-pub const LABEL_STS_POD_NAME: &str = "statefulset.kubernetes.io/pod-name";
-
 pub const HDFS_SITE_XML: &str = "hdfs-site.xml";
 pub const CORE_SITE_XML: &str = "core-site.xml";
 pub const HADOOP_POLICY_XML: &str = "hadoop-policy.xml";
@@ -27,6 +24,8 @@ pub const SERVICE_PORT_NAME_HTTPS: &str = "https";
 pub const SERVICE_PORT_NAME_DATA: &str = "data";
 pub const SERVICE_PORT_NAME_METRICS: &str = "metrics";
 
+pub const DEFAULT_LISTENER_CLASS: &str = "cluster-internal";
+
 pub const DEFAULT_NAME_NODE_METRICS_PORT: u16 = 8183;
 pub const DEFAULT_NAME_NODE_HTTP_PORT: u16 = 9870;
 pub const DEFAULT_NAME_NODE_HTTPS_PORT: u16 = 9871;
@@ -77,4 +76,7 @@ pub const JOURNALNODE_ROOT_DATA_DIR: &str = "/stackable/data/journalnode";
 pub const DATANODE_ROOT_DATA_DIR_PREFIX: &str = "/stackable/data/";
 pub const DATANODE_ROOT_DATA_DIR_SUFFIX: &str = "/datanode";
 
+pub const LISTENER_VOLUME_NAME: &str = "listener";
+pub const LISTENER_VOLUME_DIR: &str = "/stackable/listener";
+
 pub const HDFS_UID: i64 = 1000;