fix: use flow.json.gz to easily migrate to version 2.0

CHANGELOG.md

@@ -16,6 +16,10 @@ All notable changes to this project will be documented in this file.
   - `extraVolumes`
 - Increase `log` Volume size from 33 MiB to 500 MiB ([#671]).
 
+### Fixed
+
+- Switch from `flow.xml.gz` to `flow.json.gz` to allow seamless upgrades to version 2.0 ([#675]).
+
 ### Removed
 
 - Removed support for NiFi versions 1.21.0 and 1.25.0 ([#665]).
@@ -26,6 +30,7 @@ All notable changes to this project will be documented in this file.
 [#668]: https://github.com/stackabletech/nifi-operator/pull/668
 [#671]: https://github.com/stackabletech/nifi-operator/pull/671
 [#672]: https://github.com/stackabletech/nifi-operator/pull/672
+[#675]: https://github.com/stackabletech/nifi-operator/pull/675
 
 ## [24.7.0] - 2024-07-24
 

docs/modules/nifi/pages/usage_guide/updating.adoc

@@ -1,7 +1,7 @@
 = Updating NiFi
 
 Updating (or downgrading for that matter) the deployed version of NiFi is as simple as changing the version stated in the CRD.
-Continuing the example above, to change the deployed version from `1.25.0` to `1.27.0` you'd simply deploy the following CRD.
+Continuing the example above, to change the deployed version from `1.27.0` to `2.0.0-M4` you'd simply deploy the following CRD.
 
 [source,yaml]
 ----
@@ -11,11 +11,15 @@ metadata:
   name: simple-nifi
 spec:
   image:
-    productVersion: 1.27.0 # <1>
+    productVersion: 2.0.0-M4 # <1>
 ----
 
 <1> Change the NiFi version here
 
 WARNING: Due to a limitation in NiFi itself it is not possible to upgrade or downgrade a NiFi cluster in a rolling fashion.
 So any change to the NiFi version you make in this CRD will result in a full cluster restart with a short downtime.
 This does not affect the Stackable image version, this can be changed in a rolling fashion, as long as the underlying NiFi version remains unchanged.
+
+== NiFi 2.0.0-M4
+
+Before you can upgrade to `2.0.0-M4` you https://cwiki.apache.org/confluence/display/NIFI/Migration+Guidance[need to update] to at least version 1.27.x!

docs/modules/nifi/partials/supported-versions.adoc

@@ -4,3 +4,5 @@
 
 - 2.0.0-M4 (experimental) - Please note that you need to upgrade to at least 1.27.x before upgrading to 2.0.x!
 - 1.27.0 (LTS)
+
+For details on how to upgrade your NiFi version, please read on xref:usage_guide/updating.adoc[].

rust/operator-binary/src/config.rs

@@ -196,12 +196,22 @@ pub fn build_nifi_properties(
     proxy_hosts: &str,
     auth_config: &NifiAuthenticationConfig,
     overrides: BTreeMap<String, String>,
+    product_version: &str,
 ) -> Result<String, Error> {
     let mut properties = BTreeMap::new();
     // Core Properties
+    // According to https://cwiki.apache.org/confluence/display/NIFI/Migration+Guidance#MigrationGuidance-Migratingto2.0.0-M1
+    // The nifi.flow.configuration.file property in nifi.properties must be changed to reference
+    // "flow.json.gz" instead of "flow.xml.gz"
+    // TODO: Remove once we dropped support for all 1.x.x versions
+    let flow_file_name = if product_version.starts_with("1.") {
+        "flow.xml.gz"
+    } else {
+        "flow.json.gz"
+    };
     properties.insert(
         "nifi.flow.configuration.file".to_string(),
-        NifiRepository::Database.mount_path() + "/flow.xml.gz",
+        NifiRepository::Database.mount_path() + "/" + flow_file_name,
     );
     properties.insert(
         "nifi.flow.configuration.archive.enabled".to_string(),

rust/operator-binary/src/controller.rs

@@ -764,6 +764,7 @@ async fn build_node_rolegroup_config_map(
                         kind: NIFI_PROPERTIES.to_string(),
                     })?
                     .clone(),
+                resolved_product_image.product_version.as_ref(),
             )
             .with_context(|_| BuildProductConfigSnafu {
                 rolegroup: rolegroup.clone(),
@@ -812,7 +813,7 @@ fn build_node_rolegroup_service(
     Ok(Service {
         metadata: ObjectMetaBuilder::new()
             .name_and_namespace(nifi)
-            .name(&rolegroup.object_name())
+            .name(rolegroup.object_name())
             .ownerreference_from_resource(nifi, None, Some(true))
             .context(ObjectMissingMetadataForOwnerRefSnafu)?
             .with_recommended_labels(build_recommended_labels(
@@ -991,24 +992,24 @@ async fn build_node_rolegroup_statefulset(
         .add_env_vars(env_vars.clone())
         .args(vec![prepare_args.join(" && ")])
         .add_volume_mount(
-            &NifiRepository::Flowfile.repository(),
-            &NifiRepository::Flowfile.mount_path(),
+            NifiRepository::Flowfile.repository(),
+            NifiRepository::Flowfile.mount_path(),
         )
         .add_volume_mount(
-            &NifiRepository::Database.repository(),
-            &NifiRepository::Database.mount_path(),
+            NifiRepository::Database.repository(),
+            NifiRepository::Database.mount_path(),
         )
         .add_volume_mount(
-            &NifiRepository::Content.repository(),
-            &NifiRepository::Content.mount_path(),
+            NifiRepository::Content.repository(),
+            NifiRepository::Content.mount_path(),
         )
         .add_volume_mount(
-            &NifiRepository::Provenance.repository(),
-            &NifiRepository::Provenance.mount_path(),
+            NifiRepository::Provenance.repository(),
+            NifiRepository::Provenance.mount_path(),
         )
         .add_volume_mount(
-            &NifiRepository::State.repository(),
-            &NifiRepository::State.mount_path(),
+            NifiRepository::State.repository(),
+            NifiRepository::State.mount_path(),
         )
         .add_volume_mount("conf", "/conf")
         .add_volume_mount(KEYSTORE_VOLUME_NAME, KEYSTORE_NIFI_CONTAINER_MOUNT)
@@ -1058,24 +1059,24 @@ async fn build_node_rolegroup_statefulset(
         .add_env_vars(env_vars)
         .add_volume_mount(KEYSTORE_VOLUME_NAME, KEYSTORE_NIFI_CONTAINER_MOUNT)
         .add_volume_mount(
-            &NifiRepository::Flowfile.repository(),
-            &NifiRepository::Flowfile.mount_path(),
+            NifiRepository::Flowfile.repository(),
+            NifiRepository::Flowfile.mount_path(),
         )
         .add_volume_mount(
-            &NifiRepository::Database.repository(),
-            &NifiRepository::Database.mount_path(),
+            NifiRepository::Database.repository(),
+            NifiRepository::Database.mount_path(),
         )
         .add_volume_mount(
-            &NifiRepository::Content.repository(),
-            &NifiRepository::Content.mount_path(),
+            NifiRepository::Content.repository(),
+            NifiRepository::Content.mount_path(),
         )
         .add_volume_mount(
-            &NifiRepository::Provenance.repository(),
-            &NifiRepository::Provenance.mount_path(),
+            NifiRepository::Provenance.repository(),
+            NifiRepository::Provenance.mount_path(),
         )
         .add_volume_mount(
-            &NifiRepository::State.repository(),
-            &NifiRepository::State.mount_path(),
+            NifiRepository::State.repository(),
+            NifiRepository::State.mount_path(),
         )
         .add_volume_mount("activeconf", NIFI_CONFIG_DIRECTORY)
         .add_volume_mount("log-config", STACKABLE_LOG_CONFIG_DIR)
@@ -1280,7 +1281,7 @@ async fn build_node_rolegroup_statefulset(
     Ok(StatefulSet {
         metadata: ObjectMetaBuilder::new()
             .name_and_namespace(nifi)
-            .name(&rolegroup_ref.object_name())
+            .name(rolegroup_ref.object_name())
             .ownerreference_from_resource(nifi, None, Some(true))
             .context(ObjectMissingMetadataForOwnerRefSnafu)?
             .with_recommended_labels(build_recommended_labels(

rust/operator-binary/src/security/sensitive_key.rs

@@ -61,7 +61,7 @@ pub(crate) async fn check_or_generate_sensitive_key(
             let new_secret = Secret {
                 metadata: ObjectMetaBuilder::new()
                     .namespace(namespace)
-                    .name(&sensitive_config.key_secret.to_string())
+                    .name(sensitive_config.key_secret.to_string())
                     .build(),
                 string_data: Some(secret_data),
                 ..Secret::default()

tests/templates/kuttl/upgrade/05-assert.yaml.j2

@@ -19,5 +19,5 @@ status:
 {% if test_scenario['values']['nifi_new'].find(",") > 0 %}
   deployed_version: "{{ test_scenario['values']['nifi_new'].split(',')[0] }}"
 {% else %}
-  deployed_version: {{ test_scenario['values']['nifi_new'].split("-")[0]  }}
+  deployed_version: {{ test_scenario['values']['nifi_new'] }}
 {% endif %}

tests/templates/kuttl/upgrade/05-upgrade-nifi.yaml.j2

@@ -11,21 +11,3 @@ spec:
 {% else %}
     productVersion: "{{ test_scenario['values']['nifi_new'] }}"
 {% endif %}
-    pullPolicy: IfNotPresent
-  clusterConfig:
-    authentication:
-      - authenticationClass: simple-nifi-users
-    sensitiveProperties:
-      keySecret: nifi-sensitive-property-key
-{% if lookup('env', 'VECTOR_AGGREGATOR') %}
-    vectorAggregatorConfigMapName: vector-aggregator-discovery
-{% endif %}
-    zookeeperConfigMapName: test-nifi-znode
-  nodes:
-    config:
-      logging:
-        enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }}
-    roleGroups:
-      default:
-        config: {}
-        replicas: 3

tests/templates/kuttl/upgrade/generate-and-log-flowfiles.xml

@@ -83,7 +83,7 @@
             <bundle>
                 <artifact>nifi-standard-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <config>
                 <backoffMechanism>PENALIZE_FLOWFILE</backoffMechanism>
@@ -222,7 +222,7 @@
             <bundle>
                 <artifact>nifi-standard-nar</artifact>
                 <group>org.apache.nifi</group>
-                <version>1.21.0</version>
+                <version>1.27.0</version>
             </bundle>
             <config>
                 <backoffMechanism>PENALIZE_FLOWFILE</backoffMechanism>

tests/test-definition.yaml

@@ -1,7 +1,3 @@
-# These tests can run against an OpenShift cluster, provided you note the following:
-#
-# 1. Set the "openshift" dimension below to "true" (with quotes)
-#
 ---
 dimensions:
   - name: zookeeper
@@ -19,16 +15,16 @@ dimensions:
       # - 1.27.0,docker.stackable.tech/sandbox/nifi:1.27.0-stackable0.0.0-dev
   - name: nifi_old
     values:
-      - 1.25.0
+      - 1.27.0
   - name: nifi_new
     values:
-      - 1.27.0
+      - 2.0.0-M4
       # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
       # as in the example below.
       # - 1.27.0,docker.stackable.tech/sandbox/nifi:1.27.0-stackable0.0.0-dev
   - name: nifi-latest
     values:
-      - 1.27.0
+      - 2.0.0-M4
       # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
       # as in the example below.
       # - 1.27.0,docker.stackable.tech/sandbox/nifi:1.27.0-stackable0.0.0-dev