From 3bcd33efc3cc6ffc865f00f1d267cf52c189f239 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Wed, 18 Sep 2024 02:50:16 -0400 Subject: [PATCH] fix: use flow.json.gz to easily migrate to version 2.0 (#675) * fix(tests): update versions for 24.11 (part 2 of #665) * fix: switch to flow.json.gz * update changelog * docs: Add some details on Nifi upgrade * Update CHANGELOG.md Co-authored-by: Sebastian Bernauer * fix: flow file name depends on the nifi version * Update rust/operator-binary/src/config.rs Co-authored-by: Sebastian Bernauer --------- Co-authored-by: Sebastian Bernauer Co-authored-by: Sebastian Bernauer --- CHANGELOG.md | 5 +++ .../nifi/pages/usage_guide/updating.adoc | 8 +++- .../nifi/partials/supported-versions.adoc | 2 + rust/operator-binary/src/config.rs | 12 ++++- rust/operator-binary/src/controller.rs | 45 ++++++++++--------- .../src/security/sensitive_key.rs | 2 +- .../templates/kuttl/upgrade/05-assert.yaml.j2 | 2 +- .../kuttl/upgrade/05-upgrade-nifi.yaml.j2 | 18 -------- .../upgrade/generate-and-log-flowfiles.xml | 4 +- tests/test-definition.yaml | 10 ++--- 10 files changed, 54 insertions(+), 54 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6ede66b5..99ac4c43 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -16,6 +16,10 @@ All notable changes to this project will be documented in this file. - `extraVolumes` - Increase `log` Volume size from 33 MiB to 500 MiB ([#671]). +### Fixed + +- Switch from `flow.xml.gz` to `flow.json.gz` to allow seamless upgrades to version 2.0 ([#675]). + ### Removed - Removed support for NiFi versions 1.21.0 and 1.25.0 ([#665]). @@ -26,6 +30,7 @@ All notable changes to this project will be documented in this file. [#668]: https://github.com/stackabletech/nifi-operator/pull/668 [#671]: https://github.com/stackabletech/nifi-operator/pull/671 [#672]: https://github.com/stackabletech/nifi-operator/pull/672 +[#675]: https://github.com/stackabletech/nifi-operator/pull/675 ## [24.7.0] - 2024-07-24 diff --git a/docs/modules/nifi/pages/usage_guide/updating.adoc b/docs/modules/nifi/pages/usage_guide/updating.adoc index 47633308..78263f10 100644 --- a/docs/modules/nifi/pages/usage_guide/updating.adoc +++ b/docs/modules/nifi/pages/usage_guide/updating.adoc @@ -1,7 +1,7 @@ = Updating NiFi Updating (or downgrading for that matter) the deployed version of NiFi is as simple as changing the version stated in the CRD. -Continuing the example above, to change the deployed version from `1.25.0` to `1.27.0` you'd simply deploy the following CRD. +Continuing the example above, to change the deployed version from `1.27.0` to `2.0.0-M4` you'd simply deploy the following CRD. [source,yaml] ---- @@ -11,7 +11,7 @@ metadata: name: simple-nifi spec: image: - productVersion: 1.27.0 # <1> + productVersion: 2.0.0-M4 # <1> ---- <1> Change the NiFi version here @@ -19,3 +19,7 @@ spec: WARNING: Due to a limitation in NiFi itself it is not possible to upgrade or downgrade a NiFi cluster in a rolling fashion. So any change to the NiFi version you make in this CRD will result in a full cluster restart with a short downtime. This does not affect the Stackable image version, this can be changed in a rolling fashion, as long as the underlying NiFi version remains unchanged. + +== NiFi 2.0.0-M4 + +Before you can upgrade to `2.0.0-M4` you https://cwiki.apache.org/confluence/display/NIFI/Migration+Guidance[need to update] to at least version 1.27.x! diff --git a/docs/modules/nifi/partials/supported-versions.adoc b/docs/modules/nifi/partials/supported-versions.adoc index 1311e8e6..fc130e55 100644 --- a/docs/modules/nifi/partials/supported-versions.adoc +++ b/docs/modules/nifi/partials/supported-versions.adoc @@ -4,3 +4,5 @@ - 2.0.0-M4 (experimental) - Please note that you need to upgrade to at least 1.27.x before upgrading to 2.0.x! - 1.27.0 (LTS) + +For details on how to upgrade your NiFi version, please read on xref:usage_guide/updating.adoc[]. diff --git a/rust/operator-binary/src/config.rs b/rust/operator-binary/src/config.rs index 3c1d71cc..fe1eb541 100644 --- a/rust/operator-binary/src/config.rs +++ b/rust/operator-binary/src/config.rs @@ -196,12 +196,22 @@ pub fn build_nifi_properties( proxy_hosts: &str, auth_config: &NifiAuthenticationConfig, overrides: BTreeMap, + product_version: &str, ) -> Result { let mut properties = BTreeMap::new(); // Core Properties + // According to https://cwiki.apache.org/confluence/display/NIFI/Migration+Guidance#MigrationGuidance-Migratingto2.0.0-M1 + // The nifi.flow.configuration.file property in nifi.properties must be changed to reference + // "flow.json.gz" instead of "flow.xml.gz" + // TODO: Remove once we dropped support for all 1.x.x versions + let flow_file_name = if product_version.starts_with("1.") { + "flow.xml.gz" + } else { + "flow.json.gz" + }; properties.insert( "nifi.flow.configuration.file".to_string(), - NifiRepository::Database.mount_path() + "/flow.xml.gz", + NifiRepository::Database.mount_path() + "/" + flow_file_name, ); properties.insert( "nifi.flow.configuration.archive.enabled".to_string(), diff --git a/rust/operator-binary/src/controller.rs b/rust/operator-binary/src/controller.rs index 7f571465..f85f31a9 100644 --- a/rust/operator-binary/src/controller.rs +++ b/rust/operator-binary/src/controller.rs @@ -764,6 +764,7 @@ async fn build_node_rolegroup_config_map( kind: NIFI_PROPERTIES.to_string(), })? .clone(), + resolved_product_image.product_version.as_ref(), ) .with_context(|_| BuildProductConfigSnafu { rolegroup: rolegroup.clone(), @@ -812,7 +813,7 @@ fn build_node_rolegroup_service( Ok(Service { metadata: ObjectMetaBuilder::new() .name_and_namespace(nifi) - .name(&rolegroup.object_name()) + .name(rolegroup.object_name()) .ownerreference_from_resource(nifi, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(build_recommended_labels( @@ -991,24 +992,24 @@ async fn build_node_rolegroup_statefulset( .add_env_vars(env_vars.clone()) .args(vec![prepare_args.join(" && ")]) .add_volume_mount( - &NifiRepository::Flowfile.repository(), - &NifiRepository::Flowfile.mount_path(), + NifiRepository::Flowfile.repository(), + NifiRepository::Flowfile.mount_path(), ) .add_volume_mount( - &NifiRepository::Database.repository(), - &NifiRepository::Database.mount_path(), + NifiRepository::Database.repository(), + NifiRepository::Database.mount_path(), ) .add_volume_mount( - &NifiRepository::Content.repository(), - &NifiRepository::Content.mount_path(), + NifiRepository::Content.repository(), + NifiRepository::Content.mount_path(), ) .add_volume_mount( - &NifiRepository::Provenance.repository(), - &NifiRepository::Provenance.mount_path(), + NifiRepository::Provenance.repository(), + NifiRepository::Provenance.mount_path(), ) .add_volume_mount( - &NifiRepository::State.repository(), - &NifiRepository::State.mount_path(), + NifiRepository::State.repository(), + NifiRepository::State.mount_path(), ) .add_volume_mount("conf", "/conf") .add_volume_mount(KEYSTORE_VOLUME_NAME, KEYSTORE_NIFI_CONTAINER_MOUNT) @@ -1058,24 +1059,24 @@ async fn build_node_rolegroup_statefulset( .add_env_vars(env_vars) .add_volume_mount(KEYSTORE_VOLUME_NAME, KEYSTORE_NIFI_CONTAINER_MOUNT) .add_volume_mount( - &NifiRepository::Flowfile.repository(), - &NifiRepository::Flowfile.mount_path(), + NifiRepository::Flowfile.repository(), + NifiRepository::Flowfile.mount_path(), ) .add_volume_mount( - &NifiRepository::Database.repository(), - &NifiRepository::Database.mount_path(), + NifiRepository::Database.repository(), + NifiRepository::Database.mount_path(), ) .add_volume_mount( - &NifiRepository::Content.repository(), - &NifiRepository::Content.mount_path(), + NifiRepository::Content.repository(), + NifiRepository::Content.mount_path(), ) .add_volume_mount( - &NifiRepository::Provenance.repository(), - &NifiRepository::Provenance.mount_path(), + NifiRepository::Provenance.repository(), + NifiRepository::Provenance.mount_path(), ) .add_volume_mount( - &NifiRepository::State.repository(), - &NifiRepository::State.mount_path(), + NifiRepository::State.repository(), + NifiRepository::State.mount_path(), ) .add_volume_mount("activeconf", NIFI_CONFIG_DIRECTORY) .add_volume_mount("log-config", STACKABLE_LOG_CONFIG_DIR) @@ -1280,7 +1281,7 @@ async fn build_node_rolegroup_statefulset( Ok(StatefulSet { metadata: ObjectMetaBuilder::new() .name_and_namespace(nifi) - .name(&rolegroup_ref.object_name()) + .name(rolegroup_ref.object_name()) .ownerreference_from_resource(nifi, None, Some(true)) .context(ObjectMissingMetadataForOwnerRefSnafu)? .with_recommended_labels(build_recommended_labels( diff --git a/rust/operator-binary/src/security/sensitive_key.rs b/rust/operator-binary/src/security/sensitive_key.rs index 191ee2b5..6c79516b 100644 --- a/rust/operator-binary/src/security/sensitive_key.rs +++ b/rust/operator-binary/src/security/sensitive_key.rs @@ -61,7 +61,7 @@ pub(crate) async fn check_or_generate_sensitive_key( let new_secret = Secret { metadata: ObjectMetaBuilder::new() .namespace(namespace) - .name(&sensitive_config.key_secret.to_string()) + .name(sensitive_config.key_secret.to_string()) .build(), string_data: Some(secret_data), ..Secret::default() diff --git a/tests/templates/kuttl/upgrade/05-assert.yaml.j2 b/tests/templates/kuttl/upgrade/05-assert.yaml.j2 index 9256259a..182ab440 100644 --- a/tests/templates/kuttl/upgrade/05-assert.yaml.j2 +++ b/tests/templates/kuttl/upgrade/05-assert.yaml.j2 @@ -19,5 +19,5 @@ status: {% if test_scenario['values']['nifi_new'].find(",") > 0 %} deployed_version: "{{ test_scenario['values']['nifi_new'].split(',')[0] }}" {% else %} - deployed_version: {{ test_scenario['values']['nifi_new'].split("-")[0] }} + deployed_version: {{ test_scenario['values']['nifi_new'] }} {% endif %} diff --git a/tests/templates/kuttl/upgrade/05-upgrade-nifi.yaml.j2 b/tests/templates/kuttl/upgrade/05-upgrade-nifi.yaml.j2 index cb23976c..ad33cfae 100644 --- a/tests/templates/kuttl/upgrade/05-upgrade-nifi.yaml.j2 +++ b/tests/templates/kuttl/upgrade/05-upgrade-nifi.yaml.j2 @@ -11,21 +11,3 @@ spec: {% else %} productVersion: "{{ test_scenario['values']['nifi_new'] }}" {% endif %} - pullPolicy: IfNotPresent - clusterConfig: - authentication: - - authenticationClass: simple-nifi-users - sensitiveProperties: - keySecret: nifi-sensitive-property-key -{% if lookup('env', 'VECTOR_AGGREGATOR') %} - vectorAggregatorConfigMapName: vector-aggregator-discovery -{% endif %} - zookeeperConfigMapName: test-nifi-znode - nodes: - config: - logging: - enableVectorAgent: {{ lookup('env', 'VECTOR_AGGREGATOR') | length > 0 }} - roleGroups: - default: - config: {} - replicas: 3 diff --git a/tests/templates/kuttl/upgrade/generate-and-log-flowfiles.xml b/tests/templates/kuttl/upgrade/generate-and-log-flowfiles.xml index 0c62692a..547e69f4 100644 --- a/tests/templates/kuttl/upgrade/generate-and-log-flowfiles.xml +++ b/tests/templates/kuttl/upgrade/generate-and-log-flowfiles.xml @@ -83,7 +83,7 @@ nifi-standard-nar org.apache.nifi - 1.21.0 + 1.27.0 PENALIZE_FLOWFILE @@ -222,7 +222,7 @@ nifi-standard-nar org.apache.nifi - 1.21.0 + 1.27.0 PENALIZE_FLOWFILE diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index 12a14d13..8769e8a1 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -1,7 +1,3 @@ -# These tests can run against an OpenShift cluster, provided you note the following: -# -# 1. Set the "openshift" dimension below to "true" (with quotes) -# --- dimensions: - name: zookeeper @@ -19,16 +15,16 @@ dimensions: # - 1.27.0,docker.stackable.tech/sandbox/nifi:1.27.0-stackable0.0.0-dev - name: nifi_old values: - - 1.25.0 + - 1.27.0 - name: nifi_new values: - - 1.27.0 + - 2.0.0-M4 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. # - 1.27.0,docker.stackable.tech/sandbox/nifi:1.27.0-stackable0.0.0-dev - name: nifi-latest values: - - 1.27.0 + - 2.0.0-M4 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. # - 1.27.0,docker.stackable.tech/sandbox/nifi:1.27.0-stackable0.0.0-dev