From 62e003c6497f82aa7b291e47c00f5434c3bfeb02 Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Wed, 29 Jan 2025 11:33:52 +0530 Subject: [PATCH 1/6] System test for mcg bucket notification Signed-off-by: Mahesh Shetty --- ocs_ci/ocs/bucket_utils.py | 17 ++ .../test_mcg_bucket_notification.py | 202 ++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 tests/cross_functional/system_test/test_mcg_bucket_notification.py diff --git a/ocs_ci/ocs/bucket_utils.py b/ocs_ci/ocs/bucket_utils.py index 13cba85f32e..e1ecf830102 100644 --- a/ocs_ci/ocs/bucket_utils.py +++ b/ocs_ci/ocs/bucket_utils.py @@ -1366,6 +1366,23 @@ def s3_delete_object(s3_obj, bucketname, object_key, versionid=None): return s3_obj.s3_client.delete_object(Bucket=bucketname, Key=object_key) +def s3_put_object_tagging(s3_obj, bucketname, object_key, tags): + """ + Boto3 client based object tagging + + Args: + s3_obj (MCG): MCG object + bucketname (str): Name of the bucket + object_key (str): Key of the object + tags (List): List of key value pair of tags + + """ + + return s3_obj.s3_client.put_object_tagging( + Bucket=bucketname, Key=object_key, Tagging={"TagSet": tags} + ) + + def s3_put_bucket_website(s3_obj, bucketname, website_config): """ Boto3 client based Put bucket website function diff --git a/tests/cross_functional/system_test/test_mcg_bucket_notification.py b/tests/cross_functional/system_test/test_mcg_bucket_notification.py new file mode 100644 index 00000000000..11fd1751d9a --- /dev/null +++ b/tests/cross_functional/system_test/test_mcg_bucket_notification.py @@ -0,0 +1,202 @@ +import pytest + +from concurrent.futures import ThreadPoolExecutor, as_completed + +from ocs_ci.ocs import constants +from ocs_ci.ocs.bucket_utils import ( + write_random_test_objects_to_bucket, + s3_delete_object, + s3_put_object_tagging, +) +from ocs_ci.ocs.resources.bucket_notifications_manager import ( + BucketNotificationsManager, + logger, +) +from ocs_ci.ocs.resources.pod import get_noobaa_pods, get_pod_node +from ocs_ci.utility.utils import TimeoutSampler + + +class TestBucketNotificationSystemTest: + + @pytest.fixture(autouse=True, scope="class") + def notify_manager(self, request, pvc_factory_class): + """ + Set up Kafka and the BucketNotificationsManager + Note that the dependency on the pvc_factory_class fixture is necessary + to guarantee the correct teardown order. Otherwise the pvc factory teardown + might fail when deleting a PVC that the BucketNotificationsManager is still using. + + Returns: + BucketNotificationsManager: An instance of the BucketNotificationsManager class + + """ + notif_manager = BucketNotificationsManager() + notif_manager.pvc_factory = pvc_factory_class + request.addfinalizer(notif_manager.cleanup) + + notif_manager.setup_kafka() + return notif_manager + + def verify_events(self, notify_manager, topic, bucket_names, event_name): + + for events in TimeoutSampler( + timeout=120, + sleep=5, + func=notify_manager.get_events, + topic=topic, + ): + for event in events: + if ( + event["eventName"] == event_name + and event["s3"]["bucket"]["name"] in bucket_names + ): + logger.info( + f'{event_name} found for the bucket {event["s3"]["bucket"]["name"]}' + ) + bucket_names.remove(event["s3"]["bucket"]["name"]) + + if len(bucket_names) == 0: + logger.info(f"Verified {event_name} for all the buckets.") + break + + def test_bucket_notification_system_test( + self, + nodes, + notify_manager, + bucket_factory, + awscli_pod, + test_directory_setup, + mcg_obj, + ): + """ + This is to test MCG bucket notification feature on multiple MCG buckets + with bucket notification enabled for multiple s3 operation. We also perform some + Noobaa specific disruptive operations to verify notification is not affected + + Steps: + 1. Enable bucket notifications on the NooBaa CR + 2. Create a Kafka topic and add a Kafka notification connection to the NooBaa CR + 3. Create 5 buckets and configure bucket notificiation for each of the s3 operation + 4. Perform Object upload and verify object upload events are received for all the buckets + + """ + + NUM_OF_BUCKETS = 5 + events_to_notify = [ + "s3:ObjectCreated:*", + "s3:ObjectRemoved:*", + "s3:LifecycleExpiration:*", + "s3:ObjectRestore:*", + "s3:ObjectTagging:*", + ] + + # 1. Enable bucket notification on Noobaa CR + notify_manager.enable_bucket_notifs_on_cr(use_provided_pvc=True) + + # 2. Create kafka topic and add it to the Noobaa CR + topic = notify_manager.create_kafka_topic() + secret, conn_file_name = notify_manager.create_kafka_conn_secret(topic) + notify_manager.add_notif_conn_to_noobaa_cr(secret) + + # 3. Create the five buckets that is needed for the testing + buckets_created = [] + for i in range(NUM_OF_BUCKETS): + bucket = bucket_factory()[0] + logger.info(f"Enabling bucket notification for the bucket {bucket.name}") + notify_manager.put_bucket_notification( + awscli_pod, + mcg_obj, + bucket.name, + events=events_to_notify, + conn_file=conn_file_name, + ) + buckets_created.append(bucket.name) + + # 4. Write some object to all the buckets and verify the events + # has occurred for all the buckets + obj_written = [] + for bucket in buckets_created: + obj_written = write_random_test_objects_to_bucket( + awscli_pod, + bucket, + file_dir=test_directory_setup.origin_dir, + amount=2, + mcg_obj=mcg_obj, + ) + self.verify_events( + notify_manager, + topic, + bucket_names=buckets_created[:], + event_name="ObjectCreated:Put", + ) + + with ThreadPoolExecutor(max_workers=5) as executor: + + # 5. Tag object from the bucket and restart noobaa pod nodes + # Verify ObjectTagging:Put events has occurred for all the buckets + noobaa_pod_nodes = [ + get_pod_node(pod_obj) + for pod_obj in get_noobaa_pods( + namespace=constants.OPENSHIFT_STORAGE_NAMESPACE + ) + ] + future_objs = [] + for bucket in buckets_created: + logger.info(f"Tagging object {obj_written[1]} from the bucket {bucket}") + future_objs.append( + executor.submit( + s3_put_object_tagging, + mcg_obj, + bucket, + obj_written[1], + [{"type": "media"}], + ) + ) + + logger.info("Stopping Noobaa pod nodes") + nodes.stop_nodes(nodes=noobaa_pod_nodes) + + for future in as_completed(future_objs): + future.result() + + logger.info( + "Verifying if ObjectTagging:Put events has occurred for all the buckets" + ) + self.verify_events( + notify_manager, + topic, + bucket_names=buckets_created[:], + event_name="ObjectTagging:Put", + ) + + logger.info("Starting Noobaa pod nodes") + nodes.start_nodes(nodes=noobaa_pod_nodes) + + # 6. Remove object from the bucket and restart kafka pods + # Verify ObjectRemoved event has occurred + future_objs = [] + for bucket in buckets_created: + logger.info(f"Deleting object {obj_written[0]} from bucket {bucket}") + future_objs.append( + executor.submit( + s3_delete_object, + mcg_obj, + bucket, + obj_written[1], + ) + ) + + logger.info("Restarting Kafka pods") + + for future in as_completed(future_objs): + future.result() + + logger.info( + "Verifying if ObjectRemoved:Delete events has occurred for all buckets" + ) + self.verify_events( + notify_manager, + topic, + bucket_names=buckets_created[:], + event_name="ObjectRemoved:Delete", + ) From 17c8ba223adc1f6f593ec3090f8148dbe138c8d8 Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Thu, 30 Jan 2025 17:16:04 +0530 Subject: [PATCH 2/6] fix up Signed-off-by: Mahesh Shetty --- .../test_mcg_bucket_notification.py | 37 ++++++++++++------- 1 file changed, 24 insertions(+), 13 deletions(-) diff --git a/tests/cross_functional/system_test/test_mcg_bucket_notification.py b/tests/cross_functional/system_test/test_mcg_bucket_notification.py index 11fd1751d9a..5cd9f38f498 100644 --- a/tests/cross_functional/system_test/test_mcg_bucket_notification.py +++ b/tests/cross_functional/system_test/test_mcg_bucket_notification.py @@ -2,7 +2,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed -from ocs_ci.ocs import constants +# from ocs_ci.ocs import constants from ocs_ci.ocs.bucket_utils import ( write_random_test_objects_to_bucket, s3_delete_object, @@ -12,7 +12,12 @@ BucketNotificationsManager, logger, ) -from ocs_ci.ocs.resources.pod import get_noobaa_pods, get_pod_node +from ocs_ci.ocs.resources.pod import ( + # get_noobaa_pods, + # get_pod_node, + get_pods_having_label, + Pod, +) from ocs_ci.utility.utils import TimeoutSampler @@ -51,7 +56,7 @@ def verify_events(self, notify_manager, topic, bucket_names, event_name): and event["s3"]["bucket"]["name"] in bucket_names ): logger.info( - f'{event_name} found for the bucket {event["s3"]["bucket"]["name"]}' + f'{event_name} event found for the bucket {event["s3"]["bucket"]["name"]}' ) bucket_names.remove(event["s3"]["bucket"]["name"]) @@ -134,12 +139,12 @@ def test_bucket_notification_system_test( # 5. Tag object from the bucket and restart noobaa pod nodes # Verify ObjectTagging:Put events has occurred for all the buckets - noobaa_pod_nodes = [ - get_pod_node(pod_obj) - for pod_obj in get_noobaa_pods( - namespace=constants.OPENSHIFT_STORAGE_NAMESPACE - ) - ] + # noobaa_pod_nodes = [ + # get_pod_node(pod_obj) + # for pod_obj in get_noobaa_pods( + # namespace=constants.OPENSHIFT_STORAGE_NAMESPACE + # ) + # ] future_objs = [] for bucket in buckets_created: logger.info(f"Tagging object {obj_written[1]} from the bucket {bucket}") @@ -154,8 +159,7 @@ def test_bucket_notification_system_test( ) logger.info("Stopping Noobaa pod nodes") - nodes.stop_nodes(nodes=noobaa_pod_nodes) - + # nodes.stop_nodes(nodes=noobaa_pod_nodes) for future in as_completed(future_objs): future.result() @@ -170,10 +174,15 @@ def test_bucket_notification_system_test( ) logger.info("Starting Noobaa pod nodes") - nodes.start_nodes(nodes=noobaa_pod_nodes) + # nodes.start_nodes(nodes=noobaa_pod_nodes) # 6. Remove object from the bucket and restart kafka pods # Verify ObjectRemoved event has occurred + kafka_kind_label = "strimzi.io/kind=Kafka" + kafka_pods = [ + Pod(**pod_info) + for pod_info in get_pods_having_label(label=kafka_kind_label) + ] future_objs = [] for bucket in buckets_created: logger.info(f"Deleting object {obj_written[0]} from bucket {bucket}") @@ -187,7 +196,9 @@ def test_bucket_notification_system_test( ) logger.info("Restarting Kafka pods") - + for pod in kafka_pods: + logger.info(f"Deleting pod {pod.name}") + pod.delete() for future in as_completed(future_objs): future.result() From a1a59f05155858874c301cc736eb258a7eb961ec Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Thu, 6 Feb 2025 13:14:06 +0530 Subject: [PATCH 3/6] More code Signed-off-by: Mahesh Shetty --- .../test_mcg_bucket_notification.py | 97 +++++++++++++++---- 1 file changed, 79 insertions(+), 18 deletions(-) diff --git a/tests/cross_functional/system_test/test_mcg_bucket_notification.py b/tests/cross_functional/system_test/test_mcg_bucket_notification.py index 5cd9f38f498..ce1d29e7cff 100644 --- a/tests/cross_functional/system_test/test_mcg_bucket_notification.py +++ b/tests/cross_functional/system_test/test_mcg_bucket_notification.py @@ -2,21 +2,26 @@ from concurrent.futures import ThreadPoolExecutor, as_completed -# from ocs_ci.ocs import constants from ocs_ci.ocs.bucket_utils import ( write_random_test_objects_to_bucket, s3_delete_object, s3_put_object_tagging, + expire_objects_in_bucket, ) from ocs_ci.ocs.resources.bucket_notifications_manager import ( BucketNotificationsManager, logger, ) +from ocs_ci.ocs.resources.mcg_lifecycle_policies import ( + LifecyclePolicy, + ExpirationRule, + LifecycleFilter, +) from ocs_ci.ocs.resources.pod import ( - # get_noobaa_pods, - # get_pod_node, + get_pod_node, get_pods_having_label, Pod, + get_noobaa_core_pod, ) from ocs_ci.utility.utils import TimeoutSampler @@ -42,11 +47,13 @@ def notify_manager(self, request, pvc_factory_class): notif_manager.setup_kafka() return notif_manager - def verify_events(self, notify_manager, topic, bucket_names, event_name): + def verify_events( + self, notify_manager, topic, bucket_names, event_name, timeout=120, sleep=5 + ): for events in TimeoutSampler( - timeout=120, - sleep=5, + timeout=timeout, + sleep=sleep, func=notify_manager.get_events, topic=topic, ): @@ -69,9 +76,14 @@ def test_bucket_notification_system_test( nodes, notify_manager, bucket_factory, + reduce_expiration_interval, awscli_pod, test_directory_setup, mcg_obj, + noobaa_db_backup_and_recovery, + snapshot_factory, + setup_mcg_bg_features, + validate_mcg_bg_features, ): """ This is to test MCG bucket notification feature on multiple MCG buckets @@ -81,8 +93,14 @@ def test_bucket_notification_system_test( Steps: 1. Enable bucket notifications on the NooBaa CR 2. Create a Kafka topic and add a Kafka notification connection to the NooBaa CR - 3. Create 5 buckets and configure bucket notificiation for each of the s3 operation + 3. Create 5 buckets and configure bucket notification for each of the s3 operation 4. Perform Object upload and verify object upload events are received for all the buckets + 5. Tag object from the bucket and restart noobaa pod nodes. + Verify ObjectTagging:Put events has occurred for all the buckets. + 6. Remove object from the bucket and restart kafka pods. Verify ObjectRemoved:Delete event has occurred + 7. Enable object expiration for 1 day and expire the objects manually. + Perform Noobaa db backup and recovery. + Make sure notifications are updated for LifecycleExpiration:Delete event """ @@ -95,12 +113,19 @@ def test_bucket_notification_system_test( "s3:ObjectTagging:*", ] + feature_setup_map = setup_mcg_bg_features( + num_of_buckets=5, + object_amount=5, + is_disruptive=True, + skip_any_features=["nsfs", "rgw kafka", "caching"], + ) + # 1. Enable bucket notification on Noobaa CR notify_manager.enable_bucket_notifs_on_cr(use_provided_pvc=True) # 2. Create kafka topic and add it to the Noobaa CR topic = notify_manager.create_kafka_topic() - secret, conn_file_name = notify_manager.create_kafka_conn_secret(topic) + secret, conn_config_path = notify_manager.create_kafka_conn_secret(topic) notify_manager.add_notif_conn_to_noobaa_cr(secret) # 3. Create the five buckets that is needed for the testing @@ -113,7 +138,7 @@ def test_bucket_notification_system_test( mcg_obj, bucket.name, events=events_to_notify, - conn_file=conn_file_name, + conn_config_path=conn_config_path, ) buckets_created.append(bucket.name) @@ -139,12 +164,7 @@ def test_bucket_notification_system_test( # 5. Tag object from the bucket and restart noobaa pod nodes # Verify ObjectTagging:Put events has occurred for all the buckets - # noobaa_pod_nodes = [ - # get_pod_node(pod_obj) - # for pod_obj in get_noobaa_pods( - # namespace=constants.OPENSHIFT_STORAGE_NAMESPACE - # ) - # ] + noobaa_pod_nodes = [get_pod_node(get_noobaa_core_pod())] future_objs = [] for bucket in buckets_created: logger.info(f"Tagging object {obj_written[1]} from the bucket {bucket}") @@ -154,12 +174,12 @@ def test_bucket_notification_system_test( mcg_obj, bucket, obj_written[1], - [{"type": "media"}], + [{"Key": "type", "Value": "media"}], ) ) logger.info("Stopping Noobaa pod nodes") - # nodes.stop_nodes(nodes=noobaa_pod_nodes) + nodes.stop_nodes(nodes=noobaa_pod_nodes) for future in as_completed(future_objs): future.result() @@ -174,7 +194,7 @@ def test_bucket_notification_system_test( ) logger.info("Starting Noobaa pod nodes") - # nodes.start_nodes(nodes=noobaa_pod_nodes) + nodes.start_nodes(nodes=noobaa_pod_nodes) # 6. Remove object from the bucket and restart kafka pods # Verify ObjectRemoved event has occurred @@ -211,3 +231,44 @@ def test_bucket_notification_system_test( bucket_names=buckets_created[:], event_name="ObjectRemoved:Delete", ) + + # 7. Enable object expiration for 1 day and expire the objects manually + # Perform noobaa db backup and recovery. Make sure notifications are updated + # for LifecycleExpiration:Delete event + + for bucket in buckets_created: + logger.info(f"Setting object expiration on bucket {bucket}") + lifecycle_policy = LifecyclePolicy( + ExpirationRule(days=1, filter=LifecycleFilter()) + ) + mcg_obj.s3_client.put_bucket_lifecycle_configuration( + Bucket=bucket, LifecycleConfiguration=lifecycle_policy.as_dict() + ) + expire_objects_in_bucket(bucket) + + reduce_expiration_interval(interval=5) + logger.info("Reduced the object expiration interval to 5 minutes") + + # Backup noobaa db and recovery + noobaa_db_backup_and_recovery(snapshot_factory=snapshot_factory) + logger.info("Noobaa db backup and recovery is completed") + + logger.info( + "Verifying if LifecycleExpiration:Delete events has occurred for all the buckets" + ) + self.verify_events( + notify_manager, + topic, + bucket_names=buckets_created[:], + event_name="LifecycleExpiration:Delete", + timeout=600, + sleep=30, + ) + + validate_mcg_bg_features( + feature_setup_map, + run_in_bg=False, + skip_any_features=["nsfs", "rgw kafka", "caching"], + object_amount=5, + ) + logger.info("No issues seen with the MCG bg feature validation") From e943259bc273ea96f37d8349d49280b609a190a3 Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Thu, 6 Feb 2025 16:52:13 +0530 Subject: [PATCH 4/6] fix up Signed-off-by: Mahesh Shetty --- .../system_test/test_mcg_bucket_notification.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/cross_functional/system_test/test_mcg_bucket_notification.py b/tests/cross_functional/system_test/test_mcg_bucket_notification.py index ce1d29e7cff..80ca0b774f0 100644 --- a/tests/cross_functional/system_test/test_mcg_bucket_notification.py +++ b/tests/cross_functional/system_test/test_mcg_bucket_notification.py @@ -2,6 +2,8 @@ from concurrent.futures import ThreadPoolExecutor, as_completed +from ocs_ci.ocs import constants +from ocs_ci.framework.pytest_customization.marks import ignore_leftover_label from ocs_ci.ocs.bucket_utils import ( write_random_test_objects_to_bucket, s3_delete_object, @@ -26,6 +28,7 @@ from ocs_ci.utility.utils import TimeoutSampler +@ignore_leftover_label(constants.CUSTOM_MCG_LABEL) class TestBucketNotificationSystemTest: @pytest.fixture(autouse=True, scope="class") @@ -191,6 +194,8 @@ def test_bucket_notification_system_test( topic, bucket_names=buckets_created[:], event_name="ObjectTagging:Put", + timeout=600, + sleep=30, ) logger.info("Starting Noobaa pod nodes") @@ -230,6 +235,8 @@ def test_bucket_notification_system_test( topic, bucket_names=buckets_created[:], event_name="ObjectRemoved:Delete", + timeout=300, + sleep=30, ) # 7. Enable object expiration for 1 day and expire the objects manually From 083640b5712b7fc79043e51ab7dc738596651ac2 Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Thu, 6 Feb 2025 17:50:26 +0530 Subject: [PATCH 5/6] fix up Signed-off-by: Mahesh Shetty --- tests/cross_functional/conftest.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/cross_functional/conftest.py b/tests/cross_functional/conftest.py index 56bb6b2c63a..92509da4d7d 100644 --- a/tests/cross_functional/conftest.py +++ b/tests/cross_functional/conftest.py @@ -510,7 +510,8 @@ def finalizer(): ), f"Failed to scale up the statefulset {constants.NOOBAA_DB_STATEFULSET}" try: - restore_pvc_objs[0].delete() + for restore_pvc_obj in restore_pvc_objs: + restore_pvc_obj.delete() except CommandFailed as ex: if f'"{restore_pvc_objs[0].name}" not found' not in str(ex): raise ex From 8f985edb56cc6564fd66985f389124d0ba24ed40 Mon Sep 17 00:00:00 2001 From: Mahesh Shetty Date: Fri, 7 Feb 2025 12:47:33 +0530 Subject: [PATCH 6/6] fix up Signed-off-by: Mahesh Shetty --- .../test_mcg_bucket_notification.py | 28 +++++++++---------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/cross_functional/system_test/test_mcg_bucket_notification.py b/tests/cross_functional/system_test/test_mcg_bucket_notification.py index 80ca0b774f0..9c192675e42 100644 --- a/tests/cross_functional/system_test/test_mcg_bucket_notification.py +++ b/tests/cross_functional/system_test/test_mcg_bucket_notification.py @@ -116,12 +116,12 @@ def test_bucket_notification_system_test( "s3:ObjectTagging:*", ] - feature_setup_map = setup_mcg_bg_features( - num_of_buckets=5, - object_amount=5, - is_disruptive=True, - skip_any_features=["nsfs", "rgw kafka", "caching"], - ) + # feature_setup_map = setup_mcg_bg_features( + # num_of_buckets=5, + # object_amount=5, + # is_disruptive=True, + # skip_any_features=["nsfs", "rgw kafka", "caching"], + # ) # 1. Enable bucket notification on Noobaa CR notify_manager.enable_bucket_notifs_on_cr(use_provided_pvc=True) @@ -194,7 +194,7 @@ def test_bucket_notification_system_test( topic, bucket_names=buckets_created[:], event_name="ObjectTagging:Put", - timeout=600, + timeout=1200, sleep=30, ) @@ -272,10 +272,10 @@ def test_bucket_notification_system_test( sleep=30, ) - validate_mcg_bg_features( - feature_setup_map, - run_in_bg=False, - skip_any_features=["nsfs", "rgw kafka", "caching"], - object_amount=5, - ) - logger.info("No issues seen with the MCG bg feature validation") + # validate_mcg_bg_features( + # feature_setup_map, + # run_in_bg=False, + # skip_any_features=["nsfs", "rgw kafka", "caching"], + # object_amount=5, + # ) + # logger.info("No issues seen with the MCG bg feature validation")