From e435cdc27a729b9759e3dc22671eca15392fe0e8 Mon Sep 17 00:00:00 2001 From: Amit Prinz Setter Date: Tue, 4 Mar 2025 17:56:31 -0800 Subject: [PATCH] bucket notifications - add notif storage test to health Signed-off-by: Amit Prinz Setter --- config.js | 2 +- docs/NooBaaNonContainerized/Health.md | 16 +++++++++++- src/endpoint/s3/s3_bucket_logging.js | 4 +-- src/manage_nsfs/health.js | 33 ++++++++++++++++++++++-- src/manage_nsfs/manage_nsfs_constants.js | 3 ++- src/test/unit_tests/test_nc_health.js | 9 +++++++ src/util/notifications_util.js | 11 ++++++-- 7 files changed, 69 insertions(+), 9 deletions(-) diff --git a/config.js b/config.js index 248fa655dc..516c51dd22 100644 --- a/config.js +++ b/config.js @@ -723,7 +723,7 @@ config.NOTIFICATION_LOG_NS = 'notification_logging'; config.NOTIFICATION_LOG_DIR = process.env.NOTIFICATION_LOG_DIR; config.NOTIFICATION_BATCH = process.env.BATCH || 10; config.NOTIFICATION_REQ_PER_SPACE_CHECK = process.env.NOTIFICATION_REQ_PER_SPACE_CHECK || 0; -config.NOTIFICATION_SPACE_CHECK_THRESHOLD = parseInt(process.env.NOTIFICATION_SPACE_CHECK_THRESHOLD, 10) || 0.1; +config.NOTIFICATION_SPACE_CHECK_THRESHOLD = parseFloat(process.env.NOTIFICATION_SPACE_CHECK_THRESHOLD) || 0.2; /////////////////////////// // KEY ROTATOR // diff --git a/docs/NooBaaNonContainerized/Health.md b/docs/NooBaaNonContainerized/Health.md index e39f1907a4..042e7cf4fe 100644 --- a/docs/NooBaaNonContainerized/Health.md +++ b/docs/NooBaaNonContainerized/Health.md @@ -55,7 +55,7 @@ The `health` command is used to analyze NooBaa health with customizable options. ```sh noobaa-cli diagnose health [--deployment_type][--https_port] -[--all_account_details][--all_bucket_details][--config_root][--debug] +[--all_account_details][--all_bucket_details][--all_connection_details][--notif_storage_threshold][--config_root][--debug] ``` ### Flags - @@ -84,6 +84,11 @@ noobaa-cli diagnose health [--deployment_type][--https_port] - Default: false - Description: Indicates if health output should contain connection test result. +- `notif_storage_threshold` + - Type: Boolean + - Default: false + - Description: Whether health ouput should check if notification storage FS is below threshold. + - `config_root` - Type: String - Description: Indicates the config directory (default config.NSFS_NC_DEFAULT_CONF_DIR). config_root flag should be used only for dev/tests envs. @@ -269,6 +274,15 @@ Output: } ] }, + "notif_storage_threshold_details": { + "invalid_storages": [], + "valid_storages": [ + { + "name": "notification storage", + "config_path": "/notification/storage/path" + } + ], + } "config_directory": { "phase": "CONFIG_DIR_UNLOCKED", "config_dir_version": "1.0.0", diff --git a/src/endpoint/s3/s3_bucket_logging.js b/src/endpoint/s3/s3_bucket_logging.js index d1e64601bf..43d0bdada2 100644 --- a/src/endpoint/s3/s3_bucket_logging.js +++ b/src/endpoint/s3/s3_bucket_logging.js @@ -6,7 +6,7 @@ const http_utils = require('../../util/http_utils'); const dgram = require('node:dgram'); const { Buffer } = require('node:buffer'); const config = require('../../../config'); -const {compose_notification_req, check_notif_relevant, check_free_space} = require('../../util/notifications_util'); +const {compose_notification_req, check_notif_relevant, check_free_space_if_needed} = require('../../util/notifications_util'); async function send_bucket_op_logs(req, res) { if (req.params && req.params.bucket && @@ -44,7 +44,7 @@ async function send_bucket_op_logs(req, res) { buffer: JSON.stringify(notif) }); - check_free_space(req); + check_free_space_if_needed(req); } } } diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js index 8dc9437fb2..8a0bb36020 100644 --- a/src/manage_nsfs/health.js +++ b/src/manage_nsfs/health.js @@ -66,6 +66,10 @@ const health_errors = { error_code: 'INVALID_CONFIG_DIR', error_message: 'Config directory is invalid', }, + LOW_STORAGE: { + error_code: 'LOW_STORAGE', + error_message: 'FS storage is low.' + }, UNKNOWN_ERROR: { error_code: 'UNKNOWN_ERROR', error_message: 'An unknown error occurred', @@ -103,6 +107,7 @@ class NSFSHealth { this.all_account_details = options.all_account_details; this.all_bucket_details = options.all_bucket_details; this.all_connection_details = options.all_connection_details; + this.notif_storage_threshold = options.notif_storage_threshold; this.config_fs = options.config_fs; } @@ -131,6 +136,7 @@ class NSFSHealth { let bucket_details; let account_details; let connection_details; + let notif_storage_threshold_details; const endpoint_response_code = (endpoint_state && endpoint_state.response?.response_code) || 'UNKNOWN_ERROR'; const health_check_params = { service_status, pid, endpoint_response_code, config_directory_status }; const service_health = this._calc_health_status(health_check_params); @@ -138,6 +144,7 @@ class NSFSHealth { if (this.all_bucket_details) bucket_details = await this.get_bucket_status(); if (this.all_account_details) account_details = await this.get_account_status(); if (this.all_connection_details) connection_details = await this.get_connection_status(); + if (this.notif_storage_threshold) notif_storage_threshold_details = await this.get_notif_storage_threshold_status(); const health = { service_name: NOOBAA_SERVICE, status: service_health, @@ -160,7 +167,8 @@ class NSFSHealth { valid_buckets: bucket_details === undefined ? undefined : bucket_details.valid_storages, error_type: health_errors_tyes.PERSISTENT, }, - connections_status: connection_details + connections_status: connection_details, + notif_storage_threshold_details } }; if (!this.all_account_details) delete health.checks.accounts_status; @@ -438,6 +446,25 @@ class NSFSHealth { }; } + get_notif_storage_threshold_status() { + const invalid_storages = []; + const valid_storages = []; + const below_threshold = notifications_util.check_free_space(); + if(below_threshold) { + invalid_storages.push(get_invalid_object('notification storage', config.NOTIFICATION_LOG_DIR, undefined, health_errors.LOW_STORAGE.error_code).invalid_storage); + } else { + valid_storages.push(get_valid_object('notification storage', config.NOTIFICATION_LOG_DIR, undefined).valid_storage); + } + const res = { + invalid_storages: invalid_storages, + valid_storages: valid_storages, + }; + if (below_threshold) { + res.error_type = health_errors_tyes.PERSISTENT; + } + return res; + } + /** * get_config_file_data_or_error_object return an object containing config_data or err_obj if error occurred * @param {string} type @@ -604,9 +631,11 @@ async function get_health_status(argv, config_fs) { const all_account_details = get_boolean_or_string_value(argv.all_account_details); const all_bucket_details = get_boolean_or_string_value(argv.all_bucket_details); const all_connection_details = get_boolean_or_string_value(argv.all_connection_details); + const notif_storage_threshold = get_boolean_or_string_value(argv.notif_storage_threshold); if (deployment_type === 'nc') { - const health = new NSFSHealth({ https_port, all_account_details, all_bucket_details, all_connection_details, config_fs }); + const health = new NSFSHealth({ https_port, + all_account_details, all_bucket_details, all_connection_details, notif_storage_threshold, config_fs }); const health_status = await health.nc_nsfs_health(); write_stdout_response(ManageCLIResponse.HealthStatus, health_status); } else { diff --git a/src/manage_nsfs/manage_nsfs_constants.js b/src/manage_nsfs/manage_nsfs_constants.js index f5d960629d..2c3553b10f 100644 --- a/src/manage_nsfs/manage_nsfs_constants.js +++ b/src/manage_nsfs/manage_nsfs_constants.js @@ -74,7 +74,7 @@ const VALID_OPTIONS_GLACIER = { }; const VALID_OPTIONS_DIAGNOSE = { - 'health': new Set([ 'https_port', 'deployment_type', 'all_account_details', 'all_bucket_details', 'all_connection_details', ...CLI_MUTUAL_OPTIONS]), + 'health': new Set([ 'https_port', 'deployment_type', 'all_account_details', 'all_bucket_details', 'all_connection_details', 'notif_storage_threshold', ...CLI_MUTUAL_OPTIONS]), 'gather-logs': new Set([ CONFIG_ROOT_FLAG]), 'metrics': new Set([CONFIG_ROOT_FLAG]) }; @@ -144,6 +144,7 @@ const OPTION_TYPE = { all_account_details: 'boolean', all_bucket_details: 'boolean', all_connection_details: 'boolean', + notif_storage_threshold: 'boolean', https_port: 'number', debug: 'number', // upgrade options diff --git a/src/test/unit_tests/test_nc_health.js b/src/test/unit_tests/test_nc_health.js index 016a10b846..82f8f70176 100644 --- a/src/test/unit_tests/test_nc_health.js +++ b/src/test/unit_tests/test_nc_health.js @@ -233,6 +233,15 @@ mocha.describe('nsfs nc health', function() { assert.strictEqual(health_status.checks.connections_status.invalid_storages[0].name, connection_from_file.name); }); + mocha.it('health should test notification storage', async function() { + Health.notif_storage_threshold = true; + config.NOTIFICATION_LOG_DIR = TMP_PATH; + const health_status = await Health.nc_nsfs_health(); + Health.notif_storage_limit = false; + + assert.strictEqual(health_status.checks.notif_storage_threshold_details.valid_storages[0].name, 'notification storage'); + }); + mocha.it('NooBaa service is inactive', async function() { set_mock_functions(Health, { get_service_state: [{ service_status: 'inactive', pid: 0 }], diff --git a/src/util/notifications_util.js b/src/util/notifications_util.js index 9d1ee0f437..9aea249fea 100644 --- a/src/util/notifications_util.js +++ b/src/util/notifications_util.js @@ -564,7 +564,7 @@ function get_notification_logger(locking, namespace, poll_interval) { } //If space check is configures, create an event in case free space is below threshold. -function check_free_space(req) { +function check_free_space_if_needed(req) { if (!req.object_sdk.nsfs_config_root || !config.NOTIFICATION_REQ_PER_SPACE_CHECK) { //free space check is disabled. nothing to do. return; @@ -577,13 +577,19 @@ function check_free_space(req) { req.notification_logger.writes_counter = 0; const fs_stat = fs.statfsSync(config.NOTIFICATION_LOG_DIR); //is the ratio of available blocks less than the configures threshold? - if (fs_stat.bavail / fs_stat.blocks < config.NOTIFICATION_SPACE_CHECK_THRESHOLD) { + if (check_free_space()) { //yes. raise an event. new NoobaaEvent(NoobaaEvent.NOTIFICATION_LOW_SPACE).create_event(null, {fs_stat}); } } } +function check_free_space() { + const fs_stat = fs.statfsSync(config.NOTIFICATION_LOG_DIR); + //is the ratio of available blocks less than the configures threshold? + return (fs_stat.bavail / fs_stat.blocks < config.NOTIFICATION_SPACE_CHECK_THRESHOLD); +} + /** * add_connect_file Creates a new connection file from the given content. * If content has an auth field in it's request_options_object, it is encrypted. @@ -649,4 +655,5 @@ exports.get_notification_logger = get_notification_logger; exports.add_connect_file = add_connect_file; exports.update_connect_file = update_connect_file; exports.check_free_space = check_free_space; +exports.check_free_space_if_needed = check_free_space_if_needed; exports.OP_TO_EVENT = OP_TO_EVENT;