From e2285af884a5ae5457d1c1a71c0e6e3b83616feb Mon Sep 17 00:00:00 2001 From: Romy <35330373+romayalon@users.noreply.github.com> Date: Tue, 6 Aug 2024 11:39:26 +0300 Subject: [PATCH] NC | Config Directory Restructure Signed-off-by: Romy <35330373+romayalon@users.noreply.github.com> --- src/cmd/manage_nsfs.js | 72 +-- .../NVA_build/standalone_deploy_nsfs.sh | 5 - src/manage_nsfs/health.js | 20 +- src/manage_nsfs/manage_nsfs_validations.js | 49 +- src/sdk/accountspace_fs.js | 118 ++-- src/sdk/bucketspace_fs.js | 12 +- src/sdk/config_fs.js | 610 ++++++++++++++---- .../run_ceph_nsfs_test_on_test_container.sh | 26 +- .../test_ceph_nsfs_s3_config_setup.js | 94 ++- .../ceph_s3_tests/test_ceph_s3_constants.js | 40 +- src/test/system_tests/test_utils.js | 56 ++ .../jest_tests/test_accountspace_fs.test.js | 281 ++++---- .../jest_tests/test_config_dir_structure.js | 150 +++++ ...t_config_fs_backward_compatibility.test.js | 176 +++++ ...nc_account_invalid_mkm_integration.test.js | 5 - .../jest_tests/test_nc_master_keys.test.js | 2 +- .../test_nc_master_keys_exec.test.js | 2 +- .../test_nc_nsfs_account_cli.test.js | 258 +++----- ..._nc_nsfs_account_schema_validation.test.js | 2 +- .../test_nc_nsfs_anonymous_cli.test.js | 53 +- .../test_nc_nsfs_bucket_cli.test.js | 87 +-- ...t_nc_nsfs_bucket_schema_validation.test.js | 2 +- ...t_nc_nsfs_config_schema_validation.test.js | 2 +- ...c_nsfs_new_buckets_path_validation.test.js | 4 +- src/test/unit_tests/nc_coretest.js | 32 +- src/test/unit_tests/test_bucketspace.js | 8 +- src/test/unit_tests/test_bucketspace_fs.js | 18 +- src/test/unit_tests/test_nc_nsfs_cli.js | 146 ++--- src/test/unit_tests/test_nc_nsfs_health.js | 82 ++- src/test/unit_tests/test_s3_bucket_policy.js | 20 +- src/util/native_fs_utils.js | 26 +- 31 files changed, 1537 insertions(+), 921 deletions(-) create mode 100644 src/test/unit_tests/jest_tests/test_config_dir_structure.js create mode 100644 src/test/unit_tests/jest_tests/test_config_fs_backward_compatibility.test.js diff --git a/src/cmd/manage_nsfs.js b/src/cmd/manage_nsfs.js index 801f969cdf..de571cb2e6 100644 --- a/src/cmd/manage_nsfs.js +++ b/src/cmd/manage_nsfs.js @@ -8,7 +8,7 @@ const minimist = require('minimist'); const config = require('../../config'); const P = require('../util/promise'); const nb_native = require('../util/nb_native'); -const { ConfigFS, JSON_SUFFIX } = require('../sdk/config_fs'); +const { ConfigFS } = require('../sdk/config_fs'); const cloud_utils = require('../util/cloud_utils'); const native_fs_utils = require('../util/native_fs_utils'); const mongo_utils = require('../util/mongo_utils'); @@ -387,8 +387,8 @@ async function add_account(data) { await manage_nsfs_validations.validate_account_args(config_fs, data, ACTIONS.ADD, undefined); const access_key = has_access_keys(data.access_keys) ? data.access_keys[0].access_key : undefined; - const name_exists = await config_fs.is_account_exists({ name: data.name }); - const access_key_exists = access_key && await config_fs.is_account_exists({ access_key }); + const name_exists = await config_fs.is_account_exists_by_name(data.name); + const access_key_exists = access_key && await config_fs.is_account_exists_by_access_key(access_key); const event_arg = data.name ? data.name : access_key; if (name_exists || access_key_exists) { @@ -405,7 +405,7 @@ async function add_account(data) { // for validating against the schema we need an object, hence we parse it back to object const account = encrypted_data ? JSON.parse(encrypted_data) : data; nsfs_schema_utils.validate_account_schema(account); - await config_fs.create_account_config_file(data.name, account, true); + await config_fs.create_account_config_file(account); write_stdout_response(ManageCLIResponse.AccountCreated, data, { account: event_arg }); } @@ -434,7 +434,7 @@ async function update_account(data, is_flag_iam_operate_on_root_account) { // for validating against the schema we need an object, hence we parse it back to object const account = encrypted_data ? JSON.parse(encrypted_data) : data; nsfs_schema_utils.validate_account_schema(account); - await config_fs.update_account_config_file(data.name, account, undefined, undefined); + await config_fs.update_account_config_file(account); write_stdout_response(ManageCLIResponse.AccountUpdated, data); return; } @@ -446,9 +446,9 @@ async function update_account(data, is_flag_iam_operate_on_root_account) { secret_key: data.access_keys[0].secret_key, }; - const name_exists = update_name && await config_fs.is_account_exists({ name: data.name }); + const name_exists = update_name && await config_fs.is_account_exists_by_name(data.name, undefined); const access_key_exists = update_access_key && - await config_fs.is_account_exists({ access_key: data.access_keys[0].access_key.unwrap() }); + await config_fs.is_account_exists_by_access_key(data.access_keys[0].access_key.unwrap()); if (name_exists || access_key_exists) { const err_code = name_exists ? ManageCLIError.AccountNameAlreadyExists : ManageCLIError.AccountAccessKeyAlreadyExists; @@ -465,18 +465,17 @@ async function update_account(data, is_flag_iam_operate_on_root_account) { // for validating against the schema we need an object, hence we parse it back to object const parsed_data = JSON.parse(encrypted_data); nsfs_schema_utils.validate_account_schema(parsed_data); - if (update_name) { - await config_fs.create_account_config_file(new_name, parsed_data, true, [cur_access_key]); - await config_fs.delete_account_config_file(cur_name, data.access_keys); - } else if (update_access_key) { - await config_fs.update_account_config_file(cur_name, parsed_data, parsed_data.access_keys, [cur_access_key]); - } + await config_fs.update_account_config_file(parsed_data, { + old_name: update_name && cur_name, + new_access_keys_to_link: update_access_key && parsed_data.access_keys, + access_keys_to_delete: update_access_key && [{ access_key: cur_access_key }] + }); write_stdout_response(ManageCLIResponse.AccountUpdated, data); } async function delete_account(data) { await manage_nsfs_validations.validate_account_args(config_fs, data, ACTIONS.DELETE, undefined); - await config_fs.delete_account_config_file(data.name, data.access_keys); + await config_fs.delete_account_config_file(data); write_stdout_response(ManageCLIResponse.AccountDeleted, '', { account: data.name }); } @@ -578,19 +577,10 @@ function filter_bucket(bucket, filters) { * @param {object} [filters] */ async function list_config_files(type, wide, show_secrets, filters = {}) { - let entries; - // in case we have a filter by name, we don't need to read all the entries and iterate them - // instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance) + let entries = []; + const should_filter = Object.keys(filters).length > 0; const is_filter_by_name = filters.name !== undefined; - if (is_filter_by_name) { - entries = [{'name': filters.name + JSON_SUFFIX}]; - } else { - entries = type === TYPES.ACCOUNT ? - await config_fs.list_root_accounts() : - await config_fs.list_buckets(); - } - const should_filter = Object.keys(filters).length > 0; // decryption causing mkm initalization // decrypt only if data has access_keys and show_secrets = true (no need to decrypt if show_secrets = false but should_filter = true) const options = { @@ -599,19 +589,27 @@ async function list_config_files(type, wide, show_secrets, filters = {}) { silent_if_missing: true }; + // in case we have a filter by name, we don't need to read all the entries and iterate them + // instead we "mock" the entries array to have one entry and it is the name by the filter (we add it for performance) + if (is_filter_by_name) { + entries = [filters.name]; + } else if (type === TYPES.ACCOUNT) { + entries = await config_fs.list_accounts(); + } else if (type === TYPES.BUCKET) { + entries = await config_fs.list_buckets(); + } + let config_files_list = await P.map_with_concurrency(10, entries, async entry => { - if (entry.name.endsWith(JSON_SUFFIX)) { - if (wide || should_filter) { - const data = type === TYPES.ACCOUNT ? - await config_fs.get_account_by_name(entry.name, options) : - await config_fs.get_bucket_by_name(entry.name, options); - if (!data) return undefined; - if (should_filter && !filter_list_item(type, data, filters)) return undefined; - // remove secrets on !show_secrets && should filter - return wide ? _.omit(data, show_secrets ? [] : ['access_keys']) : { name: entry.name.slice(0, entry.name.indexOf(JSON_SUFFIX)) }; - } else { - return { name: entry.name.slice(0, entry.name.indexOf(JSON_SUFFIX)) }; - } + if (wide || should_filter) { + const data = type === TYPES.ACCOUNT ? + await config_fs.get_account_by_name(entry, options) : + await config_fs.get_bucket_by_name(entry, options); + if (!data) return undefined; + if (should_filter && !filter_list_item(type, data, filters)) return undefined; + // remove secrets on !show_secrets && should filter + return wide ? _.omit(data, show_secrets ? [] : ['access_keys']) : { name: entry }; + } else { + return { name: entry }; } }); // it inserts undefined for the entry '.noobaa-config-nsfs' and we wish to remove it diff --git a/src/deploy/NVA_build/standalone_deploy_nsfs.sh b/src/deploy/NVA_build/standalone_deploy_nsfs.sh index 6d42a4fb0e..00be389f60 100755 --- a/src/deploy/NVA_build/standalone_deploy_nsfs.sh +++ b/src/deploy/NVA_build/standalone_deploy_nsfs.sh @@ -10,13 +10,8 @@ function execute() { # Please note that the command we use here are without "sudo" because we are running from the container with Root permissions function main() { - # Add accounts to run ceph tests - execute "node src/cmd/manage_nsfs account add --name cephalt --new_buckets_path ${FS_ROOT_1} --uid 1000 --gid 1000" nsfs_cephalt.log - execute "node src/cmd/manage_nsfs account add --name cephtenant --new_buckets_path ${FS_ROOT_2} --uid 2000 --gid 2000" nsfs_cephtenant.log - # Start noobaa service execute "node src/cmd/nsfs" nsfs.log - # Wait for sometime to process to start sleep 10 } diff --git a/src/manage_nsfs/health.js b/src/manage_nsfs/health.js index b16f934ed9..9cf742ec4a 100644 --- a/src/manage_nsfs/health.js +++ b/src/manage_nsfs/health.js @@ -10,7 +10,6 @@ const nb_native = require('../util/nb_native'); const native_fs_utils = require('../util/native_fs_utils'); const { read_stream_join } = require('../util/buffer_utils'); const { make_https_request } = require('../util/http_utils'); -const { JSON_SUFFIX } = require('../sdk/config_fs'); const { TYPES } = require('./manage_nsfs_constants'); const { get_boolean_or_string_value, throw_cli_error, write_stdout_response } = require('./manage_nsfs_cli_utils'); const { ManageCLIResponse } = require('./manage_nsfs_cli_responses'); @@ -327,7 +326,7 @@ class NSFSHealth { config_root_type_exists = await this.config_fs.validate_config_dir_exists(config_dir_path); } else if (type === TYPES.ACCOUNT) { // TODO - handle iam accounts when directory structure changes - read_account_by_id - config_dir_path = this.config_fs.accounts_dir_path; + config_dir_path = this.config_fs.accounts_by_name_dir_path; config_root_type_exists = await this.config_fs.validate_config_dir_exists(config_dir_path); } // TODO - this is not a good handling for that - we need to take it to an upper level @@ -339,15 +338,16 @@ class NSFSHealth { }; } - const entries = type === TYPES.BUCKET ? - await this.config_fs.list_buckets() : - await this.config_fs.list_root_accounts(); - - const config_files = entries.filter(entree => !native_fs_utils.isDirectory(entree) && entree.name.endsWith(JSON_SUFFIX)); + let config_files; + if (type === TYPES.BUCKET) { + config_files = await this.config_fs.list_buckets(); + } else { + config_files = await this.config_fs.list_accounts(); + } for (const config_file of config_files) { // config_file get data or push error const { config_data = undefined, err_obj = undefined } = - await this.get_config_file_data_or_error_object(type, config_file.name); + await this.get_config_file_data_or_error_object(type, config_file); if (!config_data && err_obj) { invalid_storages.push(err_obj.invalid_storage); continue; @@ -395,9 +395,9 @@ class NSFSHealth { } catch (err) { let err_code; const config_file_path = type === TYPES.BUCKET ? - await this.config_fs.get_bucket_path_by_name(config_file_name) : + this.config_fs.get_bucket_path_by_name(config_file_name) : // TODO - should be changed to id when moving to new structure for supporting iam accounts - await this.config_fs.get_account_path_by_name(config_file_name); + this.config_fs.get_account_path_by_name(config_file_name); if (err.code === 'ENOENT') { dbg.log1(`Error: Config file path should be a valid path`, config_file_path, err); diff --git a/src/manage_nsfs/manage_nsfs_validations.js b/src/manage_nsfs/manage_nsfs_validations.js index 99b63ac443..6e49082bbe 100644 --- a/src/manage_nsfs/manage_nsfs_validations.js +++ b/src/manage_nsfs/manage_nsfs_validations.js @@ -3,12 +3,9 @@ const config = require('../../config'); const dbg = require('../util/debug_module')(__filename); -const path = require('path'); const net = require('net'); const P = require('../util/promise'); -const nb_native = require('../util/nb_native'); const string_utils = require('../util/string_utils'); -const { JSON_SUFFIX } = require('../sdk/config_fs'); const native_fs_utils = require('../util/native_fs_utils'); const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils'); @@ -354,7 +351,7 @@ async function validate_bucket_args(config_fs, data, action) { if (data.s3_policy) { try { await bucket_policy_utils.validate_s3_policy(data.s3_policy, data.name, - async principal => config_fs.is_account_exists({ name: principal }) + async principal => config_fs.is_account_exists_by_name(principal, account.owner) ); } catch (err) { dbg.error('validate_bucket_args invalid bucket policy err:', err); @@ -482,16 +479,14 @@ function _validate_access_keys(access_key, secret_key) { * @param {string} account_name */ async function validate_account_not_owns_buckets(config_fs, account_name) { - const entries = await config_fs.list_buckets(); - await P.map_with_concurrency(10, entries, async entry => { - if (entry.name.endsWith(JSON_SUFFIX)) { - const data = await config_fs.get_bucket_by_name(entry.name, { silent_if_missing: true }); - if (data && data.bucket_owner === account_name) { - const detail_msg = `Account ${account_name} has bucket ${data.name}`; - throw_cli_error(ManageCLIError.AccountDeleteForbiddenHasBuckets, detail_msg); - } - return data; + const bucket_names = await config_fs.list_buckets(); + await P.map_with_concurrency(10, bucket_names, async bucket_name => { + const data = await config_fs.get_bucket_by_name(bucket_name, { silent_if_missing: true }); + if (data && data.bucket_owner === account_name) { + const detail_msg = `Account ${account_name} has bucket ${data.name}`; + throw_cli_error(ManageCLIError.AccountDeleteForbiddenHasBuckets, detail_msg); } + return data; }); } @@ -503,24 +498,20 @@ async function validate_account_not_owns_buckets(config_fs, account_name) { * @param {string} action */ async function check_if_root_account_does_not_have_IAM_users(config_fs, account_to_check, action) { - const fs_context = config_fs.fs_context; - const entries = await nb_native().fs.readdir(fs_context, config_fs.accounts_dir_path); - await P.map_with_concurrency(10, entries, async entry => { - if (entry.name.endsWith(JSON_SUFFIX)) { - const full_path = path.join(config_fs.accounts_dir_path, entry.name); - const account_data = await config_fs.get_config_data(full_path); - if (entry.name.includes(config.NSFS_TEMP_CONF_DIR_NAME)) return undefined; - const is_root_account_owns_user = check_root_account_owns_user(account_to_check, account_data); - if (is_root_account_owns_user) { - const detail_msg = `Account ${account_to_check.name} has IAM account ${account_data.name}`; - if (action === ACTIONS.DELETE) { - throw_cli_error(ManageCLIError.AccountDeleteForbiddenHasIAMAccounts, detail_msg); - } - // else it is called with action ACTIONS.UPDATE - throw_cli_error(ManageCLIError.AccountCannotBeRootAccountsManager, detail_msg); + // TODO - For supporting IAM, we need to check if {config_dir}/identities/{account_id}/users/ has anything inside + const account_names = await config_fs.list_accounts(); + await P.map_with_concurrency(10, account_names, async account_name => { + const account_data = await config_fs.get_account_by_name(account_name); + const is_root_account_owns_user = check_root_account_owns_user(account_to_check, account_data); + if (is_root_account_owns_user) { + const detail_msg = `Account ${account_to_check.name} has IAM account ${account_data.name}`; + if (action === ACTIONS.DELETE) { + throw_cli_error(ManageCLIError.AccountDeleteForbiddenHasIAMAccounts, detail_msg); } - return account_data; + // else it is called with action ACTIONS.UPDATE + throw_cli_error(ManageCLIError.AccountCannotBeRootAccountsManager, detail_msg); } + return account_data; }); } diff --git a/src/sdk/accountspace_fs.js b/src/sdk/accountspace_fs.js index f56dd5c221..58f2fa49f4 100644 --- a/src/sdk/accountspace_fs.js +++ b/src/sdk/accountspace_fs.js @@ -5,7 +5,7 @@ const _ = require('lodash'); const config = require('../../config'); const dbg = require('../util/debug_module')(__filename); const P = require('../util/promise'); -const { ConfigFS, JSON_SUFFIX } = require('./config_fs'); +const { ConfigFS } = require('./config_fs'); const native_fs_utils = require('../util/native_fs_utils'); const { create_arn, get_action_message_title, check_iam_path_was_set } = require('../endpoint/iam/iam_utils'); const { IAM_ACTIONS, MAX_NUMBER_OF_ACCESS_KEYS, IAM_DEFAULT_PATH, @@ -153,7 +153,7 @@ class AccountSpaceFS { const requested_account_encrypted = await nc_mkm.encrypt_access_keys(requested_account); const account_string = JSON.stringify(requested_account_encrypted); nsfs_schema_utils.validate_account_schema(JSON.parse(account_string)); - await this.config_fs.update_account_config_file(params.username, JSON.parse(account_string)); + await this.config_fs.update_account_config_file(JSON.parse(account_string)); } this._clean_account_cache(requested_account); return { @@ -193,7 +193,7 @@ class AccountSpaceFS { this._check_if_requested_account_is_root_account_or_IAM_user(action, requesting_account, account_to_delete); this._check_if_requested_is_owned_by_root_account(action, requesting_account, account_to_delete); await this._check_if_user_does_not_have_resources_before_deletion(action, account_to_delete); - await this.config_fs.delete_account_config_file(params.username); + await this.config_fs.delete_account_config_file(account_to_delete); } catch (err) { dbg.error(`AccountSpaceFS.${action} error`, err); throw native_fs_utils.translate_error_codes(err, native_fs_utils.entity_enum.USER); @@ -268,8 +268,10 @@ class AccountSpaceFS { const requested_account_encrypted = await nc_mkm.encrypt_access_keys(requested_account); const account_to_create_access_keys_string = JSON.stringify(requested_account_encrypted); nsfs_schema_utils.validate_account_schema(JSON.parse(account_to_create_access_keys_string)); - await this.config_fs.update_account_config_file(name_for_access_key, JSON.parse(account_to_create_access_keys_string), - [created_access_key_obj]); + await this.config_fs.update_account_config_file( + JSON.parse(account_to_create_access_keys_string), + { new_access_keys_to_link: [created_access_key_obj] } + ); return { username: requested_account.name, access_key: created_access_key_obj.access_key, @@ -356,8 +358,7 @@ class AccountSpaceFS { const requested_account_encrypted = await nc_mkm.encrypt_access_keys(requested_account); const account_string = JSON.stringify(requested_account_encrypted); nsfs_schema_utils.validate_account_schema(JSON.parse(account_string)); - const name_for_access_key = params.username ?? requester.name; - await this.config_fs.update_account_config_file(name_for_access_key, JSON.parse(account_string)); + await this.config_fs.update_account_config_file(JSON.parse(account_string)); this._clean_account_cache(requested_account); } catch (err) { dbg.error(`AccountSpaceFS.${action} error`, err); @@ -387,7 +388,7 @@ class AccountSpaceFS { requesting_account, params.username); const username = params.username ?? requester.name; // username is not required await this._check_if_account_exists_by_access_key_symlink(action, access_key_id); - const requested_account = await this.config_fs.get_account_by_access_key(params.access_key, { show_secrets: true}); + const requested_account = await this.config_fs.get_account_by_access_key(access_key_id, { show_secrets: true}); this._check_username_match_to_requested_account(action, username, requested_account); this._check_access_key_belongs_to_account(action, requested_account, access_key_id); this._check_if_requested_account_same_root_account_as_requesting_account(action, @@ -400,12 +401,9 @@ class AccountSpaceFS { const requested_account_encrypted = await nc_mkm.encrypt_access_keys(requested_account); const account_string = JSON.stringify(requested_account_encrypted); nsfs_schema_utils.validate_account_schema(JSON.parse(account_string)); - const name_for_access_key = params.username ?? requester.name; await this.config_fs.update_account_config_file( - name_for_access_key, JSON.parse(account_string), - undefined, - [params.access_key] + { access_keys_to_delete: [{ access_key: access_key_id }] } ); this._clean_account_cache(requested_account); } catch (err) { @@ -560,33 +558,30 @@ class AccountSpaceFS { // based on the function from manage_nsfs async _list_config_files_for_users(requesting_account, iam_path_prefix) { // TODO - currently handles only root accounts, need to add iam_accounts - const entries = await this.config_fs.list_root_accounts(); + const account_names = await this.config_fs.list_accounts(); const should_filter_by_prefix = check_iam_path_was_set(iam_path_prefix); - const config_files_list = await P.map_with_concurrency(10, entries, async entry => { - if (entry.name.endsWith(JSON_SUFFIX)) { - const account_data = await this.config_fs.get_account_by_name(entry.name, { silent_if_missing: true }); - if (!account_data) return undefined; - if (entry.name.includes(config.NSFS_TEMP_CONF_DIR_NAME)) return undefined; - const is_root_account_owns_user = this._check_root_account_owns_user(requesting_account, account_data); - if ((!requesting_account.iam_operate_on_root_account && is_root_account_owns_user) || - (requesting_account.iam_operate_on_root_account && this._check_root_account(account_data))) { - if (should_filter_by_prefix) { - if (account_data.iam_path === undefined) return undefined; - if (!account_data.iam_path.startsWith(iam_path_prefix)) return undefined; - } - const user_data = { - user_id: account_data._id, - iam_path: account_data.iam_path || IAM_DEFAULT_PATH, - username: account_data.name, - arn: create_arn(requesting_account._id, account_data.name, account_data.iam_path), - create_date: account_data.creation_date, - password_last_used: Date.now(), // GAP - }; - return user_data; + const config_files_list = await P.map_with_concurrency(10, account_names, async account_name => { + const account_data = await this.config_fs.get_account_by_name(account_name, { silent_if_missing: true }); + if (!account_data) return undefined; + const is_root_account_owns_user = this._check_root_account_owns_user(requesting_account, account_data); + if ((!requesting_account.iam_operate_on_root_account && is_root_account_owns_user) || + (requesting_account.iam_operate_on_root_account && this._check_root_account(account_data))) { + if (should_filter_by_prefix) { + if (account_data.iam_path === undefined) return undefined; + if (!account_data.iam_path.startsWith(iam_path_prefix)) return undefined; } - return undefined; + const user_data = { + user_id: account_data._id, + iam_path: account_data.iam_path || IAM_DEFAULT_PATH, + username: account_data.name, + arn: create_arn(requesting_account._id, account_data.name, account_data.iam_path), + create_date: account_data.creation_date, + password_last_used: Date.now(), // GAP + }; + return user_data; } + return undefined; }); // remove undefined entries return config_files_list.filter(item => item); @@ -620,7 +615,7 @@ class AccountSpaceFS { } async _check_username_already_exists(action, username) { - const name_exists = await this.config_fs.is_account_exists({ name: username }); + const name_exists = await this.config_fs.is_account_exists_by_name(username); if (name_exists) { dbg.error(`AccountSpaceFS.${action} username already exists`, username); const message_with_details = `User with name ${username} already exists.`; @@ -635,12 +630,12 @@ class AccountSpaceFS { dbg.log1(`AccountSpaceFS.${action} new_account`, created_account); const new_account_string = JSON.stringify(created_account); nsfs_schema_utils.validate_account_schema(JSON.parse(new_account_string)); - await this.config_fs.create_account_config_file(params.username, JSON.parse(new_account_string), false); + await this.config_fs.create_account_config_file(JSON.parse(new_account_string)); return created_account; } async _check_if_account_config_file_exists(action, username) { - const is_user_account_exists = await this.config_fs.is_account_exists({ name: username }); + const is_user_account_exists = await this.config_fs.is_account_exists_by_name(username); if (!is_user_account_exists) { dbg.error(`AccountSpaceFS.${action} username does not exist`, username); const message_with_details = `The user with name ${username} cannot be found.`; @@ -674,15 +669,13 @@ class AccountSpaceFS { // currently, partial copy from verify_account_not_owns_bucket async _check_if_root_account_does_not_have_buckets_before_deletion(action, account_to_delete) { const resource_name = 'buckets'; - const entries = await this.config_fs.list_buckets(); - await P.map_with_concurrency(10, entries, async entry => { - if (entry.name.endsWith(JSON_SUFFIX)) { - const bucket_data = await this.config_fs.get_bucket_by_name(entry.name, { silent_if_missing: true}); - if (bucket_data && bucket_data.bucket_owner === account_to_delete.name) { - this._throw_error_delete_conflict(action, account_to_delete, resource_name); - } - return bucket_data; + const bucket_names = await this.config_fs.list_buckets(); + await P.map_with_concurrency(10, bucket_names, async bucket_name => { + const bucket_data = await this.config_fs.get_bucket_by_name(bucket_name, { silent_if_missing: true}); + if (bucket_data && bucket_data.bucket_owner === account_to_delete.name) { + this._throw_error_delete_conflict(action, account_to_delete, resource_name); } + return bucket_data; }); } @@ -690,19 +683,16 @@ class AccountSpaceFS { // currently, partial copy from _list_config_files_for_users async _check_if_root_account_does_not_have_IAM_users_before_deletion(action, account_to_delete) { const resource_name = 'IAM users'; - const entries = await this.config_fs.list_root_accounts(); - await P.map_with_concurrency(10, entries, async entry => { - if (entry.name.endsWith(JSON_SUFFIX)) { - const account_data = await this.config_fs.get_account_by_name(entry.name, { silent_if_missing: true }); - if (!account_data) return undefined; - if (entry.name.includes(config.NSFS_TEMP_CONF_DIR_NAME)) return undefined; - const is_root_account_owns_user = this._check_root_account_owns_user(account_to_delete, account_data); - if ((!account_to_delete.iam_operate_on_root_account && is_root_account_owns_user) || - (account_to_delete.iam_operate_on_root_account && this._check_root_account(account_data))) { - this._throw_error_delete_conflict(action, account_to_delete, resource_name); - } - return account_data; + const account_names = await this.config_fs.list_accounts(); + await P.map_with_concurrency(10, account_names, async account_name => { + const account_data = await this.config_fs.get_account_by_name(account_name, { silent_if_missing: true }); + if (!account_data) return undefined; + const is_root_account_owns_user = this._check_root_account_owns_user(account_to_delete, account_data); + if ((!account_to_delete.iam_operate_on_root_account && is_root_account_owns_user) || + (account_to_delete.iam_operate_on_root_account && this._check_root_account(account_data))) { + this._throw_error_delete_conflict(action, account_to_delete, resource_name); } + return account_data; }); } @@ -719,19 +709,11 @@ class AccountSpaceFS { // prepare requested_account.name = params.new_username; requested_account.email = params.new_username; // internally saved - const access_key_ids = []; - for (const access_keys of requested_account.access_keys) { - access_key_ids.push(access_keys.access_key); - } // handle account config creation const requested_account_encrypted = await nc_mkm.encrypt_access_keys(requested_account); const account_string = JSON.stringify(requested_account_encrypted); nsfs_schema_utils.validate_account_schema(JSON.parse(account_string)); - // handle account config creation and - // access keys(unlink and then create the new symbolic link) - await this.config_fs.create_account_config_file(params.new_username, JSON.parse(account_string), true, access_key_ids); - // handle account config deletion - await this.config_fs.delete_account_config_file(params.username); + await this.config_fs.update_account_config_file(JSON.parse(account_string), { old_name: params.username }); } _check_root_account_or_user(requesting_account, username) { @@ -831,7 +813,7 @@ class AccountSpaceFS { } async _check_if_account_exists_by_access_key_symlink(action, access_key_id) { - const is_user_account_exists = await this.config_fs.is_account_exists({ access_key: access_key_id }); + const is_user_account_exists = await this.config_fs.is_account_exists_by_access_key(access_key_id); if (!is_user_account_exists) { this._throw_error_no_such_entity_access_key(action, access_key_id); } diff --git a/src/sdk/bucketspace_fs.js b/src/sdk/bucketspace_fs.js index aba1f295d1..f64c712230 100644 --- a/src/sdk/bucketspace_fs.js +++ b/src/sdk/bucketspace_fs.js @@ -19,7 +19,7 @@ const { ConfigFS, JSON_SUFFIX } = require('./config_fs'); const mongo_utils = require('../util/mongo_utils'); const KeysSemaphore = require('../util/keys_semaphore'); -const { get_umasked_mode, isDirectory, validate_bucket_creation, get_bucket_tmpdir_full_path, folder_delete, +const { get_umasked_mode, validate_bucket_creation, get_bucket_tmpdir_full_path, folder_delete, entity_enum, translate_error_codes, get_process_fs_context} = require('../util/native_fs_utils'); const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent; const { anonymous_access_key } = require('./object_sdk'); @@ -178,9 +178,9 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { * @returns {Promise} */ async list_buckets(object_sdk) { - let entries; + let bucket_names; try { - entries = await this.config_fs.list_buckets(); + bucket_names = await this.config_fs.list_buckets(); } catch (err) { if (err.code === 'ENOENT') { dbg.error('BucketSpaceFS: root dir not found', err, this.config_fs.buckets_dir_path); @@ -190,11 +190,7 @@ class BucketSpaceFS extends BucketSpaceSimpleFS { } const account = object_sdk.requesting_account; - const buckets = await P.map_with_concurrency(10, entries, async entry => { - if (isDirectory(entry) || !entry.name.endsWith(JSON_SUFFIX)) { - return; - } - const bucket_name = this.get_bucket_name(entry.name); + const buckets = await P.map_with_concurrency(10, bucket_names, async bucket_name => { let bucket; try { bucket = await object_sdk.read_bucket_sdk_config_info(bucket_name); diff --git a/src/sdk/config_fs.js b/src/sdk/config_fs.js index b59d32a026..f7a6beeca1 100644 --- a/src/sdk/config_fs.js +++ b/src/sdk/config_fs.js @@ -5,14 +5,40 @@ const config = require('../../config'); const dbg = require('../util/debug_module')(__filename); const _ = require('lodash'); const path = require('path'); +const os_utils = require('../util/os_utils'); const nb_native = require('../util/nb_native'); const native_fs_utils = require('../util/native_fs_utils'); const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance(); +/* Config directory sub directory comments - + On 5.18 - + 1. accounts/ will be deprecated + 2. A new identities/ directory will be created that represents an account identity, + for example - + when an account called alice is created (id = 111) and an IAM user called bob (id = 222) + was created for alice, in the FS it will look as the folloing - + 2.1. config_dir/identities/111/identity.json -> represents the account properties (alice) + 2.2. config_dir/identities/111/users/ -> represents the account's iam users + 2.3. config_dir/identities/111/users/bob.symlink -> config_dir/identities/222/identity.json (an index to another identity) + 2.4. config_dir/identities/222/identity.json -> represents the account properties (bob) + in the future the identity directory will contain more features like policies, roles etc. + 3. A new accounts_by_name/ directory will be created that represents an index which is a + a symlink from account name to its actual identity. + For example - + config_dir/accounts_by_name/alice.symlink -> config_dir/identities/111/identity.json +*/ + const CONFIG_SUBDIRS = Object.freeze({ - ACCOUNTS: 'accounts', BUCKETS: 'buckets', - ACCESS_KEYS: 'access_keys' + ACCESS_KEYS: 'access_keys', + IDENTITIES: 'identities', + ACCOUNTS_BY_NAME: 'accounts_by_name', + ACCOUNTS: 'accounts', // deprecated on 5.18 +}); + +const CONFIG_TYPES = Object.freeze({ + ACCOUNT: 'account', + BUCKET: 'bucket', }); const JSON_SUFFIX = '.json'; @@ -34,7 +60,9 @@ class ConfigFS { constructor(config_root, config_root_backend, fs_context) { this.config_root = config_root; this.config_root_backend = config_root_backend || config.NSFS_NC_CONFIG_DIR_BACKEND; - this.accounts_dir_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS); + this.old_accounts_dir_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS); + this.accounts_by_name_dir_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS_BY_NAME); + this.identities_dir_path = path.join(config_root, CONFIG_SUBDIRS.IDENTITIES); this.access_keys_dir_path = path.join(config_root, CONFIG_SUBDIRS.ACCESS_KEYS); this.buckets_dir_path = path.join(config_root, CONFIG_SUBDIRS.BUCKETS); this.config_json_path = path.join(config_root, 'config.json'); @@ -48,7 +76,7 @@ class ConfigFS { * @returns {string} */ add_config_file_suffix(config_file_name, suffix) { - if (!config_file_name) throw new Error(`Config file name is missing - ${config_file_name}`); + if (!config_file_name) dbg.warn(`Config file name is missing - ${config_file_name}`); if (String(config_file_name).endsWith(suffix)) return config_file_name; return config_file_name + suffix; } @@ -88,7 +116,8 @@ class ConfigFS { const pre_req_dirs = [ this.config_root, this.buckets_dir_path, - this.accounts_dir_path, + this.accounts_by_name_dir_path, + this.identities_dir_path, this.access_keys_dir_path, ]; @@ -111,6 +140,32 @@ class ConfigFS { } } + /** + * get_config_json_path returns config.json file path + * @returns {String} + */ + get_config_json_path() { + return this.config_json_path; + } + + /** + * get_config_json returns config.json file data + * @returns {Promise} + */ + async get_config_json() { + const config_json_data = await this.get_config_data(this.config_json_path); + return config_json_data; + } + + /** + * create_config_json_file created the config.json file with the configuration data + * @param {object} data + * @returns {Promise} + */ + async create_config_json_file(data) { + await native_fs_utils.create_config_file(this.fs_context, this.config_root, this.config_json_path, data); + } + /** * update_config_json_file updates the config.json file with the new configuration data * @param {object} data @@ -120,6 +175,7 @@ class ConfigFS { await native_fs_utils.update_config_file(this.fs_context, this.config_root, this.config_json_path, data); } + /** * get_config_data reads a config file and returns its content * while omitting secrets if show_secrets flag was not provided @@ -130,16 +186,34 @@ class ConfigFS { * @param {{show_secrets?: boolean, decrypt_secret_key?: boolean, silent_if_missing?: boolean}} [options] * @returns {Promise} */ - async get_config_data(config_file_path, options = {}) { + async get_identity_config_data(config_file_path, options = {}) { const { show_secrets = false, decrypt_secret_key = false, silent_if_missing = false } = options; try { - const { data } = await nb_native().fs.readFile(this.fs_context, config_file_path); - const config_data = _.omit(JSON.parse(data.toString()), show_secrets ? [] : ['access_keys']); + const data = await this.get_config_data(config_file_path, options); + if (!data && silent_if_missing) return; + const config_data = _.omit(data, show_secrets ? [] : ['access_keys']); if (decrypt_secret_key) config_data.access_keys = await nc_mkm.decrypt_access_keys(config_data); return config_data; + } catch (err) { + dbg.warn('get_identity_config_data: with config_file_path', config_file_path, 'got an error', err); + if (err.code === 'ENOENT' && silent_if_missing) return; + throw err; + } + } + /** + * get_config_data reads a config file and returns its content + * @param {string} config_file_path + * @param {{ silent_if_missing?: boolean }} [options] + * @returns {Promise} + */ + async get_config_data(config_file_path, options = {}) { + try { + const { data } = await nb_native().fs.readFile(this.fs_context, config_file_path); + const config_data = JSON.parse(data.toString()); + return config_data; } catch (err) { dbg.warn('get_config_data: with config_file_path', config_file_path, 'got an error', err); - if (silent_if_missing && err.code === 'ENOENT') return; + if (err.code === 'ENOENT' && options.silent_if_missing) return; throw err; } } @@ -151,88 +225,200 @@ class ConfigFS { /** * get_account_path_by_name returns the full account path by name * @param {string} account_name - * @param {string} [owner_root_account_id] + * @param {string} [owner_account_id] * @returns {string} */ - get_account_path_by_name(account_name, owner_root_account_id) { - // TODO - change to this.symlink(account_name) on identities/ PR; - return owner_root_account_id ? - this.get_iam_account_path_by_name(account_name, owner_root_account_id) : - this.get_root_account_path_by_name(account_name); + get_account_or_user_path_by_name(account_name, owner_account_id) { + return owner_account_id ? + this.get_user_path_by_name(account_name, owner_account_id) : + this.get_account_path_by_name(account_name); } /** - * get_root_account_path_by_name returns the full account path by name + * get_account_path_by_name returns the full account path by name + * root account can be found by name under accounts_by_name/account_name.symlink * @param {string} account_name * @returns {string} */ - get_root_account_path_by_name(account_name) { - // TODO - change to this.symlink(account_name) on identities/ PR; - return path.join(this.accounts_dir_path, this.json(account_name)); + get_account_path_by_name(account_name) { + return path.join(this.accounts_by_name_dir_path, this.symlink(account_name)); } /** - * get_iam_account_path_by_name returns the full account path by name + * get_user_path_by_name returns the full iam user path by name + * iam user can be found by name under identities/owner_account_id/users/iam_account_name.symlink * @param {string} account_name - * @param {string} owner_root_account_id + * @param {string} owner_account_id * @returns {string} */ - get_iam_account_path_by_name(account_name, owner_root_account_id) { - // TODO - change to this.symlink(account_name) on identities/ PR; - // update IAM user location identities/root_id_number/iam_account_name.symlink - return path.join(this.accounts_dir_path, this.json(account_name)); + get_user_path_by_name(account_name, owner_account_id) { + // TODO - change to return path.join(this.identities_dir_path, owner_account_id, 'users', this.symlink(account_name)); + return path.join(this.accounts_by_name_dir_path, this.symlink(account_name)); } /** - * get_account_relative_path_by_name returns the full account path by name + * get_old_account_relative_path_by_name returns the old (5.17) full account path by name * @param {string} account_name * @returns {string} */ - get_account_relative_path_by_name(account_name) { - // TODO - change to this.symlink(account_name) on identities/ PR; - return path.join('../', CONFIG_SUBDIRS.ACCOUNTS, this.json(account_name)); + get_old_account_relative_path_by_name(account_name) { + return path.join('../', CONFIG_SUBDIRS.ACCOUNTS, this.json(account_name)); + } + + /** + * get_account_relative_path_by_id returns the full account path by id + * the target of symlinks will be the + * @param {string} account_id + * @returns {string} + */ + get_account_relative_path_by_id(account_id) { + return path.join('../', CONFIG_SUBDIRS.IDENTITIES, account_id, this.json('identity')); + } + + /** + * get_identity_path_by_id returns the full identity path by id as following - + * {config_dir}/identities/{id}/identity.json + * @param {string} id + * @returns {string} + */ + get_identity_path_by_id(id) { + return path.join(this.identities_dir_path, id, this.json('identity')); + } + + /** + * get_identity_by_id returns the full account/user data by id from the following path + * {config_dir}/identities/{account_id}/identity.json + * @param {string} id + * @param {string} [type] + * @param {{show_secrets?: boolean, decrypt_secret_key?: boolean, silent_if_missing?: boolean}} [options] + * @returns {Promise} + */ + async get_identity_by_id(id, type, options = {}) { + const identity_path = this.get_identity_path_by_id(id); + let identity = await this.get_identity_config_data(identity_path, { ...options, silent_if_missing: true }); + + if (!identity && type === CONFIG_TYPES.ACCOUNT) { + identity = await this.search_accounts_by_id(id, options); + } + if (!identity && !options.silent_if_missing) { + const err = new Error(`Could not find identity by id ${id}`); + err.code = 'ENOENT'; + throw err; + } + return identity; + } + + /** + * search_accounts_by_id searches old accounts directory and finds an account that its _id matches the given id param + * @param {string} id + * @param {{show_secrets?: boolean, decrypt_secret_key?: boolean, silent_if_missing?: boolean}} [options] + * @returns {Promise} + */ + async search_accounts_by_id(id, options = {}) { + const old_account_names = await this.list_old_accounts(); + for (const account_name of old_account_names) { + const account_data = await this.get_account_by_name(account_name, { ...options, silent_if_missing: true }); + if (!account_data) continue; + if (account_data._id === id) return account_data; + } + } + + /** + * get_identities_by_id returns the full account/user data by id from the following path + * {config_dir}/identities/{id}/identity.json + * @param {string[]} ids + * @returns {Promise} + */ + async get_identities_by_id(ids, options = {}) { + const res = []; + for (const id of ids) { + const id_path = this.get_identity_path_by_id(id); + const id_data = await this.get_identity_config_data(id_path, { ...options, silent_if_missing: true }); + if (!id_data) continue; + res.push(id_data); + } + return res; + } + + /** + * get_identity_path_by_id returns the account/user identity dir path by id as follows + * {config_dir}/identities/{account_id}/ + * @param {string} id + * @returns {string} + */ + get_identity_dir_path_by_id(id) { + return path.join(this.config_root, CONFIG_SUBDIRS.IDENTITIES, id, '/'); } /** - * get_account_path_by_access_key returns the full account path by access key + * get_account_or_user_path_by_access_key returns the full account path by access key as follows + * {config_dir}/access_keys/{access_key}.symlink * @param {string} access_key * @returns {string} */ - get_account_path_by_access_key(access_key) { + get_account_or_user_path_by_access_key(access_key) { return path.join(this.access_keys_dir_path, this.symlink(access_key)); } /** - * get_old_account_path_by_name returns the full account path by name based on old config dir structure + * _get_old_account_path_by_name returns the full account path by name based on old config dir structure + * as follows - {config_dir}/accounts/{access_name}.json * @param {string} account_name * @returns {string} */ - get_old_account_path_by_name(account_name) { - return path.join(this.accounts_dir_path, this.json(account_name)); + _get_old_account_path_by_name(account_name) { + return path.join(this.old_accounts_dir_path, this.json(account_name)); } /** - * is_account_exists returns true if account config path exists in config dir - * it can be determined by any account identifier - name, access_key and in the future id - * @param {{ name?: string, access_key?: string }} identifier_object + * is_account_exists_by_name returns true if account config path exists in config dir + * if account does not exist and it's a regular account (not an IAM user) + * try to locate it under the old accounts/ directory + * @param {string} account_name + * @param {string} [owner_account_id] * @returns {Promise} */ - async is_account_exists(identifier_object) { - if (!identifier_object || typeof identifier_object !== 'object') throw new Error('config_fs.is_account_exists - no identifier provided'); - let path_to_check; - let use_lstat = false; - if (identifier_object.name) { - // TODO - when users/ move to be symlink we need to use_lstat by name, id is not using lstat - path_to_check = this.get_account_path_by_name(identifier_object.name); - } else if (identifier_object.access_key) { - path_to_check = this.get_account_path_by_access_key(identifier_object.access_key); - use_lstat = true; - } else { - throw new Error(`config_fs.is_account_exists - invalid identifier provided ${identifier_object}`); + async is_account_exists_by_name(account_name, owner_account_id) { + const path_to_check = this.get_account_or_user_path_by_name(account_name, owner_account_id); + let account_exists = await native_fs_utils.is_path_exists(this.fs_context, path_to_check); + + if (!account_exists && account_name !== undefined && owner_account_id === undefined) { + const old_path_to_check = this._get_old_account_path_by_name(account_name); + account_exists = await native_fs_utils.is_path_exists(this.fs_context, old_path_to_check); } - // TODO - add is_account_exists by id when idenetities/ added - // else if (identifier_object.id) {} - return native_fs_utils.is_path_exists(this.fs_context, path_to_check, use_lstat); + return account_exists; + } + + /** + * is_identity_exists returns true if identity config path exists in config dir + * @param {string} id + * @param {string} [type] + * @param {{show_secrets?: boolean, decrypt_secret_key?: boolean, silent_if_missing?: boolean}} [options] + * @returns {Promise} + */ + async is_identity_exists(id, type, options) { + const path_to_check = this.get_identity_path_by_id(id); + let identity = await native_fs_utils.is_path_exists(this.fs_context, path_to_check); + + if (!identity && type === CONFIG_TYPES.ACCOUNT) { + identity = await this.search_accounts_by_id(id, options); + } + if (!identity && !options.silent_if_missing) { + const err = new Error(`Could not find identity by id ${id}`); + err.code = 'ENOENT'; + throw err; + } + return identity; + } + + /** + * is_account_exists_by_access_key returns true if account config path exists in config dir + * @param {string} access_key + * @returns {Promise} + */ + async is_account_exists_by_access_key(access_key) { + const path_to_check = this.get_account_or_user_path_by_access_key(access_key); + return native_fs_utils.is_path_exists(this.fs_context, path_to_check); } /** @@ -244,8 +430,8 @@ class ConfigFS { * @returns {Promise} */ async get_account_by_access_key(access_key, options = {}) { - const account_path = this.get_account_path_by_access_key(access_key); - const account = await this.get_config_data(account_path, options); + const account_path = this.get_account_or_user_path_by_access_key(access_key); + const account = await this.get_identity_config_data(account_path, options); return account; } @@ -265,104 +451,246 @@ class ConfigFS { */ async get_account_by_name(account_name, options = {}) { const account_path = this.get_account_path_by_name(account_name); - const account = await this.get_config_data(account_path, options); + let account = await this.get_identity_config_data(account_path, { ...options, silent_if_missing: true }); + if (!account) { + const old_account_path = this._get_old_account_path_by_name(account_name); + account = await this.get_identity_config_data(old_account_path, { ...options, silent_if_missing: true }); + } + if (!account && !options.silent_if_missing) { + const err = new Error(`Could not find account by name ${account_name}`); + err.code = 'ENOENT'; + throw err; + } return account; } /** - * list_root_accounts returns the root accounts array exists under the config dir - * @returns {Promise} + * list_accounts returns the account names array - + * 1. get new accounts names + * 2. check old accounts/ dir exists + * 3. if old accounts dir exists return the union of new accounts and old accounts names array + * 4. else return new account names array + * and add accounts from old accounts/ directory if exists + * during upgrade - list accounts is a very expensive operation as it's iterating old accounts and adds the entries that still do not appear in the new accounts folder + * @returns {Promise} */ - async list_root_accounts(options) { - return nb_native().fs.readdir(this.fs_context, this.accounts_dir_path); + async list_accounts() { + const new_entries = await nb_native().fs.readdir(this.fs_context, this.accounts_by_name_dir_path); + const new_accounts_names = this._get_config_entries_names(new_entries, SYMLINK_SUFFIX); + const old_accounts_names = await this.list_old_accounts(); + return this.unify_old_and_new_accounts(new_accounts_names, old_accounts_names); + } + + /** + * unify_old_and_new_accounts list the old accounts directory and add them to a set of the accounts + * this will create a union of accounts and old accounts under accounts/ directory. + * @returns {Promise} + */ + async unify_old_and_new_accounts(new_entries, old_entries) { + if (!old_entries?.length) return new_entries; + const set = new Set([...new_entries, ...old_entries]); + return Array.from(set); + } + + /** + * list_old_accounts lists the old accounts under accounts/ directory and return their names. + * @returns {Promise} + */ + async list_old_accounts() { + const old_accounts_dir_exists = await native_fs_utils.is_path_exists(this.fs_context, this.old_accounts_dir_path); + if (!old_accounts_dir_exists) return []; + + const old_entries = await nb_native().fs.readdir(this.fs_context, this.old_accounts_dir_path); + if (old_entries.length === 0) return []; + + return this._get_config_entries_names(old_entries, JSON_SUFFIX); } /** * create_account_config_file creates account config file - * if account_data.access_keys is an array that contains at least 1 item - - * link all item in account_data.access_keys to the relative path of the newly created config file - * @param {string} account_name - * @param {*} account_data - * @param {boolean} symlink_new_access_keys - * @param {String[]} old_access_keys + * 1. create /identities/account_id/ directory + * 2. create /identities/account_id/identity.json file + * 3. symlink /accounts_by_name/account_name -> /identities/account_id/identity.json + * 4. symlink new access keys if account_data.access_keys is an array that contains at least 1 item - + * link each item in account_data.access_keys to the relative path of the newly created config file + * @param {Object} account_data * @returns {Promise} */ - async create_account_config_file(account_name, account_data, symlink_new_access_keys, old_access_keys = []) { - const account_path = this.get_account_path_by_name(account_name); - await native_fs_utils.create_config_file(this.fs_context, this.accounts_dir_path, account_path, JSON.stringify(account_data)); - - if (old_access_keys.length > 0) { - for (const access_key of old_access_keys) { - const access_key_config_path = this.get_account_path_by_access_key(access_key); - await nb_native().fs.unlink(this.fs_context, access_key_config_path); - } - } - if (symlink_new_access_keys && account_data.access_keys?.length > 0) { - for (const access_key_obj of account_data.access_keys) { - const account_config_access_key_path = this.get_account_path_by_access_key(access_key_obj.access_key); - const account_config_relative_path = this.get_account_relative_path_by_name(account_name); - await native_fs_utils._create_path(this.access_keys_dir_path, this.fs_context, config.BASE_MODE_CONFIG_DIR); - await nb_native().fs.symlink(this.fs_context, account_config_relative_path, account_config_access_key_path); - } - } + async create_account_config_file(account_data) { + const { _id, name, owner = undefined } = account_data; + const data_string = JSON.stringify(account_data); + const account_path = this.get_identity_path_by_id(_id); + const account_dir_path = this.get_identity_dir_path_by_id(_id); + + await native_fs_utils._create_path(account_dir_path, this.fs_context, config.BASE_MODE_CONFIG_DIR); + await native_fs_utils.create_config_file(this.fs_context, account_dir_path, account_path, data_string); + await this.link_account_name_index(_id, name, owner); + await this.link_access_keys_index(_id, account_data.access_keys); } /** * update_account_config_file updates account config file * if old_access_keys is an array that contains at least 1 item - * unlink old access_keys and link new access_keys - * @param {string} account_name + * 1. update /identities/account_id/identity.json + * 2. if name updated - + * link /accounts_by_name/new_account_name -> /identities/account_id/identity.json + * unlink /accounts_by_name/old_account_name + * 3. if access key was updated - + * for all new_access_keys - link /access_keys/new_access_key -> /identities/account_id/identity.json + * for all old_access_keys - unlink /access_keys/old_access_key * @param {Object} account_new_data - * @param {Object[]} [new_access_keys_to_link] - * @param {String[]} [old_access_keys_to_remove] + * @param {{old_name?: string, new_access_keys_to_link?: Object[], access_keys_to_delete?: { access_key: string }[]}} [options] * @returns {Promise} */ - async update_account_config_file(account_name, account_new_data, new_access_keys_to_link = [], old_access_keys_to_remove = []) { - const account_config_path = this.get_account_path_by_name(account_name); - await native_fs_utils.update_config_file(this.fs_context, this.accounts_dir_path, - account_config_path, JSON.stringify(account_new_data)); + async update_account_config_file(account_new_data, options = {}) { + const { _id, name, owner = undefined } = account_new_data; + const data_string = JSON.stringify(account_new_data); + const account_path = this.get_identity_path_by_id(_id); + const account_dir_path = this.get_identity_dir_path_by_id(_id); + await native_fs_utils.update_config_file(this.fs_context, account_dir_path, account_path, data_string); + + if (options.old_name) { + await this.link_account_name_index(_id, name, owner); + await this.unlink_account_name_index(options.old_name, account_path); + } + await this.link_access_keys_index(_id, options.new_access_keys_to_link); + await this.unlink_access_keys_indexes(options.access_keys_to_delete, account_path); + } + + /** + * delete_account_config_file deletes account config file + * 1. unlink /access_keys/access_key if access_keys_to_delete is an array that contains at least 1 item - + * unlink all item in access_keys_to_delete + * 2. unlink /root_accounts/account_name + * 3. delete /identities/account_id/identity.json + * 4. delete /identities/account_id/ folder + * @param {Object} data + * @returns {Promise} + */ + async delete_account_config_file(data) { + const { _id, name, access_keys = [] } = data; + const account_id_config_path = this.get_identity_path_by_id(_id); + const account_dir_path = this.get_identity_dir_path_by_id(_id); + + await this.unlink_access_keys_indexes(access_keys, account_id_config_path); + await this.unlink_account_name_index(name, account_id_config_path); + await native_fs_utils.delete_config_file(this.fs_context, account_dir_path, account_id_config_path); + await native_fs_utils.folder_delete(account_dir_path, this.fs_context, undefined, true); + } - if (new_access_keys_to_link.length > 0) { - for (const access_keys of new_access_keys_to_link) { - const new_access_key_path = this.get_account_path_by_access_key(access_keys.access_key); - const account_config_relative_path = this.get_account_relative_path_by_name(account_name); - await nb_native().fs.symlink(this.fs_context, account_config_relative_path, new_access_key_path); + ///////////////////////////////////// + ////// ACCOUNT NAME INDEX ////// + ///////////////////////////////////// + + /** + * link_account_name_index links the access key to the relative path of the account id config file + * @param {string} account_id + * @param {string} account_name + * @param {string} owner_id + * @returns {Promise} + */ + async link_account_name_index(account_id, account_name, owner_id) { + const account_name_path = this.get_account_or_user_path_by_name(account_name, owner_id); + const account_id_relative_path = this.get_account_relative_path_by_id(account_id); + await nb_native().fs.symlink(this.fs_context, account_id_relative_path, account_name_path); + } + + /** + * unlink_account_name_index unlinks the access key from the config directory + * 1. get the account name path + * 2. check realpath on the account name path to make sure it belongs to the account id we meant to delete + * 3. check if the account id path is the same as the account name path + * 4. unlink the account name path + * 5. else, do nothing as the name path might already point to a new identity/deleted by concurrent calls + * @param {string} account_name + * @returns {Promise} + */ + async unlink_account_name_index(account_name, account_id_config_path) { + const account_name_path = this.get_account_path_by_name(account_name); + const should_unlink = await this._is_symlink_pointing_to_identity(account_name_path, account_id_config_path); + if (should_unlink) { + try { + await nb_native().fs.unlink(this.fs_context, account_name_path); + } catch (err) { + if (err.code === 'ENOENT') { + dbg.warn(`config_fs.unlink_account_name_index: account name already unlinked ${account_name} ${account_id_config_path}`); + return; + } + throw err; } } + } - if (old_access_keys_to_remove.length > 0) { - for (const access_key of old_access_keys_to_remove) { - const cur_access_key_path = this.get_account_path_by_access_key(access_key); - await nb_native().fs.unlink(this.fs_context, cur_access_key_path); + ////////////////////////////////////// + ////// ACCESS KEYS INDEXES ////// + ////////////////////////////////////// + + /** + * link_access_keys_index links the access keys symlinks + * @param {Object[]} access_keys_to_link + * @returns {Promise} + */ + async link_access_keys_index(account_id, access_keys_to_link = []) { + if (!access_keys_to_link?.length) return; + const account_config_relative_path = this.get_account_relative_path_by_id(account_id); + for (const access_keys of access_keys_to_link) { + const new_access_key_path = this.get_account_or_user_path_by_access_key(access_keys.access_key); + await nb_native().fs.symlink(this.fs_context, account_config_relative_path, new_access_key_path); + } + } + + /** + * unlink_access_key_index unlinks the access key from the config directory + * 1. get the account access_key path + * 2. check realpath on the account access_key path to make sure it belongs to the account id we meant to delete + * 3. check if the account id path is the same as the account name path + * 4. unlink the account name path + * 5. else, do nothing as the name path might already point to a new identity/deleted by concurrent calls + * @param {string} access_key + * @returns {Promise} + */ + async unlink_access_key_index(access_key, account_id_config_path) { + const access_key_path = this.get_account_or_user_path_by_access_key(access_key); + const should_unlink = await this._is_symlink_pointing_to_identity(access_key_path, account_id_config_path); + if (should_unlink) { + try { + await nb_native().fs.unlink(this.fs_context, access_key_path); + } catch (err) { + if (err.code === 'ENOENT') { + dbg.warn(`config_fs.unlink_access_key_index: account access_key already unlinked ${access_key} ${account_id_config_path}`); + return; + } + throw err; } } } /** - * delete_account_config_file deletes account config file - * if access_keys_to_delete is an array that contains at least 1 item - - * unlink all items in access_keys_to_delete - * @param {string} account_name - * @param {Object[]} [access_keys_to_delete] + * unlink_access_keys_index unlinks the access keys from the config directory + * iterate access_keys_to_delete array and for each call unlink_access_key_index() + * @param {Object[]} access_keys_to_delete + * @param {String} account_id_config_path * @returns {Promise} */ - async delete_account_config_file(account_name, access_keys_to_delete = []) { - const account_config_path = this.get_account_path_by_name(account_name); - await native_fs_utils.delete_config_file(this.fs_context, this.accounts_dir_path, account_config_path); + async unlink_access_keys_indexes(access_keys_to_delete, account_id_config_path) { + if (!access_keys_to_delete?.length) return; for (const access_keys of access_keys_to_delete) { - const access_key_config_path = this.get_account_path_by_access_key(access_keys.access_key); - await nb_native().fs.unlink(this.fs_context, access_key_config_path); + await this.unlink_access_key_index(access_keys.access_key, account_id_config_path); } } /** - * unlink_access_key_symlink unlinks the access key from the file system - * @param {string} access_key - * @returns {Promise} + * _is_symlink_pointing_to_identity checks if the index symlink (name/access_key) + * is pointing to the identity file path + * @param {string} symlink_path + * @param {string} identity_path + * @returns {Promise} */ - async unlink_access_key_symlink(access_key) { - const acces_key_path = this.get_account_path_by_access_key(access_key); - await nb_native().fs.unlink(this.fs_context, acces_key_path); + async _is_symlink_pointing_to_identity(symlink_path, identity_path) { + const full_path = await nb_native().fs.realpath(this.fs_context, symlink_path); + return (full_path === identity_path || + (os_utils.IS_MAC && full_path === path.join('/private/', identity_path))); } ////////////////////////////////////// @@ -404,10 +732,12 @@ class ConfigFS { /** * list_buckets returns the array of buckets that exists under the config dir - * @returns {Promise} + * @returns {Promise} */ async list_buckets() { - return nb_native().fs.readdir(this.fs_context, this.buckets_dir_path); + const bucket_entries = await nb_native().fs.readdir(this.fs_context, this.buckets_dir_path); + const bucket_names = this._get_config_entries_names(bucket_entries, JSON_SUFFIX); + return bucket_names; } /** @@ -442,6 +772,11 @@ class ConfigFS { await native_fs_utils.delete_config_file(this.fs_context, this.buckets_dir_path, bucket_config_path); } + + //////////////////////// + /// HELPERS //// + //////////////////////// + /** * adjust_bucket_with_schema_updates changes the bucket properties according to the schema * @param {object} bucket @@ -453,10 +788,51 @@ class ConfigFS { delete bucket.system_owner; } } + + /** + * @param {fs.Dirent} entry + * @param {string} suffix + * @returns {boolean} + */ + _has_config_file_name_format(entry, suffix) { + return entry.name.endsWith(suffix) && + !entry.name.includes(config.NSFS_TEMP_CONF_DIR_NAME) && + !native_fs_utils.isDirectory(entry); + } + + /** + * _get_config_entry_name returns config file entry name if it adheres a config file name format, + * else returns undefined + * @param {fs.Dirent} entry + * @param {string} suffix + * @returns {string | undefined} + */ + _get_config_entry_name(entry, suffix) { + return (this._has_config_file_name_format(entry, suffix)) ? + path.parse(entry.name).name : + undefined; + } + + /** + * _get_config_entries_names returns config file names array + * @param {fs.Dirent[]} entries + * @param {string} suffix + * @returns {string[]} + */ + _get_config_entries_names(entries, suffix) { + const config_file_names = []; + for (const entry of entries) { + if (this._has_config_file_name_format(entry, suffix)) { + config_file_names.push(path.parse(entry.name).name); + } + } + return config_file_names; + } } // EXPORTS exports.SYMLINK_SUFFIX = SYMLINK_SUFFIX; exports.JSON_SUFFIX = JSON_SUFFIX; exports.CONFIG_SUBDIRS = CONFIG_SUBDIRS; +exports.CONFIG_TYPES = CONFIG_TYPES; exports.ConfigFS = ConfigFS; diff --git a/src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh b/src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh index 621bad0345..ec502899d9 100644 --- a/src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh +++ b/src/test/system_tests/ceph_s3_tests/run_ceph_nsfs_test_on_test_container.sh @@ -21,31 +21,29 @@ export NOOBAA_MGMT_SERVICE_PROTO=wss export S3_SERVICE_HOST=localhost export CEPH_TEST_LOGS_DIR=/logs/ceph-nsfs-test-logs -export FS_ROOT_1=/tmp/nsfs_root1 -export FS_ROOT_2=/tmp/nsfs_root2 export CONFIG_DIR=/etc/noobaa.conf.d/ +export FS_ROOT_1=/tmp/nsfs_root1/ +export FS_ROOT_2=/tmp/nsfs_root2/ # ==================================================================================== -# Create the logs directory -mkdir -p ${CEPH_TEST_LOGS_DIR} - -# Create configuration directory +# 1. Create configuration directory +# 2. Create config.json file mkdir -p ${CONFIG_DIR} +config='{"ALLOW_HTTP":true}' +echo "$config" > ${CONFIG_DIR}/config.json -# Create root directory for bucket creation -mkdir -p ${FS_ROOT_1} -mkdir -p ${FS_ROOT_2} - -# Add permission to all users +# 1. Create root directory for bucket creation +# 2. Add permission to all users # this will allow the new accounts to create directories (buckets), # else we would see [Error: Permission denied] { code: 'EACCES' } +mkdir -p ${FS_ROOT_1} +mkdir -p ${FS_ROOT_2} chmod 777 ${FS_ROOT_1} chmod 777 ${FS_ROOT_2} -# Create config.json file -config='{"ALLOW_HTTP":true}' -echo "$config" > ${CONFIG_DIR}/config.json +# Create the logs directory +mkdir -p ${CEPH_TEST_LOGS_DIR} # Deploy standalone NooBaa on the test container # And create the accounts needed for the Ceph tests diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js b/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js index b14dac1293..b555015d00 100644 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js +++ b/src/test/system_tests/ceph_s3_tests/test_ceph_nsfs_s3_config_setup.js @@ -7,16 +7,14 @@ * In the past this script was a part of file test_ceph_s3. */ -const fs = require('fs'); const dbg = require('../../../util/debug_module')(__filename); dbg.set_process_name('test_ceph_s3'); +const fs = require('fs'); const os_utils = require('../../../util/os_utils'); -const config = require('../../../../config'); -const mongo_utils = require('../../../util/mongo_utils'); -const { CEPH_TEST, account_path, account_tenant_path, anonymous_account_path } = require('./test_ceph_s3_constants.js'); -const nc_mkm = require('../../../manage_nsfs/nc_master_key_manager').get_instance(); - +const test_utils = require('../../system_tests/test_utils'); +const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); +const { CEPH_TEST } = require('./test_ceph_s3_constants.js'); async function main() { try { @@ -46,62 +44,54 @@ async function ceph_test_setup() { await fs.promises.writeFile(conf_file, new_conf_file_content); console.log('conf file updated'); - console.info('CEPH TEST CONFIGURATION:', JSON.stringify(CEPH_TEST)); - let access_keys = await get_access_keys(account_path); - const access_key = access_keys.access_key; - const secret_key = access_keys.secret_key; + console.info('CEPH TEST CONFIGURATION: CREATE ACCOUNTS', JSON.stringify(CEPH_TEST)); + await create_account(CEPH_TEST.nc_cephalt_account_params); + await create_account(CEPH_TEST.nc_cephtenant_account_params); + await create_account(CEPH_TEST.nc_anonymous_account_params); - await os_utils.exec(`echo access_key = ${access_key} >> ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); - await os_utils.exec(`echo secret_key = ${secret_key} >> ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); + console.info('CEPH TEST CONFIGURATION: GET ACCESS KEYS', JSON.stringify(CEPH_TEST)); + const cephalt_access_keys = await get_access_keys(CEPH_TEST.nc_cephalt_account_params.name); + const cephalt_access_key = cephalt_access_keys.access_key; + const cephalt_secret_key = cephalt_access_keys.secret_key; - access_keys = await get_access_keys(account_tenant_path); - const access_key_tenant = access_keys.access_key; - const secret_key_tenant = access_keys.secret_key; + await os_utils.exec(`echo access_key = ${cephalt_access_key} >> ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); + await os_utils.exec(`echo secret_key = ${cephalt_secret_key} >> ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); - if (os_utils.IS_MAC) { - await os_utils.exec(`sed -i "" "s|tenant_access_key|"${access_key_tenant}"|g" ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); - await os_utils.exec(`sed -i "" "s|tenant_secret_key|${secret_key_tenant}|g" ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); + const cephtenant_access_keys = await get_access_keys(CEPH_TEST.nc_cephtenant_account_params.name); + const cephtenant_access_key = cephtenant_access_keys.access_key; + const cephtenant_secret_key = cephtenant_access_keys.secret_key; + if (os_utils.IS_MAC) { + await os_utils.exec(`sed -i "" "s|tenant_access_key|"${cephtenant_access_key}"|g" ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); + await os_utils.exec(`sed -i "" "s|tenant_secret_key|${cephtenant_secret_key}|g" ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); } else { - await os_utils.exec(`sed -i -e 's:tenant_access_key:${access_key_tenant}:g' ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); - await os_utils.exec(`sed -i -e 's:tenant_secret_key:${secret_key_tenant}:g' ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); - await os_utils.exec(`sed -i -e 's:s3_access_key:${access_key}:g' ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); - await os_utils.exec(`sed -i -e 's:s3_secret_key:${secret_key}:g' ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); + await os_utils.exec(`sed -i -e 's:tenant_access_key:${cephtenant_access_key}:g' ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); + await os_utils.exec(`sed -i -e 's:tenant_secret_key:${cephtenant_secret_key}:g' ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); + await os_utils.exec(`sed -i -e 's:s3_access_key:${cephalt_access_key}:g' ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); + await os_utils.exec(`sed -i -e 's:s3_secret_key:${cephalt_secret_key}:g' ${CEPH_TEST.test_dir}${CEPH_TEST.ceph_config}`); } - // create anonymous account - await create_anonymous_account(); - + console.info('CEPH TEST CONFIGURATION: DONE'); } -async function get_access_keys(path) { - const account_data = await fs.promises.readFile(path, 'utf8'); - const data_json = JSON.parse(account_data); - const access_key = data_json.access_keys[0].access_key; - const encrypted_secret_key = data_json.access_keys[0].encrypted_secret_key; - const secret_key = await nc_mkm.decrypt(encrypted_secret_key, data_json.master_key_id); - return {access_key, secret_key}; +/** + * get_access_keys returns account access keys using noobaa-cli + * @param {string} account_name + */ +async function get_access_keys(account_name) { + const options = { name: account_name, show_secrets: true }; + const res = await test_utils.exec_manage_cli(TYPES.ACCOUNT, ACTIONS.STATUS, options); + const json_account = JSON.parse(res); + const account_data = json_account.response.reply; + return account_data.access_keys[0]; } -// Create an anonymous account for anonymous request. Use this account UID and GID for bucket access. -async function create_anonymous_account() { - const nsfs_account_config = { - uid: process.getuid(), - gid: process.getgid(), - }; - const { master_key_id } = await nc_mkm.encrypt_access_keys({}); - const data = { - _id: mongo_utils.mongoObjectId(), - name: config.ANONYMOUS_ACCOUNT_NAME, - email: config.ANONYMOUS_ACCOUNT_NAME, - nsfs_account_config: nsfs_account_config, - access_keys: [], - allow_bucket_creation: false, - creation_date: new Date().toISOString(), - master_key_id: master_key_id, - }; - const account_data = JSON.stringify(data); - await fs.promises.writeFile(anonymous_account_path, account_data); - console.log('Anonymous account created'); +/** + * create_account creates accounts using noobaa-cli + * @param {{ name?: string, uid?: number, gid?: number, anonymous?: boolean }} [options] + */ +async function create_account(options = {}) { + const res = await test_utils.exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, options); + console.log('Account Created', res); } if (require.main === module) { diff --git a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js b/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js index 270b338de7..c7c3869d6a 100644 --- a/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js +++ b/src/test/system_tests/ceph_s3_tests/test_ceph_s3_constants.js @@ -1,30 +1,48 @@ /* Copyright (C) 2022 NooBaa */ "use strict"; +const cephalt_name = 'cephalt'; +const cephtenant_name = 'cephtenant'; + +// NC ceph tests paths for bucket creation +const FS_ROOT_1 = '/tmp/nsfs_root1/'; +const FS_ROOT_2 = '/tmp/nsfs_root2/'; + const CEPH_TEST = { test_dir: 'src/test/system_tests/ceph_s3_tests/', s3_test_dir: 's3-tests/', ceph_config: 'test_ceph_s3_config.conf', tox_config: 'tox.ini', new_account_params: { - name: 'cephalt', + name: cephalt_name, email: 'ceph.alt@noobaa.com', has_login: false, s3_access: true, }, new_account_params_tenant: { - name: 'cephtenant', + name: cephtenant_name, email: 'ceph.tenant@noobaa.com', has_login: false, s3_access: true, }, + nc_cephalt_account_params: { + name: cephalt_name, + uid: 1000, + gid: 1000, + new_buckets_path: FS_ROOT_1 + }, + nc_cephtenant_account_params: { + name: cephtenant_name, + uid: 2000, + gid: 2000, + new_buckets_path: FS_ROOT_2 + }, + nc_anonymous_account_params: { + anonymous: true, + uid: process.getuid(), + gid: process.getgid() + } }; - -// For NSFS NC path (using default values) -const account_path = '/etc/noobaa.conf.d/accounts/cephalt.json'; -const account_tenant_path = '/etc/noobaa.conf.d/accounts/cephtenant.json'; -const anonymous_account_path = '/etc/noobaa.conf.d/accounts/anonymous.json'; - const DEFAULT_NUMBER_OF_WORKERS = 5; //5 was the number of workers in the previous CI/CD process const TOX_ARGS = `-c ${CEPH_TEST.test_dir}${CEPH_TEST.s3_test_dir}${CEPH_TEST.tox_config}`; @@ -32,9 +50,9 @@ const TOX_ARGS = `-c ${CEPH_TEST.test_dir}${CEPH_TEST.s3_test_dir}${CEPH_TEST.to const AWS4_TEST_SUFFIX = '_aws4'; exports.CEPH_TEST = CEPH_TEST; -exports.account_path = account_path; -exports.anonymous_account_path = anonymous_account_path; -exports.account_tenant_path = account_tenant_path; exports.DEFAULT_NUMBER_OF_WORKERS = DEFAULT_NUMBER_OF_WORKERS; exports.TOX_ARGS = TOX_ARGS; exports.AWS4_TEST_SUFFIX = AWS4_TEST_SUFFIX; +exports.FS_ROOT_1 = FS_ROOT_1; +exports.FS_ROOT_2 = FS_ROOT_2; + diff --git a/src/test/system_tests/test_utils.js b/src/test/system_tests/test_utils.js index 9bcc924af3..8a024d0208 100644 --- a/src/test/system_tests/test_utils.js +++ b/src/test/system_tests/test_utils.js @@ -6,11 +6,13 @@ const _ = require('lodash'); const http = require('http'); const P = require('../../util/promise'); const os_utils = require('../../util/os_utils'); +const nb_native = require('../../util/nb_native'); const native_fs_utils = require('../../util/native_fs_utils'); const config = require('../../../config'); const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); const path = require('path'); +const { CONFIG_TYPES } = require('../../sdk/config_fs'); /** * TMP_PATH is a path to the tmp path based on the process platform @@ -410,6 +412,7 @@ async function generate_nsfs_account(rpc_client, EMAIL, default_new_buckets_path email: `${random_name}@noobaa.com` }; } + /** * get_new_buckets_path_by_test_env returns new_buckets_path value * on NC - new_buckets_path is full absolute path @@ -425,6 +428,57 @@ function get_new_buckets_path_by_test_env(new_buckets_full_path, new_buckets_dir return is_nc_coretest ? path.join(new_buckets_full_path, new_buckets_dir) : new_buckets_dir; } +/** + * write_manual_config_file writes config file directly to the file system without using config FS + * used for creating backward compatibility tests, invalid config files etc + * @param {import('../../sdk/config_fs').ConfigFS} config_fs + * @param {Object} config_data + * @param {String} [invalid_str] + * @returns {Promise} + */ +async function write_manual_config_file(type, config_fs, config_data, invalid_str = '') { + const config_path = type === CONFIG_TYPES.BUCKET ? + config_fs.get_bucket_path_by_name(config_data.name) : + config_fs.get_identity_path_by_id(config_data._id); + if (type === CONFIG_TYPES.ACCOUNT) { + const dir_path = config_fs.get_identity_dir_path_by_id(config_data._id); + await nb_native().fs.mkdir(config_fs.fs_context, dir_path, native_fs_utils.get_umasked_mode(config.BASE_MODE_DIR)); + } + await nb_native().fs.writeFile( + config_fs.fs_context, + config_path, + Buffer.from(JSON.stringify(config_data) + invalid_str), + { + mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_FILE) + } + ); + + if (type === CONFIG_TYPES.ACCOUNT) { + const id_relative_path = config_fs.get_account_relative_path_by_id(config_data._id); + const name_symlink_path = config_fs.get_account_or_user_path_by_name(config_data.name); + await nb_native().fs.symlink(config_fs.fs_context, id_relative_path, name_symlink_path); + } +} + + +/** + * write_manual_old_account_config_file writes account config file directly to the old file system account path without using config FS + * @param {import('../../sdk/config_fs').ConfigFS} config_fs + * @param {Object} config_data + * @returns {Promise} + */ +async function write_manual_old_account_config_file(config_fs, config_data) { + const config_path = config_fs._get_old_account_path_by_name(config_data.name); + await nb_native().fs.writeFile( + config_fs.fs_context, + config_path, + Buffer.from(JSON.stringify(config_data)), + { + mode: native_fs_utils.get_umasked_mode(config.BASE_MODE_FILE) + } + ); +} + exports.blocks_exist_on_cloud = blocks_exist_on_cloud; exports.create_hosts_pool = create_hosts_pool; exports.delete_hosts_pool = delete_hosts_pool; @@ -444,4 +498,6 @@ exports.TMP_PATH = TMP_PATH; exports.is_nc_coretest = is_nc_coretest; exports.generate_nsfs_account = generate_nsfs_account; exports.get_new_buckets_path_by_test_env = get_new_buckets_path_by_test_env; +exports.write_manual_config_file = write_manual_config_file; +exports.write_manual_old_account_config_file = write_manual_old_account_config_file; diff --git a/src/test/unit_tests/jest_tests/test_accountspace_fs.test.js b/src/test/unit_tests/jest_tests/test_accountspace_fs.test.js index d620fb5ecf..f914828569 100644 --- a/src/test/unit_tests/jest_tests/test_accountspace_fs.test.js +++ b/src/test/unit_tests/jest_tests/test_accountspace_fs.test.js @@ -9,22 +9,16 @@ const _ = require('lodash'); const fs = require('fs'); const path = require('path'); -const nb_native = require('../../../util/nb_native'); -const { JSON_SUFFIX, SYMLINK_SUFFIX } = require('../../../sdk/config_fs'); const SensitiveString = require('../../../util/sensitive_string'); const AccountSpaceFS = require('../../../sdk/accountspace_fs'); const { TMP_PATH } = require('../../system_tests/test_utils'); -const { get_process_fs_context } = require('../../../util/native_fs_utils'); const { IAM_DEFAULT_PATH, ACCESS_KEY_STATUS_ENUM } = require('../../../endpoint/iam/iam_constants'); const fs_utils = require('../../../util/fs_utils'); const { IamError } = require('../../../endpoint/iam/iam_errors'); -const nc_mkm = require('../../../manage_nsfs/nc_master_key_manager').get_instance(); -const native_fs_utils = require('../../../util/native_fs_utils'); const nsfs_schema_utils = require('../../../manage_nsfs/nsfs_schema_utils'); class NoErrorThrownError extends Error {} -const DEFAULT_FS_CONFIG = get_process_fs_context(); const tmp_fs_path = path.join(TMP_PATH, 'test_accountspace_fs'); const config_root = path.join(tmp_fs_path, 'config_root'); const new_buckets_path1 = path.join(tmp_fs_path, 'new_buckets_path1', '/'); @@ -32,6 +26,7 @@ const new_buckets_path2 = path.join(tmp_fs_path, 'new_buckets_path2', '/'); const new_buckets_path3 = path.join(tmp_fs_path, 'new_buckets_path3', '/'); const accountspace_fs = new AccountSpaceFS({ config_root }); +const config_fs_account_options = { show_secrets: true, decrypt_secret_key: true }; const root_user_account = { _id: '65a8edc9bc5d5bbf9db71b91', @@ -184,9 +179,7 @@ function make_dummy_account_sdk_root_accounts_manager() { describe('Accountspace_FS tests', () => { beforeAll(async () => { - await fs_utils.create_fresh_path(accountspace_fs.config_fs.accounts_dir_path); - await fs_utils.create_fresh_path(accountspace_fs.config_fs.access_keys_dir_path); - await fs_utils.create_fresh_path(accountspace_fs.config_fs.buckets_dir_path); + await accountspace_fs.config_fs.create_config_dirs_if_missing(); await fs_utils.create_fresh_path(new_buckets_path1); await fs_utils.create_fresh_path(new_buckets_path3); await fs.promises.chown(new_buckets_path1, @@ -196,12 +189,7 @@ describe('Accountspace_FS tests', () => { for (const account of [root_user_account, root_user_account2, root_user_root_accounts_manager]) { - const account_path = accountspace_fs.config_fs.get_account_path_by_name(account.name); - // assuming that the root account has only 1 access key in the 0 index - const account_access_path = accountspace_fs.config_fs.get_account_path_by_access_key(account.access_keys[0].access_key); - await fs.promises.writeFile(account_path, JSON.stringify(account)); - await fs.promises.chmod(account_path, 0o600); - await fs.promises.symlink(account_path, account_access_path); + await accountspace_fs.config_fs.create_account_config_file(account); } }); afterAll(async () => { @@ -248,7 +236,8 @@ describe('Accountspace_FS tests', () => { expect(res.arn).toBeDefined(); expect(res.create_date).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.username); expect(user_account_config_file._id).toBeDefined(); expect(user_account_config_file.creation_date).toBeDefined(); @@ -271,7 +260,8 @@ describe('Accountspace_FS tests', () => { expect(res.arn).toBeDefined(); expect(res.create_date).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.username); expect(user_account_config_file._id).toBeDefined(); expect(user_account_config_file.creation_date).toBeDefined(); @@ -300,7 +290,8 @@ describe('Accountspace_FS tests', () => { expect(res.arn).toBeDefined(); expect(res.create_date).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.username); expect(user_account_config_file._id).toBeDefined(); expect(user_account_config_file.creation_date).toBeDefined(); @@ -330,7 +321,8 @@ describe('Accountspace_FS tests', () => { expect(res.arn).toBeDefined(); expect(res.create_date).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.username); expect(user_account_config_file._id).toBeDefined(); expect(user_account_config_file.creation_date).toBeDefined(); @@ -499,7 +491,8 @@ describe('Accountspace_FS tests', () => { expect(res.user_id).toBeDefined(); expect(res.arn).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.username); expect(user_account_config_file.iam_path).toBe(dummy_user1.iam_path); }); @@ -515,7 +508,8 @@ describe('Accountspace_FS tests', () => { expect(res.username).toBe(dummy_user1.username); expect(res.user_id).toBeDefined(); expect(res.arn).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.username); expect(user_account_config_file.iam_path).toBe(dummy_iam_path2); // back as it was @@ -537,7 +531,8 @@ describe('Accountspace_FS tests', () => { expect(res.username).toBe(dummy_user_root_account.username); expect(res.user_id).toBeDefined(); expect(res.arn).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.username); expect(user_account_config_file.iam_path).toBe(dummy_iam_path); }); @@ -631,7 +626,8 @@ describe('Accountspace_FS tests', () => { expect(res.username).toBe(params.new_username); expect(res.user_id).toBeDefined(); expect(res.arn).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.new_username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.new_username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.new_username); // back as it was params = { @@ -665,12 +661,13 @@ describe('Accountspace_FS tests', () => { expect(res.username).toBe(params.new_username); expect(res.user_id).toBeDefined(); expect(res.arn).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.new_username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.new_username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.new_username); - const symlink_config_path = path.join(accountspace_fs.config_fs.access_keys_dir_path, access_key + SYMLINK_SUFFIX); - await fs_utils.file_must_exist(symlink_config_path); - const user_account_config_file_from_symlink = await read_config_file( - accountspace_fs.config_fs.access_keys_dir_path, access_key, true); + const is_path_exists = await accountspace_fs.config_fs.is_account_exists_by_access_key(access_key); + expect(is_path_exists).toBe(true); + const user_account_config_file_from_symlink = await accountspace_fs.config_fs.get_account_by_access_key(access_key, + config_fs_account_options); expect(user_account_config_file_from_symlink.name).toBe(params.new_username); }); }); @@ -683,8 +680,8 @@ describe('Accountspace_FS tests', () => { const account_sdk = make_dummy_account_sdk(); const res = await accountspace_fs.delete_user(params, account_sdk); expect(res).toBeUndefined(); - const user_account_config_path = path.join(accountspace_fs.config_fs.accounts_dir_path, params.username + JSON_SUFFIX); - await fs_utils.file_must_not_exist(user_account_config_path); + const is_path_exists = await accountspace_fs.config_fs.is_account_exists_by_name(params.username); + expect(is_path_exists).toBe(false); }); it('delete_user does not return any params (requesting account is root accounts manager to create root account user)', async function() { @@ -694,8 +691,8 @@ describe('Accountspace_FS tests', () => { const account_sdk = make_dummy_account_sdk_root_accounts_manager(); const res = await accountspace_fs.delete_user(params, account_sdk); expect(res).toBeUndefined(); - const user_account_config_path = path.join(accountspace_fs.config_fs.accounts_dir_path, params.username + JSON_SUFFIX); - await fs_utils.file_must_not_exist(user_account_config_path); + const is_path_exists = await accountspace_fs.config_fs.is_account_exists_by_name(params.username); + expect(is_path_exists).toBe(false); }); it('delete_user should return an error if requesting user is not a root account user', async function() { @@ -770,8 +767,8 @@ describe('Accountspace_FS tests', () => { expect(err).toHaveProperty('code', IamError.DeleteConflict.code); expect(err).toHaveProperty('message'); expect(err.message).toMatch(/must delete access keys first/i); - const user_account_config_path = path.join(accountspace_fs.config_fs.accounts_dir_path, params.username + JSON_SUFFIX); - await fs_utils.file_must_exist(user_account_config_path); + const is_path_exists = await accountspace_fs.config_fs.is_account_exists_by_name(params.username); + expect(is_path_exists).toBe(true); } }); @@ -788,8 +785,8 @@ describe('Accountspace_FS tests', () => { // same params await accountspace_fs.create_access_key(params, account_sdk); // create a user with the root account - const account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, - username_for_root_account); + const account_config_file = await accountspace_fs.config_fs.get_account_by_name(username_for_root_account, + config_fs_account_options); const root_account_manager_id = account_sdk.requesting_account._id; const account_sdk_root = make_dummy_account_sdk_from_root_accounts_manager( account_config_file, root_account_manager_id); @@ -807,8 +804,8 @@ describe('Accountspace_FS tests', () => { expect(err).toHaveProperty('code', IamError.DeleteConflict.code); expect(err).toHaveProperty('message'); expect(err.message).toMatch(/must delete IAM users first/i); - const user_account_config_path = path.join(accountspace_fs.config_fs.accounts_dir_path, params.username + JSON_SUFFIX); - await fs_utils.file_must_exist(user_account_config_path); + const is_path_exists = await accountspace_fs.config_fs.is_account_exists_by_name(params.username); + expect(is_path_exists).toBe(true); } }); @@ -823,7 +820,8 @@ describe('Accountspace_FS tests', () => { await accountspace_fs.create_user(params, account_sdk); // create a dummy bucket const bucket_name = `my-bucket-${params.username}`; - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); await create_dummy_bucket(user_account_config_file, bucket_name); await accountspace_fs.delete_user(params, account_sdk); throw new NoErrorThrownError(); @@ -832,8 +830,8 @@ describe('Accountspace_FS tests', () => { expect(err).toHaveProperty('code', IamError.DeleteConflict.code); expect(err).toHaveProperty('message'); expect(err.message).toMatch(/must delete buckets first/i); - const user_account_config_path = path.join(accountspace_fs.config_fs.accounts_dir_path, params.username + JSON_SUFFIX); - await fs_utils.file_must_exist(user_account_config_path); + const is_path_exists = await accountspace_fs.config_fs.is_account_exists_by_name(params.username); + expect(is_path_exists).toBe(true); } }); }); @@ -933,7 +931,8 @@ describe('Accountspace_FS tests', () => { }; beforeAll(async () => { - await fs_utils.create_fresh_path(accountspace_fs.config_fs.accounts_dir_path); + await fs_utils.create_fresh_path(accountspace_fs.config_fs.identities_dir_path); + await fs_utils.create_fresh_path(accountspace_fs.config_fs.accounts_by_name_dir_path); await fs_utils.create_fresh_path(accountspace_fs.config_fs.access_keys_dir_path); await fs_utils.create_fresh_path(accountspace_fs.config_fs.buckets_dir_path); await fs_utils.create_fresh_path(new_buckets_path1); @@ -941,12 +940,7 @@ describe('Accountspace_FS tests', () => { root_user_root_accounts_manager.nsfs_account_config.uid, root_user_root_accounts_manager.nsfs_account_config.gid); for (const account of [root_user_root_accounts_manager]) { - const account_path = accountspace_fs.config_fs.get_account_path_by_name(account.name); - // assuming that the root account has only 1 access key in the 0 index - const account_access_path = accountspace_fs.config_fs.get_account_path_by_access_key(account.access_keys[0].access_key); - await fs.promises.writeFile(account_path, JSON.stringify(account)); - await fs.promises.chmod(account_path, 0o600); - await fs.promises.symlink(account_path, account_access_path); + await accountspace_fs.config_fs.create_account_config_file(account); } }); afterAll(async () => { @@ -1016,18 +1010,16 @@ describe('Accountspace_FS tests', () => { expect(res.status).toBe('Active'); expect(res.secret_key).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.username); expect(user_account_config_file.access_keys).toBeDefined(); expect(Array.isArray(user_account_config_file.access_keys)).toBe(true); expect(user_account_config_file.access_keys.length).toBe(1); const access_key = res.access_key; - const user_account_config_file_from_symlink = await read_config_file( - accountspace_fs.config_fs.access_keys_dir_path, - access_key, - true - ); + const user_account_config_file_from_symlink = await accountspace_fs.config_fs.get_account_by_access_key(access_key, + config_fs_account_options); expect(user_account_config_file_from_symlink.name).toBe(params.username); expect(user_account_config_file_from_symlink.access_keys).toBeDefined(); expect(Array.isArray(user_account_config_file_from_symlink.access_keys)).toBe(true); @@ -1046,18 +1038,16 @@ describe('Accountspace_FS tests', () => { expect(res.status).toBe('Active'); expect(res.secret_key).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.username); expect(user_account_config_file.access_keys).toBeDefined(); expect(Array.isArray(user_account_config_file.access_keys)).toBe(true); expect(user_account_config_file.access_keys.length).toBe(2); const access_key = res.access_key; - const user_account_config_file_from_symlink = await read_config_file( - accountspace_fs.config_fs.access_keys_dir_path, - access_key, - true - ); + const user_account_config_file_from_symlink = await accountspace_fs.config_fs.get_account_by_access_key(access_key, + config_fs_account_options); expect(user_account_config_file_from_symlink.name).toBe(params.username); expect(user_account_config_file_from_symlink.access_keys).toBeDefined(); expect(Array.isArray(user_account_config_file_from_symlink.access_keys)).toBe(true); @@ -1092,7 +1082,8 @@ describe('Accountspace_FS tests', () => { username: dummy_username5, }; await accountspace_fs.create_access_key(params_for_access_key_creation, account_sdk); - let user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username5); + let user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username5, + config_fs_account_options); // create the second access key // by the IAM user account_sdk = make_dummy_account_sdk_created_from_another_account(user_account_config_file, @@ -1104,18 +1095,16 @@ describe('Accountspace_FS tests', () => { expect(res.status).toBe('Active'); expect(res.secret_key).toBeDefined(); - user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username5); + user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username5, + config_fs_account_options); expect(user_account_config_file.name).toBe(dummy_username5); expect(user_account_config_file.access_keys).toBeDefined(); expect(Array.isArray(user_account_config_file.access_keys)).toBe(true); expect(user_account_config_file.access_keys.length).toBe(2); const access_key = res.access_key; - const user_account_config_file_from_symlink = await read_config_file( - accountspace_fs.config_fs.access_keys_dir_path, - access_key, - true - ); + const user_account_config_file_from_symlink = await accountspace_fs.config_fs.get_account_by_access_key(access_key, + config_fs_account_options); expect(user_account_config_file_from_symlink.name).toBe(dummy_username5); expect(user_account_config_file_from_symlink.access_keys).toBeDefined(); expect(Array.isArray(user_account_config_file_from_symlink.access_keys)).toBe(true); @@ -1126,7 +1115,8 @@ describe('Accountspace_FS tests', () => { try { // both IAM users are under the same root account (owner property) let account_sdk = make_dummy_account_sdk(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username5); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username5, + config_fs_account_options); // create the second access key // by the IAM user account_sdk = make_dummy_account_sdk_created_from_another_account(user_account_config_file, @@ -1157,18 +1147,16 @@ describe('Accountspace_FS tests', () => { expect(res.status).toBe('Active'); expect(res.secret_key).toBeDefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, params.username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(params.username, + config_fs_account_options); expect(user_account_config_file.name).toBe(params.username); expect(user_account_config_file.access_keys).toBeDefined(); expect(Array.isArray(user_account_config_file.access_keys)).toBe(true); expect(user_account_config_file.access_keys.length).toBe(1); const access_key = res.access_key; - const user_account_config_file_from_symlink = await read_config_file( - accountspace_fs.config_fs.access_keys_dir_path, - access_key, - true - ); + const user_account_config_file_from_symlink = await accountspace_fs.config_fs.get_account_by_access_key(access_key, + config_fs_account_options); expect(user_account_config_file_from_symlink.name).toBe(params.username); expect(user_account_config_file_from_symlink.access_keys).toBeDefined(); expect(Array.isArray(user_account_config_file_from_symlink.access_keys)).toBe(true); @@ -1265,7 +1253,8 @@ describe('Accountspace_FS tests', () => { it('get_access_key_last_used should return user access key params (requester is an IAM user)', async function() { const username = dummy_user2.username; let account_sdk = make_dummy_account_sdk(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(username, + config_fs_account_options); // by the IAM user account_sdk = make_dummy_account_sdk_created_from_another_account(user_account_config_file, user_account_config_file.owner); const access_key = user_account_config_file.access_keys[0].access_key; @@ -1282,17 +1271,13 @@ describe('Accountspace_FS tests', () => { it('get_access_key_last_used return an error if user is not owned by the root account (requester is an IAM user)', async function() { try { let account_sdk = make_dummy_account_sdk(); - const requester_account_config_file = await read_config_file( - accountspace_fs.config_fs.accounts_dir_path, - dummy_user2.username - ); + const requester_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_user2.username, + config_fs_account_options); // by the IAM user account_sdk = make_dummy_account_sdk_created_from_another_account(requester_account_config_file, requester_account_config_file.owner); - const user_account_config_file = await read_config_file( - accountspace_fs.config_fs.accounts_dir_path, - dummy_user_root_account.username - ); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_user_root_account.username, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; const params = { access_key: access_key, @@ -1363,7 +1348,8 @@ describe('Accountspace_FS tests', () => { }); it('update_access_key should return an error if user account does not exist', async function() { - const user_account = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + const user_account = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); const dummy_access_key = user_account.access_keys[0].access_key; try { const params = { @@ -1383,7 +1369,8 @@ describe('Accountspace_FS tests', () => { it('update_access_key should return an error if access key belongs to another account ' + 'without passing the username flag', async function() { - const user_account = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + const user_account = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); const dummy_access_key = user_account.access_keys[0].access_key; try { const params = { @@ -1403,7 +1390,8 @@ describe('Accountspace_FS tests', () => { it('update_access_key should return an error if access key is on another root account', async function() { try { const account_sdk = make_dummy_account_sdk_not_for_creating_resources(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; const params = { username: dummy_username1, @@ -1420,7 +1408,8 @@ describe('Accountspace_FS tests', () => { it('update_access_key should not return any param (update status to Inactive) (requesting account is root account to create IAM user)', async function() { const account_sdk = make_dummy_account_sdk(); - let user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + let user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; const params = { username: dummy_username1, @@ -1429,13 +1418,15 @@ describe('Accountspace_FS tests', () => { }; const res = await accountspace_fs.update_access_key(params, account_sdk); expect(res).toBeUndefined(); - user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); expect(user_account_config_file.access_keys[0].deactivated).toBe(true); }); it('update_access_key should not return any param (update status to Active) (requesting account is root account to create IAM user)', async function() { const account_sdk = make_dummy_account_sdk(); - let user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + let user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; const params = { username: dummy_username1, @@ -1444,13 +1435,15 @@ describe('Accountspace_FS tests', () => { }; const res = await accountspace_fs.update_access_key(params, account_sdk); expect(res).toBeUndefined(); - user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); expect(user_account_config_file.access_keys[0].deactivated).toBe(false); }); it('update_access_key should not return any param (update status to Active, already was Active) (requesting account is root account to create IAM user)', async function() { const account_sdk = make_dummy_account_sdk(); - let user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + let user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; const params = { username: dummy_username1, @@ -1459,14 +1452,16 @@ describe('Accountspace_FS tests', () => { }; const res = await accountspace_fs.update_access_key(params, account_sdk); expect(res).toBeUndefined(); - user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); expect(user_account_config_file.access_keys[0].deactivated).toBe(false); }); it('update_access_key should not return any param (requester is an IAM user)', async function() { const dummy_username = dummy_username5; let account_sdk = make_dummy_account_sdk(); - let user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username); + let user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username, + config_fs_account_options); // by the IAM user account_sdk = make_dummy_account_sdk_created_from_another_account(user_account_config_file, user_account_config_file.owner); const access_key = user_account_config_file.access_keys[1].access_key; @@ -1476,7 +1471,8 @@ describe('Accountspace_FS tests', () => { }; const res = await accountspace_fs.update_access_key(params, account_sdk); expect(res).toBeUndefined(); - user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username); + user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username, + config_fs_account_options); expect(user_account_config_file.access_keys[1].deactivated).toBe(true); }); @@ -1484,7 +1480,8 @@ describe('Accountspace_FS tests', () => { try { // both IAM users are under the same root account (owner property) let account_sdk = make_dummy_account_sdk(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username5); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username5, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; // create the second access key // by the IAM user @@ -1506,7 +1503,8 @@ describe('Accountspace_FS tests', () => { it('update_access_key should not return any param (update status to Inactive) (requesting account is root accounts manager requested account is root account)', async function() { const username = dummy_user_root_account.username; const account_sdk = make_dummy_account_sdk_root_accounts_manager(); - let user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, username); + let user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(username, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; const params = { username: username, @@ -1515,7 +1513,8 @@ describe('Accountspace_FS tests', () => { }; const res = await accountspace_fs.update_access_key(params, account_sdk); expect(res).toBeUndefined(); - user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, username); + user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(username, + config_fs_account_options); expect(user_account_config_file.access_keys[0].deactivated).toBe(true); }); }); @@ -1554,7 +1553,8 @@ describe('Accountspace_FS tests', () => { }); it('delete_access_key should return an error if user account does not exist', async function() { - const user_account = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + const user_account = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); const dummy_access_key = user_account.access_keys[0].access_key; try { const params = { @@ -1573,7 +1573,8 @@ describe('Accountspace_FS tests', () => { it('delete_access_key should return an error if access key belongs to another account ' + 'without passing the username flag', async function() { - const user_account = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + const user_account = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); const dummy_access_key = user_account.access_keys[0].access_key; try { const params = { @@ -1592,7 +1593,8 @@ describe('Accountspace_FS tests', () => { it('delete_access_key should not return an error if access key is on another root account', async function() { try { const account_sdk = make_dummy_account_sdk_not_for_creating_resources(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; const params = { username: dummy_username1, @@ -1608,7 +1610,9 @@ describe('Accountspace_FS tests', () => { it('delete_access_key should not return any param (requesting account is root account to create IAM user)', async function() { const account_sdk = make_dummy_account_sdk(); - let user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + let user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); + const access_key = user_account_config_file.access_keys[0].access_key; const params = { username: dummy_username1, @@ -1616,10 +1620,11 @@ describe('Accountspace_FS tests', () => { }; const res = await accountspace_fs.delete_access_key(params, account_sdk); expect(res).toBeUndefined(); - user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); expect(user_account_config_file.access_keys.length).toBe(1); - const symlink_config_path = path.join(accountspace_fs.config_fs.access_keys_dir_path, access_key + SYMLINK_SUFFIX); - await fs_utils.file_must_not_exist(symlink_config_path); + const is_path_exists = await accountspace_fs.config_fs.is_account_exists_by_access_key(access_key); + expect(is_path_exists).toBe(false); }); it('delete_access_key should not return any param (account with 2 access keys) (requesting account is root account to create IAM user)', async function() { @@ -1646,19 +1651,21 @@ describe('Accountspace_FS tests', () => { }; const res = await accountspace_fs.delete_access_key(params, account_sdk); expect(res).toBeUndefined(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, username); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(username, + config_fs_account_options); expect(user_account_config_file.access_keys.length).toBe(1); expect(user_account_config_file.access_keys[0].access_key).toBe(access_key); expect(user_account_config_file.access_keys[0].access_key).not.toBe(access_key_to_delete); - let symlink_config_path = path.join(accountspace_fs.config_fs.access_keys_dir_path, access_key_to_delete + SYMLINK_SUFFIX); - await fs_utils.file_must_not_exist(symlink_config_path); - symlink_config_path = path.join(accountspace_fs.config_fs.access_keys_dir_path, access_key + SYMLINK_SUFFIX); - await fs_utils.file_must_exist(symlink_config_path); + const is_path_exists1 = await accountspace_fs.config_fs.is_account_exists_by_access_key(access_key_to_delete); + expect(is_path_exists1).toBe(false); + const is_path_exists2 = await accountspace_fs.config_fs.is_account_exists_by_access_key(access_key); + expect(is_path_exists2).toBe(true); }); it('delete_access_key should not return any param (requester is an IAM user)', async function() { let account_sdk = make_dummy_account_sdk(); - let user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username5); + let user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username5, + config_fs_account_options); // by the IAM user account_sdk = make_dummy_account_sdk_created_from_another_account(user_account_config_file, user_account_config_file.owner); const access_key = user_account_config_file.access_keys[1].access_key; @@ -1667,18 +1674,20 @@ describe('Accountspace_FS tests', () => { }; const res = await accountspace_fs.delete_access_key(params, account_sdk); expect(res).toBeUndefined(); - user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username1); + user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username1, + config_fs_account_options); expect(user_account_config_file.access_keys.length).toBe(1); expect(user_account_config_file.access_keys[0].access_key).not.toBe(access_key); - const symlink_config_path = path.join(accountspace_fs.config_fs.access_keys_dir_path, access_key + SYMLINK_SUFFIX); - await fs_utils.file_must_not_exist(symlink_config_path); + const is_path_exists = await accountspace_fs.config_fs.is_account_exists_by_access_key(access_key); + expect(is_path_exists).toBe(false); }); it('delete_access_key should return an error if user is not owned by the root account (requester is an IAM user)', async function() { try { // both IAM users are under the same root account (owner property) let account_sdk = make_dummy_account_sdk(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username5); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username5, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; // create the second access key // by the IAM user @@ -1699,7 +1708,8 @@ describe('Accountspace_FS tests', () => { it('delete_access_key should not return any param (requesting account is root accounts manager requested account is root account)', async function() { const username = dummy_user_root_account.username; const account_sdk = make_dummy_account_sdk_root_accounts_manager(); - let user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, username); + let user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(username, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; const params = { username: username, @@ -1707,10 +1717,11 @@ describe('Accountspace_FS tests', () => { }; const res = await accountspace_fs.delete_access_key(params, account_sdk); expect(res).toBeUndefined(); - user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, username); + user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(username, + config_fs_account_options); expect(user_account_config_file.access_keys.length).toBe(1); - const symlink_config_path = path.join(accountspace_fs.config_fs.access_keys_dir_path, access_key + SYMLINK_SUFFIX); - await fs_utils.file_must_not_exist(symlink_config_path); + const is_path_exists = await accountspace_fs.config_fs.is_account_exists_by_access_key(access_key); + expect(is_path_exists).toBe(false); }); }); @@ -1779,7 +1790,8 @@ describe('Accountspace_FS tests', () => { it('list_access_keys return array of access_keys and value of is_truncated (requester is an IAM user)', async function() { let account_sdk = make_dummy_account_sdk(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username5); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username5, + config_fs_account_options); // by the IAM user account_sdk = make_dummy_account_sdk_created_from_another_account(user_account_config_file, user_account_config_file.owner); const params = {}; @@ -1793,7 +1805,8 @@ describe('Accountspace_FS tests', () => { try { // both IAM users are under the same root account (owner property) let account_sdk = make_dummy_account_sdk(); - const user_account_config_file = await read_config_file(accountspace_fs.config_fs.accounts_dir_path, dummy_username5); + const user_account_config_file = await accountspace_fs.config_fs.get_account_by_name(dummy_username5, + config_fs_account_options); const access_key = user_account_config_file.access_keys[0].access_key; // create the second access key // by the IAM user @@ -1831,38 +1844,14 @@ describe('Accountspace_FS tests', () => { }); -/** - * read_config_file will read the config files - * @param {string} account_path full account path - * @param {string} config_file_name the name of the config file - * @param {boolean} [is_symlink] a flag to set the suffix as a symlink instead of json - */ - -async function read_config_file(account_path, config_file_name, is_symlink) { - const config_path = path.join(account_path, config_file_name + (is_symlink ? SYMLINK_SUFFIX : JSON_SUFFIX)); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, config_path); - const config_data = JSON.parse(data.toString()); - if (config_data.access_keys.length > 0) { - const encrypted_secret_key = config_data.access_keys[0].encrypted_secret_key; - config_data.access_keys[0].secret_key = await nc_mkm.decrypt(encrypted_secret_key, config_data.master_key_id); - delete config_data.access_keys[0].encrypted_secret_key; - } - return config_data; -} - async function create_dummy_bucket(account, bucket_name) { const bucket_storage_path = path.join(account.nsfs_account_config.new_buckets_path, bucket_name); const bucket = _new_bucket_defaults(account, bucket_name, bucket_storage_path); const bucket_config = JSON.stringify(bucket); const bucket_to_validate = JSON.parse(bucket_config); nsfs_schema_utils.validate_bucket_schema(bucket_to_validate); - const bucket_config_path = path.join(accountspace_fs.config_fs.buckets_dir_path, bucket_name + JSON_SUFFIX); - await native_fs_utils.create_config_file( - accountspace_fs.fs_context, - accountspace_fs.config_fs.buckets_dir_path, - bucket_config_path, - bucket_config - ); + await accountspace_fs.config_fs.create_bucket_config_file(bucket_name, bucket_config); + const bucket_config_path = accountspace_fs.config_fs.get_bucket_path_by_name(bucket_name); return bucket_config_path; } diff --git a/src/test/unit_tests/jest_tests/test_config_dir_structure.js b/src/test/unit_tests/jest_tests/test_config_dir_structure.js new file mode 100644 index 0000000000..5a2ae836b2 --- /dev/null +++ b/src/test/unit_tests/jest_tests/test_config_dir_structure.js @@ -0,0 +1,150 @@ +/* Copyright (C) 2024 NooBaa */ +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const nb_native = require('../../../util/nb_native'); +const config = require('../../../../config'); +const fs_utils = require('../../../util/fs_utils'); +const { ConfigFS } = require('../../../sdk/config_fs'); +const { TMP_PATH } = require('../../system_tests/test_utils'); +const { get_process_fs_context, is_path_exists } = require('../../../util/native_fs_utils'); + +const tmp_fs_path = path.join(TMP_PATH, 'test_config_fs'); +const config_root = path.join(tmp_fs_path, 'config_root'); +const config_root_backend = config.NSFS_NC_CONFIG_DIR_BACKEND; +const fs_context = get_process_fs_context(config_root_backend); + + +const DEFAULT_CONF_DIR_PATH = config.NSFS_NC_DEFAULT_CONF_DIR; +const CUSTOM_CONF_DIR_PATH = path.join(TMP_PATH, 'redirected_config_dir'); +const REDIRECT_FILE_PATH = path.join(DEFAULT_CONF_DIR_PATH, config.NSFS_NC_CONF_DIR_REDIRECT_FILE); +const default_config_fs = new ConfigFS(DEFAULT_CONF_DIR_PATH, config_root_backend, fs_context); +const custom_config_fs = new ConfigFS(CUSTOM_CONF_DIR_PATH, config_root_backend, fs_context); +const config_fs = new ConfigFS(config_root, config_root_backend, fs_context); + +const buckets_dir_name = '/buckets/'; +const identities_dir_name = '/identities/'; +const accounts_by_name = '/accounts_by_name/'; +const access_keys_dir_name = '/access_keys/'; +const old_accounts_dir_name = '/accounts/'; + +// WARNING: +// The following test file will check the directory structure created using create_config_dirs_if_missing() +// which is called when using noobaa-cli, for having an accurate test, it'll be blocked from running on an +// env having an existing default config directory and the test executer will be asked to remove the default +// config directory before running the test again, do not run on production env or on any env where the existing config directory is being used +describe('create_config_dirs_if_missing', () => { + + beforeAll(async () => { + const config_dir_exists = await is_path_exists(fs_context, DEFAULT_CONF_DIR_PATH); + const msg = `test_config_dir_restructure found an existing default config directory ${DEFAULT_CONF_DIR_PATH},` + + `this test needs to test the creation of the config directory elements, therefore make sure ` + + `the content of the config directory is not needed and remove it for ensuring a used config directory will not get deleted`; + if (config_dir_exists) { + console.error(msg); + process.exit(1); + } + }); + + afterEach(async () => { + await clean_config_dir(); + }); + + it('create_config_dirs_if_missing() first time - nothing exists - everything should be created', async () => { + await default_config_fs.create_config_dirs_if_missing(); + await assert_config_dir(DEFAULT_CONF_DIR_PATH); + }); + + it('create_config_dirs_if_missing() first time - config_dir exists - all subdirs should be created under the default config dir', async () => { + const config_dir = DEFAULT_CONF_DIR_PATH; + await create_config_dir(config_dir); + await default_config_fs.create_config_dirs_if_missing(); + await assert_config_dir(config_dir); + }); + + it('create_config_dirs_if_missing() first time - default_config_dir exists and redirect file, redirected config_dir missing - all subdirs should be created under the redirect folder', async () => { + const default_config_dir = DEFAULT_CONF_DIR_PATH; + const config_dir = CUSTOM_CONF_DIR_PATH; + await create_config_dir(default_config_dir); + await create_redirect_file(); + await custom_config_fs.create_config_dirs_if_missing(); + await assert_config_dir(config_dir); + }); + + it('create_config_dirs_if_missing() first time - default_config_dir, redirect file, redirected config_dir exist - all subdirs should be created under the redirect folder', async () => { + const default_config_dir = DEFAULT_CONF_DIR_PATH; + const config_dir = CUSTOM_CONF_DIR_PATH; + await create_config_dir(default_config_dir); + await create_config_dir(CUSTOM_CONF_DIR_PATH); + await create_redirect_file(); + await custom_config_fs.create_config_dirs_if_missing(); + await assert_config_dir(config_dir); + }); +}); + + +/** + * assert_config_dir asserts the config dir structure was created appropriately + * 1. default config dir exists + * 2. if redirect file exists - + * redirect config dir exists + * 3. sub directories exists under the redirect config dir if exists else under the default config dir + * 4. old accounts/ folder does not exist + * @param {String} config_dir_path + */ +async function assert_config_dir(config_dir_path) { + // config dir exists + let path_exists = await is_path_exists(fs_context, config_dir_path); + expect(path_exists).toBe(true); + + // config subdirs do not exist + for (const dir of [buckets_dir_name, identities_dir_name, access_keys_dir_name, accounts_by_name]) { + const path_to_check = path.join(config_dir_path, dir); + path_exists = await is_path_exists(fs_context, path_to_check); + expect(path_exists).toBe(true); + } + + // old accounts directory does not exist - (5.17) + const path_to_check = path.join(config_dir_path, old_accounts_dir_name); + path_exists = await is_path_exists(fs_context, path_to_check); + expect(path_exists).toBe(false); +} + +/** + * clean_config_dir cleans the config directory + * @returns {Promise} + */ +async function clean_config_dir() { + for (const dir of [buckets_dir_name, identities_dir_name, access_keys_dir_name, accounts_by_name]) { + const default_path = path.join(config.NSFS_NC_DEFAULT_CONF_DIR, dir); + await fs_utils.folder_delete(default_path); + const custom_path = path.join(CUSTOM_CONF_DIR_PATH, dir); + await fs_utils.folder_delete(custom_path); + + } + await fs_utils.file_delete(REDIRECT_FILE_PATH); + await fs_utils.folder_delete(config.NSFS_NC_DEFAULT_CONF_DIR); + await fs_utils.folder_delete(CUSTOM_CONF_DIR_PATH); +} + +/** + * create_config_dir will create the config directory on the file system + * @param {String} config_dir + * @returns {Promise} + */ +async function create_config_dir(config_dir) { + await fs.promises.mkdir(config_dir); +} + +/** + * create_redirect_file will create the redirect file on the file system + * @returns {Promise} + */ +async function create_redirect_file() { + await nb_native().fs.writeFile( + config_fs.fs_context, + REDIRECT_FILE_PATH, + Buffer.from(CUSTOM_CONF_DIR_PATH) + ); +} diff --git a/src/test/unit_tests/jest_tests/test_config_fs_backward_compatibility.test.js b/src/test/unit_tests/jest_tests/test_config_fs_backward_compatibility.test.js new file mode 100644 index 0000000000..14a7afe57a --- /dev/null +++ b/src/test/unit_tests/jest_tests/test_config_fs_backward_compatibility.test.js @@ -0,0 +1,176 @@ +/* Copyright (C) 2024 NooBaa */ +'use strict'; + +const path = require('path'); +const fs_utils = require('../../../util/fs_utils'); +const { ConfigFS, CONFIG_TYPES } = require('../../../sdk/config_fs'); +const { TMP_PATH, write_manual_config_file, write_manual_old_account_config_file } = require('../../system_tests/test_utils'); + + +const tmp_fs_path = path.join(TMP_PATH, 'test_config_fs_backward_compatibility'); +const config_root = path.join(tmp_fs_path, 'config_root'); +const config_fs = new ConfigFS(config_root); + +const silent_options = { silent_if_missing: true }; +const accounts = { + '1': { + old: { _id: '1', name: 'backwards_account1', user: 'root' }, + new: { _id: '1', name: 'backwards_account1', user: 'staff' } + }, + '2': { + old: { _id: '2', name: 'backwards_account2', user: 'root' }, + new: { _id: '2', name: 'backwards_account2', user: 'staff' } + }, + '3': { + old: { _id: '3', name: 'backwards_account3', user: 'root' }, + new: { _id: '3', name: 'backwards_account3', user: 'staff' } + } +}; + +describe('ConfigFS Backwards Compatibility', () => { + const old_accounts_path = config_fs.old_accounts_dir_path; + + beforeAll(async () => { + await fs_utils.create_fresh_path(old_accounts_path); + await fs_utils.create_fresh_path(config_fs.identities_dir_path); + await fs_utils.create_fresh_path(config_fs.accounts_by_name_dir_path); + }); + + afterAll(async () => { + await fs_utils.folder_delete(config_root); + }); + + afterEach(async () => { + await clean_accounts(); + }); + + /////////////////////////// + /// GET ACCOUNT BY NAME /// + /////////////////////////// + + it('get_account_by_name() - {config_dir}/accounts_by_name/{account_name} is missing', async () => { + const old_account_data = accounts['1'].old; + await write_manual_old_account_config_file(config_fs, old_account_data); + const res = await config_fs.get_account_by_name(old_account_data.name, silent_options); + assert_account(res, old_account_data); + }); + + it('get_account_by_name() - {config_dir}/accounts_by_name/{account_name} exist', async () => { + const new_account_data = accounts['1'].new; + await write_manual_config_file(CONFIG_TYPES.ACCOUNT, config_fs, new_account_data); + const res = await config_fs.get_account_by_name(new_account_data.name, silent_options); + assert_account(res, new_account_data); + }); + + it('get_account_by_name() - both {config_dir}/accounts_by_name/{account_name}.symlink and {config_dir}/accounts/{account_name}.json missing', async () => { + const old_account_data = accounts['1'].old; + const new_account_data = accounts['1'].new; + await write_manual_old_account_config_file(config_fs, old_account_data); + await write_manual_config_file(CONFIG_TYPES.ACCOUNT, config_fs, new_account_data); + const res = await config_fs.get_account_by_name(new_account_data.name, silent_options); + assert_account(res, new_account_data); + }); + + it('get_account_by_name() - none exists', async () => { + const new_account_data = accounts['1'].new; + const res = await config_fs.get_account_by_name(new_account_data.name, silent_options); + expect(res).toBeUndefined(); + }); + + ///////////////////////// + /// IS ACCOUNT EXISTS /// + ///////////////////////// + + + it('is_account_exists() - {config_dir}/accounts_by_name/{account_name} is missing', async () => { + const old_account_data = accounts['1'].old; + await write_manual_old_account_config_file(config_fs, old_account_data); + const res = await config_fs.is_account_exists_by_name(old_account_data.name, undefined); + expect(res).toBe(true); + }); + + + it('is_account_exists() - {config_dir}/accounts_by_name/{account_name} exist', async () => { + const new_account_data = accounts['1'].new; + await write_manual_config_file(CONFIG_TYPES.ACCOUNT, config_fs, new_account_data); + const res = await config_fs.is_account_exists_by_name(new_account_data.name, undefined); + expect(res).toBe(true); + }); + + it('is_account_exists() - both {config_dir}/accounts_by_name/{account_name}.symlink and {config_dir}/accounts/{account_name}.json missing', async () => { + const old_account_data = accounts['1'].old; + const new_account_data = accounts['1'].new; + await write_manual_old_account_config_file(config_fs, old_account_data); + await write_manual_config_file(CONFIG_TYPES.ACCOUNT, config_fs, new_account_data); + const res = await config_fs.is_account_exists_by_name(new_account_data.name, undefined); + expect(res).toBe(true); + }); + + it('is_account_exists() - none exists', async () => { + const new_account_data = accounts['1'].new; + const res = await config_fs.is_account_exists_by_name(new_account_data.name, undefined); + expect(res).toBe(false); + }); + + + /////////////////////////// + /// LIST ACCOUNT EXISTS /// + /////////////////////////// + + it('list_accounts() - none exists', async () => { + const res = await config_fs.list_accounts(); + expect(res.length).toBe(0); + }); + + it('list_accounts() - only new exists', async () => { + await write_manual_config_file(CONFIG_TYPES.ACCOUNT, config_fs, accounts['1'].new); + await write_manual_config_file(CONFIG_TYPES.ACCOUNT, config_fs, accounts['2'].new); + await write_manual_config_file(CONFIG_TYPES.ACCOUNT, config_fs, accounts['3'].new); + + const res = await config_fs.list_accounts(); + expect(res.length).toBe(3); + }); + + it('list_accounts() - only old exists', async () => { + await write_manual_old_account_config_file(config_fs, accounts['1'].old); + await write_manual_old_account_config_file(config_fs, accounts['2'].old); + await write_manual_old_account_config_file(config_fs, accounts['3'].old); + const res = await config_fs.list_accounts(); + expect(res.length).toBe(3); + }); + + it('list_accounts() - both new and old exists', async () => { + await write_manual_config_file(CONFIG_TYPES.ACCOUNT, config_fs, accounts['1'].new); + await write_manual_config_file(CONFIG_TYPES.ACCOUNT, config_fs, accounts['2'].new); + await write_manual_config_file(CONFIG_TYPES.ACCOUNT, config_fs, accounts['3'].new); + await write_manual_old_account_config_file(config_fs, accounts['1'].old); + await write_manual_old_account_config_file(config_fs, accounts['2'].old); + await write_manual_old_account_config_file(config_fs, accounts['3'].old); + const res = await config_fs.list_accounts(); + expect(res.length).toBe(3); + }); +}); + +function assert_account(result_account, expected_account) { + expect(result_account._id).toBe(expected_account._id); + expect(result_account.name).toBe(expected_account.name); + expect(result_account.user).toBe(expected_account.user); +} + + +async function clean_accounts() { + for (const account of Object.values(accounts)) { + if (account.old) { + const old_account_path = config_fs._get_old_account_path_by_name(account.old.name); + await fs_utils.file_delete(old_account_path); + } + if (account.new) { + const new_account_path = config_fs.get_account_or_user_path_by_name(account.old.name); + const new_account_id_path = config_fs.get_identity_path_by_id(account.old._id); + const new_account_id_dir_path = config_fs.get_identity_dir_path_by_id(account.old._id); + await fs_utils.file_delete(new_account_path); + await fs_utils.file_delete(new_account_id_path); + await fs_utils.folder_delete(new_account_id_dir_path); + } + } +} diff --git a/src/test/unit_tests/jest_tests/test_nc_account_invalid_mkm_integration.test.js b/src/test/unit_tests/jest_tests/test_nc_account_invalid_mkm_integration.test.js index 1baab2fc14..5b8d04d84b 100644 --- a/src/test/unit_tests/jest_tests/test_nc_account_invalid_mkm_integration.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_account_invalid_mkm_integration.test.js @@ -5,11 +5,8 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const fs = require('fs'); -const _ = require('lodash'); const path = require('path'); -const P = require('../../../util/promise'); const fs_utils = require('../../../util/fs_utils'); -const { CONFIG_SUBDIRS } = require('../../../sdk/config_fs'); const { exec_manage_cli, set_path_permissions_and_owner, TMP_PATH, set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; @@ -270,8 +267,6 @@ function fail(reason) { } async function setup_nc_system_and_first_account() { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); const action = ACTIONS.ADD; diff --git a/src/test/unit_tests/jest_tests/test_nc_master_keys.test.js b/src/test/unit_tests/jest_tests/test_nc_master_keys.test.js index 6fb6177d2b..501b63cf02 100644 --- a/src/test/unit_tests/jest_tests/test_nc_master_keys.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_master_keys.test.js @@ -162,7 +162,7 @@ function validate_mkm_instance(nc_mkm_instance) { expect(nc_mkm_instance.active_master_key.id).toBeDefined(); expect(nc_mkm_instance.active_master_key.cipher_key).toBeDefined(); expect(nc_mkm_instance.active_master_key.cipher_iv).toBeDefined(); - expect(Object.entries(master_keys_cache).length === 1).toBeTruthy(); + expect(Object.entries(master_keys_cache).length === 1).toBe(true); expect(nc_mkm_instance.active_master_key).toEqual(active_master_key_by_cache); } diff --git a/src/test/unit_tests/jest_tests/test_nc_master_keys_exec.test.js b/src/test/unit_tests/jest_tests/test_nc_master_keys_exec.test.js index 4f7c871b7d..444927f836 100644 --- a/src/test/unit_tests/jest_tests/test_nc_master_keys_exec.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_master_keys_exec.test.js @@ -158,7 +158,7 @@ function validate_mkm_instance(nc_mkm_instance) { expect(nc_mkm_instance.active_master_key.id).toBeDefined(); expect(nc_mkm_instance.active_master_key.cipher_key).toBeDefined(); expect(nc_mkm_instance.active_master_key.cipher_iv).toBeDefined(); - expect(Object.entries(master_keys_cache).length === 1).toBeTruthy(); + expect(Object.entries(master_keys_cache).length === 1).toBe(true); expect(nc_mkm_instance.active_master_key).toEqual(active_master_key_by_cache); } diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js index 14becc27c8..179d5ea8b8 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_cli.test.js @@ -9,28 +9,25 @@ process.env.DISABLE_INIT_RANDOM_SEED = "true"; const fs = require('fs'); const _ = require('lodash'); const path = require('path'); -const P = require('../../../util/promise'); const os_util = require('../../../util/os_utils'); const fs_utils = require('../../../util/fs_utils'); -const nb_native = require('../../../util/nb_native'); -const { CONFIG_SUBDIRS, JSON_SUFFIX, SYMLINK_SUFFIX } = require('../../../sdk/config_fs'); +const { ConfigFS } = require('../../../sdk/config_fs'); const { set_path_permissions_and_owner, create_fs_user_by_platform, delete_fs_user_by_platform, TMP_PATH, set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); -const { get_process_fs_context, update_config_file } = require('../../../util/native_fs_utils'); const { TYPES, ACTIONS, ANONYMOUS } = require('../../../manage_nsfs/manage_nsfs_constants'); const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const ManageCLIResponse = require('../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; const tmp_fs_path = path.join(TMP_PATH, 'test_nc_nsfs_account_cli'); -const DEFAULT_FS_CONFIG = get_process_fs_context(); -const nc_mkm = require('../../../manage_nsfs/nc_master_key_manager').get_instance(); const timeout = 50000; const config = require('../../../../config'); +const config_fs_account_options = { show_secrets: true, decrypt_secret_key: true }; // eslint-disable-next-line max-lines-per-function describe('manage nsfs cli account flow', () => { describe('cli create account', () => { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); + const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); const defaults = { _id: 'account1', @@ -43,8 +40,6 @@ describe('manage nsfs cli account flow', () => { secret_key: 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsD+3oEXAMPLE', }; beforeEach(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); }); @@ -62,13 +57,13 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, false); const access_key = account.access_keys[0].access_key; const secret_key = account.access_keys[0].secret_key; expect(access_key).toBeDefined(); expect(secret_key).toBeDefined(); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); }); @@ -102,9 +97,9 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, true); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); }); @@ -148,9 +143,9 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, false); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); }); @@ -267,7 +262,7 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, false); expect(account.allow_bucket_creation).toBe(true); }); @@ -281,7 +276,7 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, false); expect(account.allow_bucket_creation).toBe(true); }); @@ -295,7 +290,7 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, false); expect(account.allow_bucket_creation).toBe(true); }); @@ -309,7 +304,7 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, false); expect(account.allow_bucket_creation).toBe(false); }); @@ -323,7 +318,7 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, false); expect(account.allow_bucket_creation).toBe(false); }); @@ -360,12 +355,12 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, false); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); - const real_path = fs.readlinkSync(path.join(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key + SYMLINK_SUFFIX)); - expect(real_path).toContain('../accounts/' + account_options.name + JSON_SUFFIX); + const real_path = fs.readlinkSync(config_fs.get_account_or_user_path_by_access_key(access_key)); + expect(real_path).toContain(config_fs.get_account_relative_path_by_id(account._id)); }); it('cli account add - use flag force_md5_etag', async function() { @@ -377,7 +372,7 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account.force_md5_etag).toBe(true); }); @@ -391,7 +386,7 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account.iam_operate_on_root_account).toBe(true); expect(account.allow_bucket_creation).toBe(true); }); @@ -406,7 +401,7 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account.iam_operate_on_root_account).toBe(false); expect(account.allow_bucket_creation).toBe(true); // by default it is inferred when we have new_buckets_path }); @@ -514,6 +509,7 @@ describe('manage nsfs cli account flow', () => { describe('cli update account', () => { describe('cli update account (has uid and gid)', () => { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs1'); + const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs1/'); const type = TYPES.ACCOUNT; const defaults = { @@ -526,8 +522,6 @@ describe('manage nsfs cli account flow', () => { }; beforeEach(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); const action = ACTIONS.ADD; @@ -547,13 +541,13 @@ describe('manage nsfs cli account flow', () => { it('cli regenerate account access_keys', async () => { const { name } = defaults; const account_options = { config_root, name, regenerate: true }; - const account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account_details = await config_fs.get_account_by_name(name, config_fs_account_options); const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + let new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account_details.access_keys[0].access_key).not.toBe(new_account_details.access_keys[0].access_key); const new_access_key = new_account_details.access_keys[0].access_key; - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, new_access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(new_access_key, config_fs_account_options); //fixing the new_account_details for compare. new_account_details = { ...new_account_details, ...new_account_details.nsfs_account_config }; assert_account(account_symlink, new_account_details); @@ -562,13 +556,13 @@ describe('manage nsfs cli account flow', () => { it('cli regenerate account access_keys with value "true"', async () => { const { name } = defaults; const account_options = { config_root, name, regenerate: 'true' }; - const account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account_details = await config_fs.get_account_by_name(name, config_fs_account_options); const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + let new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account_details.access_keys[0].access_key).not.toBe(new_account_details.access_keys[0].access_key); const new_access_key = new_account_details.access_keys[0].access_key; - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, new_access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(new_access_key, config_fs_account_options); //fixing the new_account_details for compare. new_account_details = { ...new_account_details, ...new_account_details.nsfs_account_config }; assert_account(account_symlink, new_account_details); @@ -577,13 +571,13 @@ describe('manage nsfs cli account flow', () => { it('cli regenerate account access_keys with value "TRUE" (case insensitive)', async () => { const { name } = defaults; const account_options = { config_root, name, regenerate: 'TRUE' }; - const account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account_details = await config_fs.get_account_by_name(name, config_fs_account_options); const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + let new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account_details.access_keys[0].access_key).not.toBe(new_account_details.access_keys[0].access_key); const new_access_key = new_account_details.access_keys[0].access_key; - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, new_access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(new_access_key, config_fs_account_options); //fixing the new_account_details for compare. new_account_details = { ...new_account_details, ...new_account_details.nsfs_account_config }; assert_account(account_symlink, new_account_details); @@ -592,20 +586,20 @@ describe('manage nsfs cli account flow', () => { it('cli regenerate account access_keys with value "false"', async () => { const { name } = defaults; const account_options = { config_root, name, regenerate: 'false' }; - const account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account_details = await config_fs.get_account_by_name(name, config_fs_account_options); const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - const new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account_details.access_keys[0].access_key).toBe(new_account_details.access_keys[0].access_key); }); it('cli regenerate account access_keys with value "FALSE" (case insensitive)', async () => { const { name } = defaults; const account_options = { config_root, name, regenerate: 'FALSE' }; - const account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account_details = await config_fs.get_account_by_name(name, config_fs_account_options); const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - const new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account_details.access_keys[0].access_key).toBe(new_account_details.access_keys[0].access_key); }); @@ -632,10 +626,10 @@ describe('manage nsfs cli account flow', () => { const action = ACTIONS.UPDATE; account_options.new_name = 'account1_new_name'; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, new_name); + let new_account_details = await config_fs.get_account_by_name(new_name, config_fs_account_options); expect(new_account_details.name).toBe(new_name); const access_key = new_account_details.access_keys[0].access_key; - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); //fixing the new_account_details for compare. new_account_details = { ...new_account_details, ...new_account_details.nsfs_account_config }; assert_account(account_symlink, new_account_details); @@ -648,7 +642,7 @@ describe('manage nsfs cli account flow', () => { let account_options = { config_root, name, new_name }; const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, new_name); + let new_account_details = await config_fs.get_account_by_name(new_name, config_fs_account_options); expect(new_account_details.name).toBe(new_name); // set the name as back as it was @@ -657,7 +651,7 @@ describe('manage nsfs cli account flow', () => { new_name = temp; account_options = { config_root, name, new_name }; await exec_manage_cli(type, action, account_options); - new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, new_name); + new_account_details = await config_fs.get_account_by_name(new_name, config_fs_account_options); expect(new_account_details.name).toBe(new_name); // set the name as undefined (not string) @@ -665,7 +659,7 @@ describe('manage nsfs cli account flow', () => { new_name = undefined; account_options = { config_root, name, new_name }; await exec_manage_cli(type, action, account_options); - new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, new_name); + new_account_details = await config_fs.get_account_by_name(new_name, config_fs_account_options); expect(new_account_details.name).toBe(String(new_name)); // set the name as back as it was @@ -674,7 +668,7 @@ describe('manage nsfs cli account flow', () => { new_name = temp; account_options = { config_root, name, new_name }; await exec_manage_cli(type, action, account_options); - new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, new_name); + new_account_details = await config_fs.get_account_by_name(new_name, config_fs_account_options); expect(new_account_details.name).toBe(new_name); }); @@ -684,16 +678,19 @@ describe('manage nsfs cli account flow', () => { const access_key = 'GIGiFAnjaaE7OKD5N7hB'; const secret_key = 'U3AYaMpU3zRDcRFWmvzgQr9MoHIAsD+3oEXAMPLE'; const account_options = { config_root, name, new_name, access_key, secret_key }; - const account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account_details = await config_fs.get_account_by_name(name, config_fs_account_options); + await config_fs.get_account_by_access_key(defaults.access_key, config_fs_account_options); + const action = ACTIONS.UPDATE; account_options.new_name = 'account1_new_name'; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, new_name); + let new_account_details = await config_fs.get_account_by_name(new_name, config_fs_account_options); + expect(new_account_details.name).toBe(new_name); expect(account_details.access_keys[0].access_key).not.toBe(new_account_details.access_keys[0].access_key); expect(account_details.access_keys[0].secret_key).not.toBe(new_account_details.access_keys[0].secret_key); expect(new_account_details.access_keys[0].access_key).toBe(access_key); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); //fixing the new_account_details for compare. new_account_details = { ...new_account_details, ...new_account_details.nsfs_account_config }; assert_account(account_symlink, new_account_details); @@ -704,14 +701,14 @@ describe('manage nsfs cli account flow', () => { const access_key = 'GIGiFAnjaaE7OKD5N7hB'; const secret_key = 'U3AYaMpU3zRDcRFWmvzgQr9MoHIAsD+3oEXAMPLE'; const account_options = { config_root, name, access_key, secret_key }; - const account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account_details = await config_fs.get_account_by_name(name, config_fs_account_options); const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + let new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account_details.access_keys[0].access_key).not.toBe(new_account_details.access_keys[0].access_key); expect(account_details.access_keys[0].secret_key).not.toBe(new_account_details.access_keys[0].secret_key); expect(new_account_details.access_keys[0].access_key).toBe(access_key); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); //fixing the new_account_details for compare. new_account_details = { ...new_account_details, ...new_account_details.nsfs_account_config }; assert_account(account_symlink, new_account_details); @@ -742,14 +739,14 @@ describe('manage nsfs cli account flow', () => { const account_options = { config_root, name, new_buckets_path: empty_string}; const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + let new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(new_account_details.nsfs_account_config.new_buckets_path).toBeUndefined(); expect(new_account_details.allow_bucket_creation).toBe(false); //set new_buckets_path value back to its original value account_options.new_buckets_path = defaults.new_buckets_path; await exec_manage_cli(type, action, account_options); - new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(new_account_details.nsfs_account_config.new_buckets_path).toBe(defaults.new_buckets_path); expect(new_account_details.allow_bucket_creation).toBe(true); }); @@ -763,13 +760,13 @@ describe('manage nsfs cli account flow', () => { const action = ACTIONS.UPDATE; account_options.new_name = 'account1_new_name'; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, new_name); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + let new_account_details = await config_fs.get_account_by_name(new_name, config_fs_account_options); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); //fixing the new_account_details for compare. new_account_details = { ...new_account_details, ...new_account_details.nsfs_account_config }; assert_account(account_symlink, new_account_details); - const real_path = fs.readlinkSync(path.join(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key + SYMLINK_SUFFIX)); - expect(real_path).toContain('../accounts/' + new_name + JSON_SUFFIX); + const real_path = fs.readlinkSync(config_fs.get_account_or_user_path_by_access_key(access_key)); + expect(real_path).toContain(config_fs.get_account_relative_path_by_id(new_account_details._id)); }); it('cli update account set flag force_md5_etag', async function() { @@ -777,12 +774,12 @@ describe('manage nsfs cli account flow', () => { const account_options = { config_root, name, force_md5_etag: 'true'}; const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + let new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(new_account_details.force_md5_etag).toBe(true); account_options.force_md5_etag = 'false'; await exec_manage_cli(type, action, account_options); - new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(new_account_details.force_md5_etag).toBe(false); }); @@ -792,14 +789,14 @@ describe('manage nsfs cli account flow', () => { const account_options = { config_root, name, force_md5_etag: 'true'}; const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + let new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(new_account_details.force_md5_etag).toBe(true); // unset force_md5_etag const empty_string = '\'\''; account_options.force_md5_etag = empty_string; await exec_manage_cli(type, action, account_options); - new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(new_account_details.force_md5_etag).toBeUndefined(); }); @@ -808,12 +805,12 @@ describe('manage nsfs cli account flow', () => { const account_options = { config_root, name, iam_operate_on_root_account: 'true'}; const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - let new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + let new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(new_account_details.iam_operate_on_root_account).toBe(true); account_options.iam_operate_on_root_account = 'false'; await exec_manage_cli(type, action, account_options); - new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(new_account_details.iam_operate_on_root_account).toBe(false); }); @@ -823,7 +820,7 @@ describe('manage nsfs cli account flow', () => { const account_options = { config_root, name, iam_operate_on_root_account: 'true'}; const action = ACTIONS.UPDATE; await exec_manage_cli(type, action, account_options); - const new_account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const new_account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(new_account_details.iam_operate_on_root_account).toBe(true); // unset iam_operate_on_root_account (is not allowed) @@ -853,7 +850,7 @@ describe('manage nsfs cli account flow', () => { it('should fail - cli update account iam_operate_on_root_account true when account owns IAM accounts', async function() { const { name } = defaults; - const accounts_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const accounts_details = await config_fs.get_account_by_name(name, config_fs_account_options); const account_id = accounts_details._id; const account_name = 'account-to-be-owned'; @@ -862,13 +859,9 @@ describe('manage nsfs cli account flow', () => { // update the account to have the property owner // (we use this way because now we don't have the way to create IAM users through the noobaa cli) - const account_config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_name + JSON_SUFFIX); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, account_config_path); - const config_data = JSON.parse(data.toString()); + const config_data = await config_fs.get_account_by_name(account_name, config_fs_account_options); config_data.owner = account_id; // just so we can identify this account as IAM user - await update_config_file(DEFAULT_FS_CONFIG, CONFIG_SUBDIRS.ACCOUNTS, - account_config_path, JSON.stringify(config_data)); - + await config_fs.update_account_config_file(config_data); // set the value of iam_operate_on_root_account to be true const account_options2 = { config_root, name, iam_operate_on_root_account: true}; const res = await exec_manage_cli(type, ACTIONS.UPDATE, account_options2); @@ -880,13 +873,9 @@ describe('manage nsfs cli account flow', () => { // update the account to have the property owner // (we use this way because now we don't have the way to create IAM users through the noobaa cli) const { name } = defaults; - const account_config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, name + JSON_SUFFIX); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, account_config_path); - const config_data = JSON.parse(data.toString()); + const config_data = await config_fs.get_account_by_name(name, config_fs_account_options); config_data.owner = '65a62e22ceae5e5f1a758aa9'; // just so we can identify this account as IAM user - await update_config_file(DEFAULT_FS_CONFIG, CONFIG_SUBDIRS.ACCOUNTS, - account_config_path, JSON.stringify(config_data)); - + await config_fs.update_account_config_file(config_data); // set the value of iam_operate_on_root_account to be true const account_options = { config_root, name, iam_operate_on_root_account: 'true'}; const action = ACTIONS.UPDATE; @@ -918,7 +907,7 @@ describe('manage nsfs cli account flow', () => { const account_options = { config_root, name, user: distinguished_name }; await set_path_permissions_and_owner(new_buckets_path, { uid: 0, gid: 0 }, 0o700); await exec_manage_cli(type, action, account_options); - const account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account_details.nsfs_account_config.uid).toBeUndefined(); expect(account_details.nsfs_account_config.gid).toBeUndefined(); expect(account_details.nsfs_account_config.distinguished_name).toBe(distinguished_name); @@ -950,7 +939,7 @@ describe('manage nsfs cli account flow', () => { it('cli update iam account with regenerate (only object access keys in index 0 changes)', async function() { const { name } = defaults; - const accounts_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const accounts_details = await config_fs.get_account_by_name(name, config_fs_account_options); const account_id = accounts_details._id; const account_name = 'account-to-be-owned'; @@ -959,16 +948,12 @@ describe('manage nsfs cli account flow', () => { // update the account to have the property owner // (we use this way because now we don't have the way to create IAM users through the noobaa cli) - const account_config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_name + JSON_SUFFIX); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, account_config_path); - const config_data = JSON.parse(data.toString()); + const config_data = await config_fs.get_account_by_name(account_name, config_fs_account_options); config_data.owner = account_id; // just so we can identify this account as IAM user // add additional properties to access_keys_object config_data.access_keys[0].creation_date = '2024-07-10T11:26:34.569Z'; config_data.access_keys[0].deactivated = false; - await update_config_file(DEFAULT_FS_CONFIG, CONFIG_SUBDIRS.ACCOUNTS, - account_config_path, JSON.stringify(config_data)); - + await config_fs.update_account_config_file(config_data); // regenerate (auto change the access keys in index 0 only) const account_options2 = { config_root, name, regenerate: true}; const res = await exec_manage_cli(type, ACTIONS.UPDATE, account_options2); @@ -979,7 +964,7 @@ describe('manage nsfs cli account flow', () => { it('cli update iam account with access_key and secret_key flag (only object access keys in index 0 changes)', async function() { const { name } = defaults; - const accounts_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const accounts_details = await config_fs.get_account_by_name(name, config_fs_account_options); const account_id = accounts_details._id; const account_name = 'account-to-be-owned'; @@ -988,16 +973,12 @@ describe('manage nsfs cli account flow', () => { // update the account to have the property owner // (we use this way because now we don't have the way to create IAM users through the noobaa cli) - const account_config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_name + JSON_SUFFIX); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, account_config_path); - const config_data = JSON.parse(data.toString()); + const config_data = await config_fs.get_account_by_name(account_name, config_fs_account_options); config_data.owner = account_id; // just so we can identify this account as IAM user // add additional properties to access_keys_object config_data.access_keys[0].creation_date = '2024-07-10T11:26:34.569Z'; config_data.access_keys[0].deactivated = false; - await update_config_file(DEFAULT_FS_CONFIG, CONFIG_SUBDIRS.ACCOUNTS, - account_config_path, JSON.stringify(config_data)); - + await config_fs.update_account_config_file(config_data); // update access_key and secret_key (change the access keys in index 0 only) const access_key = 'GIGiFAnjsaE7OKg5N7hB'; const secret_key = 'U3AYaMpU3zRDcRKWmvzgTr9MoHIAsD+3oEXAMPLE'; @@ -1027,6 +1008,7 @@ describe('manage nsfs cli account flow', () => { describe('cli update account (has distinguished name)', () => { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs2'); + const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs2/'); const type = TYPES.ACCOUNT; const distinguished_name = 'root'; @@ -1039,8 +1021,6 @@ describe('manage nsfs cli account flow', () => { }; beforeEach(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); const action = ACTIONS.ADD; @@ -1065,7 +1045,7 @@ describe('manage nsfs cli account flow', () => { const account_options = { config_root, name, uid, gid }; await set_path_permissions_and_owner(new_buckets_path, { uid, gid }, 0o700); await exec_manage_cli(type, action, account_options); - const account_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account_details = await config_fs.get_account_by_name(name, config_fs_account_options); expect(account_details.nsfs_account_config.uid).toBe(uid); expect(account_details.nsfs_account_config.gid).toBe(gid); expect(account_details.nsfs_account_config.distinguished_name).toBeUndefined(); @@ -1108,8 +1088,6 @@ describe('manage nsfs cli account flow', () => { }]; beforeAll(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); // Creating the account @@ -1360,6 +1338,7 @@ describe('manage nsfs cli account flow', () => { describe('cli delete account', () => { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs4'); + const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs4/'); const defaults = { type: TYPES.ACCOUNT, @@ -1372,8 +1351,6 @@ describe('manage nsfs cli account flow', () => { }; beforeEach(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS, CONFIG_SUBDIRS.BUCKETS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); }); @@ -1387,8 +1364,8 @@ describe('manage nsfs cli account flow', () => { await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(account_options.new_buckets_path, account_options, 0o700); await exec_manage_cli(type, action, account_options); - const config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, name + JSON_SUFFIX); - await fs_utils.file_must_exist(config_path); + const is_account_exists = await config_fs.is_account_exists_by_name(name); + expect(is_account_exists).toBe(true); }); afterEach(async () => { @@ -1406,10 +1383,10 @@ describe('manage nsfs cli account flow', () => { const res = await exec_manage_cli(type, action, account_options); const res_json = JSON.parse(res.trim()); expect(res_json.response.code).toBe(ManageCLIResponse.AccountDeleted.code); - const config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, name + JSON_SUFFIX); - await fs_utils.file_must_not_exist(config_path); - const symlink_config_path = path.join(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, defaults.access_key + SYMLINK_SUFFIX); - await fs_utils.file_must_not_exist(symlink_config_path); + const account_by_name_exists = await config_fs.is_account_exists_by_name(name); + expect(account_by_name_exists).toBe(false); + const account_by_access_key_exists = await config_fs.is_account_exists_by_access_key(defaults.access_key); + expect(account_by_access_key_exists).toBe(false); }); it('cli delete account (account has 2 access keys objects)', async () => { @@ -1419,7 +1396,7 @@ describe('manage nsfs cli account flow', () => { // currently we don't have the ability to create 2 access keys in the noobaa-cli // therefore, we will mock the config as there are 2 access keys objects // and also create the symlink for the one that manually added - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); const account_with_2_access_keys_objects = _.cloneDeep(account); const access_key2 = 'AIGiFAnjaaE7OKD5N7hA'; const secret_key2 = 'A2AYaMpU3zRDcRFWmvzgQr9MoHIAsD+3AEXAMPLE'; @@ -1429,12 +1406,10 @@ describe('manage nsfs cli account flow', () => { creation_date: new Date().toISOString(), deactivated: false, }); - const account_config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, name + JSON_SUFFIX); - await update_config_file(DEFAULT_FS_CONFIG, CONFIG_SUBDIRS.ACCOUNTS, - account_config_path, JSON.stringify(account_with_2_access_keys_objects)); - const account_config_relative_path = path.join(config_root, '../' + CONFIG_SUBDIRS.ACCOUNTS + '/', name + JSON_SUFFIX); - const account_config_access_key_path = path.join(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key2 + SYMLINK_SUFFIX); - await nb_native().fs.symlink(DEFAULT_FS_CONFIG, account_config_relative_path, account_config_access_key_path); + await config_fs.update_account_config_file( + account_with_2_access_keys_objects, + { new_access_keys_to_link: [{ access_key: access_key2 }] } + ); // cli delete account const action = ACTIONS.DELETE; @@ -1442,12 +1417,12 @@ describe('manage nsfs cli account flow', () => { const res = await exec_manage_cli(type, action, account_options); const res_json = JSON.parse(res.trim()); expect(res_json.response.code).toBe(ManageCLIResponse.AccountDeleted.code); - const config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, name + JSON_SUFFIX); - await fs_utils.file_must_not_exist(config_path); - const symlink_config_path1 = path.join(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, defaults.access_key + SYMLINK_SUFFIX); - await fs_utils.file_must_not_exist(symlink_config_path1); - const symlink_config_path2 = path.join(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key2 + SYMLINK_SUFFIX); - await fs_utils.file_must_not_exist(symlink_config_path2); + const name_exists = await config_fs.is_account_exists_by_name(name); + expect(name_exists).toBe(false); + const access_key1_exists = await config_fs.is_account_exists_by_access_key(defaults.access_key); + expect(access_key1_exists).toBe(false); + const access_key2_exists = await config_fs.is_account_exists_by_access_key(access_key2); + expect(access_key2_exists).toBe(false); }); it('should fail - cli delete account, account owns a bucket', async () => { @@ -1461,8 +1436,8 @@ describe('manage nsfs cli account flow', () => { const account_name = defaults.name; const bucket_options = { config_root, path: new_buckets_path, name: bucket_name, owner: account_name}; await exec_manage_cli(type, action, bucket_options); - let config_path = path.join(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_name + JSON_SUFFIX); - await fs_utils.file_must_exist(config_path); + const is_bucket_exists = await config_fs.is_bucket_exists(bucket_name); + expect(is_bucket_exists).toBe(true); // cli delete account type = TYPES.ACCOUNT; @@ -1471,8 +1446,8 @@ describe('manage nsfs cli account flow', () => { const account_options = { config_root, name }; const res = await exec_manage_cli(type, action, account_options); expect(JSON.parse(res.stdout).error.code).toBe(ManageCLIError.AccountDeleteForbiddenHasBuckets.code); - config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, name + JSON_SUFFIX); - await fs_utils.file_must_exist(config_path); + const is_account_exists = await config_fs.is_account_exists_by_name(name); + expect(is_account_exists).toBe(true); }); it('should fail - cli delete account invalid option (lala)', async () => { @@ -1502,7 +1477,7 @@ describe('manage nsfs cli account flow', () => { it('should fail - cli account delete - root account has IAM accounts', async function() { const { name, type } = defaults; - const accounts_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const accounts_details = await config_fs.get_account_by_name(name, config_fs_account_options); const account_id = accounts_details._id; const account_name = 'account-to-be-owned'; @@ -1511,13 +1486,9 @@ describe('manage nsfs cli account flow', () => { // update the account to have the property owner // (we use this way because now we don't have the way to create IAM users through the noobaa cli) - const account_config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_name + JSON_SUFFIX); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, account_config_path); - const config_data = JSON.parse(data.toString()); + const config_data = await config_fs.get_account_by_name(account_name, config_fs_account_options); config_data.owner = account_id; // just so we can identify this account as IAM user; - await update_config_file(DEFAULT_FS_CONFIG, CONFIG_SUBDIRS.ACCOUNTS, - account_config_path, JSON.stringify(config_data)); - + await config_fs.update_account_config_file(config_data); const action = ACTIONS.DELETE; const account_options2 = { config_root, name }; const res = await exec_manage_cli(type, action, account_options2); @@ -1540,8 +1511,6 @@ describe('manage nsfs cli account flow', () => { }; beforeEach(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); const action = ACTIONS.ADD; @@ -1577,6 +1546,7 @@ describe('manage nsfs cli account flow', () => { describe('cli create account using from_file', () => { const type = TYPES.ACCOUNT; const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs6'); + const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs6/'); const path_to_json_account_options_dir = path.join(tmp_fs_path, 'options'); @@ -1590,8 +1560,6 @@ describe('manage nsfs cli account flow', () => { }; beforeEach(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); set_nc_config_dir_in_config(config_root); await fs_utils.create_fresh_path(root_path); await fs_utils.create_fresh_path(path_to_json_account_options_dir); @@ -1619,7 +1587,7 @@ describe('manage nsfs cli account flow', () => { // create the account and check the details await exec_manage_cli(type, action, command_flags); // compare the details - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, false); }); @@ -1633,7 +1601,7 @@ describe('manage nsfs cli account flow', () => { // create the account and check the details await exec_manage_cli(type, action, command_flags); // compare the details - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options, true); expect(account.access_keys[0].access_key).toBe(access_key); expect(account.access_keys[0].secret_key).toBe(secret_key); @@ -1994,22 +1962,6 @@ describe('cli account flow distinguished_name - permissions', function() { }, timeout); }); -/** - * read_config_file will read the config files - * @param {string} config_root - * @param {string} schema_dir - * @param {string} config_file_name the name of the config file - * @param {boolean} [is_symlink] a flag to set the suffix as a symlink instead of json - */ -async function read_config_file(config_root, schema_dir, config_file_name, is_symlink) { - const config_path = path.join(config_root, schema_dir, config_file_name + (is_symlink ? SYMLINK_SUFFIX : JSON_SUFFIX)); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, config_path); - const config_data = JSON.parse(data.toString()); - const encrypted_secret_key = config_data.access_keys[0].encrypted_secret_key; - config_data.access_keys[0].secret_key = await nc_mkm.decrypt(encrypted_secret_key, config_data.master_key_id); - delete config_data.access_keys[0].encrypted_secret_key; - return config_data; -} /** * assert_account will verify the fields of the accounts diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js index 2327b2fdf5..5a4cb3a303 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_account_schema_validation.test.js @@ -518,7 +518,7 @@ function assert_validation(account_to_validate, reason, basic_message) { } catch (err) { expect(err).toBeInstanceOf(RpcError); expect(err).toHaveProperty('message'); - expect((err.message).includes(basic_message)).toBeTruthy(); + expect((err.message).includes(basic_message)).toBe(true); } } diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_anonymous_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_anonymous_cli.test.js index 7be0f04146..bc1f93ec08 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_anonymous_cli.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_anonymous_cli.test.js @@ -4,28 +4,25 @@ // disabling init_rand_seed as it takes longer than the actual test execution process.env.DISABLE_INIT_RANDOM_SEED = "true"; -const _ = require('lodash'); const path = require('path'); -const P = require('../../../util/promise'); const os_util = require('../../../util/os_utils'); const fs_utils = require('../../../util/fs_utils'); -const nb_native = require('../../../util/nb_native'); -const { CONFIG_SUBDIRS, JSON_SUFFIX, SYMLINK_SUFFIX } = require('../../../sdk/config_fs'); +const { ConfigFS } = require('../../../sdk/config_fs'); const { TMP_PATH, set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); -const { get_process_fs_context } = require('../../../util/native_fs_utils'); const { TYPES, ACTIONS } = require('../../../manage_nsfs/manage_nsfs_constants'); const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const ManageCLIResponse = require('../../../manage_nsfs/manage_nsfs_cli_responses').ManageCLIResponse; const tmp_fs_path = path.join(TMP_PATH, 'test_nc_nsfs_anon_account_cli'); -const DEFAULT_FS_CONFIG = get_process_fs_context(); const config = require('../../../../config'); +const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); +const config_fs = new ConfigFS(config_root); +const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); + // eslint-disable-next-line max-lines-per-function describe('manage nsfs cli anonymous account flow', () => { describe('cli create anonymous account', () => { - const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); - const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); const defaults = { _id: 'account1', type: TYPES.ACCOUNT, @@ -36,8 +33,6 @@ describe('manage nsfs cli anonymous account flow', () => { gid: 999, }; beforeAll(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); config.NSFS_NC_CONF_DIR = config_root; @@ -53,7 +48,7 @@ describe('manage nsfs cli anonymous account flow', () => { const { type, uid, gid, anonymous } = defaults; const account_options = { anonymous, config_root, uid, gid }; await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + const account = await config_fs.get_account_by_name(config.ANONYMOUS_ACCOUNT_NAME); assert_account(account, account_options); }); @@ -116,7 +111,7 @@ describe('manage nsfs cli anonymous account flow', () => { action = ACTIONS.ADD; account_options = { anonymous, config_root, user }; resp = await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + const account = await config_fs.get_account_by_name(config.ANONYMOUS_ACCOUNT_NAME); assert_account(account, account_options); }); @@ -131,8 +126,6 @@ describe('manage nsfs cli anonymous account flow', () => { }); describe('cli update anonymous account', () => { - const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); - const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); const defaults = { _id: 'account2', type: TYPES.ACCOUNT, @@ -143,8 +136,6 @@ describe('manage nsfs cli anonymous account flow', () => { gid: 999, }; beforeAll(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); config.NSFS_NC_CONF_DIR = config_root; @@ -160,14 +151,14 @@ describe('manage nsfs cli anonymous account flow', () => { let { type, uid, gid, anonymous } = defaults; const account_options = { anonymous, config_root, uid, gid }; await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + const account = await config_fs.get_account_by_name(config.ANONYMOUS_ACCOUNT_NAME); assert_account(account, account_options); action = ACTIONS.UPDATE; gid = 1001; const account_update_options = { anonymous, config_root, uid, gid }; await exec_manage_cli(type, action, account_update_options); - const update_account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + const update_account = await config_fs.get_account_by_name(config.ANONYMOUS_ACCOUNT_NAME); assert_account(update_account, account_update_options); }); @@ -191,8 +182,6 @@ describe('manage nsfs cli anonymous account flow', () => { }); describe('cli delete anonymous account', () => { - const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); - const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); const defaults = { _id: 'account2', type: TYPES.ACCOUNT, @@ -203,8 +192,6 @@ describe('manage nsfs cli anonymous account flow', () => { gid: 999, }; beforeAll(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); config.NSFS_NC_CONF_DIR = config_root; @@ -220,7 +207,7 @@ describe('manage nsfs cli anonymous account flow', () => { const { type, uid, gid, anonymous } = defaults; const account_options = { anonymous, config_root, uid, gid }; await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + const account = await config_fs.get_account_by_name(config.ANONYMOUS_ACCOUNT_NAME); assert_account(account, account_options); action = ACTIONS.DELETE; @@ -240,8 +227,6 @@ describe('manage nsfs cli anonymous account flow', () => { }); describe('cli status anonymous account', () => { - const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); - const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); const defaults = { _id: 'account4', type: TYPES.ACCOUNT, @@ -252,8 +237,6 @@ describe('manage nsfs cli anonymous account flow', () => { gid: 999, }; beforeAll(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); set_nc_config_dir_in_config(config_root); config.NSFS_NC_CONF_DIR = config_root; @@ -269,7 +252,7 @@ describe('manage nsfs cli anonymous account flow', () => { const { type, uid, gid, anonymous } = defaults; const account_options = { anonymous, config_root, uid, gid }; await exec_manage_cli(type, action, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, config.ANONYMOUS_ACCOUNT_NAME); + const account = await config_fs.get_account_by_name(config.ANONYMOUS_ACCOUNT_NAME); assert_account(account, account_options); action = ACTIONS.STATUS; @@ -282,20 +265,6 @@ describe('manage nsfs cli anonymous account flow', () => { }); -/** - * read_config_file will read the config files - * @param {string} config_root - * @param {string} schema_dir - * @param {string} config_file_name the name of the config file - * @param {boolean} [is_symlink] a flag to set the suffix as a symlink instead of json - */ -async function read_config_file(config_root, schema_dir, config_file_name, is_symlink) { - const config_path = path.join(config_root, schema_dir, config_file_name + (is_symlink ? SYMLINK_SUFFIX : JSON_SUFFIX)); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, config_path); - const config_data = JSON.parse(data.toString()); - return config_data; -} - /** * assert_account will verify the fields of the accounts * @param {object} account diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_cli.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_cli.test.js index aeb40d23bb..4e472dd182 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_cli.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_cli.test.js @@ -8,23 +8,24 @@ const fs = require('fs'); const path = require('path'); const os_util = require('../../../util/os_utils'); const fs_utils = require('../../../util/fs_utils'); -const nb_native = require('../../../util/nb_native'); -const { CONFIG_SUBDIRS, JSON_SUFFIX, SYMLINK_SUFFIX } = require('../../../sdk/config_fs'); +const { ConfigFS } = require('../../../sdk/config_fs'); const { set_path_permissions_and_owner, TMP_PATH, generate_s3_policy, set_nc_config_dir_in_config } = require('../../system_tests/test_utils'); const { ACTIONS, TYPES } = require('../../../manage_nsfs/manage_nsfs_constants'); -const { get_process_fs_context, is_path_exists, get_bucket_tmpdir_full_path, update_config_file } = require('../../../util/native_fs_utils'); +const { get_process_fs_context, is_path_exists, get_bucket_tmpdir_full_path } = require('../../../util/native_fs_utils'); const ManageCLIError = require('../../../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError; const { ManageCLIResponse } = require('../../../manage_nsfs/manage_nsfs_cli_responses'); const tmp_fs_path = path.join(TMP_PATH, 'test_nc_nsfs_bucket_cli'); const DEFAULT_FS_CONFIG = get_process_fs_context(); +const config_fs_account_options = { show_secrets: true, decrypt_secret_key: true }; // eslint-disable-next-line max-lines-per-function describe('manage nsfs cli bucket flow', () => { describe('cli create bucket', () => { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); + const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); const bucket_storage_path = path.join(tmp_fs_path, 'root_path_manage_nsfs', 'bucket1'); set_nc_config_dir_in_config(config_root); @@ -45,7 +46,6 @@ describe('manage nsfs cli bucket flow', () => { }; beforeEach(async () => { - await fs_utils.create_fresh_path(`${config_root}/${CONFIG_SUBDIRS.BUCKETS}`); await fs_utils.create_fresh_path(root_path); await fs_utils.create_fresh_path(bucket_storage_path); const action = ACTIONS.ADD; @@ -87,7 +87,7 @@ describe('manage nsfs cli bucket flow', () => { await fs_utils.file_must_exist(bucket_options.path); await set_path_permissions_and_owner(bucket_options.path, account_defaults, 0o700); await exec_manage_cli(TYPES.BUCKET, action, bucket_options); - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name); + const bucket = await config_fs.get_bucket_by_name(bucket_defaults.name); expect(bucket.force_md5_etag).toBe(true); }); @@ -128,7 +128,7 @@ describe('manage nsfs cli bucket flow', () => { await fs_utils.file_must_exist(bucket_options.path); await set_path_permissions_and_owner(bucket_options.path, account_defaults, 0o700); await exec_manage_cli(TYPES.BUCKET, action, bucket_options); - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name); + const bucket = await config_fs.get_bucket_by_name(bucket_defaults.name); assert_bucket(bucket, bucket_options); }); @@ -138,7 +138,7 @@ describe('manage nsfs cli bucket flow', () => { await fs_utils.create_fresh_path(bucket_options.path); await fs_utils.file_must_exist(bucket_options.path); set_nc_config_dir_in_config(config_root); - await create_json_file(config_root, 'config.json', { NC_DISABLE_ACCESS_CHECK: true }); + await config_fs.create_config_json_file(JSON.stringify({ NC_DISABLE_ACCESS_CHECK: true })); await set_path_permissions_and_owner(bucket_options.path, account_defaults, 0o000); const res = await exec_manage_cli(TYPES.BUCKET, action, bucket_options); expect(JSON.parse(res).response.code).toEqual(ManageCLIResponse.BucketCreated.code); @@ -170,7 +170,7 @@ describe('manage nsfs cli bucket flow', () => { it('should fail - cli create bucket - owner is an IAM account', async () => { const { name } = account_defaults; - const accounts_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const accounts_details = await config_fs.get_account_by_name(name); const account_id = accounts_details._id; const { new_buckets_path: account_path } = account_defaults; @@ -184,13 +184,9 @@ describe('manage nsfs cli bucket flow', () => { // update the account to have the property owner // (we use this way because now we don't have the way to create IAM users through the noobaa cli) - const account_config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_name + JSON_SUFFIX); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, account_config_path); - const config_data = JSON.parse(data.toString()); + const config_data = await config_fs.get_account_by_name(account_name, config_fs_account_options); config_data.owner = account_id; // just so we can identify this account as IAM user; - await update_config_file(DEFAULT_FS_CONFIG, CONFIG_SUBDIRS.ACCOUNTS, - account_config_path, JSON.stringify(config_data)); - + await config_fs.update_account_config_file(config_data); // create the bucket const action = ACTIONS.ADD; const bucket_options = { config_root, ...bucket_defaults, owner: account_name}; @@ -205,6 +201,7 @@ describe('manage nsfs cli bucket flow', () => { describe('cli create bucket using from_file', () => { const type = TYPES.BUCKET; const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs1'); + const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs1/'); const bucket_storage_path = path.join(tmp_fs_path, 'root_path_manage_nsfs1', 'bucket1'); const path_to_json_bucket_options_dir = path.join(tmp_fs_path, 'options'); @@ -225,7 +222,6 @@ describe('manage nsfs cli bucket flow', () => { }; beforeEach(async () => { - await fs_utils.create_fresh_path(`${config_root}/${CONFIG_SUBDIRS.BUCKETS}`); await fs_utils.create_fresh_path(root_path); await fs_utils.create_fresh_path(bucket_storage_path); await fs_utils.create_fresh_path(path_to_json_bucket_options_dir); @@ -259,7 +255,7 @@ describe('manage nsfs cli bucket flow', () => { // create the bucket and check the details await exec_manage_cli(type, action, command_flags); // compare the details - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name); + const bucket = await config_fs.get_bucket_by_name(bucket_defaults.name); assert_bucket(bucket, bucket_options); }); @@ -274,7 +270,7 @@ describe('manage nsfs cli bucket flow', () => { // create the bucket and check the details await exec_manage_cli(type, action, command_flags); // compare the details - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name); + const bucket = await config_fs.get_bucket_by_name(bucket_defaults.name); assert_bucket(bucket, bucket_options); expect(bucket.fs_backend).toEqual(bucket_options.fs_backend); }); @@ -290,7 +286,7 @@ describe('manage nsfs cli bucket flow', () => { // create the bucket and check the details await exec_manage_cli(type, action, command_flags); // compare the details - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name); + const bucket = await config_fs.get_bucket_by_name(bucket_defaults.name); assert_bucket(bucket, bucket_options); expect(bucket.bucket_policy).toEqual(bucket_options.s3_policy); }); @@ -375,6 +371,7 @@ describe('manage nsfs cli bucket flow', () => { describe('cli update bucket', () => { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs2'); + const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs2/'); const bucket_storage_path = path.join(tmp_fs_path, 'root_path_manage_nsfs2', 'bucket1'); set_nc_config_dir_in_config(config_root); @@ -403,7 +400,6 @@ describe('manage nsfs cli bucket flow', () => { }; beforeEach(async () => { - await fs_utils.create_fresh_path(`${config_root}/${CONFIG_SUBDIRS.BUCKETS}`); await fs_utils.create_fresh_path(root_path); await fs_utils.create_fresh_path(bucket_storage_path); const action = ACTIONS.ADD; @@ -444,12 +440,12 @@ describe('manage nsfs cli bucket flow', () => { const force_md5_etag = 'true'; const bucket_options = { config_root, name: bucket_defaults.name, force_md5_etag }; await exec_manage_cli(TYPES.BUCKET, action, bucket_options); - let bucket_config = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name); + let bucket_config = await config_fs.get_bucket_by_name(bucket_defaults.name); expect(bucket_config.force_md5_etag).toBe(true); bucket_options.force_md5_etag = 'false'; await exec_manage_cli(TYPES.BUCKET, action, bucket_options); - bucket_config = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name); + bucket_config = await config_fs.get_bucket_by_name(bucket_defaults.name); expect(bucket_config.force_md5_etag).toBe(false); }); @@ -459,14 +455,14 @@ describe('manage nsfs cli bucket flow', () => { const force_md5_etag = 'true'; const bucket_options = { config_root, name: bucket_defaults.name, force_md5_etag }; await exec_manage_cli(TYPES.BUCKET, action, bucket_options); - let bucket_config = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name); + let bucket_config = await config_fs.get_bucket_by_name(bucket_defaults.name); expect(bucket_config.force_md5_etag).toBe(true); // unset force_md5_etag const empty_string = '\'\''; bucket_options.force_md5_etag = empty_string; await exec_manage_cli(TYPES.BUCKET, action, bucket_options); - bucket_config = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name); + bucket_config = await config_fs.get_bucket_by_name(bucket_defaults.name); expect(bucket_config.force_md5_etag).toBeUndefined(); }); @@ -514,7 +510,7 @@ describe('manage nsfs cli bucket flow', () => { await fs_utils.file_must_exist(bucket_defaults.path); await set_path_permissions_and_owner(bucket_defaults.path, account_defaults2, 0o700); await exec_manage_cli(TYPES.BUCKET, action, bucket_options); - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name); + const bucket = await config_fs.get_bucket_by_name(bucket_defaults.name); expect(bucket.bucket_owner).toBe(account_defaults2.name); }); @@ -543,7 +539,7 @@ describe('manage nsfs cli bucket flow', () => { it('should fail - cli update bucket - owner is an IAM account', async () => { const { name } = account_defaults; - const accounts_details = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const accounts_details = await config_fs.get_account_by_name(name); const account_id = accounts_details._id; const { new_buckets_path: account_path } = account_defaults; @@ -557,13 +553,9 @@ describe('manage nsfs cli bucket flow', () => { // update the account to have the property owner // (we use this way because now we don't have the way to create IAM users through the noobaa cli) - const account_config_path = path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_name_iam_account + JSON_SUFFIX); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, account_config_path); - const config_data = JSON.parse(data.toString()); + const config_data = await config_fs.get_account_by_name(account_name_iam_account); config_data.owner = account_id; // just so we can identify this account as IAM user; - await update_config_file(DEFAULT_FS_CONFIG, CONFIG_SUBDIRS.ACCOUNTS, - account_config_path, JSON.stringify(config_data)); - + await config_fs.update_account_config_file(config_data); // update the bucket const action = ACTIONS.UPDATE; const bucket_options = { config_root, name: bucket_defaults.name, owner: account_name_iam_account}; @@ -577,6 +569,7 @@ describe('manage nsfs cli bucket flow', () => { describe('cli delete bucket', () => { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs3'); + const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs3/'); const bucket_storage_path = path.join(tmp_fs_path, 'root_path_manage_nsfs3', 'bucket1'); set_nc_config_dir_in_config(config_root); @@ -599,7 +592,6 @@ describe('manage nsfs cli bucket flow', () => { }; beforeEach(async () => { - await fs_utils.create_fresh_path(`${config_root}/${CONFIG_SUBDIRS.BUCKETS}`); await fs_utils.create_fresh_path(root_path); await fs_utils.create_fresh_path(bucket_storage_path); const action = ACTIONS.ADD; @@ -648,8 +640,8 @@ describe('manage nsfs cli bucket flow', () => { const delete_bucket_options = { config_root, name: bucket_defaults.name, force: true}; const resp = await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, delete_bucket_options); expect(JSON.parse(resp.trim()).response.code).toBe(ManageCLIResponse.BucketDeleted.code); - const config_path = path.join(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name + JSON_SUFFIX); - await fs_utils.file_must_not_exist(config_path); + const is_bucket_exists = await config_fs.is_bucket_exists(bucket_defaults.name); + expect(is_bucket_exists).toBe(false); }); it('should fail - cli delete bucket when bucket path is not empty', async () => { @@ -658,8 +650,8 @@ describe('manage nsfs cli bucket flow', () => { const delete_bucket_options = { config_root, name: bucket_defaults.name}; const resp = await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, delete_bucket_options); expect(JSON.parse(resp.stdout).error.code).toBe(ManageCLIError.BucketDeleteForbiddenHasObjects.code); - const config_path = path.join(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name + JSON_SUFFIX); - await fs_utils.file_must_exist(config_path); + const is_bucket_exists = await config_fs.is_bucket_exists(bucket_defaults.name); + expect(is_bucket_exists).toBe(true); }); it('cli delete bucket force flag with valid boolean string(\'true\')', async () => { @@ -669,8 +661,8 @@ describe('manage nsfs cli bucket flow', () => { const delete_bucket_options = { config_root, name: bucket_defaults.name, force: 'true'}; const resp = await exec_manage_cli(TYPES.BUCKET, ACTIONS.DELETE, delete_bucket_options); expect(JSON.parse(resp.trim()).response.code).toBe(ManageCLIResponse.BucketDeleted.code); - const config_path = path.join(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name + JSON_SUFFIX); - await fs_utils.file_must_not_exist(config_path); + const is_bucket_exists = await config_fs.is_bucket_exists(bucket_defaults.name); + expect(is_bucket_exists).toBe(false); }); it('should fail - cli delete bucket force flag with invalid boolean string(\'nottrue\')', async () => { @@ -694,8 +686,8 @@ describe('manage nsfs cli bucket flow', () => { const action = ACTIONS.DELETE; const res = await exec_manage_cli(TYPES.BUCKET, action, bucket_options); expect(JSON.parse(res.trim()).response.code).toBe(ManageCLIResponse.BucketDeleted.code); - const config_path = path.join(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_defaults.name + JSON_SUFFIX); - await fs_utils.file_must_not_exist(config_path); + const is_bucket_exists = await config_fs.is_bucket_exists(bucket_defaults.name); + expect(is_bucket_exists).toBe(false); }); }); @@ -722,7 +714,6 @@ describe('manage nsfs cli bucket flow', () => { }; beforeEach(async () => { - await fs_utils.create_fresh_path(`${config_root}/${CONFIG_SUBDIRS.BUCKETS}`); await fs_utils.create_fresh_path(root_path); await fs_utils.create_fresh_path(bucket_storage_path); const action = ACTIONS.ADD; @@ -782,7 +773,6 @@ describe('manage nsfs cli bucket flow', () => { }; beforeEach(async () => { - await fs_utils.create_fresh_path(`${config_root}/${CONFIG_SUBDIRS.BUCKETS}`); await fs_utils.create_fresh_path(root_path); await fs_utils.create_fresh_path(bucket_storage_path); const action = ACTIONS.ADD; @@ -859,19 +849,6 @@ async function exec_manage_cli(type, action, options) { return res; } -/** - * read_config_file will read the config files - * @param {string} config_root - * @param {string} schema_dir - * @param {string} config_file_name the name of the config file - * @param {boolean} [is_symlink] a flag to set the suffix as a symlink instead of json - */ -async function read_config_file(config_root, schema_dir, config_file_name, is_symlink) { - const config_path = path.join(config_root, schema_dir, config_file_name + (is_symlink ? SYMLINK_SUFFIX : JSON_SUFFIX)); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, config_path); - const config = JSON.parse(data.toString()); - return config; -} /** * create_json_bucket_options would create a JSON file with the options (key-value) inside file diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js index c1d7d9a7ad..4c105efdb4 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_bucket_schema_validation.test.js @@ -472,7 +472,7 @@ function assert_validation(bucket_to_validate, reason, basic_message) { } catch (err) { expect(err).toBeInstanceOf(RpcError); expect(err).toHaveProperty('message'); - expect((err.message).includes(basic_message)).toBeTruthy(); + expect((err.message).includes(basic_message)).toBe(true); } } diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_config_schema_validation.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_config_schema_validation.test.js index 08f93960ec..e0e8c3fffe 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_config_schema_validation.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_config_schema_validation.test.js @@ -371,7 +371,7 @@ function assert_validation(config_to_validate, reason, basic_message) { } catch (err) { expect(err).toBeInstanceOf(RpcError); expect(err).toHaveProperty('message'); - expect((err.message).includes(basic_message)).toBeTruthy(); + expect((err.message).includes(basic_message)).toBe(true); } } diff --git a/src/test/unit_tests/jest_tests/test_nc_nsfs_new_buckets_path_validation.test.js b/src/test/unit_tests/jest_tests/test_nc_nsfs_new_buckets_path_validation.test.js index 6db65639bb..50f5caecfd 100644 --- a/src/test/unit_tests/jest_tests/test_nc_nsfs_new_buckets_path_validation.test.js +++ b/src/test/unit_tests/jest_tests/test_nc_nsfs_new_buckets_path_validation.test.js @@ -239,8 +239,8 @@ async function path_accessible_to_account(path_to_check, account_data, is_access const fs_context = await get_fs_context(account_data); const accessible = await is_dir_rw_accessible(fs_context, path_to_check); if (is_accessible) { - expect(accessible).toBeTruthy(); + expect(accessible).toBe(true); } else { - expect(accessible).toBeFalsy(); + expect(accessible).toBe(false); } } diff --git a/src/test/unit_tests/nc_coretest.js b/src/test/unit_tests/nc_coretest.js index 22a2156b4b..2896a36d82 100644 --- a/src/test/unit_tests/nc_coretest.js +++ b/src/test/unit_tests/nc_coretest.js @@ -10,6 +10,7 @@ const argv = require('minimist')(process.argv); const SensitiveString = require('../../util/sensitive_string'); const { exec_manage_cli, TMP_PATH } = require('../system_tests/test_utils'); const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); +const { ConfigFS } = require('../../sdk/config_fs'); // keep me first - this is setting envs that should be set before other modules are required. const NC_CORETEST = 'nc_coretest'; @@ -54,10 +55,12 @@ const NC_CORETEST_REDIRECT_FILE_PATH = p.join(config.NSFS_NC_DEFAULT_CONF_DIR, ' const NC_CORETEST_STORAGE_PATH = p.join(TMP_PATH, '/nc_coretest_storage_root_path/'); const FIRST_BUCKET_PATH = p.join(NC_CORETEST_STORAGE_PATH, FIRST_BUCKET, '/'); const CONFIG_FILE_PATH = p.join(NC_CORETEST_CONFIG_DIR_PATH, 'config.json'); +const NC_CORETEST_CONFIG_FS = new ConfigFS(NC_CORETEST_CONFIG_DIR_PATH); const nsrs_to_root_paths = {}; let nsfs_process; + /** * setup will setup nc coretest * @param {object} options @@ -148,7 +151,6 @@ async function config_dir_setup() { VACCUM_ANALYZER_INTERVAL: 1 })); await fs.promises.mkdir(FIRST_BUCKET_PATH, { recursive: true }); - } /** @@ -253,7 +255,12 @@ function get_admin_mock_account_details() { * @return {Promise} */ async function create_account_manage(options) { - const cli_options = { + const cli_options = options.name === config.ANONYMOUS_ACCOUNT_NAME ? + { + anonymous: true, + uid: options.nsfs_account_config.uid, + gid: options.nsfs_account_config.gid, + } : { name: options.name, new_buckets_path: options.nsfs_account_config.new_buckets_path, distinguished_name: options.nsfs_account_config.distinguished_name, @@ -265,10 +272,12 @@ async function create_account_manage(options) { const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.ADD, cli_options); const json_account = JSON.parse(res); const account = json_account.response.reply; - account.access_keys[0] = { - access_key: new SensitiveString(account.access_keys[0].access_key), - secret_key: new SensitiveString(account.access_keys[0].secret_key) - }; + if (account.access_keys.length > 0) { + account.access_keys[0] = { + access_key: new SensitiveString(account.access_keys[0].access_key), + secret_key: new SensitiveString(account.access_keys[0].secret_key) + }; + } return account; } @@ -299,10 +308,12 @@ async function read_account_manage(options) { const res = await exec_manage_cli(TYPES.ACCOUNT, ACTIONS.STATUS, cli_options); const json_account = JSON.parse(res); const account = json_account.response.reply; - account.access_keys[0] = { - access_key: new SensitiveString(account.access_keys[0].access_key), - secret_key: new SensitiveString(account.access_keys[0].secret_key) - }; + if (account.access_keys.length > 0) { + account.access_keys[0] = { + access_key: new SensitiveString(account.access_keys[0].access_key), + secret_key: new SensitiveString(account.access_keys[0].secret_key) + }; + } return account; } @@ -471,3 +482,4 @@ exports.get_http_address = get_http_address; exports.get_https_address = get_https_address; exports.get_admin_mock_account_details = get_admin_mock_account_details; exports.NC_CORETEST_CONFIG_DIR_PATH = NC_CORETEST_CONFIG_DIR_PATH; +exports.NC_CORETEST_CONFIG_FS = NC_CORETEST_CONFIG_FS; diff --git a/src/test/unit_tests/test_bucketspace.js b/src/test/unit_tests/test_bucketspace.js index 95d49f645b..0551902534 100644 --- a/src/test/unit_tests/test_bucketspace.js +++ b/src/test/unit_tests/test_bucketspace.js @@ -14,7 +14,6 @@ const mocha = require('mocha'); const assert = require('assert'); const config = require('../../../config'); const fs_utils = require('../../util/fs_utils'); -const { JSON_SUFFIX } = require('../../sdk/config_fs'); const test_utils = require('../system_tests/test_utils'); const { stat, open } = require('../../util/nb_native')().fs; const { get_process_fs_context } = require('../../util/native_fs_utils'); @@ -30,11 +29,10 @@ const { S3 } = require('@aws-sdk/client-s3'); const { NodeHttpHandler } = require("@smithy/node-http-handler"); const native_fs_utils = require('../../util/native_fs_utils'); const mongo_utils = require('../../util/mongo_utils'); -const accounts_dir_name = '/accounts'; const coretest_path = get_coretest_path(); const coretest = require(coretest_path); -const { rpc_client, EMAIL, PASSWORD, SYSTEM, get_admin_mock_account_details, NC_CORETEST_CONFIG_DIR_PATH } = coretest; +const { rpc_client, EMAIL, PASSWORD, SYSTEM, get_admin_mock_account_details, NC_CORETEST_CONFIG_FS } = coretest; coretest.setup({}); let CORETEST_ENDPOINT; const inspect = (x, max_arr = 5) => util.inspect(x, { colors: true, depth: null, maxArrayLength: max_arr }); @@ -1778,8 +1776,8 @@ mocha.describe('Namespace s3_bucket_policy', function() { let accounts_dir_path; let account_config_path; if (is_nc_coretest) { - accounts_dir_path = path.join(NC_CORETEST_CONFIG_DIR_PATH, accounts_dir_name); - account_config_path = path.join(accounts_dir_path, config.ANONYMOUS_ACCOUNT_NAME + JSON_SUFFIX); + accounts_dir_path = NC_CORETEST_CONFIG_FS.accounts_by_name_dir_path; + account_config_path = NC_CORETEST_CONFIG_FS.get_account_or_user_path_by_name(config.ANONYMOUS_ACCOUNT_NAME); } mocha.before(async function() { diff --git a/src/test/unit_tests/test_bucketspace_fs.js b/src/test/unit_tests/test_bucketspace_fs.js index fe8ee0970f..e29230354e 100644 --- a/src/test/unit_tests/test_bucketspace_fs.js +++ b/src/test/unit_tests/test_bucketspace_fs.js @@ -18,7 +18,7 @@ const SensitiveString = require('../../util/sensitive_string'); const NamespaceFS = require('../../sdk/namespace_fs'); const BucketSpaceFS = require('../../sdk/bucketspace_fs'); const { TMP_PATH } = require('../system_tests/test_utils'); -const { CONFIG_SUBDIRS, JSON_SUFFIX, SYMLINK_SUFFIX } = require('../../sdk/config_fs'); +const { CONFIG_SUBDIRS, JSON_SUFFIX } = require('../../sdk/config_fs'); const nc_mkm = require('../../manage_nsfs/nc_master_key_manager').get_instance(); @@ -266,16 +266,22 @@ mocha.describe('bucketspace_fs', function() { }; mocha.before(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.ACCESS_KEYS, CONFIG_SUBDIRS.BUCKETS], async dir => + await P.all(_.map([CONFIG_SUBDIRS.IDENTITIES, + CONFIG_SUBDIRS.ACCOUNTS_BY_NAME, CONFIG_SUBDIRS.ACCESS_KEYS, CONFIG_SUBDIRS.BUCKETS], async dir => await fs_utils.create_fresh_path(`${config_root}/${dir}`)) ); await fs_utils.create_fresh_path(new_buckets_path); for (let account of [account_user1, account_user2, account_user3, account_iam_user1, account_iam_user2]) { account = await nc_mkm.encrypt_access_keys(account); - const account_path = get_config_file_path(CONFIG_SUBDIRS.ACCOUNTS, account.name); - const account_access_path = get_access_key_symlink_path(CONFIG_SUBDIRS.ACCESS_KEYS, account.access_keys[0].access_key); + const account_dir_path = bucketspace_fs.config_fs.get_identity_dir_path_by_id(account._id); + const account_path = bucketspace_fs.config_fs.get_identity_path_by_id(account._id); + const account_name_path = bucketspace_fs.config_fs.get_account_path_by_name(account.name); + const account_access_path = bucketspace_fs.config_fs.get_account_or_user_path_by_access_key(account.access_keys[0].access_key); + await fs.promises.mkdir(account_dir_path); await fs.promises.writeFile(account_path, JSON.stringify(account)); + await fs.promises.symlink(account_path, account_name_path); await fs.promises.symlink(account_path, account_access_path); + } }); mocha.after(async () => { @@ -810,8 +816,4 @@ function get_config_file_path(config_type_path, file_name) { return path.join(config_root, config_type_path, file_name + JSON_SUFFIX); } -// returns the path of the access_key symlink to the config file json -function get_access_key_symlink_path(config_type_path, file_name) { - return path.join(config_root, config_type_path, file_name + SYMLINK_SUFFIX); -} diff --git a/src/test/unit_tests/test_nc_nsfs_cli.js b/src/test/unit_tests/test_nc_nsfs_cli.js index 05c4d2a783..39e2b6281e 100644 --- a/src/test/unit_tests/test_nc_nsfs_cli.js +++ b/src/test/unit_tests/test_nc_nsfs_cli.js @@ -3,37 +3,34 @@ /*eslint max-statements: ["error", 90]*/ 'use strict'; -const _ = require('lodash'); const path = require('path'); const mocha = require('mocha'); const assert = require('assert'); -const P = require('../../util/promise'); const config = require('../../../config'); const fs_utils = require('../../util/fs_utils'); -const config_module = require('../../../config'); const nb_native = require('../../util/nb_native'); -const { CONFIG_SUBDIRS, JSON_SUFFIX, SYMLINK_SUFFIX } = require('../../sdk/config_fs'); +const { CONFIG_SUBDIRS, JSON_SUFFIX, SYMLINK_SUFFIX, ConfigFS } = require('../../sdk/config_fs'); const { get_process_fs_context } = require('../../util/native_fs_utils'); const { ManageCLIError } = require('../../manage_nsfs/manage_nsfs_cli_errors'); const { ManageCLIResponse } = require('../../manage_nsfs/manage_nsfs_cli_responses'); const { exec_manage_cli, generate_s3_policy, create_fs_user_by_platform, delete_fs_user_by_platform, set_path_permissions_and_owner, TMP_PATH, set_nc_config_dir_in_config } = require('../system_tests/test_utils'); const { TYPES, ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); -const nc_mkm = require('../../manage_nsfs/nc_master_key_manager').get_instance(); const tmp_fs_path = path.join(TMP_PATH, 'test_bucketspace_fs'); const DEFAULT_FS_CONFIG = get_process_fs_context(); +const config_fs_account_options = { show_secrets: true, decrypt_secret_key: true }; mocha.describe('manage_nsfs cli', function() { const config_root = path.join(tmp_fs_path, 'config_root_manage_nsfs'); const root_path = path.join(tmp_fs_path, 'root_path_manage_nsfs/'); set_nc_config_dir_in_config(config_root); + let config_fs; // TODO: needed for NC_CORETEST FLOW - should be handled better const nc_coretes_location = config.NC_MASTER_KEYS_FILE_LOCATION; mocha.before(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.BUCKETS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); + config_fs = new ConfigFS(config_root); await fs_utils.create_fresh_path(root_path); config.NC_MASTER_KEYS_FILE_LOCATION = ''; }); @@ -82,8 +79,6 @@ mocha.describe('manage_nsfs cli', function() { }; mocha.before(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.BUCKETS, CONFIG_SUBDIRS.ACCESS_KEYS], async dir => - fs_utils.create_fresh_path(`${config_root}/${dir}`))); await fs_utils.create_fresh_path(root_path); }); mocha.after(async () => { @@ -156,12 +151,12 @@ mocha.describe('manage_nsfs cli', function() { try { await fs_utils.create_fresh_path(bucket_path); await fs_utils.file_must_exist(bucket_path); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_name); + const account = await config_fs.get_account_by_name(account_name, config_fs_account_options); const account_options = { gid: account.nsfs_account_config.gid, uid: account.nsfs_account_config.uid, user: account.nsfs_account_config.distinguished_name, - new_buckets_path: account.nsfs_account_config.mew_buckets_path, + new_buckets_path: account.nsfs_account_config.new_buckets_path, }; await set_path_permissions_and_owner(bucket_path, account_options, 0o700); await exec_manage_cli(type, action, { ...bucket_options, bucket_policy: invalid_bucket_policy }); @@ -174,7 +169,7 @@ mocha.describe('manage_nsfs cli', function() { mocha.it('cli bucket create - bucket_with_policy', async function() { const action = ACTIONS.ADD; await exec_manage_cli(type, action, bucket_with_policy_options); - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_with_policy); + const bucket = await config_fs.get_bucket_by_name(bucket_with_policy); assert_bucket(bucket, bucket_with_policy_options); await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_with_policy); }); @@ -183,11 +178,11 @@ mocha.describe('manage_nsfs cli', function() { const action = ACTIONS.ADD; add_res = await exec_manage_cli(type, action, bucket_options); assert_response(action, type, add_res, bucket_options); - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, name); + const bucket = await config_fs.get_bucket_by_name(name); assert_bucket(bucket, bucket_options); assert(bucket._id !== undefined); // make sure that the config file includes id and owner_account (account id) - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_name); + const account = await config_fs.get_account_by_name(account_name, config_fs_account_options); assert(bucket.owner_account === account._id); await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.BUCKETS, name); }); @@ -207,7 +202,7 @@ mocha.describe('manage_nsfs cli', function() { const action = ACTIONS.STATUS; const bucket_status = await exec_manage_cli(type, action, { config_root, name }); assert_response(action, type, bucket_status, bucket_options); - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, name); + const bucket = await config_fs.get_bucket_by_name(name); assert_bucket(bucket, bucket_options); }); @@ -352,18 +347,18 @@ mocha.describe('manage_nsfs cli', function() { mocha.it('cli bucket update owner', async function() { const action = ACTIONS.UPDATE; - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_name2); + const account = await config_fs.get_account_by_name(account_name2, config_fs_account_options); const account_options = { gid: account.nsfs_account_config.gid, uid: account.nsfs_account_config.uid, user: account.nsfs_account_config.distinguished_name, - new_buckets_path: account.nsfs_account_config.mew_buckets_path, + new_buckets_path: account.nsfs_account_config.new_buckets_path, }; await set_path_permissions_and_owner(bucket_path, account_options, 0o700); const update_options = { config_root, name, owner: account_name2}; const update_res = await exec_manage_cli(type, action, update_options); bucket_options = { ...bucket_options, ...update_options }; - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, name); + const bucket = await config_fs.get_bucket_by_name(name); assert_bucket(bucket, bucket_options); await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.BUCKETS, name); assert.equal(JSON.parse(update_res).response.reply.creation_date, JSON.parse(add_res).response.reply.creation_date); @@ -384,7 +379,7 @@ mocha.describe('manage_nsfs cli', function() { const action = ACTIONS.UPDATE; bucket_options = { ...bucket_options, bucket_policy: bucket1_policy }; await exec_manage_cli(type, action, bucket_options); - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_options.name); + const bucket = await config_fs.get_bucket_by_name(bucket_options.name); assert_bucket(bucket, bucket_options); await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_options.name); }); @@ -396,7 +391,7 @@ mocha.describe('manage_nsfs cli', function() { // in the CLI we use empty string to unset the s3_policy // but as a parameter is it undefined property bucket_options.bucket_policy = undefined; - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_options.name); + const bucket = await config_fs.get_bucket_by_name(bucket_options.name); assert_bucket(bucket, bucket_options); await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_options.name); }); @@ -406,7 +401,7 @@ mocha.describe('manage_nsfs cli', function() { const update_options = { config_root, new_name: 'bucket2', name }; await exec_manage_cli(type, action, update_options); bucket_options = { ...bucket_options, ...update_options, new_name: undefined, name: update_options.new_name }; - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_options.name); + const bucket = await config_fs.get_bucket_by_name(bucket_options.name); assert_bucket(bucket, bucket_options); await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_options.name); }); @@ -429,7 +424,7 @@ mocha.describe('manage_nsfs cli', function() { try { const res = await exec_manage_cli(type, action, { config_root, name: bucket_options.name }); assert_response(action, type, res); - await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_options.name); + await config_fs.get_bucket_by_name(bucket_options.name); assert.fail('cli bucket delete failed - bucket config file exists after deletion'); } catch (err) { assert.equal(err.code, 'ENOENT'); @@ -454,7 +449,7 @@ mocha.describe('manage_nsfs cli', function() { await set_path_permissions_and_owner(bucket_on_gpfs_path, account_options1, 0o700); const bucket_status = await exec_manage_cli(type, action, gpfs_bucket_options); assert_response(action, type, bucket_status, gpfs_bucket_options); - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, gpfs_bucket_options.name); + const bucket = await config_fs.get_bucket_by_name(gpfs_bucket_options.name); assert_bucket(bucket, gpfs_bucket_options); await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.BUCKETS, gpfs_bucket_options.name); }); @@ -467,7 +462,7 @@ mocha.describe('manage_nsfs cli', function() { await set_path_permissions_and_owner(bucket_on_gpfs_path, account_options2, 0o700); const bucket_status = await exec_manage_cli(type, action, gpfs_bucket_options); assert_response(action, type, bucket_status, gpfs_bucket_options); - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, gpfs_bucket_options.name); + const bucket = await config_fs.get_bucket_by_name(gpfs_bucket_options.name); assert_bucket(bucket, gpfs_bucket_options); await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.BUCKETS, gpfs_bucket_options.name); }); @@ -480,7 +475,7 @@ mocha.describe('manage_nsfs cli', function() { // but as a parameter is it undefined property gpfs_bucket_options.fs_backend = undefined; assert_response(action, type, bucket_status, gpfs_bucket_options); - const bucket = await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, gpfs_bucket_options.name); + const bucket = await config_fs.get_bucket_by_name(gpfs_bucket_options.name); assert_bucket(bucket, gpfs_bucket_options); await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.BUCKETS, gpfs_bucket_options.name); }); @@ -490,7 +485,7 @@ mocha.describe('manage_nsfs cli', function() { try { const res = await exec_manage_cli(type, action, { config_root, name: gpfs_bucket_options.name }); assert_response(action, type, res); - await read_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, gpfs_bucket_options.name); + await config_fs.get_bucket_by_name(gpfs_bucket_options.name); assert.fail('cli bucket delete failed - bucket config file exists after deletion'); } catch (err) { assert.equal(err.code, 'ENOENT'); @@ -545,24 +540,29 @@ mocha.describe('manage_nsfs cli', function() { const gid = 999; const access_key = 'GIGiFAnjaaE7OKD5N7hA'; const secret_key = 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsDy3o+4h0oFR'; + const gpfs_access_key = 'GIGiFAnjaaE7OKD5N7h1'; + const gpfs_secret_key = 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsDy3oEXAMPLE'; let account_options = { config_root, name, new_buckets_path, uid, gid, access_key, secret_key }; - const gpfs_account_options = { ...account_options, name: gpfs_account, fs_backend: 'GPFS' }; + const gpfs_account_options = { + ...account_options, access_key: gpfs_access_key, secret_key: gpfs_secret_key, + name: gpfs_account, fs_backend: 'GPFS' + }; let updating_options = account_options; let compare_details; // we will use it for update account and compare the results let add_res; - mocha.it('cli account create', async function() { + mocha.it('cli account create 1', async function() { const action = ACTIONS.ADD; await fs_utils.create_fresh_path(new_buckets_path); await fs_utils.file_must_exist(new_buckets_path); await set_path_permissions_and_owner(new_buckets_path, { uid, gid }, 0o700); add_res = await exec_manage_cli(type, action, account_options); assert_response(action, type, add_res, account_options); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options); - await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.ACCOUNTS_BY_NAME, name, true); }); mocha.it('cli account status', async function() { @@ -571,9 +571,9 @@ mocha.describe('manage_nsfs cli', function() { assert_response(action, type, account_status, account_options); const access_key_account_status = await exec_manage_cli(type, action, { config_root, access_key: account_options.access_key }); assert_response(action, type, access_key_account_status, account_options); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options); }); @@ -591,9 +591,9 @@ mocha.describe('manage_nsfs cli', function() { const action = ACTIONS.STATUS; const account_status = await exec_manage_cli(type, action, { config_root, name: account_options.name, show_secrets: true }); assert_response(action, type, account_status, account_options, true); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options); }); @@ -601,9 +601,9 @@ mocha.describe('manage_nsfs cli', function() { const action = ACTIONS.STATUS; const account_status = await exec_manage_cli(type, action, { config_root, name: account_options.name, show_secrets: 'true' }); assert_response(action, type, account_status, account_options, true); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options); }); @@ -611,9 +611,9 @@ mocha.describe('manage_nsfs cli', function() { const action = ACTIONS.STATUS; const account_status = await exec_manage_cli(type, action, { config_root, name: account_options.name, show_secrets: 'TRUE' }); assert_response(action, type, account_status, account_options, true); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options); }); @@ -742,9 +742,9 @@ mocha.describe('manage_nsfs cli', function() { updating_options = { ...updating_options, ...update_options }; assert_response(action, type, update_response, updating_options); account_options = { ...account_options, ...update_options }; - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options); assert.equal(JSON.parse(update_response).response.reply.creation_date, JSON.parse(add_res).response.reply.creation_date); }); @@ -754,7 +754,7 @@ mocha.describe('manage_nsfs cli', function() { const res = await exec_manage_cli(type, action, { config_root, name: account_options.name }); assert_response(action, type, res); try { - await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, account_options.access_key, true); + await config_fs.get_account_by_access_key(account_options.access_key, config_fs_account_options); throw new Error('cli account delete failed - account config link file exists after deletion'); } catch (err) { if (err.code !== 'ENOENT') { @@ -762,7 +762,7 @@ mocha.describe('manage_nsfs cli', function() { } } try { - await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_options.name); + await config_fs.get_account_by_name(account_options.name, config_fs_account_options); throw new Error('cli account delete failed - account config file exists after deletion'); } catch (err) { if (err.code !== 'ENOENT') { @@ -775,9 +775,9 @@ mocha.describe('manage_nsfs cli', function() { const action = ACTIONS.ADD; const account_status = await exec_manage_cli(type, action, gpfs_account_options); assert_response(action, type, account_status, gpfs_account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, gpfs_account_options.name); + const account = await config_fs.get_account_by_name(gpfs_account_options.name, config_fs_account_options); assert_account(account, gpfs_account_options); - await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.ACCOUNTS, gpfs_account_options.name); + await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.ACCOUNTS_BY_NAME, gpfs_account_options.name, true); }); mocha.it('cli account update to non GPFS', async function() { @@ -796,9 +796,9 @@ mocha.describe('manage_nsfs cli', function() { // but as a parameter is it undefined property compare_details.fs_backend = undefined; assert_response(action, type, account_status, compare_details); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, gpfs_account_options.name); + const account = await config_fs.get_account_by_name(gpfs_account_options.name, config_fs_account_options); assert_account(account, compare_details); - await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.ACCOUNTS, gpfs_account_options.name); + await assert_config_file_permissions(config_root, CONFIG_SUBDIRS.ACCOUNTS_BY_NAME, gpfs_account_options.name, true); }); mocha.it('cli account delete', async function() { @@ -806,7 +806,7 @@ mocha.describe('manage_nsfs cli', function() { try { const res = await exec_manage_cli(type, action, { config_root, name: gpfs_account_options.name }); assert_response(action, type, res); - await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, gpfs_account_options.name); + await config_fs.get_account_by_name(gpfs_account_options.name, config_fs_account_options); assert.fail('cli account delete failed - account config file exists after deletion'); } catch (err) { assert.equal(err.code, 'ENOENT'); @@ -822,8 +822,8 @@ mocha.describe('manage_nsfs cli', function() { const new_buckets_path = `${root_path}new_buckets_path_user1/`; const uid = 999; const gid = 999; - const access_key = 'GIGiFAnjaaE7OKD5N7hA'; - const secret_key = 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsDy3o+4h0oFR'; + const access_key = 'GIGiFAnjaaE7OKD5N7h2'; + const secret_key = 'U2AYaMpU3zRDcRFWmvzgQr9MoHIAsDy3oEXAMPLE'; const account1_options = { config_root, name: name1, new_buckets_path, uid, gid, access_key, secret_key }; const account1_options_for_delete = { config_root, name: name1 }; const account2_options = { config_root, name: 'account2', new_buckets_path, uid, gid, access_key: 'BISiDSnjaaE7OKD5N7hB', secret_key }; @@ -856,7 +856,7 @@ mocha.describe('manage_nsfs cli', function() { mocha.it('cli account2 update - new_access_key already exists', async function() { const action = ACTIONS.UPDATE; const options = { ...account2_options }; - options.access_key = 'GIGiFAnjaaE7OKD5N7hA'; + options.access_key = 'GIGiFAnjaaE7OKD5N7h2'; try { await exec_manage_cli(type, action, options); assert.fail('should have failed with account access key already exists'); @@ -893,15 +893,15 @@ mocha.describe('manage_nsfs cli', function() { await delete_fs_user_by_platform(new_user); }); - mocha.it('cli account create', async function() { + mocha.it('cli account create 2', async function() { const action = ACTIONS.ADD; await fs_utils.create_fresh_path(new_buckets_path); await fs_utils.file_must_exist(new_buckets_path); const res = await exec_manage_cli(type, action, account_options); assert_response(action, type, res, account_options); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options); }); @@ -916,9 +916,9 @@ mocha.describe('manage_nsfs cli', function() { const res = await exec_manage_cli(type, action, update_options); account_options = { ...account_options, ...update_options }; assert_response(action, type, res, account_options); - const account_symlink = await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + const account_symlink = await config_fs.get_account_by_access_key(access_key, config_fs_account_options); assert_account(account_symlink, account_options); - const account = await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, name); + const account = await config_fs.get_account_by_name(name, config_fs_account_options); assert_account(account, account_options); }); @@ -927,7 +927,7 @@ mocha.describe('manage_nsfs cli', function() { const res = await exec_manage_cli(type, action, { config_root, name: account_options.name }); assert_response(action, type, res); try { - await read_config_file(config_root, CONFIG_SUBDIRS.ACCESS_KEYS, access_key, true); + await config_fs.get_account_by_access_key(access_key, config_fs_account_options); throw new Error('cli account delete failed - account config file exists after deletion'); } catch (err) { if (err.code !== 'ENOENT') { @@ -935,7 +935,7 @@ mocha.describe('manage_nsfs cli', function() { } } try { - await read_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_options.name); + await config_fs.get_account_by_name(account_options.name, config_fs_account_options); throw new Error('cli account delete failed - account config file exists after deletion'); } catch (err) { if (err.code !== 'ENOENT') { @@ -950,17 +950,18 @@ mocha.describe('manage_nsfs cli', function() { const type = TYPES.IP_WHITELIST; const config_options = { ENDPOINT_FORKS: 1, UV_THREADPOOL_SIZE: 4 }; mocha.before(async () => { - await write_config_file(config_root, '', 'config', config_options); + await config_fs.create_config_json_file(JSON.stringify(config_options)); }); mocha.after(async () => { - await fs_utils.file_delete(path.join(config_root, 'config.json')); + const config_file_path = config_fs.get_config_json_path(); + await fs_utils.file_delete(config_file_path); }); mocha.it('cli add whitelist ips first time (IPV4 format)', async function() { const ips = ['127.0.0.1']; // IPV4 format const res = await exec_manage_cli(type, '', { config_root, ips: JSON.stringify(ips) }); config_options.S3_SERVER_IP_WHITELIST = ips; - const config_data = await read_config_file(config_root, '', 'config'); + const config_data = await config_fs.get_config_json(); assert_response('', type, res, ips); assert_whitelist(config_data, config_options); }); @@ -969,7 +970,7 @@ mocha.describe('manage_nsfs cli', function() { const ips = ['0000:0000:0000:0000:0000:ffff:7f00:0002']; // IPV6 expanded format const res = await exec_manage_cli(type, '', { config_root, ips: JSON.stringify(ips) }); config_options.S3_SERVER_IP_WHITELIST = ips; - const config_data = await read_config_file(config_root, '', 'config'); + const config_data = await config_fs.get_config_json(); assert_response('', type, res, ips); assert_whitelist(config_data, config_options); }); @@ -978,7 +979,7 @@ mocha.describe('manage_nsfs cli', function() { const ips = ['::ffff:7f00:3']; // IPV6 compressed format const res = await exec_manage_cli(type, '', { config_root, ips: JSON.stringify(ips) }); config_options.S3_SERVER_IP_WHITELIST = ips; - const config_data = await read_config_file(config_root, '', 'config'); + const config_data = await config_fs.get_config_json(); assert_response('', type, res, ips); assert_whitelist(config_data, config_options); }); @@ -1042,25 +1043,6 @@ mocha.describe('manage_nsfs cli', function() { }); -async function read_config_file(config_root, schema_dir, config_file_name, is_symlink) { - const config_path = path.join(config_root, schema_dir, config_file_name + (is_symlink ? SYMLINK_SUFFIX : JSON_SUFFIX)); - const { data } = await nb_native().fs.readFile(DEFAULT_FS_CONFIG, config_path); - const config_data = JSON.parse(data.toString()); - if (config_data.access_keys) { - const encrypted_secret_key = config_data.access_keys[0].encrypted_secret_key; - config_data.access_keys[0].secret_key = await nc_mkm.decrypt(encrypted_secret_key, config_data.master_key_id); - delete config_data.access_keys[0].encrypted_secret_key; - } - return config_data; -} - -async function write_config_file(config_root, schema_dir, config_file_name, data, is_symlink) { - const config_path = path.join(config_root, schema_dir, config_file_name + (is_symlink ? SYMLINK_SUFFIX : JSON_SUFFIX)); - await nb_native().fs.writeFile(DEFAULT_FS_CONFIG, config_path, - Buffer.from(JSON.stringify(data)), { - mode: config_module.BASE_MODE_FILE, - }); -} async function assert_config_file_permissions(config_root, schema_dir, config_file_name, is_symlink) { const config_path = path.join(config_root, schema_dir, config_file_name + (is_symlink ? SYMLINK_SUFFIX : JSON_SUFFIX)); diff --git a/src/test/unit_tests/test_nc_nsfs_health.js b/src/test/unit_tests/test_nc_nsfs_health.js index e0bd6a4a86..6ea11b7c3c 100644 --- a/src/test/unit_tests/test_nc_nsfs_health.js +++ b/src/test/unit_tests/test_nc_nsfs_health.js @@ -3,20 +3,17 @@ 'use strict'; -const _ = require('lodash'); const path = require('path'); const mocha = require('mocha'); const sinon = require('sinon'); const assert = require('assert'); -const P = require('../../util/promise'); -const config = require('../../../config'); const { ConfigFS } = require('../../sdk/config_fs'); const NSFSHealth = require('../../manage_nsfs/health').NSFSHealth; const fs_utils = require('../../util/fs_utils'); +const test_utils = require('../system_tests/test_utils'); const nb_native = require('../../util/nb_native'); -const { CONFIG_SUBDIRS, JSON_SUFFIX } = require('../../sdk/config_fs'); const { TYPES, DIAGNOSE_ACTIONS } = require('../../manage_nsfs/manage_nsfs_constants'); -const { get_process_fs_context, get_umasked_mode } = require('../../util/native_fs_utils'); +const { get_process_fs_context } = require('../../util/native_fs_utils'); const { TMP_PATH, create_fs_user_by_platform, delete_fs_user_by_platform, exec_manage_cli } = require('../system_tests/test_utils'); const { ManageCLIError } = require('../../manage_nsfs/manage_nsfs_cli_errors'); @@ -28,13 +25,12 @@ const bucket_storage_path = path.join(tmp_fs_path, 'account_inaccessible'); mocha.describe('nsfs nc health', function() { const config_root = path.join(tmp_fs_path, 'config_root_nsfs_health'); + const config_fs = new ConfigFS(config_root); const root_path = path.join(tmp_fs_path, 'root_path_nsfs_health/'); const config_root_invalid = path.join(tmp_fs_path, 'config_root_nsfs_health_invalid'); let Health; mocha.before(async () => { - await P.all(_.map([CONFIG_SUBDIRS.ACCOUNTS, CONFIG_SUBDIRS.BUCKETS], async dir => - fs_utils.create_fresh_path(path.join(config_root, dir)))); await fs_utils.create_fresh_path(root_path); await fs_utils.create_fresh_path(config_root_invalid); await nb_native().fs.mkdir(DEFAULT_FS_CONFIG, bucket_storage_path, 0o770); @@ -101,16 +97,18 @@ mocha.describe('nsfs nc health', function() { const bucket_name = 'bucket1'; const new_buckets_path = `${root_path}new_buckets_path_user1/`; const account1 = { - name: acount_name, nsfs_account_config: { + _id: '1', + name: acount_name, + nsfs_account_config: { uid: process.getuid(), gid: process.getgid(), new_buckets_path: new_buckets_path } }; const bucket1 = { name: bucket_name, path: new_buckets_path + '/bucket1' }; - const account_inaccessible = { name: 'account_inaccessible', nsfs_account_config: { uid: 999, gid: 999, new_buckets_path: bucket_storage_path } }; - const account_inaccessible_dn = { name: 'account_inaccessible_dn', nsfs_account_config: { distinguished_name: 'inaccessible_dn', new_buckets_path: bucket_storage_path } }; - const invalid_account_dn = { name: 'invalid_account_dn', nsfs_account_config: { distinguished_name: 'invalid_account_dn', new_buckets_path: bucket_storage_path } }; + const account_inaccessible = { _id: '2', name: 'account_inaccessible', nsfs_account_config: { uid: 999, gid: 999, new_buckets_path: bucket_storage_path } }; + const account_inaccessible_dn = { _id: '3', name: 'account_inaccessible_dn', nsfs_account_config: { distinguished_name: 'inaccessible_dn', new_buckets_path: bucket_storage_path } }; + const invalid_account_dn = { _id: '4', name: 'invalid_account_dn', nsfs_account_config: { distinguished_name: 'invalid_account_dn', new_buckets_path: bucket_storage_path } }; const fs_users = { other_user: { distinguished_name: account_inaccessible_dn.nsfs_account_config.distinguished_name, @@ -120,14 +118,14 @@ mocha.describe('nsfs nc health', function() { }; mocha.before(async () => { const https_port = 6443; - const config_fs = new ConfigFS(config_root); Health = new NSFSHealth({ config_root, https_port, config_fs }); await fs_utils.create_fresh_path(new_buckets_path); await fs_utils.file_must_exist(new_buckets_path); await fs_utils.create_fresh_path(new_buckets_path + '/bucket1'); await fs_utils.file_must_exist(new_buckets_path + '/bucket1'); - await write_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, acount_name, account1); - await write_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_name, bucket1); + await config_fs.create_config_dirs_if_missing(); + await config_fs.create_account_config_file(account1); + await config_fs.create_bucket_config_file(bucket1.name, JSON.stringify(bucket1)); const get_service_memory_usage = sinon.stub(Health, "get_service_memory_usage"); get_service_memory_usage.onFirstCall().returns(Promise.resolve(100)); for (const user of Object.values(fs_users)) { @@ -138,8 +136,8 @@ mocha.describe('nsfs nc health', function() { mocha.after(async () => { fs_utils.folder_delete(new_buckets_path); fs_utils.folder_delete(path.join(new_buckets_path, 'bucket1')); - fs_utils.file_delete(path.join(config_root, CONFIG_SUBDIRS.BUCKETS, bucket1.name + JSON_SUFFIX)); - fs_utils.file_delete(path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account1.name + JSON_SUFFIX)); + await config_fs.delete_bucket_config_file(bucket1.name); + await config_fs.delete_account_config_file(account1); for (const user of Object.values(fs_users)) { await delete_fs_user_by_platform(user.distinguished_name); } @@ -191,8 +189,8 @@ mocha.describe('nsfs nc health', function() { mocha.it('NSFS account with invalid storage path', async function() { Health.get_service_state.restore(); Health.get_endpoint_response.restore(); - const account_invalid = { name: 'account_invalid', nsfs_account_config: { new_buckets_path: new_buckets_path + '/invalid' } }; - await write_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_invalid.name, account_invalid); + const account_invalid = { _id: '5', name: 'account_invalid', nsfs_account_config: { new_buckets_path: new_buckets_path + '/invalid' } }; + await config_fs.create_account_config_file(account_invalid); const get_service_state = sinon.stub(Health, "get_service_state"); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); @@ -202,14 +200,14 @@ mocha.describe('nsfs nc health', function() { assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, 'account_invalid'); - await fs_utils.file_delete(path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_invalid.name + JSON_SUFFIX)); + await fs_utils.file_delete(config_fs.get_account_or_user_path_by_name(account_invalid.name)); }); mocha.it('NSFS bucket with invalid storage path', async function() { Health.get_service_state.restore(); Health.get_endpoint_response.restore(); const bucket_invalid = { name: 'bucket_invalid', path: new_buckets_path + '/bucket1/invalid' }; - await write_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_invalid.name, bucket_invalid); + await config_fs.create_bucket_config_file(bucket_invalid.name, JSON.stringify(bucket_invalid)); const get_service_state = sinon.stub(Health, "get_service_state"); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); @@ -219,14 +217,14 @@ mocha.describe('nsfs nc health', function() { assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid'); - await fs_utils.file_delete(path.join(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_invalid.name + JSON_SUFFIX)); + await fs_utils.file_delete(config_fs.get_bucket_path_by_name(bucket_invalid.name)); }); mocha.it('NSFS invalid bucket schema json', async function() { Health.get_service_state.restore(); Health.get_endpoint_response.restore(); const bucket_invalid_schema = { name: 'bucket_invalid_schema', path: new_buckets_path }; - await write_config_file(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_invalid_schema.name, bucket_invalid_schema, "invalid"); + await config_fs.create_bucket_config_file(bucket_invalid_schema.name, JSON.stringify(bucket_invalid_schema) + 'invalid'); const get_service_state = sinon.stub(Health, "get_service_state"); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); @@ -235,15 +233,15 @@ mocha.describe('nsfs nc health', function() { const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.buckets_status.invalid_buckets.length, 1); - assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid_schema.json'); - await fs_utils.file_delete(path.join(config_root, CONFIG_SUBDIRS.BUCKETS, bucket_invalid_schema.name + JSON_SUFFIX)); + assert.strictEqual(health_status.checks.buckets_status.invalid_buckets[0].name, 'bucket_invalid_schema'); + await fs_utils.file_delete(config_fs.get_bucket_path_by_name(bucket_invalid_schema.name)); }); mocha.it('NSFS invalid account schema json', async function() { Health.get_service_state.restore(); Health.get_endpoint_response.restore(); - const account_invalid_schema = { name: 'account_invalid_schema', path: new_buckets_path }; - await write_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_invalid_schema.name, account_invalid_schema, "invalid"); + const account_invalid_schema = { _id: '9', name: 'account_invalid_schema', path: new_buckets_path, bla: 5 }; + await test_utils.write_manual_config_file(TYPES.ACCOUNT, config_fs, account_invalid_schema, 'invalid'); const get_service_state = sinon.stub(Health, "get_service_state"); get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 })) .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 })); @@ -252,8 +250,8 @@ mocha.describe('nsfs nc health', function() { const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.status, 'OK'); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); - assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, 'account_invalid_schema.json'); - await fs_utils.file_delete(path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_invalid_schema.name + JSON_SUFFIX)); + assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_invalid_schema.name); + await fs_utils.file_delete(config_fs.get_account_or_user_path_by_name(account_invalid_schema.name)); }); mocha.it('Health all condition is success, all_account_details is false', async function() { @@ -318,7 +316,7 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Account with inaccessible path - uid gid', async function() { - await write_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_inaccessible.name, account_inaccessible); + await config_fs.create_account_config_file(account_inaccessible); Health.get_service_state.restore(); Health.get_endpoint_response.restore(); Health.all_account_details = true; @@ -334,11 +332,11 @@ mocha.describe('nsfs nc health', function() { assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible.name); - await fs_utils.file_delete(path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_inaccessible.name + JSON_SUFFIX)); + await fs_utils.file_delete(config_fs.get_account_or_user_path_by_name(account_inaccessible.name)); }); mocha.it('Account with inaccessible path - dn', async function() { - await write_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_inaccessible_dn.name, account_inaccessible_dn); + await config_fs.create_account_config_file(account_inaccessible_dn); Health.get_service_state.restore(); Health.get_endpoint_response.restore(); Health.all_account_details = true; @@ -354,11 +352,11 @@ mocha.describe('nsfs nc health', function() { assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 1); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].code, "ACCESS_DENIED"); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts[0].name, account_inaccessible_dn.name); - await fs_utils.file_delete(path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_inaccessible_dn.name + JSON_SUFFIX)); + await fs_utils.file_delete(config_fs.get_account_or_user_path_by_name(account_inaccessible_dn.name)); }); mocha.it('Account with invalid dn', async function() { - await write_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, invalid_account_dn.name, invalid_account_dn); + await config_fs.create_account_config_file(invalid_account_dn); Health.get_service_state.restore(); Health.get_endpoint_response.restore(); Health.all_account_details = true; @@ -379,8 +377,8 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Account with new_buckets_path missing and allow_bucket_creation false, valid account', async function() { - const account_valid = { name: 'account_valid', nsfs_account_config: { uid: 999, gid: 999 }, allow_bucket_creation: false }; - await write_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_valid.name, account_valid); + const account_valid = { _id: '6', name: 'account_valid', nsfs_account_config: { uid: 999, gid: 999 }, allow_bucket_creation: false }; + await config_fs.create_account_config_file(account_valid); Health.get_service_state.restore(); Health.get_endpoint_response.restore(); Health.all_account_details = true; @@ -396,8 +394,8 @@ mocha.describe('nsfs nc health', function() { }); mocha.it('Account with new_buckets_path missing and allow_bucket_creation true, invalid account', async function() { - const account_invalid = { name: 'account_invalid', nsfs_account_config: { uid: 999, gid: 999 }, allow_bucket_creation: true }; - await write_config_file(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_invalid.name, account_invalid); + const account_invalid = { _id: '8', name: 'account_invalid', nsfs_account_config: { uid: 999, gid: 999 }, allow_bucket_creation: true }; + await config_fs.create_account_config_file(account_invalid); Health.get_service_state.restore(); Health.get_endpoint_response.restore(); Health.all_account_details = true; @@ -410,16 +408,8 @@ mocha.describe('nsfs nc health', function() { const health_status = await Health.nc_nsfs_health(); assert.strictEqual(health_status.checks.accounts_status.valid_accounts.length, 2); assert.strictEqual(health_status.checks.accounts_status.invalid_accounts.length, 2); - await fs_utils.file_delete(path.join(config_root, CONFIG_SUBDIRS.ACCOUNTS, account_invalid.name + JSON_SUFFIX)); + await fs_utils.file_delete(config_fs.get_account_or_user_path_by_name(account_invalid.name)); }); }); }); -async function write_config_file(config_root, schema_dir, config_file_name, config_data, invalid_str = '') { - const config_path = path.join(config_root, schema_dir, config_file_name + JSON_SUFFIX); - await nb_native().fs.writeFile( - DEFAULT_FS_CONFIG, - config_path, Buffer.from(JSON.stringify(config_data) + invalid_str), { - mode: get_umasked_mode(config.BASE_MODE_FILE), - }); -} diff --git a/src/test/unit_tests/test_s3_bucket_policy.js b/src/test/unit_tests/test_s3_bucket_policy.js index a39e70626b..d6d64494bd 100644 --- a/src/test/unit_tests/test_s3_bucket_policy.js +++ b/src/test/unit_tests/test_s3_bucket_policy.js @@ -71,12 +71,20 @@ async function setup() { if (process.env.NC_CORETEST) { await fs_utils.create_fresh_path(tmp_fs_root, 0o777); - await rpc_client.pool.create_namespace_resource({ - name: nsr, - nsfs_config: { - fs_root_path: tmp_fs_root, + try { + await rpc_client.account.read_account({ email: config.ANONYMOUS_ACCOUNT_NAME + '@' }); + } catch (err) { + const error = JSON.parse(err.stdout); + if (error.error.code === 'NoSuchAccountName') { + await rpc_client.account.create_account({ + name: config.ANONYMOUS_ACCOUNT_NAME, + nsfs_account_config: { + uid: 0, + gid: 0 + } + }); } - }); + } } const account = { has_login: false, @@ -87,7 +95,7 @@ async function setup() { account.nsfs_account_config = { uid: process.getuid(), gid: process.getgid(), - new_buckets_path: '/' + new_buckets_path: tmp_fs_root }; } const admin_keys = (await rpc_client.account.read_account({ diff --git a/src/util/native_fs_utils.js b/src/util/native_fs_utils.js index 2eca3ee1f5..0874d53c0e 100644 --- a/src/util/native_fs_utils.js +++ b/src/util/native_fs_utils.js @@ -336,8 +336,15 @@ async function delete_config_file(fs_context, schema_dir, config_path) { dbg.log1('native_fs_utils: delete_config_file unlinking:', config_path, 'is_gpfs=', is_gpfs); const tmp_dir_path = path.join(schema_dir, get_config_files_tmpdir()); // TODO: add retry? should we fail deletion if the config file was updated at the same time? - await safe_unlink(fs_context, config_path, stat, gpfs_options, tmp_dir_path); - + try { + await safe_unlink(fs_context, config_path, stat, gpfs_options, tmp_dir_path); + } catch (err) { + if (err.code === 'ENOENT') { + dbg.warn(`delete_config_file: config file already deleted ${config_path}`); + return; + } + throw err; + } dbg.log1('native_fs_utils: delete_config_file done', config_path); } catch (err) { dbg.log1('native_fs_utils: delete_config_file error', err); @@ -534,9 +541,10 @@ async function is_dir_rw_accessible(fs_context, dir_path) { * delete bucket specific temp folder from bucket storage path, config.NSFS_TEMP_DIR_NAME_ * @param {string} dir * @param {nb.NativeFSContext} fs_context - * @param {boolean} is_temp + * @param {boolean} [is_temp] + * @param {boolean} [silent_if_missing] */ -async function folder_delete(dir, fs_context, is_temp = false) { +async function folder_delete(dir, fs_context, is_temp, silent_if_missing) { const exists = await is_path_exists(fs_context, dir); if (!exists && is_temp) { return; @@ -552,7 +560,15 @@ async function folder_delete(dir, fs_context, is_temp = false) { // Ignore missing files/directories; bail on other errors if (result && result.error && result.error.code !== 'ENOENT') throw result.error; }); - await nb_native().fs.rmdir(fs_context, dir); + try { + await nb_native().fs.rmdir(fs_context, dir); + } catch (err) { + if (err.code === 'ENOENT' && silent_if_missing) { + dbg.warn(`native_fs_utils.folder_delete already deleted, skipping`); + return; + } + throw err; + } } /**