NC | More Refactoring config_fs

Signed-off-by: Romy <[email protected]>

Author: romayalon

src/cmd/manage_nsfs.js

@@ -129,6 +129,30 @@ function check_root_account_owns_user(root_account, account) {
 }
 
 
+/**
+ * _is_name_update returns true if a new_name flag was provided and it's not equal than 
+ * the current name, 
+ * @param {Object} data
+ * @returns {Boolean} 
+ */
+function _is_name_update(data) {
+    const cur_name = data.name;
+    const new_name = data.new_name;
+    return new_name && cur_name && new_name !== cur_name;
+}
+
+/**
+ * _is_access_key_update returns true if a new_access_key flag was provided and it's not equal than 
+ * the current access_key, 
+ * @param {Object} data
+ * @returns {Boolean} 
+ */
+function _is_access_key_update(data) {
+    const cur_access_key = has_access_keys(data.access_keys) ? data.access_keys[0].access_key.unwrap() : undefined;
+    const new_access_key = data.new_access_key;
+    return new_access_key && cur_access_key && new_access_key !== cur_access_key;
+}
+
 // EXPORTS
 exports.throw_cli_error = throw_cli_error;
 exports.write_stdout_response = write_stdout_response;
@@ -139,3 +163,5 @@ exports.has_access_keys = has_access_keys;
 exports.generate_id = generate_id;
 exports.set_debug_level = set_debug_level;
 exports.check_root_account_owns_user = check_root_account_owns_user;
+exports._is_name_update = _is_name_update;
+exports._is_access_key_update = _is_access_key_update;

src/manage_nsfs/manage_nsfs_cli_utils.js

@@ -10,7 +10,7 @@ const native_fs_utils = require('../util/native_fs_utils');
 const ManageCLIError = require('../manage_nsfs/manage_nsfs_cli_errors').ManageCLIError;
 const bucket_policy_utils = require('../endpoint/s3/s3_bucket_policy_utils');
 const { throw_cli_error, get_bucket_owner_account, get_options_from_file, get_boolean_or_string_value,
-    check_root_account_owns_user } = require('../manage_nsfs/manage_nsfs_cli_utils');
+    check_root_account_owns_user, _is_name_update, _is_access_key_update } = require('../manage_nsfs/manage_nsfs_cli_utils');
 const { TYPES, ACTIONS, VALID_OPTIONS, OPTION_TYPE, FROM_FILE, BOOLEAN_STRING_VALUES, BOOLEAN_STRING_OPTIONS,
     GLACIER_ACTIONS, LIST_UNSETABLE_OPTIONS, ANONYMOUS, DIAGNOSE_ACTIONS } = require('../manage_nsfs/manage_nsfs_constants');
 const iam_utils = require('../endpoint/iam/iam_utils');
@@ -306,6 +306,22 @@ if (action === ACTIONS.STATUS || action === ACTIONS.ADD || action === ACTIONS.UP
     // in list there is no identifier
 }
 
+/**
+ * check_new_bucket_name_exists will validate that we have the needed identifier for the command
+ * @param {import('../sdk/config_fs').ConfigFS} config_fs
+ * @param {string} action
+ * @param {object} data
+ */
+async function check_new_bucket_name_exists(config_fs, action, data) {
+    let new_bucket_name = data.name;
+    if (action === ACTIONS.UPDATE) {
+        if (!data.new_name) return;
+        new_bucket_name = data.new_name;
+    }
+    const exists = await config_fs.is_bucket_exists(new_bucket_name);
+    if (exists) throw_cli_error(ManageCLIError.BucketAlreadyExists, new_bucket_name, { bucket: new_bucket_name });
+}
+
 /**
  * validate_bucket_args will validate the cli args of the bucket command
  * @param {import('../sdk/config_fs').ConfigFS} config_fs
@@ -322,6 +338,7 @@ async function validate_bucket_args(config_fs, data, action) {
         if (data.fs_backend !== undefined && !['GPFS', 'CEPH_FS', 'NFSv4'].includes(data.fs_backend)) {
             throw_cli_error(ManageCLIError.InvalidFSBackend);
         }
+        await check_new_bucket_name_exists(config_fs, action, data);
         // in case we have the fs_backend it changes the fs_context that we use for the path
         const fs_context_fs_backend = native_fs_utils.get_process_fs_context(data.fs_backend);
         const exists = await native_fs_utils.is_path_exists(fs_context_fs_backend, data.path);
@@ -393,6 +410,15 @@ function validate_account_identifier(action, input_options) {
  */
 async function validate_account_args(config_fs, data, action, is_flag_iam_operate_on_root_account_update_action) {
     if (action === ACTIONS.ADD || action === ACTIONS.UPDATE) {
+        const update_name = _is_name_update(data);
+        const update_access_key = _is_access_key_update(data);
+
+        const name_exists = update_name && await config_fs.is_account_exists_by_name(data.new_name);
+        const access_key_exists = update_access_key && await config_fs.is_account_exists_by_access_key(data.new_access_keys);
+        if (name_exists || access_key_exists) {
+            const err_code = name_exists ? ManageCLIError.AccountNameAlreadyExists : ManageCLIError.AccountAccessKeyAlreadyExists;
+            throw_cli_error(err_code);
+        }
         if (data.nsfs_account_config.gid && data.nsfs_account_config.uid === undefined) {
             throw_cli_error(ManageCLIError.MissingAccountNSFSConfigUID, data.nsfs_account_config);
         }

src/manage_nsfs/manage_nsfs_validations.js

@@ -10,12 +10,10 @@ const native_fs_utils = require('../util/native_fs_utils');
 const { create_arn, get_action_message_title, check_iam_path_was_set } = require('../endpoint/iam/iam_utils');
 const { IAM_ACTIONS, MAX_NUMBER_OF_ACCESS_KEYS, IAM_DEFAULT_PATH,
     ACCESS_KEY_STATUS_ENUM, IDENTITY_ENUM } = require('../endpoint/iam/iam_constants');
-const nsfs_schema_utils = require('../manage_nsfs/nsfs_schema_utils');
 const IamError = require('../endpoint/iam/iam_errors').IamError;
 const cloud_utils = require('../util/cloud_utils');
 const SensitiveString = require('../util/sensitive_string');
 const { generate_id } = require('../manage_nsfs/manage_nsfs_cli_utils');
-const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();
 const { account_cache } = require('./object_sdk');
 
 
@@ -150,10 +148,7 @@ class AccountSpaceFS {
                     is_username_update);
                 await this._update_account_config_new_username(action, params, requested_account);
             } else {
-                const requested_account_encrypted = await nc_mkm.encrypt_access_keys(requested_account);
-                const account_string = JSON.stringify(requested_account_encrypted);
-                nsfs_schema_utils.validate_account_schema(JSON.parse(account_string));
-                await this.config_fs.update_account_config_file(JSON.parse(account_string));
+                await this.config_fs.update_account_config_file(requested_account);
             }
             this._clean_account_cache(requested_account);
             return {
@@ -265,11 +260,8 @@ class AccountSpaceFS {
                 deactivated: false,
             };
             requested_account.access_keys.push(created_access_key_obj);
-            const requested_account_encrypted = await nc_mkm.encrypt_access_keys(requested_account);
-            const account_to_create_access_keys_string = JSON.stringify(requested_account_encrypted);
-            nsfs_schema_utils.validate_account_schema(JSON.parse(account_to_create_access_keys_string));
             await this.config_fs.update_account_config_file(
-                JSON.parse(account_to_create_access_keys_string),
+                requested_account,
                 { new_access_keys_to_link: [created_access_key_obj] }
             );
             return {
@@ -355,10 +347,7 @@ class AccountSpaceFS {
                 return;
             }
             access_key_obj.deactivated = this._check_access_key_is_deactivated(params.status);
-            const requested_account_encrypted = await nc_mkm.encrypt_access_keys(requested_account);
-            const account_string = JSON.stringify(requested_account_encrypted);
-            nsfs_schema_utils.validate_account_schema(JSON.parse(account_string));
-            await this.config_fs.update_account_config_file(JSON.parse(account_string));
+            await this.config_fs.update_account_config_file(requested_account);
             this._clean_account_cache(requested_account);
         } catch (err) {
             dbg.error(`AccountSpaceFS.${action} error`, err);
@@ -398,11 +387,8 @@ class AccountSpaceFS {
             }
             requested_account.access_keys = requested_account.access_keys.filter(access_key_obj =>
                 access_key_obj.access_key !== access_key_id);
-            const requested_account_encrypted = await nc_mkm.encrypt_access_keys(requested_account);
-            const account_string = JSON.stringify(requested_account_encrypted);
-            nsfs_schema_utils.validate_account_schema(JSON.parse(account_string));
             await this.config_fs.update_account_config_file(
-                JSON.parse(account_string),
+                requested_account,
                 { access_keys_to_delete: [{ access_key: access_key_id }] }
             );
             this._clean_account_cache(requested_account);
@@ -625,12 +611,9 @@ class AccountSpaceFS {
     }
 
     async _copy_data_from_requesting_account_to_account_config(action, requesting_account, params) {
-        const master_key_id = await nc_mkm.get_active_master_key_id();
-        const created_account = this._new_user_defaults(requesting_account, params, master_key_id);
+        const created_account = this._new_user_defaults(requesting_account, params);
         dbg.log1(`AccountSpaceFS.${action} new_account`, created_account);
-        const new_account_string = JSON.stringify(created_account);
-        nsfs_schema_utils.validate_account_schema(JSON.parse(new_account_string));
-        await this.config_fs.create_account_config_file(JSON.parse(new_account_string));
+        await this.config_fs.create_account_config_file(created_account);
         return created_account;
     }
 
@@ -665,8 +648,6 @@ class AccountSpaceFS {
         this._check_if_user_does_not_have_access_keys_before_deletion(action, account_to_delete);
     }
 
-    // TODO - when we have the structure of config we can check easily which buckets are owned by the root account
-    // currently, partial copy from verify_account_not_owns_bucket
     async _check_if_root_account_does_not_have_buckets_before_deletion(action, account_to_delete) {
         const resource_name = 'buckets';
         const bucket_names = await this.config_fs.list_buckets();
@@ -710,10 +691,7 @@ class AccountSpaceFS {
         requested_account.name = params.new_username;
         requested_account.email = params.new_username; // internally saved
         // handle account config creation
-        const requested_account_encrypted = await nc_mkm.encrypt_access_keys(requested_account);
-        const account_string = JSON.stringify(requested_account_encrypted);
-        nsfs_schema_utils.validate_account_schema(JSON.parse(account_string));
-        await this.config_fs.update_account_config_file(JSON.parse(account_string), { old_name: params.username });
+        await this.config_fs.update_account_config_file(requested_account, { old_name: params.username });
     }
 
     _check_root_account_or_user(requesting_account, username) {

src/sdk/accountspace_fs.js

@@ -9,6 +9,8 @@ const nb_native = require('../util/nb_native');
 const native_fs_utils = require('../util/native_fs_utils');
 const nc_mkm = require('../manage_nsfs/nc_master_key_manager').get_instance();
 const { TYPES } = require('../manage_nsfs/manage_nsfs_constants');
+const nsfs_schema_utils = require('../manage_nsfs/nsfs_schema_utils');
+const { IS_MAC } = require('../util/os_utils');
 
 /* Config directory sub directory comments - 
    On 5.18 - 
@@ -517,7 +519,7 @@ class ConfigFS {
      */
     async create_account_config_file(account_data) {
         const { name, _id, owner = undefined } = account_data;
-        const data_string = JSON.stringify(account_data);
+        const data_string = await this._prepare_for_account_schema(account_data);
         const account_path = this.get_identity_path_by_id(_id);
         const account_dir_path = this.get_identity_dir_path_by_id(_id);
 
@@ -544,7 +546,7 @@ class ConfigFS {
      */
     async update_account_config_file(account_new_data, options = {}) {
         const { name, _id, owner = undefined } = account_new_data;
-        const data_string = JSON.stringify(account_new_data);
+        const data_string = await this._prepare_for_account_schema(account_new_data);
         const account_path = this.get_identity_path_by_id(_id);
         const account_dir_path = this.get_identity_dir_path_by_id(_id);
         await native_fs_utils.update_config_file(this.fs_context, account_dir_path, account_path, data_string);
@@ -607,7 +609,8 @@ class ConfigFS {
     async unlink_account_name_index(account_name, account_id_config_path) {
         const account_name_path = this.get_account_path_by_name(account_name);
         const full_path = await nb_native().fs.realpath(this.fs_context, account_name_path);
-        if (full_path === account_id_config_path) {
+        if (full_path === account_id_config_path ||
+            (IS_MAC && full_path === path.join('/private/', account_id_config_path))) {
             await nb_native().fs.unlink(this.fs_context, account_name_path);
         }
     }
@@ -639,7 +642,8 @@ class ConfigFS {
     async unlink_access_key_index(access_key, account_id_config_path) {
         const access_key_path = this.get_account_or_user_path_by_access_key(access_key);
         const full_path = await nb_native().fs.realpath(this.fs_context, access_key_path);
-        if (full_path === account_id_config_path) {
+        if (full_path === account_id_config_path ||
+            (IS_MAC && full_path === path.join('/private/', account_id_config_path))) {
             await nb_native().fs.unlink(this.fs_context, access_key_path);
         }
     }
@@ -709,24 +713,63 @@ class ConfigFS {
 
     /**
      * create_bucket_config_file creates bucket config file
-     * @param {string} bucket_name
-     * @param {*} data
-     * @returns {Promise<void>} 
+     * @param {Object} bucket_data
+     * @returns {Promise<String>} 
      */
-    async create_bucket_config_file(bucket_name, data) {
-        const bucket_path = this.get_bucket_path_by_name(bucket_name);
-        await native_fs_utils.create_config_file(this.fs_context, this.buckets_dir_path, bucket_path, data);
+    async create_bucket_config_file(bucket_data) {
+        const bucket_string_data = this._prepare_for_bucket_schema(bucket_data);
+        const bucket_path = this.get_bucket_path_by_name(bucket_data.name);
+        await native_fs_utils.create_config_file(this.fs_context, this.buckets_dir_path, bucket_path, bucket_string_data);
+        return bucket_string_data;
     }
 
     /**
-     * update_bucket_config_file updates bucket config file
-     * @param {string} bucket_name
-     * @param {*} data
-     * @returns {Promise<void>} 
+     * _prepare_for_bucket_schema takes bucket data -
+     * 1. removes API bucket properties
+     * 2. removes undefined properties, unwrap sensitive_strings and creation_data to string
+     * 3. checks bucket schema validation
+     * 4. and returns stringified data ready to be written to the config directory
+     * @param {Object} bucket_data
+     * @returns {String}
      */
-    async update_bucket_config_file(bucket_name, data) {
-        const bucket_config_path = this.get_bucket_path_by_name(bucket_name);
-        await native_fs_utils.update_config_file(this.fs_context, this.buckets_dir_path, bucket_config_path, data);
+    _prepare_for_bucket_schema(bucket_data) {
+        const api_bucket_properties_to_remove = ['new_name'];
+        const bucket_data_api_props_omitted = _.omit(bucket_data, api_bucket_properties_to_remove);
+        const bucket_string_data = JSON.stringify(bucket_data_api_props_omitted);
+        nsfs_schema_utils.validate_bucket_schema(JSON.parse(bucket_string_data));
+        return bucket_string_data;
+    }
+
+    /**
+     * _prepare_for_account_schema takes account data -
+     * 1. encrypts its access keys
+     * 2. sets the used master key on the account
+     * 3. removes API account properties
+     * 4. removes undefined properties, unwrap sensitive_strings and creation_data to string
+     * 5. checks accpimt schema validation
+     * 6. and returns stringified data ready to be written to the config directory
+     * @param {Object} account_data
+     * @returns {Promise<String>}
+     */
+    async _prepare_for_account_schema(account_data) {
+        const encrypted_account = await nc_mkm.encrypt_access_keys(account_data);
+        const api_account_properties_to_remove = ['new_name', 'new_access_key'];
+        const account_data_api_props_omitted = _.omit(encrypted_account, api_account_properties_to_remove);
+        const account_string_data = JSON.stringify(account_data_api_props_omitted);
+        nsfs_schema_utils.validate_account_schema(JSON.parse(account_string_data));
+        return account_string_data;
+    }
+
+    /**
+     * update_bucket_config_file updates bucket config file
+     * @param {Object} bucket_data
+     * @returns {Promise<String>} 
+     */
+    async update_bucket_config_file(bucket_data) {
+        const bucket_string_data = this._prepare_for_bucket_schema(bucket_data);
+        const bucket_config_path = this.get_bucket_path_by_name(bucket_data.name);
+        await native_fs_utils.update_config_file(this.fs_context, this.buckets_dir_path, bucket_config_path, bucket_string_data);
+        return bucket_string_data;
     }
 
     /**

src/sdk/config_fs.js

@@ -125,7 +125,7 @@ mocha.describe('nsfs nc health', function() {
             await fs_utils.file_must_exist(new_buckets_path + '/bucket1');
             await config_fs.create_config_dirs_if_missing();
             await config_fs.create_account_config_file(account1);
-            await config_fs.create_bucket_config_file(bucket1.name, JSON.stringify(bucket1));
+            await config_fs.create_bucket_config_file(bucket1);
             const get_service_memory_usage = sinon.stub(Health, "get_service_memory_usage");
             get_service_memory_usage.onFirstCall().returns(Promise.resolve(100));
             for (const user of Object.values(fs_users)) {
@@ -224,7 +224,7 @@ mocha.describe('nsfs nc health', function() {
             Health.get_service_state.restore();
             Health.get_endpoint_response.restore();
             const bucket_invalid_schema = { name: 'bucket_invalid_schema', path: new_buckets_path };
-            await config_fs.create_bucket_config_file(bucket_invalid_schema.name, JSON.stringify(bucket_invalid_schema) + 'invalid');
+            await config_fs.create_bucket_config_file(bucket_invalid_schema);
             const get_service_state = sinon.stub(Health, "get_service_state");
             get_service_state.onFirstCall().returns(Promise.resolve({ service_status: 'active', pid: 1000 }))
                 .onSecondCall().returns(Promise.resolve({ service_status: 'active', pid: 2000 }));