From 0bff063fe6e63d93557e13882c94afd696d085e2 Mon Sep 17 00:00:00 2001 From: Utkarsh Srivastava Date: Wed, 1 May 2024 21:06:34 +0530 Subject: [PATCH] add support for DMAPI xattr based GLACIER storage class Signed-off-by: Utkarsh Srivastava add tests for tape info parsing Signed-off-by: Utkarsh Srivastava remove auto-code formatting Signed-off-by: Utkarsh Srivastava --- .gitignore | 2 + config.js | 24 +++ src/endpoint/s3/s3_utils.js | 6 + src/manage_nsfs/manage_nsfs_glacier.js | 57 +++-- src/native/fs/fs_napi.cpp | 31 ++- .../backend.js => glacier.js} | 196 +++++++++++++++--- .../tapecloud.js => glacier_tapecloud.js} | 46 ++-- src/sdk/namespace_fs.js | 80 ++++--- src/sdk/nb.d.ts | 21 +- src/sdk/nsfs_glacier_backend/helper.js | 29 --- .../unit_tests/test_nsfs_glacier_backend.js | 165 +++++++++++---- 11 files changed, 491 insertions(+), 166 deletions(-) rename src/sdk/{nsfs_glacier_backend/backend.js => glacier.js} (56%) rename src/sdk/{nsfs_glacier_backend/tapecloud.js => glacier_tapecloud.js} (89%) delete mode 100644 src/sdk/nsfs_glacier_backend/helper.js diff --git a/.gitignore b/.gitignore index 7f8cf8730a..1c1c29bffe 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,8 @@ config-local.js *.sublime* .DS_Store heapdump-* +.cache +.clangd ## PRIVATE *.pem diff --git a/config.js b/config.js index 3735e5446e..da9e4f1f1c 100644 --- a/config.js +++ b/config.js @@ -191,6 +191,15 @@ config.DENY_UPLOAD_TO_STORAGE_CLASS_STANDARD = false; // of days an object can be restored using `restore-object` call. config.S3_RESTORE_REQUEST_MAX_DAYS = 30; +// NSFS_GLACIER_DMAPI_PMIG_DAYS controls the "virtual"/fake expiry +// days that will be shown if we detect a glacier object whose life- +// cycle NSFS doesn't controls +// +// This is initialized to be the same as S3_RESTORE_REQUEST_MAX_DAYS +// but can be overridden to any numberical value +config.NSFS_GLACIER_DMAPI_PMIG_DAYS = config.S3_RESTORE_REQUEST_MAX_DAYS; + + /** * S3_RESTORE_MAX_DAYS_BEHAVIOUR controls whether to truncate the * requested number of days in restore request or whether to deny the request. @@ -873,6 +882,21 @@ config.NSFS_GLACIER_EXPIRY_TZ = 'LOCAL'; // the request will be used config.NSFS_GLACIER_EXPIRY_TIME_OF_DAY = ''; +// If set to true then NooBaa will consider DMAPI extended attributes +// in conjuction with NooBaa's `user.storage_class` extended attribute +// to determine state of an object. +config.NSFS_GLACIER_USE_DMAPI = false; + +// NSFS_GLACIER_DMAPI_ALLOW_NOOBAA_TAKEOVER allows NooBaa to take over lifecycle +// management of an object which was originally NOT managed by NooBaa. +config.NSFS_GLACIER_DMAPI_ALLOW_NOOBAA_TAKEOVER = false; + +// NSFS_GLACIER_DMAPI_TPS_HTTP_HEADER if enabled will add additional HTTP headers +// `x-tape-meta-copy-n` based on `dmapi.IBMTPS` EA. +// +// For this to work, NSFS_GLACIER_USE_DMAPI must be set to `true`. +config.NSFS_GLACIER_DMAPI_TPS_HTTP_HEADER = config.NSFS_GLACIER_USE_DMAPI || false; + config.NSFS_STATFS_CACHE_SIZE = 10000; config.NSFS_STATFS_CACHE_EXPIRY_MS = 1 * 1000; diff --git a/src/endpoint/s3/s3_utils.js b/src/endpoint/s3/s3_utils.js index 5290383184..d900e06aa9 100644 --- a/src/endpoint/s3/s3_utils.js +++ b/src/endpoint/s3/s3_utils.js @@ -325,6 +325,12 @@ function set_response_object_md(res, object_md) { res.setHeader('x-amz-restore', restore); } + if (storage_class === STORAGE_CLASS_GLACIER) { + object_md.restore_status?.backend_meta?.forEach?.((meta, idx) => { + const header = Object.keys(meta).map(key => `${key}=${meta[key]}`).join('&'); + res.setHeader(`x-tape-meta-copy-${idx}`, header); + }); + } } /** set_response_headers_get_object_attributes is based on set_response_object_md diff --git a/src/manage_nsfs/manage_nsfs_glacier.js b/src/manage_nsfs/manage_nsfs_glacier.js index 3b8cdf8455..66a9377a63 100644 --- a/src/manage_nsfs/manage_nsfs_glacier.js +++ b/src/manage_nsfs/manage_nsfs_glacier.js @@ -5,8 +5,7 @@ const path = require('path'); const { PersistentLogger } = require('../util/persistent_logger'); const config = require('../../config'); const nb_native = require('../util/nb_native'); -const { GlacierBackend } = require('../sdk/nsfs_glacier_backend/backend'); -const { getGlacierBackend } = require('../sdk/nsfs_glacier_backend/helper'); +const { Glacier } = require('../sdk/glacier'); const native_fs_utils = require('../util/native_fs_utils'); const CLUSTER_LOCK = 'cluster.lock'; @@ -16,14 +15,14 @@ async function process_migrations() { const fs_context = native_fs_utils.get_process_fs_context(); await lock_and_run(fs_context, CLUSTER_LOCK, async () => { - const backend = getGlacierBackend(); + const backend = Glacier.getBackend(); if ( await backend.low_free_space() || - await time_exceeded(fs_context, config.NSFS_GLACIER_MIGRATE_INTERVAL, GlacierBackend.MIGRATE_TIMESTAMP_FILE) + await time_exceeded(fs_context, config.NSFS_GLACIER_MIGRATE_INTERVAL, Glacier.MIGRATE_TIMESTAMP_FILE) ) { await run_glacier_migrations(fs_context, backend); - await record_current_time(fs_context, GlacierBackend.MIGRATE_TIMESTAMP_FILE); + await record_current_time(fs_context, Glacier.MIGRATE_TIMESTAMP_FILE); } }); } @@ -32,44 +31,44 @@ async function process_migrations() { * run_tape_migrations reads the migration WALs and attempts to migrate the * files mentioned in the WAL. * @param {nb.NativeFSContext} fs_context - * @param {import('../sdk/nsfs_glacier_backend/backend').GlacierBackend} backend + * @param {import('../sdk/glacier').Glacier} backend */ async function run_glacier_migrations(fs_context, backend) { - await run_glacier_operation(fs_context, GlacierBackend.MIGRATE_WAL_NAME, backend.migrate.bind(backend)); + await run_glacier_operation(fs_context, Glacier.MIGRATE_WAL_NAME, backend.migrate.bind(backend)); } async function process_restores() { const fs_context = native_fs_utils.get_process_fs_context(); await lock_and_run(fs_context, CLUSTER_LOCK, async () => { - const backend = getGlacierBackend(); + const backend = Glacier.getBackend(); if ( await backend.low_free_space() || - !(await time_exceeded(fs_context, config.NSFS_GLACIER_RESTORE_INTERVAL, GlacierBackend.RESTORE_TIMESTAMP_FILE)) + !(await time_exceeded(fs_context, config.NSFS_GLACIER_RESTORE_INTERVAL, Glacier.RESTORE_TIMESTAMP_FILE)) ) return; await run_glacier_restore(fs_context, backend); - await record_current_time(fs_context, GlacierBackend.RESTORE_TIMESTAMP_FILE); + await record_current_time(fs_context, Glacier.RESTORE_TIMESTAMP_FILE); }); } /** * run_tape_restore reads the restore WALs and attempts to restore the * files mentioned in the WAL. - * @param {nb.NativeFSContext} fs_context - * @param {import('../sdk/nsfs_glacier_backend/backend').GlacierBackend} backend + * @param {nb.NativeFSContext} fs_context + * @param {import('../sdk/glacier').Glacier} backend */ async function run_glacier_restore(fs_context, backend) { - await run_glacier_operation(fs_context, GlacierBackend.RESTORE_WAL_NAME, backend.restore.bind(backend)); + await run_glacier_operation(fs_context, Glacier.RESTORE_WAL_NAME, backend.restore.bind(backend)); } async function process_expiry() { - const fs_context = native_fs_utils.get_process_fs_context(); + const fs_context = force_gpfs_fs_context(native_fs_utils.get_process_fs_context()); await lock_and_run(fs_context, SCAN_LOCK, async () => { - const backend = getGlacierBackend(); + const backend = Glacier.getBackend(); if ( await backend.low_free_space() || await is_desired_time( @@ -77,11 +76,11 @@ async function process_expiry() { new Date(), config.NSFS_GLACIER_EXPIRY_RUN_TIME, config.NSFS_GLACIER_EXPIRY_RUN_DELAY_LIMIT_MINS, - GlacierBackend.EXPIRY_TIMESTAMP_FILE, + Glacier.EXPIRY_TIMESTAMP_FILE, ) ) { await backend.expiry(fs_context); - await record_current_time(fs_context, GlacierBackend.EXPIRY_TIMESTAMP_FILE); + await record_current_time(fs_context, Glacier.EXPIRY_TIMESTAMP_FILE); } }); } @@ -178,6 +177,8 @@ async function record_current_time(fs_context, timestamp_file) { */ async function run_glacier_operation(fs_context, log_namespace, cb) { const log = new PersistentLogger(config.NSFS_GLACIER_LOGS_DIR, log_namespace, { locking: 'EXCLUSIVE' }); + + fs_context = force_gpfs_fs_context(fs_context); try { await log.process(async (entry, failure_recorder) => cb(fs_context, entry, failure_recorder)); } catch (error) { @@ -212,6 +213,28 @@ function get_tz_date(hours, mins, secs, tz) { return date; } +/** + * force_gpfs_fs_context returns a shallow copy of given + * fs_context with backend set to 'GPFS'. + * + * NOTE: The function will throw error if it detects that GPFS + * DL isn't loaded. + * + * @param {nb.NativeFSContext} fs_context + * @returns {nb.NativeFSContext} + */ +function force_gpfs_fs_context(fs_context) { + if (config.NSFS_GLACIER_USE_DMAPI) { + if (!nb_native().fs.gpfs) { + throw new Error('cannot use DMAPI EA: gpfs dl not loaded'); + } + + return { ...fs_context, backend: 'GPFS', use_dmapi: true }; + } + + return { ...fs_context }; +} + /** * lock_and_run acquires a flock and calls the given callback after * acquiring the lock diff --git a/src/native/fs/fs_napi.cpp b/src/native/fs/fs_napi.cpp index b5195eb5b8..cbacc99b93 100644 --- a/src/native/fs/fs_napi.cpp +++ b/src/native/fs/fs_napi.cpp @@ -46,6 +46,13 @@ #define GPFS_XATTR_PREFIX "gpfs" #define GPFS_DOT_ENCRYPTION_EA "Encryption" #define GPFS_ENCRYPTION_XATTR_NAME GPFS_XATTR_PREFIX "." GPFS_DOT_ENCRYPTION_EA +#define GPFS_DMAPI_XATTR_PREFIX "dmapi" +#define GPFS_DMAPI_DOT_IBMOBJ_EA "IBMObj" +#define GPFS_DMAPI_DOT_IBMPMIG_EA "IBMPMig" +#define GPFS_DMAPI_DOT_IBMTPS_EA "IBMTPS" +#define GPFS_DMAPI_XATTR_TAPE_INDICATOR GPFS_DMAPI_XATTR_PREFIX "." GPFS_DMAPI_DOT_IBMOBJ_EA +#define GPFS_DMAPI_XATTR_TAPE_PREMIG GPFS_DMAPI_XATTR_PREFIX "." GPFS_DMAPI_DOT_IBMPMIG_EA +#define GPFS_DMAPI_XATTR_TAPE_TPS GPFS_DMAPI_XATTR_PREFIX "." GPFS_DMAPI_DOT_IBMTPS_EA // This macro should be used after openning a file // it will autoclose the file using AutoCloser and will throw an error in case of failures @@ -244,6 +251,11 @@ parse_open_flags(std::string flags) } const static std::vector GPFS_XATTRS{ GPFS_ENCRYPTION_XATTR_NAME }; +const static std::vector GPFS_DMAPI_XATTRS{ + GPFS_DMAPI_XATTR_TAPE_INDICATOR, + GPFS_DMAPI_XATTR_TAPE_PREMIG, + GPFS_DMAPI_XATTR_TAPE_TPS, +}; const static std::vector USER_XATTRS{ "user.content_type", "user.content_md5", @@ -461,9 +473,14 @@ get_fd_xattr(int fd, XattrMap& xattr, const std::vector& xattr_keys } static int -get_fd_gpfs_xattr(int fd, XattrMap& xattr, int& gpfs_error) +get_fd_gpfs_xattr(int fd, XattrMap& xattr, int& gpfs_error, bool use_dmapi) { - for (auto const& key : GPFS_XATTRS) { + auto gpfs_xattrs { GPFS_XATTRS }; + if (use_dmapi) { + gpfs_xattrs.insert(gpfs_xattrs.end(), GPFS_DMAPI_XATTRS.begin(), GPFS_DMAPI_XATTRS.end()); + } + + for (auto const& key : gpfs_xattrs) { gpfsRequest_t gpfsGetXattrRequest; build_gpfs_get_ea_request(&gpfsGetXattrRequest, key); int r = dlsym_gpfs_fcntl(fd, &gpfsGetXattrRequest); @@ -603,6 +620,8 @@ struct FSWorker : public Napi::AsyncWorker // NOTE: If _do_ctime_check = false, then some functions will fallback to using mtime check bool _do_ctime_check; + bool _use_dmapi; + FSWorker(const Napi::CallbackInfo& info) : AsyncWorker(info.Env()) , _deferred(Napi::Promise::Deferred::New(info.Env())) @@ -616,6 +635,7 @@ struct FSWorker : public Napi::AsyncWorker , _should_add_thread_capabilities(false) , _supplemental_groups() , _do_ctime_check(false) + , _use_dmapi(false) { for (int i = 0; i < (int)info.Length(); ++i) _args_ref.Set(i, info[i]); if (info[0].ToBoolean()) { @@ -635,6 +655,7 @@ struct FSWorker : public Napi::AsyncWorker _report_fs_stats = Napi::Persistent(fs_context.Get("report_fs_stats").As()); } _do_ctime_check = fs_context.Get("do_ctime_check").ToBoolean(); + _use_dmapi = fs_context.Get("use_dmapi").ToBoolean(); } } void Begin(std::string desc) @@ -793,7 +814,7 @@ struct Stat : public FSWorker if (!_use_lstat) { SYSCALL_OR_RETURN(get_fd_xattr(fd, _xattr, _xattr_get_keys)); if (use_gpfs_lib()) { - GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error)); + GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error, _use_dmapi)); } } @@ -1221,7 +1242,7 @@ struct Readfile : public FSWorker if (_read_xattr) { SYSCALL_OR_RETURN(get_fd_xattr(fd, _xattr, _xattr_get_keys)); if (use_gpfs_lib()) { - GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error)); + GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error, _use_dmapi)); } } @@ -1752,7 +1773,7 @@ struct FileStat : public FSWrapWorker SYSCALL_OR_RETURN(fstat(fd, &_stat_res)); SYSCALL_OR_RETURN(get_fd_xattr(fd, _xattr, _xattr_get_keys)); if (use_gpfs_lib()) { - GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error)); + GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error, _use_dmapi)); } if (_do_ctime_check) { diff --git a/src/sdk/nsfs_glacier_backend/backend.js b/src/sdk/glacier.js similarity index 56% rename from src/sdk/nsfs_glacier_backend/backend.js rename to src/sdk/glacier.js index 162ba48767..361da65669 100644 --- a/src/sdk/nsfs_glacier_backend/backend.js +++ b/src/sdk/glacier.js @@ -1,12 +1,13 @@ /* Copyright (C) 2024 NooBaa */ 'use strict'; -const nb_native = require('../../util/nb_native'); -const s3_utils = require('../../endpoint/s3/s3_utils'); -const { round_up_to_next_time_of_day } = require('../../util/time_utils'); -const dbg = require('../../util/debug_module')(__filename); +const nb_native = require('../util/nb_native'); +const s3_utils = require('../endpoint/s3/s3_utils'); +const { round_up_to_next_time_of_day } = require('../util/time_utils'); +const dbg = require('../util/debug_module')(__filename); +const config = require('../../config'); -class GlacierBackend { +class Glacier { // These names start with the word 'timestamp' so as to assure // that it acts like a 'namespace' for the these kind of files. // @@ -17,9 +18,9 @@ class GlacierBackend { static EXPIRY_TIMESTAMP_FILE = 'expiry.timestamp'; /** - * XATTR_RESTORE_REQUEST is set to a NUMBER (expiry days) by `restore_object` when - * a restore request is made. This is unset by the underlying restore process when - * it finishes the request, this is to ensure that the same object is not queued + * XATTR_RESTORE_REQUEST is set to a NUMBER (expiry days) by `restore_object` when + * a restore request is made. This is unset by the underlying restore process when + * it finishes the request, this is to ensure that the same object is not queued * for restoration multiple times. */ static XATTR_RESTORE_REQUEST = 'user.noobaa.restore.request'; @@ -29,7 +30,7 @@ class GlacierBackend { * NooBaa (in case restore is issued again while the object is on disk). * This is read by the underlying "disk evict" process to determine if the object * should be evicted from the disk or not. - * + * * NooBaa will use this date to determine if the object is on disk or not, if the * expiry date is in the future, the object is on disk, if the expiry date is in * the past, the object is not on disk. This may or may not represent the actual @@ -40,6 +41,29 @@ class GlacierBackend { static STORAGE_CLASS_XATTR = 'user.storage_class'; + /** + * GPFS_DMAPI_XATTR_TAPE_INDICATOR if set on a file indicates that the file is on tape. + * + * NOTE: The existence of the xattr only indicates if a copy of the file is on the tape + * or not, it doesn't tell the file state (premigrated, etc.). + */ + static GPFS_DMAPI_XATTR_TAPE_INDICATOR = 'dmapi.IBMObj'; + + /** + * GPFS_DMAPI_XATTR_TAPE_PREMIG if set indicates if a file is not only on tape but is + * also in premigrated state (that is a copy of the file exists on the disk as well). + */ + static GPFS_DMAPI_XATTR_TAPE_PREMIG = 'dmapi.IBMPMig'; + + /** + * GPFS_DMAPI_XATTR_TAPE_TPS xattr contains tape related information. + * + * Example: `1 @@:@@...` + * + * NOTE: If IBMTPS EA exists, that means the file is either migrated or premigrated. + */ + static GPFS_DMAPI_XATTR_TAPE_TPS = 'dmapi.IBMTPS'; + static MIGRATE_WAL_NAME = 'migrate'; static RESTORE_WAL_NAME = 'restore'; @@ -117,6 +141,7 @@ class GlacierBackend { * * The caller can pass the stat data, if none is passed, stat is * called internally. + * @param {nb.NativeFSContext} fs_context * @param {string} file name of the file * @param {nb.NativeFSStats} [stat] * @returns {Promise} @@ -125,9 +150,9 @@ class GlacierBackend { if (!stat) { stat = await nb_native().fs.stat(fs_context, file, { xattr_get_keys: [ - GlacierBackend.XATTR_RESTORE_REQUEST, - GlacierBackend.XATTR_RESTORE_EXPIRY, - GlacierBackend.STORAGE_CLASS_XATTR, + Glacier.XATTR_RESTORE_REQUEST, + Glacier.XATTR_RESTORE_EXPIRY, + Glacier.STORAGE_CLASS_XATTR, ], }); } @@ -136,10 +161,10 @@ class GlacierBackend { // the migration. if (stat.blocks === 0) return false; - const restore_status = GlacierBackend.get_restore_status(stat.xattr, new Date(), file); + const restore_status = Glacier.get_restore_status(stat.xattr, new Date(), file); if (!restore_status) return false; - return restore_status.state === GlacierBackend.RESTORE_STATUS_CAN_RESTORE; + return restore_status.state === Glacier.RESTORE_STATUS_CAN_RESTORE; } /** @@ -154,13 +179,34 @@ class GlacierBackend { * @returns {nb.RestoreStatus | undefined} */ static get_restore_status(xattr, now, file_path) { - if (xattr[GlacierBackend.STORAGE_CLASS_XATTR] !== s3_utils.STORAGE_CLASS_GLACIER) return; + const storage_class = Glacier.storage_class_from_xattr(xattr); + if (storage_class !== s3_utils.STORAGE_CLASS_GLACIER) { + return; + } + + if (Glacier.is_externally_managed(xattr)) { + if (xattr[Glacier.GPFS_DMAPI_XATTR_TAPE_PREMIG]) { + const premig_expiry = new Date(); + // we do not know for how long the file is going to remain available, + // the expiry is set to now + fixed config, which means it's always appears + // to the user with the same amount of time left before it expires. + premig_expiry.setDate(premig_expiry.getDate() + config.NSFS_GLACIER_DMAPI_PMIG_DAYS); + + return { + state: Glacier.RESTORE_STATUS_RESTORED, + ongoing: false, + expiry_time: premig_expiry, + + backend_meta: Glacier.parse_tape_info(xattr), + }; + } + } // Total 8 states (2x restore_request, 4x restore_expiry) let restore_request; let restore_expiry; - const restore_request_xattr = xattr[GlacierBackend.XATTR_RESTORE_REQUEST]; + const restore_request_xattr = xattr[Glacier.XATTR_RESTORE_REQUEST]; if (restore_request_xattr) { const num = Number(restore_request_xattr); if (!isNaN(num) && num > 0) { @@ -169,8 +215,8 @@ class GlacierBackend { dbg.error('unexpected value for restore request for', file_path); } } - if (xattr[GlacierBackend.XATTR_RESTORE_EXPIRY]) { - const expiry = new Date(xattr[GlacierBackend.XATTR_RESTORE_EXPIRY]); + if (xattr[Glacier.XATTR_RESTORE_EXPIRY]) { + const expiry = new Date(xattr[Glacier.XATTR_RESTORE_EXPIRY]); if (isNaN(expiry.getTime())) { dbg.error('unexpected value for restore expiry for', file_path); } else { @@ -185,20 +231,30 @@ class GlacierBackend { return { ongoing: true, - state: GlacierBackend.RESTORE_STATUS_ONGOING, + state: Glacier.RESTORE_STATUS_ONGOING, }; } else { if (!restore_expiry || restore_expiry <= now) { return { ongoing: false, - state: GlacierBackend.RESTORE_STATUS_CAN_RESTORE, + state: Glacier.RESTORE_STATUS_CAN_RESTORE, }; } + if (config.NSFS_GLACIER_USE_DMAPI && !xattr[Glacier.GPFS_DMAPI_XATTR_TAPE_PREMIG]) { + dbg.warn( + 'NooBaa object state for file:', file_path, + 'diverged from actual file state - file not on disk', + 'allowing transparent read to converge file states' + ); + } + return { ongoing: false, expiry_time: restore_expiry, - state: GlacierBackend.RESTORE_STATUS_RESTORED, + state: Glacier.RESTORE_STATUS_RESTORED, + + backend_meta: Glacier.parse_tape_info(xattr), }; } } @@ -246,21 +302,22 @@ class GlacierBackend { * * The caller can pass the stat data, if none is passed, stat is * called internally. + * @param {nb.NativeFSContext} fs_context * @param {string} file name of the file * @param {nb.NativeFSStats} [stat] * @returns {Promise} */ - async should_restore(fs_context, file, stat) { + static async should_restore(fs_context, file, stat) { if (!stat) { stat = await nb_native().fs.stat(fs_context, file, { xattr_get_keys: [ - GlacierBackend.XATTR_RESTORE_REQUEST, - GlacierBackend.STORAGE_CLASS_XATTR, + Glacier.XATTR_RESTORE_REQUEST, + Glacier.STORAGE_CLASS_XATTR, ], }); } - const restore_status = GlacierBackend.get_restore_status(stat.xattr, new Date(), file); + const restore_status = Glacier.get_restore_status(stat.xattr, new Date(), file); if (!restore_status) return false; // We don't check for pre-existing expiry here, it can happen in 2 cases @@ -271,8 +328,93 @@ class GlacierBackend { // removing the request extended attribute. In such case, NSFS would still // report the object restore status to be `ONGOING` and we are going // to allow a retry of that entry. - return restore_status.state === GlacierBackend.RESTORE_STATUS_ONGOING; + return restore_status.state === Glacier.RESTORE_STATUS_ONGOING; + } + + /** + * storage_class_from_xattr returns a parsed storage class derived from the given + * extended attribute. It will use DMAPI EA if `use_dmapi` is set to true. + * + * NOTE: For `use_dmapi` to work, the xattr must have been retrieved using fs_context + * where backend is set to 'GPFS'. + * + * @param {nb.NativeFSXattr} xattr + * @param {Boolean} [use_dmapi] + * @returns {nb.StorageClass} + */ + static storage_class_from_xattr(xattr, use_dmapi = config.NSFS_GLACIER_USE_DMAPI) { + if ( + use_dmapi && + xattr[Glacier.GPFS_DMAPI_XATTR_TAPE_INDICATOR] && + xattr[Glacier.GPFS_DMAPI_XATTR_TAPE_INDICATOR] !== '' + ) { + return s3_utils.STORAGE_CLASS_GLACIER; + } + + return s3_utils.parse_storage_class(xattr[Glacier.STORAGE_CLASS_XATTR]); + } + + /** + * is_externally_managed returns true if given extended attributes + * have tape indicator on them or not (only if `NSFS_GLACIER_USE_DMAPI` + * is set to true) and the storage class xattr is empty (ie none assigned + * by NooBaa). + * + * @param {nb.NativeFSXattr} xattr + * @returns {boolean} + */ + static is_externally_managed(xattr) { + return Boolean( + config.NSFS_GLACIER_USE_DMAPI && + !xattr[Glacier.STORAGE_CLASS_XATTR] && + ( + xattr[Glacier.GPFS_DMAPI_XATTR_TAPE_INDICATOR] || + xattr[Glacier.GPFS_DMAPI_XATTR_TAPE_PREMIG] + ) + ); + } + + /** + * parse_tape_info takes xattr for a file and parses out tape infos + * from the `dmapi.IBMTPS` headers. + * + * @param {nb.NativeFSXattr} xattr + * @returns {nb.TapeInfo[]} + */ + static parse_tape_info(xattr) { + const tape_info_extended = xattr[Glacier.GPFS_DMAPI_XATTR_TAPE_TPS]; + if (!tape_info_extended) return []; + + const tape_info = tape_info_extended.split(" ")[1]; + if (!tape_info) return []; + + return tape_info.split(':').map(info => { + const parsed_info = info.split("@"); + if (parsed_info.length !== 3) { + dbg.warn("Glacier.parse_tape_info: found unexpected dmapi.IBMTPS xattr:", info); + return null; + } + + return { + volser: parsed_info[0], + poolid: parsed_info[1], + libid: parsed_info[2], + }; + }).filter(Boolean); + } + + /** + * getBackend returns appropriate backend for the provided type + * @param {string} [typ] + * @returns {Glacier} + */ + static getBackend(typ = config.NSFS_GLACIER_BACKEND) { + switch (typ) { + case 'TAPECLOUD': return new (require('./glacier_tapecloud').TapeCloudGlacier)(); + default: + throw new Error('invalid backend type provided'); + } } } -exports.GlacierBackend = GlacierBackend; +exports.Glacier = Glacier; diff --git a/src/sdk/nsfs_glacier_backend/tapecloud.js b/src/sdk/glacier_tapecloud.js similarity index 89% rename from src/sdk/nsfs_glacier_backend/tapecloud.js rename to src/sdk/glacier_tapecloud.js index ea3dace401..90184e6b6c 100644 --- a/src/sdk/nsfs_glacier_backend/tapecloud.js +++ b/src/sdk/glacier_tapecloud.js @@ -5,14 +5,14 @@ const { spawn } = require("child_process"); const events = require('events'); const os = require("os"); const path = require("path"); -const { LogFile } = require("../../util/persistent_logger"); -const { NewlineReader, NewlineReaderEntry } = require('../../util/file_reader'); -const { GlacierBackend } = require("./backend"); -const config = require('../../../config'); -const { exec } = require('../../util/os_utils'); -const nb_native = require("../../util/nb_native"); -const { get_process_fs_context } = require("../../util/native_fs_utils"); -const dbg = require('../../util/debug_module')(__filename); +const { LogFile } = require("../util/persistent_logger"); +const { NewlineReader, NewlineReaderEntry } = require('../util/file_reader'); +const { Glacier } = require("./glacier"); +const config = require('../../config'); +const { exec } = require('../util/os_utils'); +const nb_native = require("../util/nb_native"); +const { get_process_fs_context } = require("../util/native_fs_utils"); +const dbg = require('../util/debug_module')(__filename); const ERROR_DUPLICATE_TASK = "GLESM431E"; @@ -189,9 +189,9 @@ class TapeCloudUtils { } } -class TapeCloudGlacierBackend extends GlacierBackend { +class TapeCloudGlacier extends Glacier { async migrate(fs_context, log_file, failure_recorder) { - dbg.log2('TapeCloudGlacierBackend.migrate starting for', log_file); + dbg.log2('TapeCloudGlacier.migrate starting for', log_file); const file = new LogFile(fs_context, log_file); @@ -241,13 +241,13 @@ class TapeCloudGlacierBackend extends GlacierBackend { } async restore(fs_context, log_file, failure_recorder) { - dbg.log2('TapeCloudGlacierBackend.restore starting for', log_file); + dbg.log2('TapeCloudGlacier.restore starting for', log_file); const file = new LogFile(fs_context, log_file); try { await file.collect_and_process(async (entry, batch_recorder) => { try { - const should_restore = await this.should_restore(fs_context, entry); + const should_restore = await Glacier.should_restore(fs_context, entry); if (!should_restore) { // Skip this file return; @@ -272,11 +272,11 @@ class TapeCloudGlacierBackend extends GlacierBackend { const success = await this._recall( batch, async entry_path => { - dbg.log2('TapeCloudGlacierBackend.restore.partial_failure - entry:', entry_path); + dbg.log2('TapeCloudGlacier.restore.partial_failure - entry:', entry_path); await failure_recorder(entry_path); }, async entry_path => { - dbg.log2('TapeCloudGlacierBackend.restore.partial_success - entry:', entry_path); + dbg.log2('TapeCloudGlacier.restore.partial_success - entry:', entry_path); await this._finalize_restore(fs_context, entry_path); } ); @@ -286,7 +286,7 @@ class TapeCloudGlacierBackend extends GlacierBackend { if (success) { const batch_file = new LogFile(fs_context, batch); await batch_file.collect_and_process(async (entry_path, batch_recorder) => { - dbg.log2('TapeCloudGlacierBackend.restore.batch - entry:', entry_path); + dbg.log2('TapeCloudGlacier.restore.batch - entry:', entry_path); await this._finalize_restore(fs_context, entry_path); }); } @@ -354,7 +354,7 @@ class TapeCloudGlacierBackend extends GlacierBackend { * @param {string} entry_path */ async _finalize_restore(fs_context, entry_path) { - dbg.log2('TapeCloudGlacierBackend.restore._finalize_restore - entry:', entry_path); + dbg.log2('TapeCloudGlacier.restore._finalize_restore - entry:', entry_path); const entry = new NewlineReaderEntry(fs_context, entry_path); let fh = null; @@ -373,20 +373,20 @@ class TapeCloudGlacierBackend extends GlacierBackend { const stat = await fh.stat(fs_context, { xattr_get_keys: [ - GlacierBackend.XATTR_RESTORE_REQUEST, + Glacier.XATTR_RESTORE_REQUEST, ] }); - const days = Number(stat.xattr[GlacierBackend.XATTR_RESTORE_REQUEST]); + const days = Number(stat.xattr[Glacier.XATTR_RESTORE_REQUEST]); // In case of invocation on the same file multiple times, // this xattr will not be present hence `days` will be NaN if (isNaN(days)) { - dbg.warn("TapeCloudGlacierBackend._finalize_restore: days is NaN - skipping restore for", entry_path); + dbg.warn("TapeCloudGlacier._finalize_restore: days is NaN - skipping restore for", entry_path); return; } - const expires_on = GlacierBackend.generate_expiry( + const expires_on = Glacier.generate_expiry( new Date(), days, config.NSFS_GLACIER_EXPIRY_TIME_OF_DAY, @@ -401,10 +401,10 @@ class TapeCloudGlacierBackend extends GlacierBackend { // is submitted again). await fh.replacexattr(fs_context, { - [GlacierBackend.XATTR_RESTORE_EXPIRY]: expires_on.toISOString(), + [Glacier.XATTR_RESTORE_EXPIRY]: expires_on.toISOString(), }); - await fh.replacexattr(fs_context, undefined, GlacierBackend.XATTR_RESTORE_REQUEST); + await fh.replacexattr(fs_context, undefined, Glacier.XATTR_RESTORE_REQUEST); } catch (error) { dbg.error(`failed to process ${entry.path}`, error); throw error; @@ -414,5 +414,5 @@ class TapeCloudGlacierBackend extends GlacierBackend { } } -exports.TapeCloudGlacierBackend = TapeCloudGlacierBackend; +exports.TapeCloudGlacier = TapeCloudGlacier; exports.TapeCloudUtils = TapeCloudUtils; diff --git a/src/sdk/namespace_fs.js b/src/sdk/namespace_fs.js index e9834e8941..e4ddbf5760 100644 --- a/src/sdk/namespace_fs.js +++ b/src/sdk/namespace_fs.js @@ -27,7 +27,7 @@ const RpcError = require('../rpc/rpc_error'); const { S3Error } = require('../endpoint/s3/s3_errors'); const NoobaaEvent = require('../manage_nsfs/manage_nsfs_events_utils').NoobaaEvent; const { PersistentLogger } = require('../util/persistent_logger'); -const { GlacierBackend } = require('./nsfs_glacier_backend/backend'); +const { Glacier } = require('./glacier'); const multi_buffer_pool = new buffer_utils.MultiSizeBuffersPool({ sorted_buf_sizes: [ @@ -67,7 +67,6 @@ const XATTR_TAG = XATTR_NOOBAA_INTERNAL_PREFIX + 'tag.'; const HIDDEN_VERSIONS_PATH = '.versions'; const NULL_VERSION_ID = 'null'; const NULL_VERSION_SUFFIX = '_' + NULL_VERSION_ID; -const XATTR_STORAGE_CLASS_KEY = XATTR_USER_PREFIX + 'storage_class'; const VERSIONING_STATUS_ENUM = Object.freeze({ VER_ENABLED: 'ENABLED', @@ -88,7 +87,7 @@ const COPY_STATUS_ENUM = Object.freeze({ }); const XATTR_METADATA_IGNORE_LIST = [ - XATTR_STORAGE_CLASS_KEY, + Glacier.STORAGE_CLASS_XATTR, ]; /** @@ -507,6 +506,7 @@ class NamespaceFS { fs_context.backend = this.fs_backend || ''; fs_context.warn_threshold_ms = config.NSFS_WARN_THRESHOLD_MS; if (this.stats) fs_context.report_fs_stats = this.stats.update_fs_stats; + fs_context.use_dmapi = config.NSFS_GLACIER_USE_DMAPI; return fs_context; } @@ -1326,8 +1326,8 @@ class NamespaceFS { if (params.copy_source) { const src_file_path = await this._find_version_path(fs_context, params.copy_source); const stat = await nb_native().fs.stat(fs_context, src_file_path); - const src_storage_class = s3_utils.parse_storage_class(stat.xattr[XATTR_STORAGE_CLASS_KEY]); - const src_restore_status = GlacierBackend.get_restore_status(stat.xattr, new Date(), src_file_path); + const src_storage_class = Glacier.storage_class_from_xattr(stat.xattr); + const src_restore_status = Glacier.get_restore_status(stat.xattr, new Date(), src_file_path); if (src_storage_class === s3_utils.STORAGE_CLASS_GLACIER) { if (src_restore_status?.ongoing || !src_restore_status?.expiry_time) { @@ -1383,7 +1383,7 @@ class NamespaceFS { } if (!part_upload && params.storage_class) { fs_xattr = Object.assign(fs_xattr || {}, { - [XATTR_STORAGE_CLASS_KEY]: params.storage_class + [Glacier.STORAGE_CLASS_XATTR]: params.storage_class }); if (params.storage_class === s3_utils.STORAGE_CLASS_GLACIER) { @@ -1732,7 +1732,7 @@ class NamespaceFS { return path.join(params.mpu_path, `part-${params.num}`); } - // optimized version of upload_multipart - + // optimized version of upload_multipart - // 1. if size is pre known - // 1.1. calc offset // 1.2. upload data to by_size file in offset position @@ -2209,7 +2209,7 @@ class NamespaceFS { /** * restore_object simply sets the restore request xattr * which should be picked by another mechanism. - * + * * restore_object internally relies on 2 xattrs: * - XATTR_RESTORE_REQUEST * - XATTR_RESTORE_EXPIRY @@ -2230,7 +2230,7 @@ class NamespaceFS { const stat = await file.stat(fs_context); const now = new Date(); - const restore_status = GlacierBackend.get_restore_status(stat.xattr, now, file_path); + const restore_status = Glacier.get_restore_status(stat.xattr, now, file_path); dbg.log1( 'namespace_fs.restore_object:', file_path, 'restore_status:', restore_status, @@ -2241,40 +2241,65 @@ class NamespaceFS { throw new S3Error(S3Error.InvalidObjectStorageClass); } - if (restore_status.state === GlacierBackend.RESTORE_STATUS_CAN_RESTORE) { + /**@type {nb.NativeFSXattr}*/ + const restore_attrs = {}; + + if (Glacier.is_externally_managed(stat.xattr)) { + if (restore_status.state === Glacier.RESTORE_STATUS_RESTORED) { + // If the item is premigrated then its a no-op + // Should result in HTTP: 200 OK + return { accepted: false }; + } + + if (config.NSFS_GLACIER_DMAPI_ALLOW_NOOBAA_TAKEOVER) { + dbg.warn( + 'NSFS_GLACIER_DMAPI_ALLOW_NOOBAA_TAKEOVER is set to true - NooBaa will mark the object "GLACIER"' + ); + + // set the storage class here so that we stop treating the object as externally managed. + // + // This is important to make sure that we report correct expiry of the objects which NooBaa + // restores. + restore_attrs[Glacier.STORAGE_CLASS_XATTR] = s3_utils.STORAGE_CLASS_GLACIER; + } else { + throw new Error('cannot restore externally managed object'); + } + } + + if (restore_status.state === Glacier.RESTORE_STATUS_CAN_RESTORE) { // First add it to the log and then add the extended attribute as if we fail after // this point then the restore request can be triggered again without issue but // the reverse doesn't works. await this.append_to_restore_wal(file_path); - await file.replacexattr(fs_context, { - [GlacierBackend.XATTR_RESTORE_REQUEST]: params.days.toString(), - }); + restore_attrs[Glacier.XATTR_RESTORE_REQUEST] = params.days.toString(); + await file.replacexattr(fs_context, restore_attrs); // Should result in HTTP: 202 Accepted - return {accepted: true}; + return { accepted: true }; } - if (restore_status.state === GlacierBackend.RESTORE_STATUS_ONGOING) { + if (restore_status.state === Glacier.RESTORE_STATUS_ONGOING) { throw new S3Error(S3Error.RestoreAlreadyInProgress); } - if (restore_status.state === GlacierBackend.RESTORE_STATUS_RESTORED) { - const expires_on = GlacierBackend.generate_expiry( + if (restore_status.state === Glacier.RESTORE_STATUS_RESTORED) { + const expires_on = Glacier.generate_expiry( now, params.days, config.NSFS_GLACIER_EXPIRY_TIME_OF_DAY, config.NSFS_GLACIER_EXPIRY_TZ, ); - await file.replacexattr(fs_context, { - [GlacierBackend.XATTR_RESTORE_EXPIRY]: expires_on.toISOString(), - }); + restore_attrs[Glacier.XATTR_RESTORE_EXPIRY] = expires_on.toISOString(); + await file.replacexattr(fs_context, restore_attrs); // Should result in HTTP: 200 OK - return {accepted: false, + return { + accepted: false, expires_on, - storage_class: s3_utils.STORAGE_CLASS_GLACIER}; + storage_class: s3_utils.STORAGE_CLASS_GLACIER + }; } } catch (error) { dbg.error('namespace_fs.restore_object: failed with error: ', error, file_path); @@ -2352,7 +2377,7 @@ class NamespaceFS { } /** - * + * * @param {*} fs_context - fs context object * @param {string} file_path - path to file * @param {*} set - the xattr object to be set @@ -2494,7 +2519,8 @@ class NamespaceFS { const content_type = stat.xattr?.[XATTR_CONTENT_TYPE] || (isDir && dir_content_type) || mime.getType(key) || 'application/octet-stream'; - const storage_class = s3_utils.parse_storage_class(stat.xattr?.[XATTR_STORAGE_CLASS_KEY]); + + const storage_class = Glacier.storage_class_from_xattr(stat.xattr); const size = Number(stat.xattr?.[XATTR_DIR_CONTENT] || stat.size); const tag_count = stat.xattr ? this._number_of_tags_fs_xttr(stat.xattr) : 0; @@ -2511,7 +2537,7 @@ class NamespaceFS { is_latest, delete_marker, storage_class, - restore_status: GlacierBackend.get_restore_status(stat.xattr, new Date(), this._get_file_path({key})), + restore_status: Glacier.get_restore_status(stat.xattr, new Date(), this._get_file_path({key})), xattr: to_xattr(stat.xattr), tag_count, @@ -3477,7 +3503,7 @@ class NamespaceFS { static get migrate_wal() { if (!NamespaceFS._migrate_wal) { - NamespaceFS._migrate_wal = new PersistentLogger(config.NSFS_GLACIER_LOGS_DIR, GlacierBackend.MIGRATE_WAL_NAME, { + NamespaceFS._migrate_wal = new PersistentLogger(config.NSFS_GLACIER_LOGS_DIR, Glacier.MIGRATE_WAL_NAME, { poll_interval: config.NSFS_GLACIER_LOGS_POLL_INTERVAL, locking: 'SHARED', }); @@ -3488,7 +3514,7 @@ class NamespaceFS { static get restore_wal() { if (!NamespaceFS._restore_wal) { - NamespaceFS._restore_wal = new PersistentLogger(config.NSFS_GLACIER_LOGS_DIR, GlacierBackend.RESTORE_WAL_NAME, { + NamespaceFS._restore_wal = new PersistentLogger(config.NSFS_GLACIER_LOGS_DIR, Glacier.RESTORE_WAL_NAME, { poll_interval: config.NSFS_GLACIER_LOGS_POLL_INTERVAL, locking: 'SHARED', }); diff --git a/src/sdk/nb.d.ts b/src/sdk/nb.d.ts index 2a2e3396b8..a2cbe0c37b 100644 --- a/src/sdk/nb.d.ts +++ b/src/sdk/nb.d.ts @@ -439,7 +439,7 @@ interface ObjectInfo { content_range?: string; ns?: Namespace; storage_class?: StorageClass; - restore_status?: { ongoing?: boolean; expiry_time?: Date; }; + restore_status?: RestoreStatus; checksum?: Checksum; object_parts?: GetObjectAttributesParts; } @@ -1036,6 +1036,7 @@ interface NativeFSContext { warn_threshold_ms?: number; report_fs_stats?: Function; do_ctime_check?: boolean; + use_dmapi?: boolean, } type GPFSNooBaaArgs = { @@ -1135,10 +1136,24 @@ type NodeCallback = (err: Error | null, res?: T) => void; type RestoreState = 'CAN_RESTORE' | 'ONGOING' | 'RESTORED'; -interface RestoreStatus { +interface RestoreStatus { state: nb.RestoreState; ongoing?: boolean; expiry_time?: Date; + + /** + * backend_meta is an optional field to store additional metadata + * associated with a Glacier backend + * + * Example storing tape related info from DMAPI + */ + backend_meta?: T; +} + +interface TapeInfo { + volser: string; + poolid: string; + libid: string; } /********************************************************** @@ -1156,4 +1171,4 @@ interface GetObjectAttributesParts { MaxParts?: number; IsTruncated?: boolean; Parts?: ObjectPart[]; - } \ No newline at end of file + } diff --git a/src/sdk/nsfs_glacier_backend/helper.js b/src/sdk/nsfs_glacier_backend/helper.js deleted file mode 100644 index 526dbb8b6e..0000000000 --- a/src/sdk/nsfs_glacier_backend/helper.js +++ /dev/null @@ -1,29 +0,0 @@ -/* Copyright (C) 2024 NooBaa */ -'use strict'; - -/** - * This module exists so as to export the common function `getGlacierBackend` - * - * Keeping this in the generic.js creates cyclic dependency issue.w - */ - -const config = require('../../../config'); -const { TapeCloudGlacierBackend } = require('./tapecloud'); -// eslint-disable-next-line no-unused-vars -const { GlacierBackend } = require('./backend'); - -/** - * getGlacierBackend returns appropriate backend for the provided type - * @param {string} [typ] - * @returns {GlacierBackend} - */ -function getGlacierBackend(typ = config.NSFS_GLACIER_BACKEND) { - switch (typ) { - case 'TAPECLOUD': - return new TapeCloudGlacierBackend(); - default: - throw new Error('invalid backend type provide'); - } -} - -exports.getGlacierBackend = getGlacierBackend; diff --git a/src/test/unit_tests/test_nsfs_glacier_backend.js b/src/test/unit_tests/test_nsfs_glacier_backend.js index d5118c7597..63da32e094 100644 --- a/src/test/unit_tests/test_nsfs_glacier_backend.js +++ b/src/test/unit_tests/test_nsfs_glacier_backend.js @@ -14,9 +14,10 @@ const s3_utils = require('../../endpoint/s3/s3_utils'); const buffer_utils = require('../../util/buffer_utils'); const endpoint_stats_collector = require('../../sdk/endpoint_stats_collector'); const { NewlineReader } = require('../../util/file_reader'); -const { TapeCloudGlacierBackend, TapeCloudUtils } = require('../../sdk/nsfs_glacier_backend/tapecloud'); +const { TapeCloudGlacier, TapeCloudUtils } = require('../../sdk/glacier_tapecloud'); const { PersistentLogger } = require('../../util/persistent_logger'); -const { GlacierBackend } = require('../../sdk/nsfs_glacier_backend/backend'); +const { Glacier } = require('../../sdk/glacier'); +const { Semaphore } = require('../../util/semaphore'); const nb_native = require('../../util/nb_native'); const { handler: s3_get_bucket } = require('../../endpoint/s3/ops/s3_get_bucket'); @@ -79,8 +80,13 @@ function assert_date(date, from, expected, tz = 'LOCAL') { } } -mocha.describe('nsfs_glacier', async () => { +/* Justification: Disable max-lines-per-function for test functions +as it is not much helpful in the sense that "describe" function capture +entire test suite instead of being a logical abstraction */ +/* eslint-disable max-lines-per-function */ +mocha.describe('nsfs_glacier', function() { const src_bkt = 'nsfs_glacier_src'; + const dmapi_config_semaphore = new Semaphore(1); const dummy_object_sdk = make_dummy_object_sdk(); const upload_bkt = 'test_ns_uploads_object'; @@ -98,7 +104,18 @@ mocha.describe('nsfs_glacier', async () => { glacier_ns._is_storage_class_supported = async () => true; - mocha.before(async () => { + const safe_dmapi_surround = async (init, cb) => { + await dmapi_config_semaphore.surround(async () => { + const start_value = config.NSFS_GLACIER_USE_DMAPI; + config.NSFS_GLACIER_USE_DMAPI = init; + + await cb(); + + config.NSFS_GLACIER_USE_DMAPI = start_value; + }); + }; + + mocha.before(async function() { await fs.mkdir(ns_src_bucket_path, { recursive: true }); config.NSFS_GLACIER_LOGS_ENABLED = true; @@ -109,7 +126,7 @@ mocha.describe('nsfs_glacier', async () => { const migrate_wal = NamespaceFS._migrate_wal; NamespaceFS._migrate_wal = new PersistentLogger( config.NSFS_GLACIER_LOGS_DIR, - GlacierBackend.MIGRATE_WAL_NAME, + Glacier.MIGRATE_WAL_NAME, { locking: 'EXCLUSIVE', poll_interval: 10 } ); @@ -118,27 +135,27 @@ mocha.describe('nsfs_glacier', async () => { const restore_wal = NamespaceFS._restore_wal; NamespaceFS._restore_wal = new PersistentLogger( config.NSFS_GLACIER_LOGS_DIR, - GlacierBackend.RESTORE_WAL_NAME, + Glacier.RESTORE_WAL_NAME, { locking: 'EXCLUSIVE', poll_interval: 10 } ); if (restore_wal) await restore_wal.close(); }); - mocha.describe('nsfs_glacier_tapecloud', async () => { + mocha.describe('nsfs_glacier_tapecloud', async function() { const upload_key = 'upload_key_1'; const restore_key = 'restore_key_1'; const xattr = { key: 'value', key2: 'value2' }; xattr[s3_utils.XATTR_SORT_SYMBOL] = true; - const backend = new TapeCloudGlacierBackend(); + const backend = new TapeCloudGlacier(); // Patch backend for test backend._migrate = async () => true; backend._recall = async () => true; backend._process_expired = async () => { /**noop*/ }; - mocha.it('upload to GLACIER should work', async () => { + mocha.it('upload to GLACIER should work', async function() { const data = crypto.randomBytes(100); const upload_res = await glacier_ns.upload_object({ bucket: upload_bkt, @@ -175,7 +192,7 @@ mocha.describe('nsfs_glacier', async () => { assert(found); }); - mocha.it('restore-object should successfully restore', async () => { + mocha.it('restore-object should successfully restore', async function() { const now = Date.now(); const data = crypto.randomBytes(100); const params = { @@ -207,12 +224,12 @@ mocha.describe('nsfs_glacier', async () => { assert(!md.restore_status.ongoing); - const expected_expiry = GlacierBackend.generate_expiry(new Date(), params.days, '', config.NSFS_GLACIER_EXPIRY_TZ); + const expected_expiry = Glacier.generate_expiry(new Date(), params.days, '', config.NSFS_GLACIER_EXPIRY_TZ); assert(expected_expiry.getTime() >= md.restore_status.expiry_time.getTime()); assert(now <= md.restore_status.expiry_time.getTime()); }); - mocha.it('restore-object should not restore failed item', async () => { + mocha.it('restore-object should not restore failed item', async function() { const now = Date.now(); const data = crypto.randomBytes(100); const failed_restore_key = `${restore_key}_failured`; @@ -239,7 +256,7 @@ mocha.describe('nsfs_glacier', async () => { const failed_file_path = glacier_ns._get_file_path(failed_params); const success_file_path = glacier_ns._get_file_path(success_params); - const failure_backend = new TapeCloudGlacierBackend(); + const failure_backend = new TapeCloudGlacier(); failure_backend._migrate = async () => true; failure_backend._process_expired = async () => { /**noop*/ }; failure_backend._recall = async (_file, failure_recorder, success_recorder) => { @@ -279,7 +296,7 @@ mocha.describe('nsfs_glacier', async () => { assert(!success_md.restore_status.ongoing); - const expected_expiry = GlacierBackend.generate_expiry(new Date(), success_params.days, '', config.NSFS_GLACIER_EXPIRY_TZ); + const expected_expiry = Glacier.generate_expiry(new Date(), success_params.days, '', config.NSFS_GLACIER_EXPIRY_TZ); assert(expected_expiry.getTime() >= success_md.restore_status.expiry_time.getTime()); assert(now <= success_md.restore_status.expiry_time.getTime()); @@ -289,26 +306,26 @@ mocha.describe('nsfs_glacier', async () => { failed_file_path, ); - assert(!failure_stats.xattr[GlacierBackend.XATTR_RESTORE_EXPIRY] || failure_stats.xattr[GlacierBackend.XATTR_RESTORE_EXPIRY] === ''); - assert(failure_stats.xattr[GlacierBackend.XATTR_RESTORE_REQUEST]); + assert(!failure_stats.xattr[Glacier.XATTR_RESTORE_EXPIRY] || failure_stats.xattr[Glacier.XATTR_RESTORE_EXPIRY] === ''); + assert(failure_stats.xattr[Glacier.XATTR_RESTORE_REQUEST]); }); - mocha.it('_finalize_restore should tolerate deleted objects', async () => { + mocha.it('_finalize_restore should tolerate deleted objects', async function() { // should not throw error if the path does not exist await backend._finalize_restore(glacier_ns.prepare_fs_context(dummy_object_sdk), '/path/does/not/exist'); }); - mocha.it('generate_expiry should round up the expiry', () => { + mocha.it('generate_expiry should round up the expiry', function() { const now = new Date(); const pivot_time = new Date(now); - const exp1 = GlacierBackend.generate_expiry(now, 1, '', 'UTC'); + const exp1 = Glacier.generate_expiry(now, 1, '', 'UTC'); assert_date(exp1, now, { day_offset: 1 }, 'UTC'); - const exp2 = GlacierBackend.generate_expiry(now, 10, '', 'UTC'); + const exp2 = Glacier.generate_expiry(now, 10, '', 'UTC'); assert_date(exp2, now, { day_offset: 10 }, 'UTC'); - const exp3 = GlacierBackend.generate_expiry(now, 10, '02:05:00', 'UTC'); + const exp3 = Glacier.generate_expiry(now, 10, '02:05:00', 'UTC'); pivot_time.setUTCHours(2, 5, 0, 0); if (now <= pivot_time) { assert_date(exp3, now, { day_offset: 10, hour: 2, min: 5, sec: 0 }, 'UTC'); @@ -316,7 +333,7 @@ mocha.describe('nsfs_glacier', async () => { assert_date(exp3, now, { day_offset: 10 + 1, hour: 2, min: 5, sec: 0 }, 'UTC'); } - const exp4 = GlacierBackend.generate_expiry(now, 1, '02:05:00', 'LOCAL'); + const exp4 = Glacier.generate_expiry(now, 1, '02:05:00', 'LOCAL'); pivot_time.setHours(2, 5, 0, 0); if (now <= pivot_time) { assert_date(exp4, now, { day_offset: 1, hour: 2, min: 5, sec: 0 }, 'LOCAL'); @@ -324,24 +341,63 @@ mocha.describe('nsfs_glacier', async () => { assert_date(exp4, now, { day_offset: 1 + 1, hour: 2, min: 5, sec: 0 }, 'LOCAL'); } - const exp5 = GlacierBackend.generate_expiry(now, 1, `${now.getHours()}:${now.getMinutes()}:${now.getSeconds()}`, 'LOCAL'); + const exp5 = Glacier.generate_expiry(now, 1, `${now.getHours()}:${now.getMinutes()}:${now.getSeconds()}`, 'LOCAL'); assert_date(exp5, now, { day_offset: 1 }, 'LOCAL'); const some_date = new Date("2004-05-08"); - const exp6 = GlacierBackend.generate_expiry(some_date, 1.5, `02:05:00`, 'UTC'); + const exp6 = Glacier.generate_expiry(some_date, 1.5, `02:05:00`, 'UTC'); assert_date(exp6, some_date, { day_offset: 1 + 1, hour: 2, min: 5, sec: 0 }, 'UTC'); }); - }); - mocha.describe('nsfs_glacier_s3_flow', async () => { - mocha.it('list_objects should throw error with incorrect optional object attributes', async () => { + mocha.it('object should be marked externally managed when DMAPI is enabled and xattrs are present', async function() { + await safe_dmapi_surround(true, async () => { + let is_external = Glacier.is_externally_managed({ + [Glacier.GPFS_DMAPI_XATTR_TAPE_PREMIG]: 'some-value', + }); + assert.strictEqual(is_external, true); + + is_external = Glacier.is_externally_managed({ + [Glacier.GPFS_DMAPI_XATTR_TAPE_INDICATOR]: 'some-value', + }); + assert.strictEqual(is_external, true); + + is_external = Glacier.is_externally_managed({ + [Glacier.GPFS_DMAPI_XATTR_TAPE_INDICATOR]: 'some-value', + [Glacier.STORAGE_CLASS_XATTR]: s3_utils.STORAGE_CLASS_GLACIER, + }); + assert.strictEqual(is_external, false); + + is_external = Glacier.is_externally_managed({}); + assert.strictEqual(is_external, false); + }); + }); + + mocha.it('restore_status should return max expiry when DMAPI is enabled and DMAPI xattr is set', async function() { + const now = new Date(); + const expected_expiry = new Date(now); + expected_expiry.setDate(expected_expiry.getDate() + config.NSFS_GLACIER_DMAPI_PMIG_DAYS); + + await safe_dmapi_surround(true, async () => { + const status = Glacier.get_restore_status({ + [Glacier.GPFS_DMAPI_XATTR_TAPE_INDICATOR]: 'some-value', + [Glacier.GPFS_DMAPI_XATTR_TAPE_PREMIG]: 'some-value', + }, now, ''); + assert(status.state === 'RESTORED'); + assert(status.ongoing === false); + assert(status.expiry_time.getDate() === expected_expiry.getDate()); + }); + }); + }); + + mocha.describe('nsfs_glacier_s3_flow', async function() { + mocha.it('list_objects should throw error with incorrect optional object attributes', async function() { const req = generate_noobaa_req_obj(); req.params.bucket = src_bkt; req.headers['x-amz-optional-object-attributes'] = 'restorestatus'; assert.rejects(async () => s3_get_bucket(req)); }); - mocha.it('list_objects should not return restore status when optional object attr header isn\'t given', async () => { + mocha.it('list_objects should not return restore status when optional object attr header isn\'t given', async function() { const req = generate_noobaa_req_obj(); req.params.bucket = src_bkt; req.object_sdk.list_objects = params => glacier_ns.list_objects(params, dummy_object_sdk); @@ -356,7 +412,7 @@ mocha.describe('nsfs_glacier', async () => { }); }); - mocha.it('list_objects should return restore status for the objects when requested', async () => { + mocha.it('list_objects should return restore status for the objects when requested', async function() { const req = generate_noobaa_req_obj(); req.params.bucket = src_bkt; req.headers['x-amz-optional-object-attributes'] = 'RestoreStatus'; @@ -376,7 +432,8 @@ mocha.describe('nsfs_glacier', async () => { const file_path = glacier_ns._get_file_path({ key: obj.Contents.Key }); const stat = await nb_native().fs.stat(fs_context, file_path); - const glacier_status = GlacierBackend.get_restore_status(stat.xattr, new Date(), file_path); + // @ts-ignore + const glacier_status = Glacier.get_restore_status(stat.xattr, new Date(), file_path); if (glacier_status === undefined) { assert.strictEqual(obj.Contents.RestoreStatus, undefined); } else { @@ -390,14 +447,14 @@ mocha.describe('nsfs_glacier', async () => { }); }); - mocha.after(async () => { + mocha.after(async function() { await Promise.all([ fs.rm(ns_src_bucket_path, { recursive: true, force: true }), fs.rm(config.NSFS_GLACIER_LOGS_DIR, { recursive: true, force: true }), ]); }); - mocha.describe('tapecloud_utils', () => { + mocha.describe('tapecloud_utils', function() { const MOCK_TASK_SHOW_DATA = `Random irrelevant data to Result Failure Code Failed time Node -- File name Fail GLESM451W 2023/11/08T02:38:47 1 -- /ibm/gpfs/NoobaaTest/file.aaai @@ -415,7 +472,7 @@ EOF`; const init_tapedir_bin = config.NSFS_GLACIER_TAPECLOUD_BIN_DIR; const tapecloud_bin_temp = path.join(os.tmpdir(), 'tapecloud-bin-dir-'); - mocha.before(async () => { + mocha.before(async function() { config.NSFS_GLACIER_TAPECLOUD_BIN_DIR = await fs.mkdtemp(tapecloud_bin_temp); await fs.writeFile( @@ -426,7 +483,7 @@ EOF`; await fs.chmod(path.join(config.NSFS_GLACIER_TAPECLOUD_BIN_DIR, TapeCloudUtils.TASK_SHOW_SCRIPT), 0o777); }); - mocha.it('record_task_status', async () => { + mocha.it('record_task_status', async function() { const expected_failed_records = [ '/ibm/gpfs/NoobaaTest/file.aaai', '/ibm/gpfs/NoobaaTest/file.aaaj', @@ -469,10 +526,48 @@ EOF`; assert.deepStrictEqual(success_records, []); }); - mocha.after(async () => { + mocha.after(async function() { config.NSFS_GLACIER_TAPECLOUD_BIN_DIR = init_tapedir_bin; await fs.rm(tapecloud_bin_temp, { recursive: true, force: true }); }); }); + + mocha.describe('glacier_utils', function() { + const sample_tape_info = [ + "1 TX0005L9@a83383e7-76d8-4c63-a875-f0d20ec80422@3608e794-53ab-4ba1-aafe-37d044162c17", + "1 TX0005L9@a83383e7-76d8-4c63-a875-f0d20ec80422@3608e794-53ab-4ba1-aafe-37d044162c17:TX0005L0@a83383e7-76d8-4c63-a875-f0d20ec80423@3608e794-53ab-4ba1-aafe-37d044162c19" + ]; + + mocha.it('parse_tape_info with one pool', async function() { + /** @type {nb.NativeFSXattr} */ + const xattr = { + [Glacier.GPFS_DMAPI_XATTR_TAPE_TPS]: sample_tape_info[0], + }; + + const tape_infos = Glacier.parse_tape_info(xattr); + assert.strictEqual(tape_infos.length, 1); + assert.strictEqual(tape_infos[0].volser, "TX0005L9"); + assert.strictEqual(tape_infos[0].poolid, "a83383e7-76d8-4c63-a875-f0d20ec80422"); + assert.strictEqual(tape_infos[0].libid, "3608e794-53ab-4ba1-aafe-37d044162c17"); + }); + + mocha.it('parse_tape_info with two pools', async function() { + /** @type {nb.NativeFSXattr} */ + const xattr = { + [Glacier.GPFS_DMAPI_XATTR_TAPE_TPS]: sample_tape_info[1], + }; + + const tape_infos = Glacier.parse_tape_info(xattr); + assert.strictEqual(tape_infos.length, 2); + + assert.strictEqual(tape_infos[0].volser, "TX0005L9"); + assert.strictEqual(tape_infos[0].poolid, "a83383e7-76d8-4c63-a875-f0d20ec80422"); + assert.strictEqual(tape_infos[0].libid, "3608e794-53ab-4ba1-aafe-37d044162c17"); + + assert.strictEqual(tape_infos[1].volser, "TX0005L0"); + assert.strictEqual(tape_infos[1].poolid, "a83383e7-76d8-4c63-a875-f0d20ec80423"); + assert.strictEqual(tape_infos[1].libid, "3608e794-53ab-4ba1-aafe-37d044162c19"); + }); + }); });