add support for DMAPI xattr based GLACIER storage class

Signed-off-by: Utkarsh Srivastava <[email protected]> add tests for tape info parsing Signed-off-by: Utkarsh Srivastava <[email protected]> remove auto-code formatting Signed-off-by: Utkarsh Srivastava <[email protected]>
noobaa · Feb 21, 2025 · 0bff063 · 0bff063
1 parent f9ea301
commit 0bff063
Show file tree

Hide file tree

Showing 11 changed files with 491 additions and 166 deletions.
diff --git a/.gitignore b/.gitignore
@@ -23,6 +23,8 @@ config-local.js
 *.sublime*
 .DS_Store
 heapdump-*
+.cache
+.clangd
 
 ## PRIVATE
 *.pem

diff --git a/config.js b/config.js
@@ -191,6 +191,15 @@ config.DENY_UPLOAD_TO_STORAGE_CLASS_STANDARD = false;
 // of days an object can be restored using `restore-object` call.
 config.S3_RESTORE_REQUEST_MAX_DAYS = 30;
 
+// NSFS_GLACIER_DMAPI_PMIG_DAYS controls the "virtual"/fake expiry
+// days that will be shown if we detect a glacier object whose life-
+// cycle NSFS doesn't controls
+//
+// This is initialized to be the same as S3_RESTORE_REQUEST_MAX_DAYS
+// but can be overridden to any numberical value
+config.NSFS_GLACIER_DMAPI_PMIG_DAYS = config.S3_RESTORE_REQUEST_MAX_DAYS;
+
+
 /**
  * S3_RESTORE_MAX_DAYS_BEHAVIOUR controls whether to truncate the
  * requested number of days in restore request or whether to deny the request.
@@ -873,6 +882,21 @@ config.NSFS_GLACIER_EXPIRY_TZ = 'LOCAL';
 // the request will be used
 config.NSFS_GLACIER_EXPIRY_TIME_OF_DAY = '';
 
+// If set to true then NooBaa will consider DMAPI extended attributes
+// in conjuction with NooBaa's `user.storage_class` extended attribute
+// to determine state of an object.
+config.NSFS_GLACIER_USE_DMAPI = false;
+
+// NSFS_GLACIER_DMAPI_ALLOW_NOOBAA_TAKEOVER allows NooBaa to take over lifecycle
+// management of an object which was originally NOT managed by NooBaa.
+config.NSFS_GLACIER_DMAPI_ALLOW_NOOBAA_TAKEOVER = false;
+
+// NSFS_GLACIER_DMAPI_TPS_HTTP_HEADER if enabled will add additional HTTP headers
+// `x-tape-meta-copy-n` based on `dmapi.IBMTPS` EA.
+//
+// For this to work, NSFS_GLACIER_USE_DMAPI must be set to `true`.
+config.NSFS_GLACIER_DMAPI_TPS_HTTP_HEADER = config.NSFS_GLACIER_USE_DMAPI || false;
+
 config.NSFS_STATFS_CACHE_SIZE = 10000;
 config.NSFS_STATFS_CACHE_EXPIRY_MS = 1 * 1000;
 

diff --git a/src/endpoint/s3/s3_utils.js b/src/endpoint/s3/s3_utils.js
@@ -325,6 +325,12 @@ function set_response_object_md(res, object_md) {
 
         res.setHeader('x-amz-restore', restore);
     }
+    if (storage_class === STORAGE_CLASS_GLACIER) {
+        object_md.restore_status?.backend_meta?.forEach?.((meta, idx) => {
+            const header = Object.keys(meta).map(key => `${key}=${meta[key]}`).join('&');
+            res.setHeader(`x-tape-meta-copy-${idx}`, header);
+        });
+    }
 }
 
 /** set_response_headers_get_object_attributes is based on set_response_object_md

diff --git a/src/manage_nsfs/manage_nsfs_glacier.js b/src/manage_nsfs/manage_nsfs_glacier.js
@@ -5,8 +5,7 @@ const path = require('path');
 const { PersistentLogger } = require('../util/persistent_logger');
 const config = require('../../config');
 const nb_native = require('../util/nb_native');
-const { GlacierBackend } = require('../sdk/nsfs_glacier_backend/backend');
-const { getGlacierBackend } = require('../sdk/nsfs_glacier_backend/helper');
+const { Glacier } = require('../sdk/glacier');
 const native_fs_utils = require('../util/native_fs_utils');
 
 const CLUSTER_LOCK = 'cluster.lock';
@@ -16,14 +15,14 @@ async function process_migrations() {
     const fs_context = native_fs_utils.get_process_fs_context();
 
     await lock_and_run(fs_context, CLUSTER_LOCK, async () => {
-        const backend = getGlacierBackend();
+        const backend = Glacier.getBackend();
 
         if (
             await backend.low_free_space() ||
-            await time_exceeded(fs_context, config.NSFS_GLACIER_MIGRATE_INTERVAL, GlacierBackend.MIGRATE_TIMESTAMP_FILE)
+            await time_exceeded(fs_context, config.NSFS_GLACIER_MIGRATE_INTERVAL, Glacier.MIGRATE_TIMESTAMP_FILE)
         ) {
             await run_glacier_migrations(fs_context, backend);
-            await record_current_time(fs_context, GlacierBackend.MIGRATE_TIMESTAMP_FILE);
+            await record_current_time(fs_context, Glacier.MIGRATE_TIMESTAMP_FILE);
         }
     });
 }
@@ -32,56 +31,56 @@ async function process_migrations() {
  * run_tape_migrations reads the migration WALs and attempts to migrate the
  * files mentioned in the WAL.
  * @param {nb.NativeFSContext} fs_context
- * @param {import('../sdk/nsfs_glacier_backend/backend').GlacierBackend} backend
+ * @param {import('../sdk/glacier').Glacier} backend
  */
 async function run_glacier_migrations(fs_context, backend) {
-    await run_glacier_operation(fs_context, GlacierBackend.MIGRATE_WAL_NAME, backend.migrate.bind(backend));
+    await run_glacier_operation(fs_context, Glacier.MIGRATE_WAL_NAME, backend.migrate.bind(backend));
 }
 
 async function process_restores() {
     const fs_context = native_fs_utils.get_process_fs_context();
 
     await lock_and_run(fs_context, CLUSTER_LOCK, async () => {
-        const backend = getGlacierBackend();
+        const backend = Glacier.getBackend();
 
         if (
             await backend.low_free_space() ||
-            !(await time_exceeded(fs_context, config.NSFS_GLACIER_RESTORE_INTERVAL, GlacierBackend.RESTORE_TIMESTAMP_FILE))
+            !(await time_exceeded(fs_context, config.NSFS_GLACIER_RESTORE_INTERVAL, Glacier.RESTORE_TIMESTAMP_FILE))
         ) return;
 
 
         await run_glacier_restore(fs_context, backend);
-        await record_current_time(fs_context, GlacierBackend.RESTORE_TIMESTAMP_FILE);
+        await record_current_time(fs_context, Glacier.RESTORE_TIMESTAMP_FILE);
     });
 }
 
 /**
  * run_tape_restore reads the restore WALs and attempts to restore the
  * files mentioned in the WAL.
- * @param {nb.NativeFSContext} fs_context 
- * @param {import('../sdk/nsfs_glacier_backend/backend').GlacierBackend} backend
+ * @param {nb.NativeFSContext} fs_context
+ * @param {import('../sdk/glacier').Glacier} backend
  */
 async function run_glacier_restore(fs_context, backend) {
-    await run_glacier_operation(fs_context, GlacierBackend.RESTORE_WAL_NAME, backend.restore.bind(backend));
+    await run_glacier_operation(fs_context, Glacier.RESTORE_WAL_NAME, backend.restore.bind(backend));
 }
 
 async function process_expiry() {
-    const fs_context = native_fs_utils.get_process_fs_context();
+    const fs_context = force_gpfs_fs_context(native_fs_utils.get_process_fs_context());
 
     await lock_and_run(fs_context, SCAN_LOCK, async () => {
-        const backend = getGlacierBackend();
+        const backend = Glacier.getBackend();
         if (
             await backend.low_free_space() ||
             await is_desired_time(
                     fs_context,
                     new Date(),
                     config.NSFS_GLACIER_EXPIRY_RUN_TIME,
                     config.NSFS_GLACIER_EXPIRY_RUN_DELAY_LIMIT_MINS,
-                    GlacierBackend.EXPIRY_TIMESTAMP_FILE,
+                    Glacier.EXPIRY_TIMESTAMP_FILE,
             )
         ) {
             await backend.expiry(fs_context);
-            await record_current_time(fs_context, GlacierBackend.EXPIRY_TIMESTAMP_FILE);
+            await record_current_time(fs_context, Glacier.EXPIRY_TIMESTAMP_FILE);
         }
     });
 }
@@ -178,6 +177,8 @@ async function record_current_time(fs_context, timestamp_file) {
  */
 async function run_glacier_operation(fs_context, log_namespace, cb) {
     const log = new PersistentLogger(config.NSFS_GLACIER_LOGS_DIR, log_namespace, { locking: 'EXCLUSIVE' });
+
+    fs_context = force_gpfs_fs_context(fs_context);
     try {
         await log.process(async (entry, failure_recorder) => cb(fs_context, entry, failure_recorder));
     } catch (error) {
@@ -212,6 +213,28 @@ function get_tz_date(hours, mins, secs, tz) {
     return date;
 }
 
+/**
+ * force_gpfs_fs_context returns a shallow copy of given
+ * fs_context with backend set to 'GPFS'.
+ *
+ * NOTE: The function will throw error if it detects that GPFS
+ * DL isn't loaded.
+ *
+ * @param {nb.NativeFSContext} fs_context
+ * @returns {nb.NativeFSContext}
+ */
+function force_gpfs_fs_context(fs_context) {
+    if (config.NSFS_GLACIER_USE_DMAPI) {
+        if (!nb_native().fs.gpfs) {
+            throw new Error('cannot use DMAPI EA: gpfs dl not loaded');
+        }
+
+        return { ...fs_context, backend: 'GPFS', use_dmapi: true };
+    }
+
+    return { ...fs_context };
+}
+
 /**
  * lock_and_run acquires a flock and calls the given callback after
  * acquiring the lock

diff --git a/src/native/fs/fs_napi.cpp b/src/native/fs/fs_napi.cpp
@@ -46,6 +46,13 @@
 #define GPFS_XATTR_PREFIX "gpfs"
 #define GPFS_DOT_ENCRYPTION_EA "Encryption"
 #define GPFS_ENCRYPTION_XATTR_NAME GPFS_XATTR_PREFIX "." GPFS_DOT_ENCRYPTION_EA
+#define GPFS_DMAPI_XATTR_PREFIX "dmapi"
+#define GPFS_DMAPI_DOT_IBMOBJ_EA "IBMObj"
+#define GPFS_DMAPI_DOT_IBMPMIG_EA "IBMPMig"
+#define GPFS_DMAPI_DOT_IBMTPS_EA "IBMTPS"
+#define GPFS_DMAPI_XATTR_TAPE_INDICATOR GPFS_DMAPI_XATTR_PREFIX "." GPFS_DMAPI_DOT_IBMOBJ_EA
+#define GPFS_DMAPI_XATTR_TAPE_PREMIG GPFS_DMAPI_XATTR_PREFIX "." GPFS_DMAPI_DOT_IBMPMIG_EA
+#define GPFS_DMAPI_XATTR_TAPE_TPS GPFS_DMAPI_XATTR_PREFIX "." GPFS_DMAPI_DOT_IBMTPS_EA
 
 // This macro should be used after openning a file
 // it will autoclose the file using AutoCloser and will throw an error in case of failures
@@ -244,6 +251,11 @@ parse_open_flags(std::string flags)
 }
 
 const static std::vector<std::string> GPFS_XATTRS{ GPFS_ENCRYPTION_XATTR_NAME };
+const static std::vector<std::string> GPFS_DMAPI_XATTRS{
+    GPFS_DMAPI_XATTR_TAPE_INDICATOR,
+    GPFS_DMAPI_XATTR_TAPE_PREMIG,
+    GPFS_DMAPI_XATTR_TAPE_TPS,
+};
 const static std::vector<std::string> USER_XATTRS{
     "user.content_type",
     "user.content_md5",
@@ -461,9 +473,14 @@ get_fd_xattr(int fd, XattrMap& xattr, const std::vector<std::string>& xattr_keys
 }
 
 static int
-get_fd_gpfs_xattr(int fd, XattrMap& xattr, int& gpfs_error)
+get_fd_gpfs_xattr(int fd, XattrMap& xattr, int& gpfs_error, bool use_dmapi)
 {
-    for (auto const& key : GPFS_XATTRS) {
+    auto gpfs_xattrs { GPFS_XATTRS };
+    if (use_dmapi) {
+        gpfs_xattrs.insert(gpfs_xattrs.end(), GPFS_DMAPI_XATTRS.begin(), GPFS_DMAPI_XATTRS.end());
+    }
+
+    for (auto const& key : gpfs_xattrs) {
         gpfsRequest_t gpfsGetXattrRequest;
         build_gpfs_get_ea_request(&gpfsGetXattrRequest, key);
         int r = dlsym_gpfs_fcntl(fd, &gpfsGetXattrRequest);
@@ -603,6 +620,8 @@ struct FSWorker : public Napi::AsyncWorker
     // NOTE: If _do_ctime_check = false, then some functions will fallback to using mtime check
     bool _do_ctime_check;
 
+    bool _use_dmapi;
+
     FSWorker(const Napi::CallbackInfo& info)
         : AsyncWorker(info.Env())
         , _deferred(Napi::Promise::Deferred::New(info.Env()))
@@ -616,6 +635,7 @@ struct FSWorker : public Napi::AsyncWorker
         , _should_add_thread_capabilities(false)
         , _supplemental_groups()
         , _do_ctime_check(false)
+        , _use_dmapi(false)
     {
         for (int i = 0; i < (int)info.Length(); ++i) _args_ref.Set(i, info[i]);
         if (info[0].ToBoolean()) {
@@ -635,6 +655,7 @@ struct FSWorker : public Napi::AsyncWorker
                 _report_fs_stats = Napi::Persistent(fs_context.Get("report_fs_stats").As<Napi::Function>());
             }
             _do_ctime_check = fs_context.Get("do_ctime_check").ToBoolean();
+            _use_dmapi = fs_context.Get("use_dmapi").ToBoolean();
         }
     }
     void Begin(std::string desc)
@@ -793,7 +814,7 @@ struct Stat : public FSWorker
         if (!_use_lstat) {
             SYSCALL_OR_RETURN(get_fd_xattr(fd, _xattr, _xattr_get_keys));
             if (use_gpfs_lib()) {
-                GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error));
+                GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error, _use_dmapi));
             }
         }
 
@@ -1221,7 +1242,7 @@ struct Readfile : public FSWorker
         if (_read_xattr) {
             SYSCALL_OR_RETURN(get_fd_xattr(fd, _xattr, _xattr_get_keys));
             if (use_gpfs_lib()) {
-                GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error));
+                GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error, _use_dmapi));
             }
         }
 
@@ -1752,7 +1773,7 @@ struct FileStat : public FSWrapWorker<FileWrap>
         SYSCALL_OR_RETURN(fstat(fd, &_stat_res));
         SYSCALL_OR_RETURN(get_fd_xattr(fd, _xattr, _xattr_get_keys));
         if (use_gpfs_lib()) {
-            GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error));
+            GPFS_FCNTL_OR_RETURN(get_fd_gpfs_xattr(fd, _xattr, gpfs_error, _use_dmapi));
         }
 
         if (_do_ctime_check) {