Merge pull request #8830 from romayalon/romy-5.17-backports1

NC | 5.17.z Backport

src/endpoint/s3/ops/s3_get_object.js

@@ -48,7 +48,7 @@ async function get_object(req, res) {
             throw new S3Error(S3Error.InvalidObjectState);
         }
     }
-
+    http_utils.set_response_headers_from_request(req, res);
     const obj_size = object_md.size;
     const params = {
         object_md,

src/endpoint/s3/ops/s3_head_object.js

@@ -28,6 +28,7 @@ async function head_object(req, res) {
 
     s3_utils.set_response_object_md(res, object_md);
     s3_utils.set_encryption_response_headers(req, res, object_md.encryption);
+    http_utils.set_response_headers_from_request(req, res);
 }
 
 module.exports = {

src/test/unit_tests/test_bucketspace.js

@@ -62,6 +62,14 @@ const tmp_fs_root = path.join(TMP_PATH, 'test_bucket_namespace_fs');
 // on Containerized - new_buckets_path is the directory
 const new_bucket_path_param = get_new_buckets_path_by_test_env(tmp_fs_root, '/');
 
+// response headers
+const response_content_disposition = 'attachment';
+const response_content_language = 'hebrew';
+const response_content_type = 'application/json';
+const response_cache_control = 'no-cache';
+const response_expires = new Date();
+response_expires.setMilliseconds(0);
+
 // currently will pass only when running locally
 mocha.describe('bucket operations - namespace_fs', function() {
     const nsr = 'nsr';
@@ -2233,6 +2241,115 @@ mocha.describe('Presigned URL tests', function() {
         const expected_err = new S3Error(S3Error.AuthorizationQueryParametersError);
         await assert_throws_async(fetchData(invalid_expiry_presigned_url), expected_err.message);
     });
+
+    it('get-object - fetch valid presigned URL - 604800 seconds - epoch expiry - should return object data + return response headers', async () => {
+        const response_queries = {
+            ResponseContentDisposition: response_content_disposition,
+            ResponseContentLanguage: response_content_language,
+            ResponseContentType: response_content_type,
+            ResponseCacheControl: response_cache_control,
+            ResponseExpires: response_expires
+        };
+        const presigned_url_params_with_response_headers = { ...presigned_url_params, response_queries };
+        const url_with_response_headers = cloud_utils.get_signed_url(presigned_url_params_with_response_headers, 604800);
+        const headers = await fetchHeaders(url_with_response_headers);
+        assert.equal(headers.get('content-disposition'), response_content_disposition);
+        assert.equal(headers.get('content-language'), response_content_language);
+        assert.equal(headers.get('content-type'), response_content_type);
+        assert.equal(headers.get('cache-control'), response_cache_control);
+        assert.deepStrictEqual(headers.get('expires'), response_expires.toUTCString());
+    });
+
+    it('head-object - fetch valid presigned URL - 604800 seconds - epoch expiry - should return object data + return response headers', async () => {
+        const response_queries = {
+            ResponseContentDisposition: response_content_disposition,
+            ResponseContentLanguage: response_content_language,
+            ResponseContentType: response_content_type,
+            ResponseCacheControl: response_cache_control,
+            ResponseExpires: response_expires
+        };
+        const presigned_url_params_with_response_headers = { ...presigned_url_params, response_queries };
+        const url_with_response_headers = cloud_utils.get_signed_url(presigned_url_params_with_response_headers, 604800, 'headObject');
+        const headers = await fetchHeaders(url_with_response_headers, { method: 'HEAD' });
+        assert.equal(headers.get('content-disposition'), response_content_disposition);
+        assert.equal(headers.get('content-language'), response_content_language);
+        assert.equal(headers.get('content-type'), response_content_type);
+        assert.equal(headers.get('cache-control'), response_cache_control);
+        assert.deepStrictEqual(headers.get('expires'), response_expires.toUTCString());
+    });
+});
+
+mocha.describe('response headers test - regular request', function() {
+    this.timeout(50000); // eslint-disable-line no-invalid-this
+    const nsr = 'response_headers_nsr';
+    const account_name = 'response_header_account';
+    const fs_path = path.join(TMP_PATH, 'response_header_tests/');
+    const response_header_bucket = 'response-headerbucket';
+    const response_header_object = 'response-header-object.txt';
+    const response_header_body = 'response_header_body';
+    let s3_client;
+    let access_key;
+    let secret_key;
+    CORETEST_ENDPOINT = coretest.get_http_address();
+
+    mocha.before(async function() {
+        await fs_utils.create_fresh_path(fs_path);
+        await rpc_client.pool.create_namespace_resource({ name: nsr, nsfs_config: { fs_root_path: fs_path } });
+        const new_buckets_path = is_nc_coretest ? fs_path : '/';
+        const nsfs_account_config = {
+            uid: process.getuid(), gid: process.getgid(), new_buckets_path, nsfs_only: true
+        };
+        const account_params = { ...new_account_params, email: `${account_name}@noobaa.io`, name: account_name, default_resource: nsr, nsfs_account_config };
+        const res = await rpc_client.account.create_account(account_params);
+        access_key = res.access_keys[0].access_key;
+        secret_key = res.access_keys[0].secret_key;
+        s3_client = generate_s3_client(access_key.unwrap(), secret_key.unwrap(), CORETEST_ENDPOINT);
+        await s3_client.createBucket({ Bucket: response_header_bucket });
+        await s3_client.putObject({ Bucket: response_header_bucket, Key: response_header_object, Body: response_header_body });
+    });
+
+    mocha.after(async function() {
+        if (!is_nc_coretest) return;
+        await s3_client.deleteObject({ Bucket: response_header_bucket, Key: response_header_object });
+        await s3_client.deleteBucket({ Bucket: response_header_bucket });
+        await rpc_client.account.delete_account({ email: `${account_name}@noobaa.io` });
+        await fs_utils.folder_delete(fs_path);
+    });
+
+    it('get-object - response headers', async () => {
+        const res = await s3_client.getObject({
+            Bucket: response_header_bucket,
+            Key: response_header_object,
+            ResponseContentDisposition: response_content_disposition,
+            ResponseContentLanguage: response_content_language,
+            ResponseContentType: response_content_type,
+            ResponseCacheControl: response_cache_control,
+            ResponseExpires: response_expires,
+        });
+        assert.equal(res.ContentDisposition, response_content_disposition);
+        assert.equal(res.ContentLanguage, response_content_language);
+        assert.equal(res.ContentType, response_content_type);
+        assert.equal(res.CacheControl, response_cache_control);
+        assert.deepStrictEqual(res.Expires, response_expires);
+    });
+
+    it('head-object - response headers', async () => {
+        const res = await s3_client.headObject({
+            Bucket: response_header_bucket,
+            Key: response_header_object,
+            ResponseContentDisposition: response_content_disposition,
+            ResponseContentLanguage: response_content_language,
+            ResponseContentType: response_content_type,
+            ResponseCacheControl: response_cache_control,
+            ResponseExpires: response_expires,
+        });
+        assert.equal(res.ContentDisposition, response_content_disposition);
+        assert.equal(res.ContentLanguage, response_content_language);
+        assert.equal(res.ContentType, response_content_type);
+        assert.equal(res.CacheControl, response_cache_control);
+        assert.deepStrictEqual(res.Expires, response_expires);
+    });
+
 });
 
 async function fetchData(presigned_url) {
@@ -2248,3 +2365,17 @@ async function fetchData(presigned_url) {
     data = await response.text();
     return data.trim();
 }
+
+async function fetchHeaders(presigned_url, options) {
+    const response = await fetch(presigned_url, { ...options, agent: new http.Agent({ keepAlive: false }) });
+    let data;
+    if (!response.ok) {
+        data = (await response.text()).trim();
+        const err_json = (await http_utils.parse_xml_to_js(data)).Error;
+        const err = new Error(err_json.Message);
+        err.code = err_json.Code;
+        throw err;
+    }
+    return response.headers;
+}
+

src/util/cloud_utils.js

@@ -58,7 +58,8 @@ async function generate_aws_sts_creds(params, roleSessionName) {
     );
 }
 
-function get_signed_url(params, expiry = 604800) {
+function get_signed_url(params, expiry = 604800, custom_operation = 'getObject') {
+    const op = custom_operation;
     const s3 = new AWS.S3({
         endpoint: params.endpoint,
         credentials: {
@@ -76,12 +77,14 @@ function get_signed_url(params, expiry = 604800) {
             agent: http_utils.get_unsecured_agent(params.endpoint)
         }
     });
+    const response_queries = params.response_queries || {};
     return s3.getSignedUrl(
-        'getObject', {
+        op, {
             Bucket: params.bucket.unwrap(),
             Key: params.key,
             VersionId: params.version_id,
-            Expires: expiry
+            Expires: expiry,
+            ...response_queries
         }
     );
 }

src/util/http_utils.js

@@ -4,6 +4,7 @@
 
 const _ = require('lodash');
 const ip = require('ip');
+const util = require('util');
 const url = require('url');
 const http = require('http');
 const https = require('https');
@@ -724,6 +725,21 @@ function http_get(uri, options) {
 }
 
 
+/**
+ * set_response_headers_from_request sets the response headers based on the request headers
+ * gap - response-content-encoding needs to be added with a more complex logic
+ * @param {http.IncomingMessage} req 
+ * @param {http.ServerResponse} res 
+ */
+function set_response_headers_from_request(req, res) {
+    dbg.log2(`set_response_headers_from_request req.query ${util.inspect(req.query)}`);
+    if (req.query['response-cache-control']) res.setHeader('Cache-Control', req.query['response-cache-control']);
+    if (req.query['response-content-disposition']) res.setHeader('Content-Disposition', req.query['response-content-disposition']);
+    if (req.query['response-content-language']) res.setHeader('Content-Language', req.query['response-content-language']);
+    if (req.query['response-content-type']) res.setHeader('Content-Type', req.query['response-content-type']);
+    if (req.query['response-expires']) res.setHeader('Expires', req.query['response-expires']);
+}
+
 exports.parse_url_query = parse_url_query;
 exports.parse_client_ip = parse_client_ip;
 exports.get_md_conditions = get_md_conditions;
@@ -757,3 +773,4 @@ exports.CONTENT_TYPE_APP_OCTET_STREAM = CONTENT_TYPE_APP_OCTET_STREAM;
 exports.CONTENT_TYPE_APP_JSON = CONTENT_TYPE_APP_JSON;
 exports.CONTENT_TYPE_APP_XML = CONTENT_TYPE_APP_XML;
 exports.CONTENT_TYPE_APP_FORM_URLENCODED = CONTENT_TYPE_APP_FORM_URLENCODED;
+exports.set_response_headers_from_request = set_response_headers_from_request;

src/util/native_fs_utils.js

@@ -291,10 +291,9 @@ function get_config_files_tmpdir() {
  * @param {string} config_data 
  */
 async function create_config_file(fs_context, schema_dir, config_path, config_data) {
-    const is_gpfs = _is_gpfs(fs_context);
-    const open_mode = is_gpfs ? 'wt' : 'w';
+    const open_mode = 'w';
+    let open_path;
     let upload_tmp_file;
-    let gpfs_dst_file;
     try {
         // validate config file doesn't exist
         try {
@@ -308,37 +307,30 @@ async function create_config_file(fs_context, schema_dir, config_path, config_da
         dbg.log1('create_config_file:: config_path:', config_path, 'config_data:', config_data, 'is_gpfs:', open_mode);
         // create config dir if it does not exist
         await _create_path(schema_dir, fs_context, config.BASE_MODE_CONFIG_DIR);
-        // when using GPFS open dst file as soon as possible for later linkat validation
-        if (is_gpfs) gpfs_dst_file = await open_file(fs_context, schema_dir, config_path, 'w*', config.BASE_MODE_CONFIG_FILE);
 
         // open tmp file (in GPFS we open the parent dir using wt open mode)
         const tmp_dir_path = path.join(schema_dir, get_config_files_tmpdir());
-        let open_path = is_gpfs ? config_path : await _generate_unique_path(fs_context, tmp_dir_path);
+        open_path = await _generate_unique_path(fs_context, tmp_dir_path);
         upload_tmp_file = await open_file(fs_context, schema_dir, open_path, open_mode, config.BASE_MODE_CONFIG_FILE);
 
         // write tmp file data
         await upload_tmp_file.writev(fs_context, [Buffer.from(config_data)], 0);
 
-        // moving tmp file to config path atomically
-        let src_stat;
-        let gpfs_options;
-        if (is_gpfs) {
-            gpfs_options = { dst_file: gpfs_dst_file, dir_file: upload_tmp_file };
-            // open path in GPFS is the parent dir 
-            open_path = schema_dir;
-        } else {
-            src_stat = await nb_native().fs.stat(fs_context, open_path);
-        }
-        dbg.log1('create_config_file:: moving from:', open_path, 'to:', config_path, 'is_gpfs=', is_gpfs);
+        dbg.log1('create_config_file:: moving from:', open_path, 'to:', config_path);
 
-        await safe_move(fs_context, open_path, config_path, src_stat, gpfs_options, tmp_dir_path);
+        await nb_native().fs.link(fs_context, open_path, config_path);
 
         dbg.log1('create_config_file:: done', config_path);
     } catch (err) {
         dbg.error('create_config_file:: error', err);
         throw err;
     } finally {
-        await finally_close_files(fs_context, [upload_tmp_file, gpfs_dst_file]);
+        await finally_close_files(fs_context, [upload_tmp_file]);
+        try {
+            await nb_native().fs.unlink(fs_context, open_path);
+        } catch (error) {
+            dbg.log0(`create_config_file:: unlink tmp file failed with error ${error}, skipping`);
+        }
     }
 }