Merge pull request #867 from versity/test_cmdline_rest_retention

test: REST retention, versioning testing, misc cleanup

tests/.env.default

@@ -27,3 +27,4 @@ USERNAME_TWO=HIJKLMN
 PASSWORD_TWO=OPQRSTU
 TEST_FILE_FOLDER=$PWD/versity-gwtest-files
 REMOVE_TEST_FILE_FOLDER=true
+VERSIONING_DIR=/tmp/versioning

tests/commands/delete_object.sh

@@ -58,6 +58,30 @@ delete_object_bypass_retention() {
   return 0
 }
 
+delete_object_version() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'delete_object_version' requires bucket, key, version ID"
+    return 1
+  fi
+  if ! delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" 2>&1); then
+    log 2 "error deleting object version: $delete_object_error"
+    return 1
+  fi
+  return 0
+}
+
+delete_object_version_bypass_retention() {
+  if [[ $# -ne 3 ]]; then
+    log 2 "'delete_object_version_bypass_retention' requires bucket, key, version ID"
+    return 1
+  fi
+  if ! delete_object_error=$(aws --no-verify-ssl s3api delete-object --bucket "$1" --key "$2" --version-id "$3" --bypass-governance-retention 2>&1); then
+    log 2 "error deleting object version with bypass retention: $delete_object_error"
+    return 1
+  fi
+  return 0
+}
+
 delete_object_with_user() {
   record_command "delete-object" "client:$1"
   if [ $# -ne 5 ]; then

tests/commands/get_bucket_versioning.sh

@@ -17,15 +17,60 @@
 get_bucket_versioning() {
   record_command "get-bucket-versioning" "client:s3api"
   if [[ $# -ne 2 ]]; then
-    log 2 "put bucket versioning command requires command type, bucket name"
+    log 2 "get bucket versioning command requires command type, bucket name"
     return 1
   fi
   local get_result=0
   if [[ $1 == 's3api' ]]; then
-    error=$(aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$?
+    versioning=$(aws --no-verify-ssl s3api get-bucket-versioning --bucket "$2" 2>&1) || get_result=$?
   fi
   if [[ $get_result -ne 0 ]]; then
-    log 2 "error getting bucket versioning: $error"
+    log 2 "error getting bucket versioning: $versioning"
+    return 1
+  fi
+  return 0
+}
+
+get_bucket_versioning_rest() {
+  log 6 "get_object_rest"
+  if [ $# -ne 1 ]; then
+    log 2 "'get_bucket_versioning_rest' requires bucket name"
+    return 1
+  fi
+
+  #generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/$1
+versioning=
+host:$aws_endpoint_url_address
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  if ! reply=$(curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?versioning" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+    -H "x-amz-date: $current_date_time" \
+    -o "$TEST_FILE_FOLDER/versioning.txt" 2>&1); then
+      log 2 "error retrieving curl reply: $reply"
+      return 1
+  fi
+  log 5 "reply: $reply"
+  if [[ "$reply" != "200" ]]; then
+    log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/versioning.txt")"
     return 1
   fi
   return 0

tests/commands/get_object_lock_configuration.sh

@@ -28,4 +28,44 @@ get_object_lock_configuration() {
   fi
   lock_config=$(echo "$lock_config" | grep -v "InsecureRequestWarning")
   return 0
+}
+
+get_object_lock_configuration_rest() {
+  log 6 "get_object_lock_configuration_rest"
+  if [ $# -ne 1 ]; then
+    log 2 "'get_object_lock_configuration_rest' requires bucket name"
+    return 1
+  fi
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/$1
+object-lock=
+host:$aws_endpoint_url_address
+x-amz-content-sha256:UNSIGNED-PAYLOAD
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+UNSIGNED-PAYLOAD"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
+    -H "x-amz-date: $current_date_time" \
+    -o "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1)
+  log 5 "reply: $reply"
+  if [[ "$reply" != "200" ]]; then
+    log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
+    return 1
+  fi
+  return 0
 }

tests/commands/get_object_retention.sh

@@ -27,4 +27,50 @@ get_object_retention() {
     return 1
   fi
   return 0
-}
+}
+
+get_object_retention_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'get_object_tagging_rest' requires bucket, key"
+    return 1
+  fi
+
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/$1/$2
+retention=
+host:$aws_endpoint_url_address
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+  # shellcheck disable=SC2154
+  reply=$(curl -ks -w "%{http_code}" "$header://$aws_endpoint_url_address/$1/$2?retention" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: $payload_hash" \
+    -H "x-amz-date: $current_date_time" \
+    -o "$TEST_FILE_FOLDER"/object_retention.txt 2>&1)
+  log 5 "reply status code: $reply"
+  if [[ "$reply" != "200" ]]; then
+    if [ "$reply" == "404" ]; then
+      return 1
+    fi
+    log 2 "reply error: $reply"
+    log 2 "get object retention command returned error: $(cat "$TEST_FILE_FOLDER"/object_retention.txt)"
+    return 2
+  fi
+  log 5 "object tags:  $(cat "$TEST_FILE_FOLDER"/object_retention.txt)"
+  return 0
+}

tests/commands/list_object_versions.sh

@@ -20,10 +20,44 @@ list_object_versions() {
     log 2 "'list object versions' command requires bucket name"
     return 1
   fi
-  versions=$(aws --no-verify-ssl s3api list-object-versions --bucket "$1") || local list_result=$?
+  versions=$(aws --no-verify-ssl s3api list-object-versions --bucket "$1" 2>&1) || local list_result=$?
   if [[ $list_result -ne 0 ]]; then
     log 2 "error listing object versions: $versions"
     return 1
   fi
+  versions=$(echo "$versions" | grep -v "InsecureRequestWarning")
   return 0
+}
+
+list_object_versions_rest() {
+  if [ $# -ne 1 ]; then
+    log 2 "'list_object_versions_rest' requires bucket name"
+    return 1
+  fi
+  generate_hash_for_payload ""
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  # shellcheck disable=SC2154
+  canonical_request="GET
+/$1
+versions=
+host:${AWS_ENDPOINT_URL#*//}
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+
+  get_signature
+  # shellcheck disable=SC2034,SC2154
+  reply=$(curl -ks "$AWS_ENDPOINT_URL/$1?versions" \
+         -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+         -H "x-amz-content-sha256: $payload_hash" \
+         -H "x-amz-date: $current_date_time" \
+         -o "$TEST_FILE_FOLDER/object_versions.txt" 2>&1)
 }

tests/commands/put_bucket_versioning.sh

@@ -29,4 +29,55 @@ put_bucket_versioning() {
     return 1
   fi
   return 0
+}
+
+put_bucket_versioning_rest() {
+  if [ $# -ne 2 ]; then
+    log 2 "'put_bucket_versioning_rest' requires bucket, 'Enabled' or 'Suspended'"
+    return 1
+  fi
+
+  versioning="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
+<VersioningConfiguration xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
+  <Status>$2</Status>
+</VersioningConfiguration>"
+
+  generate_hash_for_payload "$versioning"
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  content_md5=$(echo -n "$versioning" | openssl dgst -binary -md5 | openssl base64)
+  # shellcheck disable=SC2154
+  canonical_request="PUT
+/$1
+versioning=
+content-md5:$content_md5
+host:$aws_endpoint_url_address
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+content-md5;host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+
+  # shellcheck disable=SC2154
+  reply=$(curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1?versioning" \
+    -H "Content-MD5: $content_md5" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: $payload_hash" \
+    -H "x-amz-date: $current_date_time" \
+    -d "$versioning" -o "$TEST_FILE_FOLDER"/put_versioning_error.txt 2>&1)
+  log 5 "reply status code: $reply"
+  if [[ "$reply" != "200" ]]; then
+    log 2 "reply error: $reply"
+    log 2 "put bucket versioning command returned error: $(cat "$TEST_FILE_FOLDER"/put_versioning_error.txt)"
+    return 1
+  fi
+  return 0
 }

tests/commands/put_object_retention.sh

@@ -26,4 +26,57 @@ put_object_retention() {
     return 1
   fi
   return 0
-}
+}
+
+put_object_retention_rest() {
+  if [ $# -ne 4 ]; then
+    log 2 "'put_object_retention_rest' requires bucket, key, retention mode, retention date"
+    return 1
+  fi
+
+  retention="<?xml version=\"1.0\" encoding=\"UTF-8\"?>
+<Retention xmlns=\"http://s3.amazonaws.com/doc/2006-03-01/\">
+  <Mode>$3</Mode>
+  <RetainUntilDate>$4</RetainUntilDate>
+</Retention>"
+
+  log 5 "retention payload: $retention"
+  generate_hash_for_payload "$retention"
+
+  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
+  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
+  content_md5=$(echo -n "$retention" | openssl dgst -binary -md5 | openssl base64)
+  # shellcheck disable=SC2154
+  canonical_request="PUT
+/$1/$2
+retention=
+content-md5:$content_md5
+host:$aws_endpoint_url_address
+x-amz-content-sha256:$payload_hash
+x-amz-date:$current_date_time
+
+content-md5;host;x-amz-content-sha256;x-amz-date
+$payload_hash"
+
+  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
+    log 2 "error generating sts string"
+    return 1
+  fi
+  get_signature
+
+  # shellcheck disable=SC2154
+  reply=$(curl -ks -w "%{http_code}" -X PUT "$header://$aws_endpoint_url_address/$1/$2?retention" \
+    -H "Content-MD5: $content_md5" \
+    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=content-md5;host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
+    -H "x-amz-content-sha256: $payload_hash" \
+    -H "x-amz-date: $current_date_time" \
+    -d "$retention" -o "$TEST_FILE_FOLDER"/put_object_retention_error.txt 2>&1)
+  log 5 "reply status code: $reply"
+  if [[ "$reply" != "200" ]]; then
+    log 2 "reply error: $reply"
+    log 2 "put object retention command returned error: $(cat "$TEST_FILE_FOLDER"/put_object_retention_error.txt)"
+    return 1
+  fi
+  return 0
+}

tests/env.sh

@@ -120,6 +120,12 @@ check_universal_vars() {
     log 1 "TEST_FILE_FOLDER missing"
     exit 1
   fi
+  if [ ! -d "$TEST_FILE_FOLDER" ]; then
+    if ! error=$(mkdir -p "$TEST_FILE_FOLDER"); then
+      log 2 "error creating test folder: $error"
+      exit 1
+    fi
+  fi
   # exporting these since they're needed for subshells
   export AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY AWS_REGION AWS_PROFILE AWS_ENDPOINT_URL
 }
@@ -129,6 +135,18 @@ check_versity_vars() {
     log 1 "LOCAL_FOLDER missing"
     exit 1
   fi
+  if [ ! -d "$LOCAL_FOLDER" ]; then
+    if ! error=$(mkdir -p "$LOCAL_FOLDER"); then
+      log 2 "error creating local posix folder: $error"
+      exit 1
+    fi
+  fi
+  if [ -n "$VERSIONING_DIR" ] && [ ! -d "$VERSIONING_DIR" ]; then
+    if ! error=$(mkdir -p "$VERSIONING_DIR"); then
+      log 2 "error creating versioning folder: $error"
+      return 1
+    fi
+  fi
   if [ -z "$VERSITY_EXE" ]; then
     log 1 "VERSITY_EXE missing"
     exit 1