test: initial checksum tests, REST script additions

tests/commands/get_object_lock_configuration.sh

@@ -36,35 +36,12 @@ get_object_lock_configuration_rest() {
     log 2 "'get_object_lock_configuration_rest' requires bucket name"
     return 1
   fi
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-object-lock=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object-lock-config.txt" ./tests/rest_scripts/get_object_lock_config.sh); then
+    log 2 "error getting lock configuration: $result"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1)
-  log 5 "reply: $reply"
-  if [[ "$reply" != "200" ]]; then
-    log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
+  if [[ "$result" != "200" ]]; then
+    log 2 "expected '200', returned '$result': $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
     return 1
   fi
   return 0

tests/commands/list_objects.sh

@@ -138,35 +138,19 @@ list_objects_rest() {
     log 2 "'list_objects_rest' requires bucket name"
     return 1
   fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-
-host:$aws_endpoint_url_address
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  log 5 "canonical request: $canonical_request"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  log 5 "bucket name: $1"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+    log 2 "error listing objects: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/objects.txt"))"
+    return 1
+  fi
+  # shellcheck disable=SC2034
+  reply=$(cat "$TEST_FILE_FOLDER/objects.txt")
+  if ! parse_objects_list_rest; then
+    log 2 "error parsing list objects"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks "$header://$aws_endpoint_url_address/$1" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: $payload_hash" \
-    -H "x-amz-date: $current_date_time" 2>&1)
-  log 5 "reply: $reply"
-  parse_objects_list_rest
 }

tests/rest_scripts/get_object_lock_config.sh

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Fields
+
+source ./tests/rest_scripts/rest.sh
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+canonical_request_data=("GET" "/$bucket_name" "object-lock=" "host:$host")
+canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
+if ! build_canonical_request "${canonical_request_data[@]}"; then
+  log_rest 2 "error building request"
+  exit 1
+fi
+echo "$canonical_request" > "cr.txt"
+
+# shellcheck disable=SC2119
+create_canonical_hash_sts_and_signature
+
+# shellcheck disable=SC2154
+curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?object-lock")
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$year_month_day/$AWS_REGION/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
+curl_command+=("${header_fields[@]}")
+curl_command+=(-o "$OUTPUT_FILE")
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/head_object.sh

@@ -22,21 +22,18 @@ source ./tests/rest_scripts/rest.sh
 bucket_name="$BUCKET_NAME"
 # shellcheck disable=SC2154
 key="$OBJECT_KEY"
+# shellcheck disable=SC2153
+version_id="$VERSION_ID"
 
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
 
-#x-amz-object-attributes:ETag
-canonical_request="HEAD
-/$bucket_name/$key
-
-host:$host
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-canonical_request_data=("HEAD" "/$bucket_name/$key" "" "host:$host")
+canonical_request_data=("HEAD" "/$bucket_name/$key")
+if [ "$version_id" != "" ]; then
+  canonical_request_data+=("versionId=$version_id")
+else
+  canonical_request_data+=("")
+fi
+canonical_request_data+=("host:$host")
 if [ "$CHECKSUM" == "true" ]; then
   canonical_request_data+=("x-amz-checksum-mode:ENABLED")
 fi
@@ -48,7 +45,11 @@ fi
 # shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
-curl_command+=(curl -ksI -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key"
+url="$AWS_ENDPOINT_URL/$bucket_name/$key"
+if [ "$version_id" != "" ]; then
+  url+="?versionId=$version_id"
+fi
+curl_command+=(curl -ksI -w "\"%{http_code}\"" "$url"
 -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
 curl_command+=("${header_fields[@]}")
 curl_command+=(-o "$OUTPUT_FILE")

tests/rest_scripts/put_object.sh

@@ -25,15 +25,32 @@ bucket_name="$BUCKET_NAME"
 # shellcheck disable=SC2153
 key="$OBJECT_KEY"
 # shellcheck disable=SC2153,SC2154
-checksum="$CHECKSUM"
+checksum_type="$CHECKSUM_TYPE"
+# shellcheck disable=SC2153
+payload="$PAYLOAD"
+
+# use this parameter to check incorrect checksums
+# shellcheck disable=SC2153,SC2154
+checksum_hash="$CHECKSUM"
 
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-payload_hash="$(sha256sum "$data_file" | awk '{print $1}')"
+if [ "$payload" == "" ]; then
+  payload_hash="$(sha256sum "$data_file" | awk '{print $1}')"
+else
+  payload_hash="$payload"
+fi
 
 cr_data=("PUT" "/$bucket_name/$key" "" "host:$host")
-if [ "$checksum" == "true" ]; then
-  checksum_hash="$(echo -n "$payload_hash" | xxd -r -p | base64)"
+if [ "$checksum_type" == "sha256" ]; then
+  if [ -z "$checksum_hash" ]; then
+    checksum_hash="$(sha256sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
+  fi
   cr_data+=("x-amz-checksum-sha256:$checksum_hash")
+elif [ "$checksum_type" == "crc32" ]; then
+  if [ -z "$checksum_hash" ]; then
+    checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
+  fi
+  cr_data+=("x-amz-checksum-crc32:$checksum_hash")
 fi
 cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
 build_canonical_request "${cr_data[@]}"

tests/run.sh

@@ -149,6 +149,8 @@ run_suite() {
         exit_code=1
       elif ! "$HOME"/bin/bats ./tests/test_rest_checksum.sh; then
         exit_code=1
+      elif ! "$HOME"/bin/bats ./tests/test_rest_versioning.sh; then
+        exit_code=1
       fi
       ;;
     s3api-user)

tests/test_rest.sh

@@ -147,28 +147,6 @@ export RUN_USERS=true
   assert_success
 }
 
-@test "REST - check, enable, suspend versioning" {
-  run setup_bucket "s3api" "$BUCKET_ONE_NAME"
-  assert_success
-
-  log 5 "get versioning"
-
-  run check_versioning_status_rest "$BUCKET_ONE_NAME" ""
-  assert_success
-
-  run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Enabled"
-  assert_success
-
-  run check_versioning_status_rest "$BUCKET_ONE_NAME" "Enabled"
-  assert_success
-
-  run put_bucket_versioning_rest "$BUCKET_ONE_NAME" "Suspended"
-  assert_success
-
-  run check_versioning_status_rest "$BUCKET_ONE_NAME" "Suspended"
-  assert_success
-}
-
 @test "test_rest_set_get_lock_config" {
   run setup_bucket "s3api" "$BUCKET_ONE_NAME"
   assert_success
@@ -189,66 +167,6 @@ export RUN_USERS=true
   assert_success
 }
 
-@test "test_rest_versioning" {
-  test_file="test_file"
-  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true"
-  assert_success
-
-  run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
-  assert_success
-
-  run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "1" "true" "true"
-  assert_success
-
-  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run get_and_check_versions_rest "$BUCKET_ONE_NAME" "$test_file" "2" "true" "false" "false" "true"
-  assert_success
-}
-
-@test "versioning - add version, then delete and check for marker" {
-  test_file="test_file"
-  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run put_object "rest" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
-  assert_success
-
-  run delete_object_rest "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run check_versions_after_file_deletion "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-}
-
-@test "versioning - retrieve after delete" {
-  test_file="test_file"
-  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run put_bucket_versioning "s3api" "$BUCKET_ONE_NAME" "Enabled"
-  assert_success
-
-  run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file"
-  assert_success
-
-  run get_object "s3api" "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file-copy"
-  assert_failure
-}
-
 @test "REST - legal hold, get without config" {
   test_file="test_file"
   run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
@@ -534,3 +452,72 @@ export RUN_USERS=true
   run verify_object_not_found "$BUCKET_ONE_NAME" "$test_file_two"
   assert_success
 }
+
+@test "REST - put object w/STREAMING-AWS4-HMAC-SHA256-PAYLOAD without content length" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1043"
+  fi
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object_rest_chunked_payload_type_without_content_length "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - HeadObject does not return 405 with versioning, after file deleted" {
+  if [ "$DIRECT" != "true" ]; then
+    skip "https://github.com/versity/versitygw/issues/1029"
+  fi
+  if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then
+    skip
+  fi
+  run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  # in static bucket config, bucket will still exist
+  if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
+    run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"
+    assert_success
+  fi
+
+  test_file="test_file"
+  run create_test_files "$test_file"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run verify_object_not_found "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}
+
+@test "REST - HeadObject returns 405 when querying DeleteMarker" {
+  if [ "$RECREATE_BUCKETS" == "false" ] || [[ ( -z "$VERSIONING_DIR" ) && ( "$DIRECT" != "true" ) ]]; then
+    skip
+  fi
+  run bucket_cleanup_if_bucket_exists "s3api" "$BUCKET_ONE_NAME"
+  assert_success
+
+  # in static bucket config, bucket will still exist
+  if ! bucket_exists "s3api" "$BUCKET_ONE_NAME"; then
+    run create_bucket_object_lock_enabled "$BUCKET_ONE_NAME"
+    assert_success
+  fi
+
+  test_file="test_file"
+  run create_test_files "$test_file"
+  assert_success
+
+  run put_object "s3api" "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run delete_object "s3api" "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run get_delete_marker_and_verify_405 "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+}