Test/rest checksum crc64nvme

tests/commands/get_object_lock_configuration.sh

@@ -36,35 +36,12 @@ get_object_lock_configuration_rest() {
     log 2 "'get_object_lock_configuration_rest' requires bucket name"
     return 1
   fi
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-object-lock=
-host:$aws_endpoint_url_address
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/object-lock-config.txt" ./tests/rest_scripts/get_object_lock_config.sh); then
+    log 2 "error getting lock configuration: $result"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -w "%{http_code}" -ks "$header://$aws_endpoint_url_address/$1?object-lock" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: UNSIGNED-PAYLOAD" \
-    -H "x-amz-date: $current_date_time" \
-    -o "$TEST_FILE_FOLDER/object-lock-config.txt" 2>&1)
-  log 5 "reply: $reply"
-  if [[ "$reply" != "200" ]]; then
-    log 2 "get object command returned error: $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
+  if [[ "$result" != "200" ]]; then
+    log 2 "expected '200', returned '$result': $(cat "$TEST_FILE_FOLDER/object-lock-config.txt")"
     return 1
   fi
   return 0

tests/commands/list_objects.sh

@@ -138,35 +138,19 @@ list_objects_rest() {
     log 2 "'list_objects_rest' requires bucket name"
     return 1
   fi
-
-  generate_hash_for_payload ""
-
-  current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-  aws_endpoint_url_address=${AWS_ENDPOINT_URL#*//}
-  header=$(echo "$AWS_ENDPOINT_URL" | awk -F: '{print $1}')
-  # shellcheck disable=SC2154
-  canonical_request="GET
-/$1
-
-host:$aws_endpoint_url_address
-x-amz-content-sha256:$payload_hash
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-$payload_hash"
-
-  log 5 "canonical request: $canonical_request"
-
-  if ! generate_sts_string "$current_date_time" "$canonical_request"; then
-    log 2 "error generating sts string"
+  log 5 "bucket name: $1"
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" BUCKET_NAME="$1" OUTPUT_FILE="$TEST_FILE_FOLDER/objects.txt" ./tests/rest_scripts/list_objects.sh); then
+    log 2 "error listing objects: $result"
+    return 1
+  fi
+  if [ "$result" != "200" ]; then
+    log 2 "expected '200', was '$result' ($(cat "$TEST_FILE_FOLDER/objects.txt"))"
+    return 1
+  fi
+  # shellcheck disable=SC2034
+  reply=$(cat "$TEST_FILE_FOLDER/objects.txt")
+  if ! parse_objects_list_rest; then
+    log 2 "error parsing list objects"
     return 1
   fi
-  get_signature
-  # shellcheck disable=SC2154
-  reply=$(send_command curl -ks "$header://$aws_endpoint_url_address/$1" \
-    -H "Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$ymd/$AWS_REGION/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date,Signature=$signature" \
-    -H "x-amz-content-sha256: $payload_hash" \
-    -H "x-amz-date: $current_date_time" 2>&1)
-  log 5 "reply: $reply"
-  parse_objects_list_rest
 }

tests/rest_scripts/calculate_crc64nvme.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+if [ -z "$TEST_FILE_FOLDER" ] || [ -z "$DATA_FILE" ]; then
+  log_rest 2 "TEST_FILE_FOLDER and DATA_FILE params required for crc64nvme checksum (for python virtual env)"
+  exit 1
+fi
+if ! python3 -m venv "$TEST_FILE_FOLDER/env" 1>/dev/null; then
+  log_rest 2 "error creating python virtual environment"
+  exit 1
+fi
+if ! source "$TEST_FILE_FOLDER/env/bin/activate" 1>/dev/null; then
+  log_rest 2 "error activating virtual environment"
+  exit 1
+fi
+if ! python3 -m pip install awscrt 1>/dev/null 2>&1; then
+  log_rest 2 "error installing awscrt"
+  exit 1
+fi
+if ! checksum_decimal=$(python3 -c "
+import sys
+from awscrt import checksums
+
+with open(sys.argv[1], 'rb') as f:
+  print(checksums.crc64nvme(f.read()))" "$DATA_FILE" 2>&1); then
+  log_rest 2 "error calculating checksum: $checksum_decimal"
+  exit 1
+fi
+log 5 "decimal checksum: $checksum_decimal"
+if ! deactivate 1>/dev/null; then
+  log_rest 2 "error deactivating virtual environment"
+  exit 1
+fi
+checksum_hash=$(printf "%016x" "$checksum_decimal" | xxd -r -p | base64)
+echo "$checksum_hash"

tests/rest_scripts/get_object_lock_config.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Fields
+
+source ./tests/rest_scripts/rest.sh
+
+# shellcheck disable=SC2153
+bucket_name="$BUCKET_NAME"
+
+current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
+canonical_request_data=("GET" "/$bucket_name" "object-lock=" "host:$host")
+canonical_request_data+=("x-amz-content-sha256:UNSIGNED-PAYLOAD" "x-amz-date:$current_date_time")
+if ! build_canonical_request "${canonical_request_data[@]}"; then
+  log_rest 2 "error building request"
+  exit 1
+fi
+
+# shellcheck disable=SC2119
+create_canonical_hash_sts_and_signature
+
+# shellcheck disable=SC2154
+curl_command+=(curl -ks -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name?object-lock")
+curl_command+=(-H "\"Authorization: AWS4-HMAC-SHA256 Credential=$AWS_ACCESS_KEY_ID/$year_month_day/$AWS_REGION/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
+curl_command+=("${header_fields[@]}")
+curl_command+=(-o "$OUTPUT_FILE")
+eval "${curl_command[*]}" 2>&1

tests/rest_scripts/head_object.sh

@@ -22,21 +22,18 @@ source ./tests/rest_scripts/rest.sh
 bucket_name="$BUCKET_NAME"
 # shellcheck disable=SC2154
 key="$OBJECT_KEY"
+# shellcheck disable=SC2153
+version_id="$VERSION_ID"
 
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
 
-#x-amz-object-attributes:ETag
-canonical_request="HEAD
-/$bucket_name/$key
-
-host:$host
-x-amz-content-sha256:UNSIGNED-PAYLOAD
-x-amz-date:$current_date_time
-
-host;x-amz-content-sha256;x-amz-date
-UNSIGNED-PAYLOAD"
-
-canonical_request_data=("HEAD" "/$bucket_name/$key" "" "host:$host")
+canonical_request_data=("HEAD" "/$bucket_name/$key")
+if [ "$version_id" != "" ]; then
+  canonical_request_data+=("versionId=$version_id")
+else
+  canonical_request_data+=("")
+fi
+canonical_request_data+=("host:$host")
 if [ "$CHECKSUM" == "true" ]; then
   canonical_request_data+=("x-amz-checksum-mode:ENABLED")
 fi
@@ -48,7 +45,11 @@ fi
 # shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature
 
-curl_command+=(curl -ksI -w "\"%{http_code}\"" "$AWS_ENDPOINT_URL/$bucket_name/$key"
+url="$AWS_ENDPOINT_URL/$bucket_name/$key"
+if [ "$version_id" != "" ]; then
+  url+="?versionId=$version_id"
+fi
+curl_command+=(curl -ksI -w "\"%{http_code}\"" "$url"
 -H "\"Authorization: AWS4-HMAC-SHA256 Credential=$aws_access_key_id/$year_month_day/$aws_region/s3/aws4_request,SignedHeaders=$param_list,Signature=$signature\"")
 curl_command+=("${header_fields[@]}")
 curl_command+=(-o "$OUTPUT_FILE")

tests/rest_scripts/put_object.sh

@@ -25,15 +25,40 @@ bucket_name="$BUCKET_NAME"
 # shellcheck disable=SC2153
 key="$OBJECT_KEY"
 # shellcheck disable=SC2153,SC2154
-checksum="$CHECKSUM"
+checksum_type="$CHECKSUM_TYPE"
+# shellcheck disable=SC2153
+payload="$PAYLOAD"
+
+# use this parameter to check incorrect checksums
+# shellcheck disable=SC2153,SC2154
+checksum_hash="$CHECKSUM"
 
 current_date_time=$(date -u +"%Y%m%dT%H%M%SZ")
-payload_hash="$(sha256sum "$data_file" | awk '{print $1}')"
+if [ "$payload" == "" ]; then
+  payload_hash="$(sha256sum "$data_file" | awk '{print $1}')"
+else
+  payload_hash="$payload"
+fi
 
 cr_data=("PUT" "/$bucket_name/$key" "" "host:$host")
-if [ "$checksum" == "true" ]; then
-  checksum_hash="$(echo -n "$payload_hash" | xxd -r -p | base64)"
+if [ "$checksum_type" == "sha256" ]; then
+  if [ -z "$checksum_hash" ]; then
+    checksum_hash="$(sha256sum "$data_file" | awk '{print $1}' | xxd -r -p | base64)"
+  fi
   cr_data+=("x-amz-checksum-sha256:$checksum_hash")
+elif [ "$checksum_type" == "crc32" ]; then
+  if [ -z "$checksum_hash" ]; then
+    checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
+  fi
+  cr_data+=("x-amz-checksum-crc32:$checksum_hash")
+elif [ "$checksum_type" == "crc64nvme" ]; then
+  if [ -z "$checksum_hash" ]; then
+    if ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
+      log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
+      exit 1
+    fi
+  fi
+  cr_data+=("x-amz-checksum-crc64nvme:$checksum_hash")
 fi
 cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
 build_canonical_request "${cr_data[@]}"

tests/run.sh

@@ -149,6 +149,8 @@ run_suite() {
         exit_code=1
       elif ! "$HOME"/bin/bats ./tests/test_rest_checksum.sh; then
         exit_code=1
+      elif ! "$HOME"/bin/bats ./tests/test_rest_versioning.sh; then
+        exit_code=1
       fi
       ;;
     s3api-user)