test: crc64nvme checksum

tests/rest_scripts/calculate_crc64nvme.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+
+# Copyright 2024 Versity Software
+# This file is licensed under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http:#www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+source ./tests/rest_scripts/rest.sh
+
+if [ -z "$TEST_FILE_FOLDER" ] || [ -z "$DATA_FILE" ]; then
+  log_rest 2 "TEST_FILE_FOLDER and DATA_FILE params required for crc64nvme checksum (for python virtual env)"
+  exit 1
+fi
+if ! python3 -m venv "$TEST_FILE_FOLDER/env" 1>/dev/null; then
+  log_rest 2 "error creating python virtual environment"
+  exit 1
+fi
+if ! source "$TEST_FILE_FOLDER/env/bin/activate" 1>/dev/null; then
+  log_rest 2 "error activating virtual environment"
+  exit 1
+fi
+if ! python3 -m pip install awscrt 1>/dev/null 2>&1; then
+  log_rest 2 "error installing awscrt"
+  exit 1
+fi
+if ! checksum_decimal=$(python3 -c "
+import sys
+from awscrt import checksums
+
+with open(sys.argv[1], 'rb') as f:
+  print(checksums.crc64nvme(f.read()))" "$DATA_FILE" 2>&1); then
+  log_rest 2 "error calculating checksum: $checksum_decimal"
+  exit 1
+fi
+log 5 "decimal checksum: $checksum_decimal"
+if ! deactivate 1>/dev/null; then
+  log_rest 2 "error deactivating virtual environment"
+  exit 1
+fi
+checksum_hash=$(printf "%016x" "$checksum_decimal" | xxd -r -p | base64)
+echo "$checksum_hash"

tests/rest_scripts/get_object_lock_config.sh

@@ -28,7 +28,6 @@ if ! build_canonical_request "${canonical_request_data[@]}"; then
   log_rest 2 "error building request"
   exit 1
 fi
-echo "$canonical_request" > "cr.txt"
 
 # shellcheck disable=SC2119
 create_canonical_hash_sts_and_signature

tests/rest_scripts/put_object.sh

@@ -51,6 +51,14 @@ elif [ "$checksum_type" == "crc32" ]; then
     checksum_hash="$(gzip -c -1 "$data_file" | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
   fi
   cr_data+=("x-amz-checksum-crc32:$checksum_hash")
+elif [ "$checksum_type" == "crc64nvme" ]; then
+  if [ -z "$checksum_hash" ]; then
+    if ! checksum_hash=$(DATA_FILE="$data_file" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
+      log_rest 2 "error calculating crc64nvme checksum: $checksum_hash"
+      exit 1
+    fi
+  fi
+  cr_data+=("x-amz-checksum-crc64nvme:$checksum_hash")
 fi
 cr_data+=("x-amz-content-sha256:$payload_hash" "x-amz-date:$current_date_time")
 build_canonical_request "${cr_data[@]}"

tests/test_rest_checksum.sh

@@ -51,23 +51,41 @@ source ./tests/util/util_setup.sh
   assert_success
 }
 
-@test "REST - crc32 checksum - success" {
+@test "REST - crc32 checksum - correct" {
   test_file="test_file"
   run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
   assert_success
 
   run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc32"
   assert_success
+
+  run check_checksum_rest_crc32 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
+  assert_success
 }
 
-@test "REST - crc32 checksum - correct" {
+@test "REST - crc32 checksum - incorrect" {
   test_file="test_file"
   run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
   assert_success
 
-  run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc32"
+  run put_object_rest_crc32_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
   assert_success
+}
 
-  run check_checksum_rest_crc32 "$BUCKET_ONE_NAME" "$test_file" "$TEST_FILE_FOLDER/$test_file"
+@test "REST - crc64nvme checksum - correct" {
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object_rest_checksum "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file" "crc64nvme"
+  assert_success
+}
+
+@test "REST - crc64nvme checksum - incorrect" {
+  test_file="test_file"
+  run setup_bucket_and_file "$BUCKET_ONE_NAME" "$test_file"
+  assert_success
+
+  run put_object_rest_crc64nvme_incorrect "$TEST_FILE_FOLDER/$test_file" "$BUCKET_ONE_NAME" "$test_file"
   assert_success
 }

tests/util/util_acl.sh

@@ -216,6 +216,17 @@ get_check_acl_after_policy() {
   fi
 }
 
+check_direct_display_name() {
+  if ! display_name=$(echo "$owner" | xmllint --xpath '//*[local-name()="DisplayName"]/text()' - 2>&1); then
+    log 2 "error getting display name: $display_name"
+    return 1
+  fi
+  if [ "$display_name" != "$DIRECT_DISPLAY_NAME" ]; then
+    log 2 "display name mismatch (expected '$DIRECT_DISPLAY_NAME', actual '$display_name')"
+    return 1
+  fi
+}
+
 get_and_check_acl_rest() {
   if [ $# -ne 1 ]; then
     log 2 "'get_and_check_acl_rest' requires bucket name"
@@ -239,12 +250,8 @@ get_and_check_acl_rest() {
     return 1
   fi
   if [ "$DIRECT" == "true" ]; then
-    if ! display_name=$(echo "$owner" | xmllint --xpath '//*[local-name()="DisplayName"]/text()' - 2>&1); then
-      log 2 "error getting display name: $display_name"
-      return 1
-    fi
-    if [ "$display_name" != "$DIRECT_DISPLAY_NAME" ]; then
-      log 2 "display name mismatch (expected '$DIRECT_DISPLAY_NAME', actual '$display_name')"
+    if ! check_direct_display_name; then
+      log 2 "error checking direct display name"
       return 1
     fi
   else

tests/util/util_head_object.sh

@@ -166,3 +166,13 @@ check_checksum_rest_crc32() {
   fi
   return 0
 }
+
+crc64_file() {
+  source ./env/bin/activate
+  python3 -m pip install awscrt
+  if ! checksum=$(python3 -c "import sys;from awscrt import checksums;crc64nvme_value = checksums.crc64nvme(sys.argv[1]);print(crc64nvme_value)" "$1"); then
+    return 1
+  fi
+  log 5 "checksum: $checksum"
+  deactivate
+}

tests/util/util_object.sh

@@ -297,20 +297,20 @@ list_and_check_directory_obj() {
   return 0
 }
 
-check_sha256_invalid_or_incorrect() {
-  if [ $# -ne 5 ]; then
-    log 2 "'check_sha256_invalid_or_incorrect' requires data file, bucket name, key, checksum, expected error"
+check_checksum_invalid_or_incorrect() {
+  if [ $# -ne 6 ]; then
+    log 2 "'check_sha256_invalid_or_incorrect' requires data file, bucket name, key, checksum type, checksum, expected error"
     return 1
   fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="sha256" CHECKSUM="$4" ./tests/rest_scripts/put_object.sh 2>&1); then
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="$4" CHECKSUM="$5" ./tests/rest_scripts/put_object.sh 2>&1); then
     log 2 "error: $result"
     return 1
   fi
   if [ "$result" != "400" ]; then
     log 2 "expected response code of '400', was '$result' (response: $(cat "$TEST_FILE_FOLDER/result.txt")"
     return 1
   fi
-  if ! check_xml_element "$TEST_FILE_FOLDER/result.txt" "$5" "Error" "Message"; then
+  if ! check_xml_element "$TEST_FILE_FOLDER/result.txt" "$6" "Error" "Message"; then
     log 2 "xml error message mismatch"
     return 1
   fi
@@ -322,7 +322,8 @@ put_object_rest_checksum() {
     log 2 "'put_object_rest_sha256_checksum' requires data file, bucket name, key, checksum type"
     return 1
   fi
-  if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="$4" ./tests/rest_scripts/put_object.sh 2>&1); then
+  # shellcheck disable=SC2097,SC2098
+  if ! result=$(COMMAND_LOG="$COMMAND_LOG" DATA_FILE="$1" BUCKET_NAME="$2" OBJECT_KEY="$3" TEST_FILE_FOLDER="$TEST_FILE_FOLDER" OUTPUT_FILE="$TEST_FILE_FOLDER/result.txt" CHECKSUM_TYPE="$4" ./tests/rest_scripts/put_object.sh 2>&1); then
     log 2 "error: $result"
     return 1
   fi
@@ -339,7 +340,7 @@ put_object_rest_sha256_invalid() {
     log 2 "'put_object_rest_sha256_invalid' requires data file, bucket name, key"
     return 1
   fi
-  if ! check_sha256_invalid_or_incorrect "$1" "$2" "$3" "dummy" "Value for x-amz-checksum-sha256 header is invalid."; then
+  if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "sha256" "dummy" "Value for x-amz-checksum-sha256 header is invalid."; then
     log 2 "error checking checksum"
     return 1
   fi
@@ -357,7 +358,7 @@ put_object_rest_sha256_incorrect() {
     error_message="The sha256 you specified did not match the calculated checksum."
   fi
   incorrect_checksum="$(echo -n "dummy" | sha256sum | awk '{print $1}' | xxd -r -p | base64)"
-  if ! check_sha256_invalid_or_incorrect "$1" "$2" "$3" "$incorrect_checksum" "$error_message"; then
+  if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "sha256" "$incorrect_checksum" "$error_message"; then
     log 2 "error checking checksum"
     return 1
   fi
@@ -379,3 +380,42 @@ put_object_rest_chunked_payload_type_without_content_length() {
   fi
   return 0
 }
+
+put_object_rest_crc32_incorrect() {
+  if [ $# -ne 3 ]; then
+    log 2 "'put_object_rest_crc32_incorrect' requires data file, bucket name, key"
+    return 1
+  fi
+  if [ "$DIRECT" == "true" ]; then
+    error_message="The CRC32 you specified did not match the calculated checksum."
+  else
+    error_message="The crc32 you specified did not match the calculated checksum."
+  fi
+  incorrect_checksum="$(echo -n "dummy" | gzip -c -1 | tail -c8 | od -t x4 -N 4 -A n | awk '{print $1}' | xxd -r -p | base64)"
+  if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "crc32" "$incorrect_checksum" "$error_message"; then
+    log 2 "error checking checksum"
+    return 1
+  fi
+  return 0
+}
+
+put_object_rest_crc64nvme_incorrect() {
+  if [ $# -ne 3 ]; then
+    log 2 "'put_object_rest_crc64nvme_incorrect' requires data file, bucket name, key"
+    return 1
+  fi
+  if [ "$DIRECT" == "true" ]; then
+    error_message="The CRC64NVME you specified did not match the calculated checksum."
+  else
+    error_message="The crc64nvme you specified did not match the calculated checksum."
+  fi
+  if ! incorrect_checksum=$(DATA_FILE=<(echo -n "dummy") TEST_FILE_FOLDER="$TEST_FILE_FOLDER" ./tests/rest_scripts/calculate_crc64nvme.sh 2>&1); then
+    log 2 "error calculating checksum: $incorrect_checksum"
+    return 1
+  fi
+  if ! check_checksum_invalid_or_incorrect "$1" "$2" "$3" "crc64nvme" "$incorrect_checksum" "$error_message"; then
+    log 2 "error checking checksum"
+    return 1
+  fi
+  return 0
+}

tests/util/util_policy.sh

@@ -42,6 +42,14 @@ check_for_empty_policy() {
   return 0
 }
 
+add_direct_user_to_principal() {
+  if [ "${principals[$idx]}" == "*" ]; then
+    modified_principal+="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/$DIRECT_S3_ROOT_ACCOUNT_NAME\"}"
+  else
+    modified_principal+="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/${principals[$idx]}\"}"
+  fi
+}
+
 get_modified_principal() {
   log 6 "get_modified_principal"
   if [ $# -ne 1 ]; then
@@ -55,11 +63,7 @@ get_modified_principal() {
   fi
   for ((idx=0; idx<${#principals[@]}; idx++)); do
     if [ "$DIRECT" == "true" ]; then
-      if [ "${principals[$idx]}" == "*" ]; then
-        modified_principal+="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/$DIRECT_S3_ROOT_ACCOUNT_NAME\"}"
-      else
-        modified_principal+="{\"AWS\": \"arn:aws:iam::$DIRECT_AWS_USER_ID:user/${principals[$idx]}\"}"
-      fi
+      add_direct_user_to_principal
     else
       # shellcheck disable=SC2089
       modified_principal+="\"${principals[$idx]}\""