diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 1ded60e8..e84f7198 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -1317,6 +1317,7 @@ func (p *Posix) CreateMultipartUpload(ctx context.Context, mpu *s3.CreateMultipa Algorithms: []types.ChecksumAlgorithm{ mpu.ChecksumAlgorithm, }, + Type: &mpu.ChecksumType, }) if err != nil { // cleanup object if returning error @@ -1352,6 +1353,29 @@ func (p *Posix) getChownIDs(acct auth.Account) (int, int, bool) { return uid, gid, needsChown } +func getChecksumType(t *types.ChecksumType) types.ChecksumType { + if t == nil { + return "" + } + + return *t +} + +func getPartChecksum(algo types.ChecksumAlgorithm, part types.CompletedPart) string { + switch algo { + case types.ChecksumAlgorithmCrc32: + return *part.ChecksumCRC32 + case types.ChecksumAlgorithmCrc32c: + return *part.ChecksumCRC32C + case types.ChecksumAlgorithmSha1: + return *part.ChecksumSHA1 + case types.ChecksumAlgorithmSha256: + return *part.ChecksumSHA256 + default: + return "" + } +} + func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteMultipartUploadInput) (*s3.CompleteMultipartUploadOutput, error) { acct, ok := ctx.Value("account").(auth.Account) if !ok { @@ -1395,23 +1419,20 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return nil, fmt.Errorf("get mp checksums: %w", err) } + var checksumAlgorithm types.ChecksumAlgorithm + if len(checksums.Algorithms) != 0 { + checksumAlgorithm = checksums.Algorithms[0] + } - // if len(checksums.Algorithms) != 0 { - // algorithm := checksums.Algorithms[0] + checksumType := getChecksumType(checksums.Type) + // ChecksumType should be the same as specified on CreateMultipartUpload + if checksumType != input.ChecksumType { + if checksumType == "" { + checksumType = types.ChecksumType("null") + } - // if input.ChecksumCRC32 != nil && algorithm != types.ChecksumAlgorithmCrc32 { - // return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32") - // } - // if input.ChecksumCRC32C != nil && algorithm != types.ChecksumAlgorithmCrc32c { - // return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-crc32c") - // } - // if input.ChecksumSHA1 != nil && algorithm != types.ChecksumAlgorithmSha1 { - // return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-sha1") - // } - // if input.ChecksumSHA256 != nil && algorithm != types.ChecksumAlgorithmSha256 { - // return nil, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-sha256") - // } - // } + return nil, s3err.GetChecksumTypeMismatchOnMpErr(checksumType) + } // check all parts ok last := len(parts) - 1 @@ -1457,11 +1478,18 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM } var hashRdr *utils.HashReader - if len(checksums.Algorithms) != 0 { - hashRdr, err = utils.NewHashReader(nil, "", utils.HashType(strings.ToLower(string(checksums.Algorithms[0])))) + var compositeChecksumRdr *utils.CompositeChecksumReader + switch checksumType { + case types.ChecksumTypeFullObject: + hashRdr, err = utils.NewHashReader(nil, "", utils.HashType(strings.ToLower(string(checksumAlgorithm)))) if err != nil { return nil, fmt.Errorf("initialize hash reader: %w", err) } + case types.ChecksumTypeComposite: + compositeChecksumRdr, err = utils.NewCompositeChecksumReader(utils.HashType(strings.ToLower(string(checksumAlgorithm)))) + if err != nil { + return nil, fmt.Errorf("initialize composite checksum reader: %w", err) + } } f, err := p.openTmpFile(filepath.Join(bucket, metaTmpDir), bucket, object, @@ -1487,9 +1515,14 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM } var rdr io.Reader = pf - if hashRdr != nil { + if checksumType == types.ChecksumTypeFullObject { hashRdr.SetReader(rdr) rdr = hashRdr + } else if checksumType == types.ChecksumTypeComposite { + err := compositeChecksumRdr.Process(getPartChecksum(checksumAlgorithm, part)) + if err != nil { + return nil, fmt.Errorf("process %v part checksum: %w", *part.PartNumber, err) + } } _, err = io.Copy(f.File(), rdr) @@ -1594,42 +1627,56 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM var crc32c *string var sha1 *string var sha256 *string + var crc64nvme *string - // set checksum - if hashRdr != nil { - algo := checksums.Algorithms[0] + // Calculate, compare with the provided checksum and store them + if checksumType != "" { checksum := s3response.Checksum{ Algorithms: []types.ChecksumAlgorithm{ - algo, + checksumAlgorithm, }, + Type: &checksumType, } - sum := hashRdr.Sum() - switch hashRdr.Type() { - case utils.HashTypeCRC32: + var sum string + switch checksumType { + case types.ChecksumTypeComposite: + sum = compositeChecksumRdr.Sum() + case types.ChecksumTypeFullObject: + sum = hashRdr.Sum() + } + + switch checksumAlgorithm { + case types.ChecksumAlgorithmCrc32: if input.ChecksumCRC32 != nil && *input.ChecksumCRC32 != sum { - return nil, s3err.GetChecksumBadDigestErr(algo) + return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm) } checksum.CRC32 = &sum crc32 = &sum - case utils.HashTypeCRC32C: + case types.ChecksumAlgorithmCrc32c: if input.ChecksumCRC32C != nil && *input.ChecksumCRC32C != sum { - return nil, s3err.GetChecksumBadDigestErr(algo) + return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm) } checksum.CRC32C = &sum crc32c = &sum - case utils.HashTypeSha1: + case types.ChecksumAlgorithmSha1: if input.ChecksumSHA1 != nil && *input.ChecksumSHA1 != sum { - return nil, s3err.GetChecksumBadDigestErr(algo) + return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm) } checksum.SHA1 = &sum sha1 = &sum - case utils.HashTypeSha256: + case types.ChecksumAlgorithmSha256: if input.ChecksumSHA256 != nil && *input.ChecksumSHA256 != sum { - return nil, s3err.GetChecksumBadDigestErr(algo) + return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm) } checksum.SHA256 = &sum sha256 = &sum + case types.ChecksumAlgorithmCrc64nvme: + if input.ChecksumCRC64NVME != nil && *input.ChecksumCRC64NVME != sum { + return nil, s3err.GetChecksumBadDigestErr(checksumAlgorithm) + } + checksum.CRC64NVME = &sum + crc64nvme = &sum } err := p.storeChecksums(f.File(), bucket, object, checksum) if err != nil { @@ -1669,14 +1716,16 @@ func (p *Posix) CompleteMultipartUpload(ctx context.Context, input *s3.CompleteM os.Remove(filepath.Join(bucket, objdir)) return &s3.CompleteMultipartUploadOutput{ - Bucket: &bucket, - ETag: &s3MD5, - Key: &object, - VersionId: &versionID, - ChecksumCRC32: crc32, - ChecksumCRC32C: crc32c, - ChecksumSHA1: sha1, - ChecksumSHA256: sha256, + Bucket: &bucket, + ETag: &s3MD5, + Key: &object, + VersionId: &versionID, + ChecksumCRC32: crc32, + ChecksumCRC32C: crc32c, + ChecksumSHA1: sha1, + ChecksumSHA256: sha256, + ChecksumCRC64NVME: crc64nvme, + ChecksumType: checksumType, }, nil } @@ -1764,6 +1813,23 @@ func validatePartChecksum(checksum s3response.Checksum, part types.CompletedPart } } } + if part.ChecksumCRC64NVME != nil { + if ok := utils.IsValidChecksum(*part.ChecksumCRC64NVME, types.ChecksumAlgorithmCrc64nvme); !ok { + return s3err.GetAPIError(s3err.ErrInvalidChecksumPart) + } + + if *part.ChecksumCRC64NVME != getString(checksum.CRC64NVME) { + if algo == types.ChecksumAlgorithmCrc64nvme { + return s3err.GetAPIError(s3err.ErrInvalidPart) + } else { + return s3err.APIError{ + Code: "BadDigest", + Description: fmt.Sprintf("The crc64nvme you specified for part %v did not match what we received.", *part.PartNumber), + HTTPStatusCode: http.StatusBadRequest, + } + } + } + } return nil } @@ -1789,6 +1855,9 @@ func numberOfChecksums(part types.CompletedPart) int { if getString(part.ChecksumSHA256) != "" { counter++ } + if getString(part.ChecksumCRC64NVME) != "" { + counter++ + } return counter } @@ -1999,6 +2068,7 @@ func (p *Posix) ListMultipartUploads(_ context.Context, mpu *s3.ListMultipartUpl StorageClass: types.StorageClassStandard, Initiated: fi.ModTime(), ChecksumAlgorithm: algo, + ChecksumType: checksum.Type, }) } } @@ -2156,14 +2226,15 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon } parts = append(parts, s3response.Part{ - PartNumber: pn, - ETag: etag, - LastModified: fi.ModTime(), - Size: fi.Size(), - ChecksumCRC32: checksum.CRC32, - ChecksumCRC32C: checksum.CRC32C, - ChecksumSHA1: checksum.SHA1, - ChecksumSHA256: checksum.SHA256, + PartNumber: pn, + ETag: etag, + LastModified: fi.ModTime(), + Size: fi.Size(), + ChecksumCRC32: checksum.CRC32, + ChecksumCRC32C: checksum.CRC32C, + ChecksumSHA1: checksum.SHA1, + ChecksumSHA256: checksum.SHA256, + ChecksumCRC64NVME: checksum.CRC64NVME, }) } @@ -2196,6 +2267,7 @@ func (p *Posix) ListParts(_ context.Context, input *s3.ListPartsInput) (s3respon UploadID: uploadID, StorageClass: types.StorageClassStandard, ChecksumAlgorithm: algo, + ChecksumType: checksum.Type, }, nil } @@ -2290,6 +2362,14 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3. tr = hashRdr } + if input.ChecksumCRC64NVME != nil { + hashRdr, err = utils.NewHashReader(tr, *input.ChecksumCRC64NVME, utils.HashTypeCRC64NVME) + if err != nil { + return nil, fmt.Errorf("initialize hash reader: %w", err) + } + + tr = hashRdr + } // If only the checksum algorithm is provided register // a new HashReader to calculate the object checksum @@ -2315,16 +2395,8 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3. // Check if the provided checksum algorithm match // the one specified on mp initialization - if hashRdr != nil { + if hashRdr != nil && chErr == nil && getChecksumType(checksums.Type) != "" { algo := types.ChecksumAlgorithm(strings.ToUpper(string(hashRdr.Type()))) - if chErr != nil { - return nil, s3err.GetChecksumTypeMismatchErr("null", algo) - } - - if len(checksums.Algorithms) == 0 { - return nil, s3err.GetChecksumTypeMismatchErr("null", algo) - } - if checksums.Algorithms[0] != algo { return nil, s3err.GetChecksumTypeMismatchErr(checksums.Algorithms[0], algo) } @@ -2349,11 +2421,12 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3. ETag: &etag, } - // Store the calculated checksum in the object metadata if hashRdr != nil { checksum := s3response.Checksum{ Algorithms: []types.ChecksumAlgorithm{input.ChecksumAlgorithm}, } + + // Validate the provided checksum sum := hashRdr.Sum() switch hashRdr.Type() { case utils.HashTypeCRC32: @@ -2368,11 +2441,18 @@ func (p *Posix) UploadPart(ctx context.Context, input *s3.UploadPartInput) (*s3. case utils.HashTypeSha256: checksum.SHA256 = &sum res.ChecksumSHA256 = &sum + case utils.HashTypeCRC64NVME: + checksum.CRC64NVME = &sum + res.ChecksumCRC64NVME = &sum } - err := p.storeChecksums(f.File(), bucket, partPath, checksum) - if err != nil { - return nil, fmt.Errorf("store checksum: %w", err) + // Store the checksums if the checksum type has been + // specified on mp initialization + if getChecksumType(checksums.Type) != "" { + err := p.storeChecksums(f.File(), bucket, partPath, checksum) + if err != nil { + return nil, fmt.Errorf("store checksum: %w", err) + } } } @@ -2519,6 +2599,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) return s3response.CopyPartResult{}, fmt.Errorf("retreive object part checksums: %w", err) } + // TODO: Should the checksum be recalculated or just copied ? var hashRdr *utils.HashReader if len(mpChecksums.Algorithms) != 0 { if len(checksums.Algorithms) == 0 || (mpChecksums.Algorithms[0] != checksums.Algorithms[0]) { @@ -2567,6 +2648,8 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) checksums.SHA1 = &sum case types.ChecksumAlgorithmSha256: checksums.SHA256 = &sum + case types.ChecksumAlgorithmCrc64nvme: + checksums.CRC64NVME = &sum } err := p.storeChecksums(f.File(), objPath, "", checksums) @@ -2600,6 +2683,7 @@ func (p *Posix) UploadPartCopy(ctx context.Context, upi *s3.UploadPartCopyInput) ChecksumCRC32C: checksums.CRC32C, ChecksumSHA1: checksums.SHA1, ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, }, nil } @@ -2771,6 +2855,14 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons rdr = hashRdr } + if po.ChecksumCRC64NVME != nil { + hashRdr, err = utils.NewHashReader(rdr, *po.ChecksumCRC64NVME, utils.HashTypeCRC64NVME) + if err != nil { + return s3response.PutObjectOutput{}, fmt.Errorf("initialize hash reader: %w", err) + } + + rdr = hashRdr + } // If only the checksum algorithm is provided register // a new HashReader to calculate the object checksum @@ -2827,25 +2919,33 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons } } + checksum := s3response.Checksum{ + Algorithms: []types.ChecksumAlgorithm{}, + } + // Store the calculated checksum in the object metadata if hashRdr != nil { - checksum := s3response.Checksum{ - Algorithms: []types.ChecksumAlgorithm{po.ChecksumAlgorithm}, - } + // The checksum type is always FULL_OBJECT for PutObject + chType := types.ChecksumTypeFullObject + checksum.Type = &chType + sum := hashRdr.Sum() switch hashRdr.Type() { case utils.HashTypeCRC32: checksum.CRC32 = &sum - po.ChecksumCRC32 = &sum + checksum.Algorithms = append(checksum.Algorithms, types.ChecksumAlgorithmCrc32) case utils.HashTypeCRC32C: checksum.CRC32C = &sum - po.ChecksumCRC32C = &sum + checksum.Algorithms = append(checksum.Algorithms, types.ChecksumAlgorithmCrc32c) case utils.HashTypeSha1: checksum.SHA1 = &sum - po.ChecksumSHA1 = &sum + checksum.Algorithms = append(checksum.Algorithms, types.ChecksumAlgorithmSha1) case utils.HashTypeSha256: checksum.SHA256 = &sum - po.ChecksumSHA256 = &sum + checksum.Algorithms = append(checksum.Algorithms, types.ChecksumAlgorithmSha256) + case utils.HashTypeCRC64NVME: + checksum.CRC64NVME = &sum + checksum.Algorithms = append(checksum.Algorithms, types.ChecksumAlgorithmCrc64nvme) } err := p.storeChecksums(f.File(), *po.Bucket, *po.Key, checksum) @@ -2934,12 +3034,14 @@ func (p *Posix) PutObject(ctx context.Context, po *s3.PutObjectInput) (s3respons } return s3response.PutObjectOutput{ - ETag: etag, - VersionID: versionID, - ChecksumCRC32: po.ChecksumCRC32, - ChecksumCRC32C: po.ChecksumCRC32C, - ChecksumSHA1: po.ChecksumSHA1, - ChecksumSHA256: po.ChecksumSHA256, + ETag: etag, + VersionID: versionID, + ChecksumCRC32: checksum.CRC32, + ChecksumCRC32C: checksum.CRC32C, + ChecksumSHA1: checksum.SHA1, + ChecksumSHA256: checksum.SHA256, + ChecksumCRC64NVME: checksum.CRC64NVME, + ChecksumType: checksum.Type, }, nil } @@ -3476,11 +3578,16 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO } var checksums s3response.Checksum - if input.ChecksumMode == types.ChecksumModeEnabled { + var cType types.ChecksumType + // Skip the checksums retreival if object isn't requested fully + if input.ChecksumMode == types.ChecksumModeEnabled && length-startOffset == objSize { checksums, err = p.retreiveChecksums(f, bucket, object) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return nil, fmt.Errorf("get object checksums: %w", err) } + if checksums.Type != nil { + cType = *checksums.Type + } } // using an os.File allows zero-copy sendfile via io.Copy(os.File, net.Conn) @@ -3491,22 +3598,24 @@ func (p *Posix) GetObject(_ context.Context, input *s3.GetObjectInput) (*s3.GetO } return &s3.GetObjectOutput{ - AcceptRanges: &acceptRange, - ContentLength: &length, - ContentEncoding: &contentEncoding, - ContentType: &contentType, - ETag: &etag, - LastModified: backend.GetTimePtr(fi.ModTime()), - Metadata: userMetaData, - TagCount: tagCount, - ContentRange: &contentRange, - StorageClass: types.StorageClassStandard, - VersionId: &versionId, - Body: body, - ChecksumCRC32: checksums.CRC32, - ChecksumCRC32C: checksums.CRC32C, - ChecksumSHA1: checksums.SHA1, - ChecksumSHA256: checksums.SHA256, + AcceptRanges: &acceptRange, + ContentLength: &length, + ContentEncoding: &contentEncoding, + ContentType: &contentType, + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + Metadata: userMetaData, + TagCount: tagCount, + ContentRange: &contentRange, + StorageClass: types.StorageClassStandard, + VersionId: &versionId, + Body: body, + ChecksumCRC32: checksums.CRC32, + ChecksumCRC32C: checksums.CRC32C, + ChecksumSHA1: checksums.SHA1, + ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumType: cType, }, nil } @@ -3681,11 +3790,15 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. } var checksums s3response.Checksum + var cType types.ChecksumType if input.ChecksumMode == types.ChecksumModeEnabled { checksums, err = p.retreiveChecksums(nil, bucket, object) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return nil, fmt.Errorf("get object checksums: %w", err) } + if checksums.Type != nil { + cType = *checksums.Type + } } return &s3.HeadObjectOutput{ @@ -3704,6 +3817,8 @@ func (p *Posix) HeadObject(ctx context.Context, input *s3.HeadObjectInput) (*s3. ChecksumCRC32C: checksums.CRC32C, ChecksumSHA1: checksums.SHA1, ChecksumSHA256: checksums.SHA256, + ChecksumCRC64NVME: checksums.CRC64NVME, + ChecksumType: cType, }, nil } @@ -3733,10 +3848,12 @@ func (p *Posix) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAttr VersionId: data.VersionId, DeleteMarker: data.DeleteMarker, Checksum: &types.Checksum{ - ChecksumCRC32: data.ChecksumCRC32, - ChecksumCRC32C: data.ChecksumCRC32C, - ChecksumSHA1: data.ChecksumSHA1, - ChecksumSHA256: data.ChecksumSHA256, + ChecksumCRC32: data.ChecksumCRC32, + ChecksumCRC32C: data.ChecksumCRC32C, + ChecksumSHA1: data.ChecksumSHA1, + ChecksumSHA256: data.ChecksumSHA256, + ChecksumCRC64NVME: data.ChecksumCRC64NVME, + ChecksumType: data.ChecksumType, }, }, nil } @@ -3838,6 +3955,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. var crc32c *string var sha1 *string var sha256 *string + var crc64nvme *string dstObjdPath := filepath.Join(dstBucket, dstObject) if dstObjdPath == objPath { @@ -3907,6 +4025,9 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. case utils.HashTypeSha256: checksums.SHA256 = &sum sha256 = &sum + case utils.HashTypeCRC64NVME: + checksums.CRC64NVME = &sum + crc64nvme = &sum } err = p.storeChecksums(f, dstBucket, dstObject, checksums) @@ -3959,6 +4080,7 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. crc32c = res.ChecksumCRC32C sha1 = res.ChecksumSHA1 sha256 = res.ChecksumSHA256 + crc64nvme = res.ChecksumCRC64NVME } fi, err = os.Stat(dstObjdPath) @@ -3968,12 +4090,13 @@ func (p *Posix) CopyObject(ctx context.Context, input *s3.CopyObjectInput) (*s3. return &s3.CopyObjectOutput{ CopyObjectResult: &types.CopyObjectResult{ - ETag: &etag, - LastModified: backend.GetTimePtr(fi.ModTime()), - ChecksumCRC32: crc32, - ChecksumCRC32C: crc32c, - ChecksumSHA1: sha1, - ChecksumSHA256: sha256, + ETag: &etag, + LastModified: backend.GetTimePtr(fi.ModTime()), + ChecksumCRC32: crc32, + ChecksumCRC32C: crc32c, + ChecksumSHA1: sha1, + ChecksumSHA256: sha256, + ChecksumCRC64NVME: crc64nvme, }, VersionId: version, CopySourceVersionId: &srcVersionId, @@ -4107,6 +4230,7 @@ func (p *Posix) fileToObj(bucket string) backend.GetObjFunc { Size: &size, StorageClass: types.ObjectStorageClassStandard, ChecksumAlgorithm: checksums.Algorithms, + ChecksumType: checksums.Type, }, nil } } diff --git a/backend/s3proxy/s3.go b/backend/s3proxy/s3.go index d3fdf348..1189c788 100644 --- a/backend/s3proxy/s3.go +++ b/backend/s3proxy/s3.go @@ -256,8 +256,10 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip ID: *u.Owner.ID, DisplayName: *u.Owner.DisplayName, }, - StorageClass: u.StorageClass, - Initiated: *u.Initiated, + StorageClass: u.StorageClass, + Initiated: *u.Initiated, + ChecksumAlgorithm: u.ChecksumAlgorithm, + ChecksumType: &u.ChecksumType, }) } @@ -293,10 +295,15 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re var parts []s3response.Part for _, p := range output.Parts { parts = append(parts, s3response.Part{ - PartNumber: int(*p.PartNumber), - LastModified: *p.LastModified, - ETag: *p.ETag, - Size: *p.Size, + PartNumber: int(*p.PartNumber), + LastModified: *p.LastModified, + ETag: *p.ETag, + Size: *p.Size, + ChecksumCRC32: p.ChecksumCRC32, + ChecksumCRC32C: p.ChecksumCRC32C, + ChecksumCRC64NVME: p.ChecksumCRC64NVME, + ChecksumSHA1: p.ChecksumSHA1, + ChecksumSHA256: p.ChecksumSHA256, }) } pnm, err := strconv.Atoi(*output.PartNumberMarker) @@ -329,6 +336,8 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re MaxParts: int(*output.MaxParts), IsTruncated: *output.IsTruncated, Parts: parts, + ChecksumAlgorithm: output.ChecksumAlgorithm, + ChecksumType: &output.ChecksumType, }, nil } @@ -348,8 +357,13 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn } return s3response.CopyPartResult{ - LastModified: *output.CopyPartResult.LastModified, - ETag: output.CopyPartResult.ETag, + LastModified: *output.CopyPartResult.LastModified, + ETag: output.CopyPartResult.ETag, + ChecksumCRC32: output.CopyPartResult.ChecksumCRC32, + ChecksumCRC32C: output.CopyPartResult.ChecksumCRC32C, + ChecksumCRC64NVME: output.CopyPartResult.ChecksumCRC64NVME, + ChecksumSHA1: output.CopyPartResult.ChecksumSHA1, + ChecksumSHA256: output.CopyPartResult.ChecksumSHA256, }, nil } @@ -369,8 +383,13 @@ func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3re } return s3response.PutObjectOutput{ - ETag: *output.ETag, - VersionID: versionID, + ETag: *output.ETag, + VersionID: versionID, + ChecksumCRC32: output.ChecksumCRC32, + ChecksumCRC32C: output.ChecksumCRC32C, + ChecksumCRC64NVME: output.ChecksumCRC64NVME, + ChecksumSHA1: output.ChecksumSHA1, + ChecksumSHA256: output.ChecksumSHA256, }, nil } @@ -421,6 +440,7 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt ObjectSize: out.ObjectSize, StorageClass: out.StorageClass, ObjectParts: &parts, + Checksum: out.Checksum, }, handleError(err) } @@ -833,13 +853,15 @@ func convertObjects(objs []types.Object) []s3response.Object { for _, obj := range objs { result = append(result, s3response.Object{ - ETag: obj.ETag, - Key: obj.Key, - LastModified: obj.LastModified, - Owner: obj.Owner, - Size: obj.Size, - RestoreStatus: obj.RestoreStatus, - StorageClass: obj.StorageClass, + ETag: obj.ETag, + Key: obj.Key, + LastModified: obj.LastModified, + Owner: obj.Owner, + Size: obj.Size, + RestoreStatus: obj.RestoreStatus, + StorageClass: obj.StorageClass, + ChecksumAlgorithm: obj.ChecksumAlgorithm, + ChecksumType: &obj.ChecksumType, }) } diff --git a/s3api/controllers/base.go b/s3api/controllers/base.go index 4c80502d..8f2fcaff 100644 --- a/s3api/controllers/base.go +++ b/s3api/controllers/base.go @@ -607,6 +607,18 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error { Value: *res.ChecksumSHA256, }) } + if res.ChecksumCRC64NVME != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc64nvme", + Value: *res.ChecksumCRC64NVME, + }) + } + if res.ChecksumType != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-type", + Value: string(res.ChecksumType), + }) + } // Set x-amz-meta-... headers utils.SetMetaHeaders(ctx, res.Metadata) @@ -2079,6 +2091,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), }) if err == nil { headers := []utils.CustomHeader{} @@ -2112,6 +2125,12 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Value: *res.ChecksumSHA256, }) } + if res.ChecksumCRC64NVME != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc64nvme", + Value: *res.ChecksumCRC64NVME, + }) + } utils.SetResponseHeaders(ctx, headers) } @@ -2511,6 +2530,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), }) if err != nil { return SendResponse(ctx, err, @@ -2562,6 +2582,18 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error { Value: getstring(res.ChecksumSHA256), }) } + if getstring(res.ChecksumCRC64NVME) != "" { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-crc64nvme", + Value: getstring(res.ChecksumCRC64NVME), + }) + } + if res.ChecksumType != nil { + hdrs = append(hdrs, utils.CustomHeader{ + Key: "x-amz-checksum-type", + Value: string(*res.ChecksumType), + }) + } utils.SetResponseHeaders(ctx, hdrs) @@ -3206,6 +3238,18 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error { Value: *res.ChecksumSHA256, }) } + if res.ChecksumCRC64NVME != nil { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-crc64nvme", + Value: *res.ChecksumCRC64NVME, + }) + } + if res.ChecksumType != "" { + headers = append(headers, utils.CustomHeader{ + Key: "x-amz-checksum-type", + Value: string(res.ChecksumType), + }) + } contentType := getstring(res.ContentType) if contentType == "" { @@ -3430,6 +3474,21 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { }) } + checksumType := types.ChecksumType(ctx.Get("x-amz-checksum-type")) + err = utils.IsChecksumTypeValid(checksumType) + if err != nil { + if c.debug { + log.Printf("invalid checksum type: %v", err) + } + return SendXMLResponse(ctx, nil, err, + &MetaOpts{ + Logger: c.logger, + MetricsMng: c.mm, + Action: metrics.ActionCompleteMultipartUpload, + BucketOwner: parsedAcl.Owner, + }) + } + res, err := c.be.CompleteMultipartUpload(ctx.Context(), &s3.CompleteMultipartUploadInput{ Bucket: &bucket, @@ -3438,10 +3497,12 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { MultipartUpload: &types.CompletedMultipartUpload{ Parts: data.Parts, }, - ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), - ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), - ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), - ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]), + ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]), + ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]), + ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]), + ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]), + ChecksumType: checksumType, }) if err == nil { if getstring(res.VersionId) != "" { @@ -3507,11 +3568,10 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { metadata := utils.GetUserMetaData(&ctx.Request().Header) - checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) - err = utils.IsChecksumAlgorithmValid(checksumAlgorithm) + checksumAlgorithm, checksumType, err := utils.ParseCreateMpChecksumHeaders(ctx) if err != nil { if c.debug { - log.Printf("invalid checksum algorithm: %v", checksumAlgorithm) + log.Printf("err parsing checksum headers: %v", err) } return SendXMLResponse(ctx, nil, err, &MetaOpts{ @@ -3534,6 +3594,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error { ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus, Metadata: metadata, ChecksumAlgorithm: checksumAlgorithm, + ChecksumType: checksumType, }) if err == nil { if checksumAlgorithm != "" { diff --git a/s3api/utils/csum-reader.go b/s3api/utils/csum-reader.go index a36da388..ce2e35e7 100644 --- a/s3api/utils/csum-reader.go +++ b/s3api/utils/csum-reader.go @@ -21,9 +21,12 @@ import ( "encoding/base64" "encoding/hex" "errors" + "fmt" "hash" "hash/crc32" + "hash/crc64" "io" + "math/bits" "github.com/aws/aws-sdk-go-v2/service/s3/types" "github.com/versity/versitygw/s3err" @@ -45,6 +48,8 @@ const ( HashTypeCRC32 HashType = "crc32" // HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream HashTypeCRC32C HashType = "crc32c" + // HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream + HashTypeCRC64NVME HashType = "crc64nvme" // HashTypeNone is a no-op checksum for the data stream HashTypeNone HashType = "none" ) @@ -83,6 +88,8 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e hash = crc32.NewIEEE() case HashTypeCRC32C: hash = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case HashTypeCRC64NVME: + hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659))) case HashTypeNone: hash = noop{} default: @@ -136,6 +143,11 @@ func (hr *HashReader) Read(p []byte) (int, error) { if sum != hr.sum { return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256) } + case HashTypeCRC64NVME: + sum := hr.Sum() + if sum != hr.sum { + return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme) + } default: return n, errInvalidHashType } @@ -162,6 +174,8 @@ func (hr *HashReader) Sum() string { return Base64SumString(hr.hash.Sum(nil)) case HashTypeSha256: return Base64SumString(hr.hash.Sum(nil)) + case HashTypeCRC64NVME: + return Base64SumString(hr.hash.Sum(nil)) default: return "" } @@ -183,3 +197,59 @@ func (n noop) Sum(b []byte) []byte { return []byte{} } func (n noop) Reset() {} func (n noop) Size() int { return 0 } func (n noop) BlockSize() int { return 1 } + +// NewCompositeChecksumReader initializes a composite checksum +// processor, which decodes and validates the provided +// checksums and returns the final checksum based on +// the previous processings. +// +// The supported checksum types are: +// - CRC32 +// - CRC32C +// - SHA1 +// - SHA256 +func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) { + var hasher hash.Hash + switch ht { + case HashTypeSha256: + hasher = sha256.New() + case HashTypeSha1: + hasher = sha1.New() + case HashTypeCRC32: + hasher = crc32.NewIEEE() + case HashTypeCRC32C: + hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case HashTypeNone: + hasher = noop{} + default: + return nil, errInvalidHashType + } + + return &CompositeChecksumReader{ + hasher: hasher, + }, nil +} + +type CompositeChecksumReader struct { + hasher hash.Hash +} + +// Decodes and writes the checksum in the hasher +func (ccr *CompositeChecksumReader) Process(checksum string) error { + data, err := base64.StdEncoding.DecodeString(checksum) + if err != nil { + return fmt.Errorf("base64 decode: %w", err) + } + + _, err = ccr.hasher.Write(data) + if err != nil { + return fmt.Errorf("hash write: %w", err) + } + + return nil +} + +// Returns the base64 encoded composite checksum +func (ccr *CompositeChecksumReader) Sum() string { + return Base64SumString(ccr.hasher.Sum(nil)) +} diff --git a/s3api/utils/utils.go b/s3api/utils/utils.go index 2b65e208..c1ace5b1 100644 --- a/s3api/utils/utils.go +++ b/s3api/utils/utils.go @@ -461,7 +461,7 @@ func shouldEscape(c byte) bool { } func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.ChecksumAlgorithm]string, error) { - sdkAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-sdk-checksum-algorithm")) + sdkAlgorithm := types.ChecksumAlgorithm(ctx.Get("X-Amz-Sdk-Checksum-Algorithm")) err := IsChecksumAlgorithmValid(sdkAlgorithm) if err != nil { @@ -469,10 +469,11 @@ func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.Ch } checksums := map[types.ChecksumAlgorithm]string{ - types.ChecksumAlgorithmCrc32: ctx.Get("x-amz-checksum-crc32"), - types.ChecksumAlgorithmCrc32c: ctx.Get("x-amz-checksum-crc32c"), - types.ChecksumAlgorithmSha1: ctx.Get("x-amz-checksum-sha1"), - types.ChecksumAlgorithmSha256: ctx.Get("x-amz-checksum-sha256"), + types.ChecksumAlgorithmCrc32: ctx.Get("X-Amz-Checksum-Crc32"), + types.ChecksumAlgorithmCrc32c: ctx.Get("X-Amz-Checksum-Crc32c"), + types.ChecksumAlgorithmSha1: ctx.Get("X-Amz-Checksum-Sha1"), + types.ChecksumAlgorithmSha256: ctx.Get("X-Amz-Checksum-Sha256"), + types.ChecksumAlgorithmCrc64nvme: ctx.Get("X-Amz-Checksum-Crc64nvme"), } headerCtr := 0 @@ -487,6 +488,7 @@ func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.Ch return sdkAlgorithm, checksums, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders) } if val != "" { + sdkAlgorithm = al headerCtr++ } @@ -499,10 +501,11 @@ func ParseChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, map[types.Ch } var checksumLengths = map[types.ChecksumAlgorithm]int{ - types.ChecksumAlgorithmCrc32: 4, - types.ChecksumAlgorithmCrc32c: 4, - types.ChecksumAlgorithmSha1: 20, - types.ChecksumAlgorithmSha256: 32, + types.ChecksumAlgorithmCrc32: 4, + types.ChecksumAlgorithmCrc32c: 4, + types.ChecksumAlgorithmCrc64nvme: 8, + types.ChecksumAlgorithmSha1: 20, + types.ChecksumAlgorithmSha256: 32, } func IsValidChecksum(checksum string, algorithm types.ChecksumAlgorithm) bool { @@ -524,9 +527,102 @@ func IsChecksumAlgorithmValid(alg types.ChecksumAlgorithm) error { alg != types.ChecksumAlgorithmCrc32 && alg != types.ChecksumAlgorithmCrc32c && alg != types.ChecksumAlgorithmSha1 && - alg != types.ChecksumAlgorithmSha256 { + alg != types.ChecksumAlgorithmSha256 && + alg != types.ChecksumAlgorithmCrc64nvme { return s3err.GetAPIError(s3err.ErrInvalidChecksumAlgorithm) } return nil } + +// Validates the provided checksum type +func IsChecksumTypeValid(t types.ChecksumType) error { + if t != "" && + t != types.ChecksumTypeComposite && + t != types.ChecksumTypeFullObject { + return s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type") + } + return nil +} + +type checksumTypeSchema map[types.ChecksumType]struct{} +type checksumSchema map[types.ChecksumAlgorithm]checksumTypeSchema + +// A table defining the checksum algorithm/type support +var checksumMap checksumSchema = checksumSchema{ + types.ChecksumAlgorithmCrc32: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + types.ChecksumTypeFullObject: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmCrc32c: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + types.ChecksumTypeFullObject: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmSha1: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmSha256: checksumTypeSchema{ + types.ChecksumTypeComposite: struct{}{}, + "": struct{}{}, + }, + types.ChecksumAlgorithmCrc64nvme: checksumTypeSchema{ + types.ChecksumTypeFullObject: struct{}{}, + "": struct{}{}, + }, + // Both could be empty + "": checksumTypeSchema{ + "": struct{}{}, + }, +} + +// Checks if checksum type and algorithm are supported together +func checkChecksumTypeAndAlgo(algo types.ChecksumAlgorithm, t types.ChecksumType) error { + typeSchema := checksumMap[algo] + _, ok := typeSchema[t] + if !ok { + return s3err.GetChecksumSchemaMismatchErr(algo, t) + } + + return nil +} + +// Parses and validates the x-amz-checksum-algorithm and x-amz-checksum-type headers +func ParseCreateMpChecksumHeaders(ctx *fiber.Ctx) (types.ChecksumAlgorithm, types.ChecksumType, error) { + algo := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm")) + if err := IsChecksumAlgorithmValid(algo); err != nil { + return "", "", err + } + + chType := types.ChecksumType(ctx.Get("x-amz-checksum-type")) + if err := IsChecksumTypeValid(chType); err != nil { + return "", "", err + } + + // Verify if checksum algorithm is provided, if + // checksum type is specified + if chType != "" && algo == "" { + return algo, chType, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo) + } + + // Verify if the checksum type is supported for + // the provided checksum algorithm + if err := checkChecksumTypeAndAlgo(algo, chType); err != nil { + return algo, chType, err + } + + // x-amz-checksum-type defaults to COMPOSITE + // if x-amz-checksum-algorithm is set except + // for the CRC64NVME algorithm: it defaults to FULL_OBJECT + if algo != "" && chType == "" { + if algo == types.ChecksumAlgorithmCrc64nvme { + chType = types.ChecksumTypeFullObject + } else { + chType = types.ChecksumTypeComposite + } + } + + return algo, chType, nil +} diff --git a/s3api/utils/utils_test.go b/s3api/utils/utils_test.go index 87f38f62..6aa09078 100644 --- a/s3api/utils/utils_test.go +++ b/s3api/utils/utils_test.go @@ -572,6 +572,13 @@ func TestIsChecksumAlgorithmValid(t *testing.T) { }, wantErr: false, }, + { + name: "crc64nvme", + args: args{ + alg: types.ChecksumAlgorithmCrc64nvme, + }, + wantErr: false, + }, { name: "invalid", args: args{ @@ -680,3 +687,165 @@ func TestIsValidChecksum(t *testing.T) { }) } } + +func TestIsChecksumTypeValid(t *testing.T) { + type args struct { + t types.ChecksumType + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "valid_FULL_OBJECT", + args: args{ + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "valid_COMPOSITE", + args: args{ + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "invalid", + args: args{ + t: types.ChecksumType("invalid_checksum_type"), + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := IsChecksumTypeValid(tt.args.t); (err != nil) != tt.wantErr { + t.Errorf("IsChecksumTypeValid() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func Test_checkChecksumTypeAndAlgo(t *testing.T) { + type args struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "full_object-crc32", + args: args{ + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "full_object-crc32c", + args: args{ + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "full_object-sha1", + args: args{ + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-sha256", + args: args{ + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + { + name: "full_object-crc64nvme", + args: args{ + algo: types.ChecksumAlgorithmCrc64nvme, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "full_object-crc32", + args: args{ + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeFullObject, + }, + wantErr: false, + }, + { + name: "composite-crc32", + args: args{ + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-crc32c", + args: args{ + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-sha1", + args: args{ + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-sha256", + args: args{ + algo: types.ChecksumAlgorithmSha256, + t: types.ChecksumTypeComposite, + }, + wantErr: false, + }, + { + name: "composite-crc64nvme", + args: args{ + algo: types.ChecksumAlgorithmCrc64nvme, + t: types.ChecksumTypeComposite, + }, + wantErr: true, + }, + { + name: "composite-empty", + args: args{ + t: types.ChecksumTypeComposite, + }, + wantErr: true, + }, + { + name: "full_object-empty", + args: args{ + t: types.ChecksumTypeFullObject, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := checkChecksumTypeAndAlgo(tt.args.algo, tt.args.t); (err != nil) != tt.wantErr { + t.Errorf("checkChecksumTypeAndAlgo() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/s3err/s3err.go b/s3err/s3err.go index 9f673556..b8c75716 100644 --- a/s3err/s3err.go +++ b/s3err/s3err.go @@ -147,6 +147,7 @@ const ( ErrMultipleChecksumHeaders ErrInvalidChecksumAlgorithm ErrInvalidChecksumPart + ErrChecksumTypeWithAlgo // Non-AWS errors ErrExistingObjectIsDirectory @@ -606,6 +607,11 @@ var errorCodeResponse = map[ErrorCode]APIError{ Description: "Invalid Base64 or multiple checksums present in request", HTTPStatusCode: http.StatusBadRequest, }, + ErrChecksumTypeWithAlgo: { + Code: "InvalidRequest", + Description: "The x-amz-checksum-type header can only be used with the x-amz-checksum-algorithm header.", + HTTPStatusCode: http.StatusBadRequest, + }, // non aws errors ErrExistingObjectIsDirectory: { @@ -719,7 +725,7 @@ func GetChecksumTypeMismatchErr(expected, actual types.ChecksumAlgorithm) APIErr } } -// Return incorrect checksum APIError +// Returns incorrect checksum APIError func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError { return APIError{ Code: "BadDigest", @@ -727,3 +733,21 @@ func GetChecksumBadDigestErr(algo types.ChecksumAlgorithm) APIError { HTTPStatusCode: http.StatusBadRequest, } } + +// Returns checksum type mismatch error with checksum algorithm +func GetChecksumSchemaMismatchErr(algo types.ChecksumAlgorithm, t types.ChecksumType) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("The %v checksum type cannot be used with the %v checksum algorithm.", algo, strings.ToLower(string(t))), + HTTPStatusCode: http.StatusBadRequest, + } +} + +// Returns checksum type mismatch error for multipart uploads +func GetChecksumTypeMismatchOnMpErr(t types.ChecksumType) APIError { + return APIError{ + Code: "InvalidRequest", + Description: fmt.Sprintf("The upload was created using the %v checksum mode. The complete request must use the same checksum mode.", t), + HTTPStatusCode: http.StatusBadRequest, + } +} diff --git a/s3response/s3response.go b/s3response/s3response.go index 1b0eeb89..0ac9b59e 100644 --- a/s3response/s3response.go +++ b/s3response/s3response.go @@ -29,24 +29,27 @@ const ( ) type PutObjectOutput struct { - ETag string - VersionID string - ChecksumCRC32 *string - ChecksumCRC32C *string - ChecksumSHA1 *string - ChecksumSHA256 *string + ETag string + VersionID string + ChecksumCRC32 *string + ChecksumCRC32C *string + ChecksumSHA1 *string + ChecksumSHA256 *string + ChecksumCRC64NVME *string + ChecksumType *types.ChecksumType } // Part describes part metadata. type Part struct { - PartNumber int - LastModified time.Time - ETag string - Size int64 - ChecksumCRC32 *string - ChecksumCRC32C *string - ChecksumSHA1 *string - ChecksumSHA256 *string + PartNumber int + LastModified time.Time + ETag string + Size int64 + ChecksumCRC32 *string + ChecksumCRC32C *string + ChecksumSHA1 *string + ChecksumSHA256 *string + ChecksumCRC64NVME *string } func (p Part) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -71,6 +74,7 @@ type ListPartsResult struct { Key string UploadID string `xml:"UploadId"` ChecksumAlgorithm types.ChecksumAlgorithm + ChecksumType *types.ChecksumType Initiator Initiator Owner Owner @@ -180,6 +184,7 @@ type ListObjectsV2Result struct { type Object struct { ChecksumAlgorithm []types.ChecksumAlgorithm + ChecksumType *types.ChecksumType ETag *string Key *string LastModified *time.Time @@ -215,6 +220,7 @@ type Upload struct { StorageClass types.StorageClass Initiated time.Time ChecksumAlgorithm types.ChecksumAlgorithm + ChecksumType *types.ChecksumType } func (u Upload) MarshalXML(e *xml.Encoder, start xml.StartElement) error { @@ -345,13 +351,14 @@ type CopyObjectResult struct { } type CopyPartResult struct { - XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"` - LastModified time.Time - ETag *string - ChecksumCRC32 *string - ChecksumCRC32C *string - ChecksumSHA1 *string - ChecksumSHA256 *string + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"` + LastModified time.Time + ETag *string + ChecksumCRC32 *string + ChecksumCRC32C *string + ChecksumSHA1 *string + ChecksumSHA256 *string + ChecksumCRC64NVME *string // not included in the body CopySourceVersionId string `xml:"-"` @@ -495,9 +502,11 @@ type ListBucketsResult struct { type Checksum struct { Algorithms []types.ChecksumAlgorithm + Type *types.ChecksumType - CRC32 *string - CRC32C *string - SHA1 *string - SHA256 *string + CRC32 *string + CRC32C *string + SHA1 *string + SHA256 *string + CRC64NVME *string } diff --git a/tests/integration/group-tests.go b/tests/integration/group-tests.go index 0a39df8f..cc1c420f 100644 --- a/tests/integration/group-tests.go +++ b/tests/integration/group-tests.go @@ -138,11 +138,14 @@ func TestPutObject(s *S3Conf) { PutObject_invalid_long_tags(s) PutObject_missing_object_lock_retention_config(s) PutObject_with_object_lock(s) - PutObject_checksum_algorithm_and_header_mismatch(s) - PutObject_multiple_checksum_headers(s) - PutObject_invalid_checksum_header(s) - PutObject_incorrect_checksums(s) - PutObject_checksums_success(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + PutObject_checksum_algorithm_and_header_mismatch(s) + PutObject_multiple_checksum_headers(s) + PutObject_invalid_checksum_header(s) + PutObject_incorrect_checksums(s) + PutObject_checksums_success(s) + } PutObject_success(s) if !s.versioningEnabled { PutObject_racey_success(s) @@ -159,8 +162,11 @@ func TestHeadObject(s *S3Conf) { HeadObject_non_existing_dir_object(s) HeadObject_with_contenttype(s) HeadObject_invalid_parent_dir(s) - HeadObject_not_enabled_checksum_mode(s) - HeadObject_checksums(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + HeadObject_not_enabled_checksum_mode(s) + HeadObject_checksums(s) + } HeadObject_success(s) } @@ -171,7 +177,11 @@ func TestGetObjectAttributes(s *S3Conf) { GetObjectAttributes_invalid_parent(s) GetObjectAttributes_empty_attrs(s) GetObjectAttributes_existing_object(s) - GetObjectAttributes_checksums(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + + GetObjectAttributes_checksums(s) + } } func TestGetObject(s *S3Conf) { @@ -181,8 +191,10 @@ func TestGetObject(s *S3Conf) { GetObject_invalid_parent(s) GetObject_with_meta(s) GetObject_large_object(s) - GetObject_not_enabled_checksum_mode(s) - GetObject_checksums(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + GetObject_checksums(s) + } GetObject_success(s) GetObject_directory_success(s) GetObject_by_range_success(s) @@ -201,7 +213,10 @@ func TestListObjects(s *S3Conf) { ListObjects_max_keys_none(s) ListObjects_marker_not_from_obj_list(s) ListObjects_list_all_objs(s) - ListObjects_with_checksum(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + ListObjects_with_checksum(s) + } } func TestListObjectsV2(s *S3Conf) { @@ -214,7 +229,10 @@ func TestListObjectsV2(s *S3Conf) { ListObjectsV2_truncated_common_prefixes(s) ListObjectsV2_all_objs_max_keys(s) ListObjectsV2_list_all_objs(s) - ListObjectsV2_with_checksum(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + ListObjectsV2_with_checksum(s) + } ListObjectsV2_invalid_parent_prefix(s) } @@ -245,11 +263,14 @@ func TestCopyObject(s *S3Conf) { CopyObject_to_itself_with_new_metadata(s) CopyObject_CopySource_starting_with_slash(s) CopyObject_non_existing_dir_object(s) - CopyObject_invalid_checksum_algorithm(s) - CopyObject_create_checksum_on_copy(s) - CopyObject_should_copy_the_existing_checksum(s) - CopyObject_should_replace_the_existing_checksum(s) - CopyObject_to_itself_by_replacing_the_checksum(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + CopyObject_invalid_checksum_algorithm(s) + CopyObject_create_checksum_on_copy(s) + CopyObject_should_copy_the_existing_checksum(s) + CopyObject_should_replace_the_existing_checksum(s) + CopyObject_to_itself_by_replacing_the_checksum(s) + } CopyObject_success(s) } @@ -282,8 +303,13 @@ func TestCreateMultipartUpload(s *S3Conf) { CreateMultipartUpload_with_object_lock_not_enabled(s) CreateMultipartUpload_with_object_lock_invalid_retention(s) CreateMultipartUpload_past_retain_until_date(s) - CreateMultipartUpload_invalid_checksum_algorithm(s) - CreateMultipartUpload_valid_checksum_algorithm(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + CreateMultipartUpload_invalid_checksum_algorithm(s) + CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type(s) + CreateMultipartUpload_invalid_checksum_type(s) + CreateMultipartUpload_valid_checksum_algorithm(s) + } CreateMultipartUpload_success(s) } @@ -292,15 +318,16 @@ func TestUploadPart(s *S3Conf) { UploadPart_invalid_part_number(s) UploadPart_non_existing_key(s) UploadPart_non_existing_mp_upload(s) - UploadPart_checksum_algorithm_and_header_mismatch(s) - UploadPart_multiple_checksum_headers(s) - UploadPart_invalid_checksum_header(s) - UploadPart_checksum_algorithm_mistmatch_on_initialization(s) - UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value(s) - UploadPart_required_checksum(s) - UploadPart_null_checksum(s) - UploadPart_incorrect_checksums(s) - UploadPart_with_checksums_success(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + UploadPart_checksum_algorithm_and_header_mismatch(s) + UploadPart_multiple_checksum_headers(s) + UploadPart_invalid_checksum_header(s) + UploadPart_checksum_algorithm_mistmatch_on_initialization(s) + UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value(s) + UploadPart_incorrect_checksums(s) + UploadPart_with_checksums_success(s) + } UploadPart_success(s) } @@ -316,9 +343,12 @@ func TestUploadPartCopy(s *S3Conf) { UploadPartCopy_by_range_invalid_range(s) UploadPartCopy_greater_range_than_obj_size(s) UploadPartCopy_by_range_success(s) - UploadPartCopy_should_copy_the_checksum(s) - UploadPartCopy_should_not_copy_the_checksum(s) - UploadPartCopy_should_calculate_the_checksum(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + UploadPartCopy_should_copy_the_checksum(s) + UploadPartCopy_should_not_copy_the_checksum(s) + UploadPartCopy_should_calculate_the_checksum(s) + } } func TestListParts(s *S3Conf) { @@ -327,7 +357,10 @@ func TestListParts(s *S3Conf) { ListParts_invalid_max_parts(s) ListParts_default_max_parts(s) ListParts_truncated(s) - ListParts_with_checksums(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + ListParts_with_checksums(s) + } ListParts_success(s) } @@ -338,7 +371,10 @@ func TestListMultipartUploads(s *S3Conf) { ListMultipartUploads_max_uploads(s) ListMultipartUploads_incorrect_next_key_marker(s) ListMultipartUploads_ignore_upload_id_marker(s) - ListMultipartUploads_with_checksums(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + ListMultipartUploads_with_checksums(s) + } ListMultipartUploads_success(s) } @@ -356,6 +392,22 @@ func TestCompleteMultipartUpload(s *S3Conf) { CompleteMultipartUpload_invalid_ETag(s) CompleteMultipartUpload_small_upload_size(s) CompleteMultipartUpload_empty_parts(s) + //TODO: remove the condition after implementing checksums in azure + if !s.azureTests { + CompleteMultipartUpload_invalid_checksum_type(s) + CompleteMultipartUpload_invalid_checksum_part(s) + CompleteMultipartUpload_multiple_checksum_part(s) + CompleteMultipartUpload_incorrect_checksum_part(s) + CompleteMultipartUpload_different_checksum_part(s) + CompleteMultipartUpload_missing_part_checksum(s) + CompleteMultipartUpload_multiple_final_checksums(s) + CompleteMultipartUpload_invalid_final_checksums(s) + CompleteMultipartUpload_incorrect_final_checksums(s) + CompleteMultipartUpload_should_calculate_the_final_checksum_full_object(s) + CompleteMultipartUpload_should_verify_the_final_checksum(s) + CompleteMultipartUpload_checksum_type_mismatch(s) + CompleteMultipartUpload_should_ignore_the_final_checksum(s) + } CompleteMultipartUpload_success(s) if !s.azureTests { CompleteMultipartUpload_racey_success(s) @@ -677,396 +729,441 @@ type IntTests map[string]func(s *S3Conf) error func GetIntTests() IntTests { return IntTests{ - "Authentication_empty_auth_header": Authentication_empty_auth_header, - "Authentication_invalid_auth_header": Authentication_invalid_auth_header, - "Authentication_unsupported_signature_version": Authentication_unsupported_signature_version, - "Authentication_malformed_credentials": Authentication_malformed_credentials, - "Authentication_malformed_credentials_invalid_parts": Authentication_malformed_credentials_invalid_parts, - "Authentication_credentials_terminated_string": Authentication_credentials_terminated_string, - "Authentication_credentials_incorrect_service": Authentication_credentials_incorrect_service, - "Authentication_credentials_incorrect_region": Authentication_credentials_incorrect_region, - "Authentication_credentials_invalid_date": Authentication_credentials_invalid_date, - "Authentication_credentials_future_date": Authentication_credentials_future_date, - "Authentication_credentials_past_date": Authentication_credentials_past_date, - "Authentication_credentials_non_existing_access_key": Authentication_credentials_non_existing_access_key, - "Authentication_invalid_signed_headers": Authentication_invalid_signed_headers, - "Authentication_missing_date_header": Authentication_missing_date_header, - "Authentication_invalid_date_header": Authentication_invalid_date_header, - "Authentication_date_mismatch": Authentication_date_mismatch, - "Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash, - "Authentication_incorrect_md5": Authentication_incorrect_md5, - "Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key, - "PresignedAuth_missing_algo_query_param": PresignedAuth_missing_algo_query_param, - "PresignedAuth_unsupported_algorithm": PresignedAuth_unsupported_algorithm, - "PresignedAuth_missing_credentials_query_param": PresignedAuth_missing_credentials_query_param, - "PresignedAuth_malformed_creds_invalid_parts": PresignedAuth_malformed_creds_invalid_parts, - "PresignedAuth_creds_invalid_terminator": PresignedAuth_creds_invalid_terminator, - "PresignedAuth_creds_incorrect_service": PresignedAuth_creds_incorrect_service, - "PresignedAuth_creds_incorrect_region": PresignedAuth_creds_incorrect_region, - "PresignedAuth_creds_invalid_date": PresignedAuth_creds_invalid_date, - "PresignedAuth_missing_date_query": PresignedAuth_missing_date_query, - "PresignedAuth_dates_mismatch": PresignedAuth_dates_mismatch, - "PresignedAuth_non_existing_access_key_id": PresignedAuth_non_existing_access_key_id, - "PresignedAuth_missing_signed_headers_query_param": PresignedAuth_missing_signed_headers_query_param, - "PresignedAuth_missing_expiration_query_param": PresignedAuth_missing_expiration_query_param, - "PresignedAuth_invalid_expiration_query_param": PresignedAuth_invalid_expiration_query_param, - "PresignedAuth_negative_expiration_query_param": PresignedAuth_negative_expiration_query_param, - "PresignedAuth_exceeding_expiration_query_param": PresignedAuth_exceeding_expiration_query_param, - "PresignedAuth_expired_request": PresignedAuth_expired_request, - "PresignedAuth_incorrect_secret_key": PresignedAuth_incorrect_secret_key, - "PresignedAuth_PutObject_success": PresignedAuth_PutObject_success, - "PutObject_missing_object_lock_retention_config": PutObject_missing_object_lock_retention_config, - "PutObject_name_too_long": PutObject_name_too_long, - "PutObject_with_object_lock": PutObject_with_object_lock, - "PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data, - "PresignedAuth_Put_GetObject_with_UTF8_chars": PresignedAuth_Put_GetObject_with_UTF8_chars, - "PresignedAuth_UploadPart": PresignedAuth_UploadPart, - "CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name, - "CreateBucket_existing_bucket": CreateBucket_existing_bucket, - "CreateBucket_owned_by_you": CreateBucket_owned_by_you, - "CreateBucket_invalid_ownership": CreateBucket_invalid_ownership, - "CreateBucket_ownership_with_acl": CreateBucket_ownership_with_acl, - "CreateBucket_as_user": CreateBucket_as_user, - "CreateDeleteBucket_success": CreateDeleteBucket_success, - "CreateBucket_default_acl": CreateBucket_default_acl, - "CreateBucket_non_default_acl": CreateBucket_non_default_acl, - "CreateBucket_default_object_lock": CreateBucket_default_object_lock, - "HeadBucket_non_existing_bucket": HeadBucket_non_existing_bucket, - "HeadBucket_success": HeadBucket_success, - "ListBuckets_as_user": ListBuckets_as_user, - "ListBuckets_as_admin": ListBuckets_as_admin, - "ListBuckets_with_prefix": ListBuckets_with_prefix, - "ListBuckets_invalid_max_buckets": ListBuckets_invalid_max_buckets, - "ListBuckets_truncated": ListBuckets_truncated, - "ListBuckets_success": ListBuckets_success, - "DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket, - "DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket, - "DeleteBucket_success_status_code": DeleteBucket_success_status_code, - "PutBucketOwnershipControls_non_existing_bucket": PutBucketOwnershipControls_non_existing_bucket, - "PutBucketOwnershipControls_multiple_rules": PutBucketOwnershipControls_multiple_rules, - "PutBucketOwnershipControls_invalid_ownership": PutBucketOwnershipControls_invalid_ownership, - "PutBucketOwnershipControls_success": PutBucketOwnershipControls_success, - "GetBucketOwnershipControls_non_existing_bucket": GetBucketOwnershipControls_non_existing_bucket, - "GetBucketOwnershipControls_default_ownership": GetBucketOwnershipControls_default_ownership, - "GetBucketOwnershipControls_success": GetBucketOwnershipControls_success, - "DeleteBucketOwnershipControls_non_existing_bucket": DeleteBucketOwnershipControls_non_existing_bucket, - "DeleteBucketOwnershipControls_success": DeleteBucketOwnershipControls_success, - "PutBucketTagging_non_existing_bucket": PutBucketTagging_non_existing_bucket, - "PutBucketTagging_long_tags": PutBucketTagging_long_tags, - "PutBucketTagging_success": PutBucketTagging_success, - "PutBucketTagging_success_status": PutBucketTagging_success_status, - "GetBucketTagging_non_existing_bucket": GetBucketTagging_non_existing_bucket, - "GetBucketTagging_unset_tags": GetBucketTagging_unset_tags, - "GetBucketTagging_success": GetBucketTagging_success, - "DeleteBucketTagging_non_existing_object": DeleteBucketTagging_non_existing_object, - "DeleteBucketTagging_success_status": DeleteBucketTagging_success_status, - "DeleteBucketTagging_success": DeleteBucketTagging_success, - "PutObject_non_existing_bucket": PutObject_non_existing_bucket, - "PutObject_special_chars": PutObject_special_chars, - "PutObject_invalid_long_tags": PutObject_invalid_long_tags, - "PutObject_success": PutObject_success, - "PutObject_racey_success": PutObject_racey_success, - "HeadObject_non_existing_object": HeadObject_non_existing_object, - "HeadObject_invalid_part_number": HeadObject_invalid_part_number, - "HeadObject_non_existing_mp": HeadObject_non_existing_mp, - "HeadObject_mp_success": HeadObject_mp_success, - "HeadObject_directory_object_noslash": HeadObject_directory_object_noslash, - "HeadObject_non_existing_dir_object": HeadObject_non_existing_dir_object, - "HeadObject_name_too_long": HeadObject_name_too_long, - "HeadObject_with_contenttype": HeadObject_with_contenttype, - "HeadObject_invalid_parent_dir": HeadObject_invalid_parent_dir, - "HeadObject_success": HeadObject_success, - "GetObjectAttributes_non_existing_bucket": GetObjectAttributes_non_existing_bucket, - "GetObjectAttributes_non_existing_object": GetObjectAttributes_non_existing_object, - "GetObjectAttributes_invalid_attrs": GetObjectAttributes_invalid_attrs, - "GetObjectAttributes_invalid_parent": GetObjectAttributes_invalid_parent, - "GetObjectAttributes_empty_attrs": GetObjectAttributes_empty_attrs, - "GetObjectAttributes_existing_object": GetObjectAttributes_existing_object, - "GetObject_non_existing_key": GetObject_non_existing_key, - "GetObject_directory_object_noslash": GetObject_directory_object_noslash, - "GetObject_invalid_ranges": GetObject_invalid_ranges, - "GetObject_invalid_parent": GetObject_invalid_parent, - "GetObject_with_meta": GetObject_with_meta, - "GetObject_large_object": GetObject_large_object, - "GetObject_success": GetObject_success, - "GetObject_directory_success": GetObject_directory_success, - "GetObject_by_range_success": GetObject_by_range_success, - "GetObject_by_range_resp_status": GetObject_by_range_resp_status, - "GetObject_non_existing_dir_object": GetObject_non_existing_dir_object, - "ListObjects_non_existing_bucket": ListObjects_non_existing_bucket, - "ListObjects_with_prefix": ListObjects_with_prefix, - "ListObjects_truncated": ListObjects_truncated, - "ListObjects_paginated": ListObjects_paginated, - "ListObjects_invalid_max_keys": ListObjects_invalid_max_keys, - "ListObjects_max_keys_0": ListObjects_max_keys_0, - "ListObjects_delimiter": ListObjects_delimiter, - "ListObjects_max_keys_none": ListObjects_max_keys_none, - "ListObjects_marker_not_from_obj_list": ListObjects_marker_not_from_obj_list, - "ListObjects_list_all_objs": ListObjects_list_all_objs, - "ListObjectsV2_start_after": ListObjectsV2_start_after, - "ListObjectsV2_both_start_after_and_continuation_token": ListObjectsV2_both_start_after_and_continuation_token, - "ListObjectsV2_start_after_not_in_list": ListObjectsV2_start_after_not_in_list, - "ListObjectsV2_start_after_empty_result": ListObjectsV2_start_after_empty_result, - "ListObjectsV2_both_delimiter_and_prefix": ListObjectsV2_both_delimiter_and_prefix, - "ListObjectsV2_single_dir_object_with_delim_and_prefix": ListObjectsV2_single_dir_object_with_delim_and_prefix, - "ListObjectsV2_truncated_common_prefixes": ListObjectsV2_truncated_common_prefixes, - "ListObjectsV2_all_objs_max_keys": ListObjectsV2_all_objs_max_keys, - "ListObjectsV2_list_all_objs": ListObjectsV2_list_all_objs, - "ListObjectVersions_VD_success": ListObjectVersions_VD_success, - "DeleteObject_non_existing_object": DeleteObject_non_existing_object, - "DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash, - "DeleteObject_name_too_long": DeleteObject_name_too_long, - "DeleteObject_non_existing_dir_object": DeleteObject_non_existing_dir_object, - "DeleteObject_success": DeleteObject_success, - "DeleteObject_success_status_code": DeleteObject_success_status_code, - "DeleteObjects_empty_input": DeleteObjects_empty_input, - "DeleteObjects_non_existing_objects": DeleteObjects_non_existing_objects, - "DeleteObjects_success": DeleteObjects_success, - "CopyObject_non_existing_dst_bucket": CopyObject_non_existing_dst_bucket, - "CopyObject_not_owned_source_bucket": CopyObject_not_owned_source_bucket, - "CopyObject_copy_to_itself": CopyObject_copy_to_itself, - "CopyObject_copy_to_itself_invalid_directive": CopyObject_copy_to_itself_invalid_directive, - "CopyObject_to_itself_with_new_metadata": CopyObject_to_itself_with_new_metadata, - "CopyObject_CopySource_starting_with_slash": CopyObject_CopySource_starting_with_slash, - "CopyObject_non_existing_dir_object": CopyObject_non_existing_dir_object, - "CopyObject_success": CopyObject_success, - "PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object, - "PutObjectTagging_long_tags": PutObjectTagging_long_tags, - "PutObjectTagging_success": PutObjectTagging_success, - "GetObjectTagging_non_existing_object": GetObjectTagging_non_existing_object, - "GetObjectTagging_unset_tags": GetObjectTagging_unset_tags, - "GetObjectTagging_invalid_parent": GetObjectTagging_invalid_parent, - "GetObjectTagging_success": GetObjectTagging_success, - "DeleteObjectTagging_non_existing_object": DeleteObjectTagging_non_existing_object, - "DeleteObjectTagging_success_status": DeleteObjectTagging_success_status, - "DeleteObjectTagging_success": DeleteObjectTagging_success, - "CreateMultipartUpload_non_existing_bucket": CreateMultipartUpload_non_existing_bucket, - "CreateMultipartUpload_with_metadata": CreateMultipartUpload_with_metadata, - "CreateMultipartUpload_with_invalid_tagging": CreateMultipartUpload_with_invalid_tagging, - "CreateMultipartUpload_with_tagging": CreateMultipartUpload_with_tagging, - "CreateMultipartUpload_with_content_type": CreateMultipartUpload_with_content_type, - "CreateMultipartUpload_with_object_lock": CreateMultipartUpload_with_object_lock, - "CreateMultipartUpload_with_object_lock_not_enabled": CreateMultipartUpload_with_object_lock_not_enabled, - "CreateMultipartUpload_with_object_lock_invalid_retention": CreateMultipartUpload_with_object_lock_invalid_retention, - "CreateMultipartUpload_past_retain_until_date": CreateMultipartUpload_past_retain_until_date, - "CreateMultipartUpload_success": CreateMultipartUpload_success, - "UploadPart_non_existing_bucket": UploadPart_non_existing_bucket, - "UploadPart_invalid_part_number": UploadPart_invalid_part_number, - "UploadPart_non_existing_key": UploadPart_non_existing_key, - "UploadPart_non_existing_mp_upload": UploadPart_non_existing_mp_upload, - "UploadPart_success": UploadPart_success, - "UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket, - "UploadPartCopy_incorrect_uploadId": UploadPartCopy_incorrect_uploadId, - "UploadPartCopy_incorrect_object_key": UploadPartCopy_incorrect_object_key, - "UploadPartCopy_invalid_part_number": UploadPartCopy_invalid_part_number, - "UploadPartCopy_invalid_copy_source": UploadPartCopy_invalid_copy_source, - "UploadPartCopy_non_existing_source_bucket": UploadPartCopy_non_existing_source_bucket, - "UploadPartCopy_non_existing_source_object_key": UploadPartCopy_non_existing_source_object_key, - "UploadPartCopy_success": UploadPartCopy_success, - "UploadPartCopy_by_range_invalid_range": UploadPartCopy_by_range_invalid_range, - "UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size, - "UploadPartCopy_by_range_success": UploadPartCopy_by_range_success, - "ListParts_incorrect_uploadId": ListParts_incorrect_uploadId, - "ListParts_incorrect_object_key": ListParts_incorrect_object_key, - "ListParts_invalid_max_parts": ListParts_invalid_max_parts, - "ListParts_default_max_parts": ListParts_default_max_parts, - "ListParts_truncated": ListParts_truncated, - "ListParts_success": ListParts_success, - "ListMultipartUploads_non_existing_bucket": ListMultipartUploads_non_existing_bucket, - "ListMultipartUploads_empty_result": ListMultipartUploads_empty_result, - "ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads, - "ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads, - "ListMultipartUploads_incorrect_next_key_marker": ListMultipartUploads_incorrect_next_key_marker, - "ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker, - "ListMultipartUploads_success": ListMultipartUploads_success, - "AbortMultipartUpload_non_existing_bucket": AbortMultipartUpload_non_existing_bucket, - "AbortMultipartUpload_incorrect_uploadId": AbortMultipartUpload_incorrect_uploadId, - "AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key, - "AbortMultipartUpload_success": AbortMultipartUpload_success, - "AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code, - "CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket, - "CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number, - "CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag, - "CompleteMultipartUpload_success": CompleteMultipartUpload_success, - "CompleteMultipartUpload_racey_success": CompleteMultipartUpload_racey_success, - "PutBucketAcl_non_existing_bucket": PutBucketAcl_non_existing_bucket, - "PutBucketAcl_disabled": PutBucketAcl_disabled, - "PutBucketAcl_none_of_the_options_specified": PutBucketAcl_none_of_the_options_specified, - "PutBucketAcl_invalid_acl_canned_and_acp": PutBucketAcl_invalid_acl_canned_and_acp, - "PutBucketAcl_invalid_acl_canned_and_grants": PutBucketAcl_invalid_acl_canned_and_grants, - "PutBucketAcl_invalid_acl_acp_and_grants": PutBucketAcl_invalid_acl_acp_and_grants, - "PutBucketAcl_invalid_owner": PutBucketAcl_invalid_owner, - "PutBucketAcl_invalid_owner_not_in_body": PutBucketAcl_invalid_owner_not_in_body, - "PutBucketAcl_invalid_empty_owner_id_in_body": PutBucketAcl_invalid_empty_owner_id_in_body, - "PutBucketAcl_invalid_permission_in_body": PutBucketAcl_invalid_permission_in_body, - "PutBucketAcl_invalid_grantee_type_in_body": PutBucketAcl_invalid_grantee_type_in_body, - "PutBucketAcl_empty_grantee_ID_in_body": PutBucketAcl_empty_grantee_ID_in_body, - "PutBucketAcl_success_access_denied": PutBucketAcl_success_access_denied, - "PutBucketAcl_success_grants": PutBucketAcl_success_grants, - "PutBucketAcl_success_canned_acl": PutBucketAcl_success_canned_acl, - "PutBucketAcl_success_acp": PutBucketAcl_success_acp, - "GetBucketAcl_non_existing_bucket": GetBucketAcl_non_existing_bucket, - "GetBucketAcl_translation_canned_public_read": GetBucketAcl_translation_canned_public_read, - "GetBucketAcl_translation_canned_public_read_write": GetBucketAcl_translation_canned_public_read_write, - "GetBucketAcl_translation_canned_private": GetBucketAcl_translation_canned_private, - "GetBucketAcl_access_denied": GetBucketAcl_access_denied, - "GetBucketAcl_success": GetBucketAcl_success, - "PutBucketPolicy_non_existing_bucket": PutBucketPolicy_non_existing_bucket, - "PutBucketPolicy_empty_statement": PutBucketPolicy_empty_statement, - "PutBucketPolicy_invalid_effect": PutBucketPolicy_invalid_effect, - "PutBucketPolicy_empty_actions_string": PutBucketPolicy_empty_actions_string, - "PutBucketPolicy_empty_actions_array": PutBucketPolicy_empty_actions_array, - "PutBucketPolicy_invalid_action": PutBucketPolicy_invalid_action, - "PutBucketPolicy_unsupported_action": PutBucketPolicy_unsupported_action, - "PutBucketPolicy_incorrect_action_wildcard_usage": PutBucketPolicy_incorrect_action_wildcard_usage, - "PutBucketPolicy_empty_principals_string": PutBucketPolicy_empty_principals_string, - "PutBucketPolicy_empty_principals_array": PutBucketPolicy_empty_principals_array, - "PutBucketPolicy_principals_aws_struct_empty_string": PutBucketPolicy_principals_aws_struct_empty_string, - "PutBucketPolicy_principals_aws_struct_empty_string_slice": PutBucketPolicy_principals_aws_struct_empty_string_slice, - "PutBucketPolicy_principals_incorrect_wildcard_usage": PutBucketPolicy_principals_incorrect_wildcard_usage, - "PutBucketPolicy_non_existing_principals": PutBucketPolicy_non_existing_principals, - "PutBucketPolicy_empty_resources_string": PutBucketPolicy_empty_resources_string, - "PutBucketPolicy_empty_resources_array": PutBucketPolicy_empty_resources_array, - "PutBucketPolicy_invalid_resource_prefix": PutBucketPolicy_invalid_resource_prefix, - "PutBucketPolicy_invalid_resource_with_starting_slash": PutBucketPolicy_invalid_resource_with_starting_slash, - "PutBucketPolicy_duplicate_resource": PutBucketPolicy_duplicate_resource, - "PutBucketPolicy_incorrect_bucket_name": PutBucketPolicy_incorrect_bucket_name, - "PutBucketPolicy_object_action_on_bucket_resource": PutBucketPolicy_object_action_on_bucket_resource, - "PutBucketPolicy_bucket_action_on_object_resource": PutBucketPolicy_bucket_action_on_object_resource, - "PutBucketPolicy_success": PutBucketPolicy_success, - "GetBucketPolicy_non_existing_bucket": GetBucketPolicy_non_existing_bucket, - "GetBucketPolicy_not_set": GetBucketPolicy_not_set, - "GetBucketPolicy_success": GetBucketPolicy_success, - "DeleteBucketPolicy_non_existing_bucket": DeleteBucketPolicy_non_existing_bucket, - "DeleteBucketPolicy_remove_before_setting": DeleteBucketPolicy_remove_before_setting, - "DeleteBucketPolicy_success": DeleteBucketPolicy_success, - "PutObjectLockConfiguration_non_existing_bucket": PutObjectLockConfiguration_non_existing_bucket, - "PutObjectLockConfiguration_empty_config": PutObjectLockConfiguration_empty_config, - "PutObjectLockConfiguration_not_enabled_on_bucket_creation": PutObjectLockConfiguration_not_enabled_on_bucket_creation, - "PutObjectLockConfiguration_invalid_status": PutObjectLockConfiguration_invalid_status, - "PutObjectLockConfiguration_invalid_mode": PutObjectLockConfiguration_invalid_mode, - "PutObjectLockConfiguration_both_years_and_days": PutObjectLockConfiguration_both_years_and_days, - "PutObjectLockConfiguration_invalid_years_days": PutObjectLockConfiguration_invalid_years_days, - "PutObjectLockConfiguration_success": PutObjectLockConfiguration_success, - "GetObjectLockConfiguration_non_existing_bucket": GetObjectLockConfiguration_non_existing_bucket, - "GetObjectLockConfiguration_unset_config": GetObjectLockConfiguration_unset_config, - "GetObjectLockConfiguration_success": GetObjectLockConfiguration_success, - "PutObjectRetention_non_existing_bucket": PutObjectRetention_non_existing_bucket, - "PutObjectRetention_non_existing_object": PutObjectRetention_non_existing_object, - "PutObjectRetention_unset_bucket_object_lock_config": PutObjectRetention_unset_bucket_object_lock_config, - "PutObjectRetention_disabled_bucket_object_lock_config": PutObjectRetention_disabled_bucket_object_lock_config, - "PutObjectRetention_expired_retain_until_date": PutObjectRetention_expired_retain_until_date, - "PutObjectRetention_invalid_mode": PutObjectRetention_invalid_mode, - "PutObjectRetention_overwrite_compliance_mode": PutObjectRetention_overwrite_compliance_mode, - "PutObjectRetention_overwrite_governance_without_bypass_specified": PutObjectRetention_overwrite_governance_without_bypass_specified, - "PutObjectRetention_overwrite_governance_with_permission": PutObjectRetention_overwrite_governance_with_permission, - "PutObjectRetention_success": PutObjectRetention_success, - "GetObjectRetention_non_existing_bucket": GetObjectRetention_non_existing_bucket, - "GetObjectRetention_non_existing_object": GetObjectRetention_non_existing_object, - "GetObjectRetention_disabled_lock": GetObjectRetention_disabled_lock, - "GetObjectRetention_unset_config": GetObjectRetention_unset_config, - "GetObjectRetention_success": GetObjectRetention_success, - "PutObjectLegalHold_non_existing_bucket": PutObjectLegalHold_non_existing_bucket, - "PutObjectLegalHold_non_existing_object": PutObjectLegalHold_non_existing_object, - "PutObjectLegalHold_invalid_body": PutObjectLegalHold_invalid_body, - "PutObjectLegalHold_invalid_status": PutObjectLegalHold_invalid_status, - "PutObjectLegalHold_unset_bucket_object_lock_config": PutObjectLegalHold_unset_bucket_object_lock_config, - "PutObjectLegalHold_disabled_bucket_object_lock_config": PutObjectLegalHold_disabled_bucket_object_lock_config, - "PutObjectLegalHold_success": PutObjectLegalHold_success, - "GetObjectLegalHold_non_existing_bucket": GetObjectLegalHold_non_existing_bucket, - "GetObjectLegalHold_non_existing_object": GetObjectLegalHold_non_existing_object, - "GetObjectLegalHold_disabled_lock": GetObjectLegalHold_disabled_lock, - "GetObjectLegalHold_unset_config": GetObjectLegalHold_unset_config, - "GetObjectLegalHold_success": GetObjectLegalHold_success, - "WORMProtection_bucket_object_lock_configuration_compliance_mode": WORMProtection_bucket_object_lock_configuration_compliance_mode, - "WORMProtection_bucket_object_lock_configuration_governance_mode": WORMProtection_bucket_object_lock_configuration_governance_mode, - "WORMProtection_bucket_object_lock_governance_bypass_delete": WORMProtection_bucket_object_lock_governance_bypass_delete, - "WORMProtection_bucket_object_lock_governance_bypass_delete_multiple": WORMProtection_bucket_object_lock_governance_bypass_delete_multiple, - "WORMProtection_object_lock_retention_compliance_locked": WORMProtection_object_lock_retention_compliance_locked, - "WORMProtection_object_lock_retention_governance_locked": WORMProtection_object_lock_retention_governance_locked, - "WORMProtection_object_lock_retention_governance_bypass_overwrite": WORMProtection_object_lock_retention_governance_bypass_overwrite, - "WORMProtection_object_lock_retention_governance_bypass_delete": WORMProtection_object_lock_retention_governance_bypass_delete, - "WORMProtection_object_lock_retention_governance_bypass_delete_mul": WORMProtection_object_lock_retention_governance_bypass_delete_mul, - "WORMProtection_object_lock_legal_hold_locked": WORMProtection_object_lock_legal_hold_locked, - "WORMProtection_root_bypass_governance_retention_delete_object": WORMProtection_root_bypass_governance_retention_delete_object, - "PutObject_overwrite_dir_obj": PutObject_overwrite_dir_obj, - "PutObject_overwrite_file_obj": PutObject_overwrite_file_obj, - "PutObject_overwrite_file_obj_with_nested_obj": PutObject_overwrite_file_obj_with_nested_obj, - "PutObject_dir_obj_with_data": PutObject_dir_obj_with_data, - "CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj, - "IAM_user_access_denied": IAM_user_access_denied, - "IAM_userplus_access_denied": IAM_userplus_access_denied, - "IAM_userplus_CreateBucket": IAM_userplus_CreateBucket, - "IAM_admin_ChangeBucketOwner": IAM_admin_ChangeBucketOwner, - "IAM_ChangeBucketOwner_back_to_root": IAM_ChangeBucketOwner_back_to_root, - "AccessControl_default_ACL_user_access_denied": AccessControl_default_ACL_user_access_denied, - "AccessControl_default_ACL_userplus_access_denied": AccessControl_default_ACL_userplus_access_denied, - "AccessControl_default_ACL_admin_successful_access": AccessControl_default_ACL_admin_successful_access, - "AccessControl_bucket_resource_single_action": AccessControl_bucket_resource_single_action, - "AccessControl_bucket_resource_all_action": AccessControl_bucket_resource_all_action, - "AccessControl_single_object_resource_actions": AccessControl_single_object_resource_actions, - "AccessControl_multi_statement_policy": AccessControl_multi_statement_policy, - "AccessControl_bucket_ownership_to_user": AccessControl_bucket_ownership_to_user, - "AccessControl_root_PutBucketAcl": AccessControl_root_PutBucketAcl, - "AccessControl_user_PutBucketAcl_with_policy_access": AccessControl_user_PutBucketAcl_with_policy_access, - "AccessControl_copy_object_with_starting_slash_for_user": AccessControl_copy_object_with_starting_slash_for_user, - "PutBucketVersioning_non_existing_bucket": PutBucketVersioning_non_existing_bucket, - "PutBucketVersioning_invalid_status": PutBucketVersioning_invalid_status, - "PutBucketVersioning_success_enabled": PutBucketVersioning_success_enabled, - "PutBucketVersioning_success_suspended": PutBucketVersioning_success_suspended, - "GetBucketVersioning_non_existing_bucket": GetBucketVersioning_non_existing_bucket, - "GetBucketVersioning_empty_response": GetBucketVersioning_empty_response, - "GetBucketVersioning_success": GetBucketVersioning_success, - "Versioning_DeleteBucket_not_empty": Versioning_DeleteBucket_not_empty, - "Versioning_PutObject_suspended_null_versionId_obj": Versioning_PutObject_suspended_null_versionId_obj, - "Versioning_PutObject_null_versionId_obj": Versioning_PutObject_null_versionId_obj, - "Versioning_PutObject_overwrite_null_versionId_obj": Versioning_PutObject_overwrite_null_versionId_obj, - "Versioning_PutObject_success": Versioning_PutObject_success, - "Versioning_CopyObject_success": Versioning_CopyObject_success, - "Versioning_CopyObject_non_existing_version_id": Versioning_CopyObject_non_existing_version_id, - "Versioning_CopyObject_from_an_object_version": Versioning_CopyObject_from_an_object_version, - "Versioning_CopyObject_special_chars": Versioning_CopyObject_special_chars, - "Versioning_HeadObject_invalid_versionId": Versioning_HeadObject_invalid_versionId, - "Versioning_HeadObject_invalid_parent": Versioning_HeadObject_invalid_parent, - "Versioning_HeadObject_success": Versioning_HeadObject_success, - "Versioning_HeadObject_without_versionId": Versioning_HeadObject_without_versionId, - "Versioning_HeadObject_delete_marker": Versioning_HeadObject_delete_marker, - "Versioning_GetObject_invalid_versionId": Versioning_GetObject_invalid_versionId, - "Versioning_GetObject_success": Versioning_GetObject_success, - "Versioning_GetObject_delete_marker_without_versionId": Versioning_GetObject_delete_marker_without_versionId, - "Versioning_GetObject_delete_marker": Versioning_GetObject_delete_marker, - "Versioning_GetObject_null_versionId_obj": Versioning_GetObject_null_versionId_obj, - "Versioning_GetObjectAttributes_object_version": Versioning_GetObjectAttributes_object_version, - "Versioning_GetObjectAttributes_delete_marker": Versioning_GetObjectAttributes_delete_marker, - "Versioning_DeleteObject_delete_object_version": Versioning_DeleteObject_delete_object_version, - "Versioning_DeleteObject_non_existing_object": Versioning_DeleteObject_non_existing_object, - "Versioning_DeleteObject_delete_a_delete_marker": Versioning_DeleteObject_delete_a_delete_marker, - "Versioning_Delete_null_versionId_object": Versioning_Delete_null_versionId_object, - "Versioning_DeleteObject_suspended": Versioning_DeleteObject_suspended, - "Versioning_DeleteObjects_success": Versioning_DeleteObjects_success, - "Versioning_DeleteObjects_delete_deleteMarkers": Versioning_DeleteObjects_delete_deleteMarkers, - "ListObjectVersions_non_existing_bucket": ListObjectVersions_non_existing_bucket, - "ListObjectVersions_list_single_object_versions": ListObjectVersions_list_single_object_versions, - "ListObjectVersions_list_multiple_object_versions": ListObjectVersions_list_multiple_object_versions, - "ListObjectVersions_multiple_object_versions_truncated": ListObjectVersions_multiple_object_versions_truncated, - "ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers, - "ListObjectVersions_containing_null_versionId_obj": ListObjectVersions_containing_null_versionId_obj, - "ListObjectVersions_single_null_versionId_object": ListObjectVersions_single_null_versionId_object, - "Versioning_Multipart_Upload_success": Versioning_Multipart_Upload_success, - "Versioning_Multipart_Upload_overwrite_an_object": Versioning_Multipart_Upload_overwrite_an_object, - "Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId, - "Versioning_UploadPartCopy_from_an_object_version": Versioning_UploadPartCopy_from_an_object_version, - "Versioning_Enable_object_lock": Versioning_Enable_object_lock, - "Versioning_status_switch_to_suspended_with_object_lock": Versioning_status_switch_to_suspended_with_object_lock, - "Versioning_PutObjectRetention_invalid_versionId": Versioning_PutObjectRetention_invalid_versionId, - "Versioning_GetObjectRetention_invalid_versionId": Versioning_GetObjectRetention_invalid_versionId, - "Versioning_Put_GetObjectRetention_success": Versioning_Put_GetObjectRetention_success, - "Versioning_PutObjectLegalHold_invalid_versionId": Versioning_PutObjectLegalHold_invalid_versionId, - "Versioning_GetObjectLegalHold_invalid_versionId": Versioning_GetObjectLegalHold_invalid_versionId, - "Versioning_Put_GetObjectLegalHold_success": Versioning_Put_GetObjectLegalHold_success, - "Versioning_WORM_obj_version_locked_with_legal_hold": Versioning_WORM_obj_version_locked_with_legal_hold, - "Versioning_WORM_obj_version_locked_with_governance_retention": Versioning_WORM_obj_version_locked_with_governance_retention, - "Versioning_WORM_obj_version_locked_with_compliance_retention": Versioning_WORM_obj_version_locked_with_compliance_retention, - "Versioning_concurrent_upload_object": Versioning_concurrent_upload_object, + "Authentication_empty_auth_header": Authentication_empty_auth_header, + "Authentication_invalid_auth_header": Authentication_invalid_auth_header, + "Authentication_unsupported_signature_version": Authentication_unsupported_signature_version, + "Authentication_malformed_credentials": Authentication_malformed_credentials, + "Authentication_malformed_credentials_invalid_parts": Authentication_malformed_credentials_invalid_parts, + "Authentication_credentials_terminated_string": Authentication_credentials_terminated_string, + "Authentication_credentials_incorrect_service": Authentication_credentials_incorrect_service, + "Authentication_credentials_incorrect_region": Authentication_credentials_incorrect_region, + "Authentication_credentials_invalid_date": Authentication_credentials_invalid_date, + "Authentication_credentials_future_date": Authentication_credentials_future_date, + "Authentication_credentials_past_date": Authentication_credentials_past_date, + "Authentication_credentials_non_existing_access_key": Authentication_credentials_non_existing_access_key, + "Authentication_invalid_signed_headers": Authentication_invalid_signed_headers, + "Authentication_missing_date_header": Authentication_missing_date_header, + "Authentication_invalid_date_header": Authentication_invalid_date_header, + "Authentication_date_mismatch": Authentication_date_mismatch, + "Authentication_incorrect_payload_hash": Authentication_incorrect_payload_hash, + "Authentication_incorrect_md5": Authentication_incorrect_md5, + "Authentication_signature_error_incorrect_secret_key": Authentication_signature_error_incorrect_secret_key, + "PresignedAuth_missing_algo_query_param": PresignedAuth_missing_algo_query_param, + "PresignedAuth_unsupported_algorithm": PresignedAuth_unsupported_algorithm, + "PresignedAuth_missing_credentials_query_param": PresignedAuth_missing_credentials_query_param, + "PresignedAuth_malformed_creds_invalid_parts": PresignedAuth_malformed_creds_invalid_parts, + "PresignedAuth_creds_invalid_terminator": PresignedAuth_creds_invalid_terminator, + "PresignedAuth_creds_incorrect_service": PresignedAuth_creds_incorrect_service, + "PresignedAuth_creds_incorrect_region": PresignedAuth_creds_incorrect_region, + "PresignedAuth_creds_invalid_date": PresignedAuth_creds_invalid_date, + "PresignedAuth_missing_date_query": PresignedAuth_missing_date_query, + "PresignedAuth_dates_mismatch": PresignedAuth_dates_mismatch, + "PresignedAuth_non_existing_access_key_id": PresignedAuth_non_existing_access_key_id, + "PresignedAuth_missing_signed_headers_query_param": PresignedAuth_missing_signed_headers_query_param, + "PresignedAuth_missing_expiration_query_param": PresignedAuth_missing_expiration_query_param, + "PresignedAuth_invalid_expiration_query_param": PresignedAuth_invalid_expiration_query_param, + "PresignedAuth_negative_expiration_query_param": PresignedAuth_negative_expiration_query_param, + "PresignedAuth_exceeding_expiration_query_param": PresignedAuth_exceeding_expiration_query_param, + "PresignedAuth_expired_request": PresignedAuth_expired_request, + "PresignedAuth_incorrect_secret_key": PresignedAuth_incorrect_secret_key, + "PresignedAuth_PutObject_success": PresignedAuth_PutObject_success, + "PutObject_missing_object_lock_retention_config": PutObject_missing_object_lock_retention_config, + "PutObject_name_too_long": PutObject_name_too_long, + "PutObject_with_object_lock": PutObject_with_object_lock, + "PutObject_checksum_algorithm_and_header_mismatch": PutObject_checksum_algorithm_and_header_mismatch, + "PutObject_multiple_checksum_headers": PutObject_multiple_checksum_headers, + "PutObject_invalid_checksum_header": PutObject_invalid_checksum_header, + "PutObject_incorrect_checksums": PutObject_incorrect_checksums, + "PutObject_checksums_success": PutObject_checksums_success, + "PresignedAuth_Put_GetObject_with_data": PresignedAuth_Put_GetObject_with_data, + "PresignedAuth_Put_GetObject_with_UTF8_chars": PresignedAuth_Put_GetObject_with_UTF8_chars, + "PresignedAuth_UploadPart": PresignedAuth_UploadPart, + "CreateBucket_invalid_bucket_name": CreateBucket_invalid_bucket_name, + "CreateBucket_existing_bucket": CreateBucket_existing_bucket, + "CreateBucket_owned_by_you": CreateBucket_owned_by_you, + "CreateBucket_invalid_ownership": CreateBucket_invalid_ownership, + "CreateBucket_ownership_with_acl": CreateBucket_ownership_with_acl, + "CreateBucket_as_user": CreateBucket_as_user, + "CreateDeleteBucket_success": CreateDeleteBucket_success, + "CreateBucket_default_acl": CreateBucket_default_acl, + "CreateBucket_non_default_acl": CreateBucket_non_default_acl, + "CreateBucket_default_object_lock": CreateBucket_default_object_lock, + "HeadBucket_non_existing_bucket": HeadBucket_non_existing_bucket, + "HeadBucket_success": HeadBucket_success, + "ListBuckets_as_user": ListBuckets_as_user, + "ListBuckets_as_admin": ListBuckets_as_admin, + "ListBuckets_with_prefix": ListBuckets_with_prefix, + "ListBuckets_invalid_max_buckets": ListBuckets_invalid_max_buckets, + "ListBuckets_truncated": ListBuckets_truncated, + "ListBuckets_success": ListBuckets_success, + "DeleteBucket_non_existing_bucket": DeleteBucket_non_existing_bucket, + "DeleteBucket_non_empty_bucket": DeleteBucket_non_empty_bucket, + "DeleteBucket_success_status_code": DeleteBucket_success_status_code, + "PutBucketOwnershipControls_non_existing_bucket": PutBucketOwnershipControls_non_existing_bucket, + "PutBucketOwnershipControls_multiple_rules": PutBucketOwnershipControls_multiple_rules, + "PutBucketOwnershipControls_invalid_ownership": PutBucketOwnershipControls_invalid_ownership, + "PutBucketOwnershipControls_success": PutBucketOwnershipControls_success, + "GetBucketOwnershipControls_non_existing_bucket": GetBucketOwnershipControls_non_existing_bucket, + "GetBucketOwnershipControls_default_ownership": GetBucketOwnershipControls_default_ownership, + "GetBucketOwnershipControls_success": GetBucketOwnershipControls_success, + "DeleteBucketOwnershipControls_non_existing_bucket": DeleteBucketOwnershipControls_non_existing_bucket, + "DeleteBucketOwnershipControls_success": DeleteBucketOwnershipControls_success, + "PutBucketTagging_non_existing_bucket": PutBucketTagging_non_existing_bucket, + "PutBucketTagging_long_tags": PutBucketTagging_long_tags, + "PutBucketTagging_success": PutBucketTagging_success, + "PutBucketTagging_success_status": PutBucketTagging_success_status, + "GetBucketTagging_non_existing_bucket": GetBucketTagging_non_existing_bucket, + "GetBucketTagging_unset_tags": GetBucketTagging_unset_tags, + "GetBucketTagging_success": GetBucketTagging_success, + "DeleteBucketTagging_non_existing_object": DeleteBucketTagging_non_existing_object, + "DeleteBucketTagging_success_status": DeleteBucketTagging_success_status, + "DeleteBucketTagging_success": DeleteBucketTagging_success, + "PutObject_non_existing_bucket": PutObject_non_existing_bucket, + "PutObject_special_chars": PutObject_special_chars, + "PutObject_invalid_long_tags": PutObject_invalid_long_tags, + "PutObject_success": PutObject_success, + "PutObject_racey_success": PutObject_racey_success, + "HeadObject_non_existing_object": HeadObject_non_existing_object, + "HeadObject_invalid_part_number": HeadObject_invalid_part_number, + "HeadObject_non_existing_mp": HeadObject_non_existing_mp, + "HeadObject_mp_success": HeadObject_mp_success, + "HeadObject_directory_object_noslash": HeadObject_directory_object_noslash, + "HeadObject_non_existing_dir_object": HeadObject_non_existing_dir_object, + "HeadObject_name_too_long": HeadObject_name_too_long, + "HeadObject_with_contenttype": HeadObject_with_contenttype, + "HeadObject_invalid_parent_dir": HeadObject_invalid_parent_dir, + "HeadObject_not_enabled_checksum_mode": HeadObject_not_enabled_checksum_mode, + "HeadObject_checksums": HeadObject_checksums, + "HeadObject_success": HeadObject_success, + "GetObjectAttributes_non_existing_bucket": GetObjectAttributes_non_existing_bucket, + "GetObjectAttributes_non_existing_object": GetObjectAttributes_non_existing_object, + "GetObjectAttributes_invalid_attrs": GetObjectAttributes_invalid_attrs, + "GetObjectAttributes_invalid_parent": GetObjectAttributes_invalid_parent, + "GetObjectAttributes_empty_attrs": GetObjectAttributes_empty_attrs, + "GetObjectAttributes_existing_object": GetObjectAttributes_existing_object, + "GetObjectAttributes_checksums": GetObjectAttributes_checksums, + "GetObject_non_existing_key": GetObject_non_existing_key, + "GetObject_directory_object_noslash": GetObject_directory_object_noslash, + "GetObject_invalid_ranges": GetObject_invalid_ranges, + "GetObject_invalid_parent": GetObject_invalid_parent, + "GetObject_with_meta": GetObject_with_meta, + "GetObject_large_object": GetObject_large_object, + "GetObject_checksums": GetObject_checksums, + "GetObject_success": GetObject_success, + "GetObject_directory_success": GetObject_directory_success, + "GetObject_by_range_success": GetObject_by_range_success, + "GetObject_by_range_resp_status": GetObject_by_range_resp_status, + "GetObject_non_existing_dir_object": GetObject_non_existing_dir_object, + "ListObjects_non_existing_bucket": ListObjects_non_existing_bucket, + "ListObjects_with_prefix": ListObjects_with_prefix, + "ListObjects_truncated": ListObjects_truncated, + "ListObjects_paginated": ListObjects_paginated, + "ListObjects_invalid_max_keys": ListObjects_invalid_max_keys, + "ListObjects_max_keys_0": ListObjects_max_keys_0, + "ListObjects_delimiter": ListObjects_delimiter, + "ListObjects_max_keys_none": ListObjects_max_keys_none, + "ListObjects_marker_not_from_obj_list": ListObjects_marker_not_from_obj_list, + "ListObjects_list_all_objs": ListObjects_list_all_objs, + "ListObjects_with_checksum": ListObjects_with_checksum, + "ListObjectsV2_start_after": ListObjectsV2_start_after, + "ListObjectsV2_both_start_after_and_continuation_token": ListObjectsV2_both_start_after_and_continuation_token, + "ListObjectsV2_start_after_not_in_list": ListObjectsV2_start_after_not_in_list, + "ListObjectsV2_start_after_empty_result": ListObjectsV2_start_after_empty_result, + "ListObjectsV2_both_delimiter_and_prefix": ListObjectsV2_both_delimiter_and_prefix, + "ListObjectsV2_single_dir_object_with_delim_and_prefix": ListObjectsV2_single_dir_object_with_delim_and_prefix, + "ListObjectsV2_truncated_common_prefixes": ListObjectsV2_truncated_common_prefixes, + "ListObjectsV2_all_objs_max_keys": ListObjectsV2_all_objs_max_keys, + "ListObjectsV2_list_all_objs": ListObjectsV2_list_all_objs, + "ListObjectsV2_with_checksum": ListObjectsV2_with_checksum, + "ListObjectVersions_VD_success": ListObjectVersions_VD_success, + "DeleteObject_non_existing_object": DeleteObject_non_existing_object, + "DeleteObject_directory_object_noslash": DeleteObject_directory_object_noslash, + "DeleteObject_name_too_long": DeleteObject_name_too_long, + "DeleteObject_non_existing_dir_object": DeleteObject_non_existing_dir_object, + "DeleteObject_success": DeleteObject_success, + "DeleteObject_success_status_code": DeleteObject_success_status_code, + "DeleteObjects_empty_input": DeleteObjects_empty_input, + "DeleteObjects_non_existing_objects": DeleteObjects_non_existing_objects, + "DeleteObjects_success": DeleteObjects_success, + "CopyObject_non_existing_dst_bucket": CopyObject_non_existing_dst_bucket, + "CopyObject_not_owned_source_bucket": CopyObject_not_owned_source_bucket, + "CopyObject_copy_to_itself": CopyObject_copy_to_itself, + "CopyObject_copy_to_itself_invalid_directive": CopyObject_copy_to_itself_invalid_directive, + "CopyObject_to_itself_with_new_metadata": CopyObject_to_itself_with_new_metadata, + "CopyObject_CopySource_starting_with_slash": CopyObject_CopySource_starting_with_slash, + "CopyObject_non_existing_dir_object": CopyObject_non_existing_dir_object, + "CopyObject_invalid_checksum_algorithm": CopyObject_invalid_checksum_algorithm, + "CopyObject_create_checksum_on_copy": CopyObject_create_checksum_on_copy, + "CopyObject_should_copy_the_existing_checksum": CopyObject_should_copy_the_existing_checksum, + "CopyObject_should_replace_the_existing_checksum": CopyObject_should_replace_the_existing_checksum, + "CopyObject_to_itself_by_replacing_the_checksum": CopyObject_to_itself_by_replacing_the_checksum, + "CopyObject_success": CopyObject_success, + "PutObjectTagging_non_existing_object": PutObjectTagging_non_existing_object, + "PutObjectTagging_long_tags": PutObjectTagging_long_tags, + "PutObjectTagging_success": PutObjectTagging_success, + "GetObjectTagging_non_existing_object": GetObjectTagging_non_existing_object, + "GetObjectTagging_unset_tags": GetObjectTagging_unset_tags, + "GetObjectTagging_invalid_parent": GetObjectTagging_invalid_parent, + "GetObjectTagging_success": GetObjectTagging_success, + "DeleteObjectTagging_non_existing_object": DeleteObjectTagging_non_existing_object, + "DeleteObjectTagging_success_status": DeleteObjectTagging_success_status, + "DeleteObjectTagging_success": DeleteObjectTagging_success, + "CreateMultipartUpload_non_existing_bucket": CreateMultipartUpload_non_existing_bucket, + "CreateMultipartUpload_with_metadata": CreateMultipartUpload_with_metadata, + "CreateMultipartUpload_with_invalid_tagging": CreateMultipartUpload_with_invalid_tagging, + "CreateMultipartUpload_with_tagging": CreateMultipartUpload_with_tagging, + "CreateMultipartUpload_with_content_type": CreateMultipartUpload_with_content_type, + "CreateMultipartUpload_with_object_lock": CreateMultipartUpload_with_object_lock, + "CreateMultipartUpload_with_object_lock_not_enabled": CreateMultipartUpload_with_object_lock_not_enabled, + "CreateMultipartUpload_with_object_lock_invalid_retention": CreateMultipartUpload_with_object_lock_invalid_retention, + "CreateMultipartUpload_past_retain_until_date": CreateMultipartUpload_past_retain_until_date, + "CreateMultipartUpload_invalid_checksum_algorithm": CreateMultipartUpload_invalid_checksum_algorithm, + "CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type": CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type, + "CreateMultipartUpload_invalid_checksum_type": CreateMultipartUpload_invalid_checksum_type, + "CreateMultipartUpload_valid_checksum_algorithm": CreateMultipartUpload_valid_checksum_algorithm, + "CreateMultipartUpload_success": CreateMultipartUpload_success, + "UploadPart_non_existing_bucket": UploadPart_non_existing_bucket, + "UploadPart_invalid_part_number": UploadPart_invalid_part_number, + "UploadPart_non_existing_key": UploadPart_non_existing_key, + "UploadPart_non_existing_mp_upload": UploadPart_non_existing_mp_upload, + "UploadPart_checksum_algorithm_and_header_mismatch": UploadPart_checksum_algorithm_and_header_mismatch, + "UploadPart_multiple_checksum_headers": UploadPart_multiple_checksum_headers, + "UploadPart_invalid_checksum_header": UploadPart_invalid_checksum_header, + "UploadPart_checksum_algorithm_mistmatch_on_initialization": UploadPart_checksum_algorithm_mistmatch_on_initialization, + "UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value": UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value, + "UploadPart_incorrect_checksums": UploadPart_incorrect_checksums, + "UploadPart_with_checksums_success": UploadPart_with_checksums_success, + "UploadPart_success": UploadPart_success, + "UploadPartCopy_non_existing_bucket": UploadPartCopy_non_existing_bucket, + "UploadPartCopy_incorrect_uploadId": UploadPartCopy_incorrect_uploadId, + "UploadPartCopy_incorrect_object_key": UploadPartCopy_incorrect_object_key, + "UploadPartCopy_invalid_part_number": UploadPartCopy_invalid_part_number, + "UploadPartCopy_invalid_copy_source": UploadPartCopy_invalid_copy_source, + "UploadPartCopy_non_existing_source_bucket": UploadPartCopy_non_existing_source_bucket, + "UploadPartCopy_non_existing_source_object_key": UploadPartCopy_non_existing_source_object_key, + "UploadPartCopy_success": UploadPartCopy_success, + "UploadPartCopy_by_range_invalid_range": UploadPartCopy_by_range_invalid_range, + "UploadPartCopy_greater_range_than_obj_size": UploadPartCopy_greater_range_than_obj_size, + "UploadPartCopy_by_range_success": UploadPartCopy_by_range_success, + "UploadPartCopy_should_copy_the_checksum": UploadPartCopy_should_copy_the_checksum, + "UploadPartCopy_should_not_copy_the_checksum": UploadPartCopy_should_not_copy_the_checksum, + "UploadPartCopy_should_calculate_the_checksum": UploadPartCopy_should_calculate_the_checksum, + "ListParts_incorrect_uploadId": ListParts_incorrect_uploadId, + "ListParts_incorrect_object_key": ListParts_incorrect_object_key, + "ListParts_invalid_max_parts": ListParts_invalid_max_parts, + "ListParts_default_max_parts": ListParts_default_max_parts, + "ListParts_truncated": ListParts_truncated, + "ListParts_with_checksums": ListParts_with_checksums, + "ListParts_success": ListParts_success, + "ListMultipartUploads_non_existing_bucket": ListMultipartUploads_non_existing_bucket, + "ListMultipartUploads_empty_result": ListMultipartUploads_empty_result, + "ListMultipartUploads_invalid_max_uploads": ListMultipartUploads_invalid_max_uploads, + "ListMultipartUploads_max_uploads": ListMultipartUploads_max_uploads, + "ListMultipartUploads_incorrect_next_key_marker": ListMultipartUploads_incorrect_next_key_marker, + "ListMultipartUploads_ignore_upload_id_marker": ListMultipartUploads_ignore_upload_id_marker, + "ListMultipartUploads_with_checksums": ListMultipartUploads_with_checksums, + "ListMultipartUploads_success": ListMultipartUploads_success, + "AbortMultipartUpload_non_existing_bucket": AbortMultipartUpload_non_existing_bucket, + "AbortMultipartUpload_incorrect_uploadId": AbortMultipartUpload_incorrect_uploadId, + "AbortMultipartUpload_incorrect_object_key": AbortMultipartUpload_incorrect_object_key, + "AbortMultipartUpload_success": AbortMultipartUpload_success, + "AbortMultipartUpload_success_status_code": AbortMultipartUpload_success_status_code, + "CompletedMultipartUpload_non_existing_bucket": CompletedMultipartUpload_non_existing_bucket, + "CompleteMultipartUpload_invalid_part_number": CompleteMultipartUpload_invalid_part_number, + "CompleteMultipartUpload_invalid_ETag": CompleteMultipartUpload_invalid_ETag, + "CompleteMultipartUpload_invalid_checksum_type": CompleteMultipartUpload_invalid_checksum_type, + "CompleteMultipartUpload_invalid_checksum_part": CompleteMultipartUpload_invalid_checksum_part, + "CompleteMultipartUpload_multiple_checksum_part": CompleteMultipartUpload_multiple_checksum_part, + "CompleteMultipartUpload_incorrect_checksum_part": CompleteMultipartUpload_incorrect_checksum_part, + "CompleteMultipartUpload_different_checksum_part": CompleteMultipartUpload_different_checksum_part, + "CompleteMultipartUpload_missing_part_checksum": CompleteMultipartUpload_missing_part_checksum, + "CompleteMultipartUpload_multiple_final_checksums": CompleteMultipartUpload_multiple_final_checksums, + "CompleteMultipartUpload_invalid_final_checksums": CompleteMultipartUpload_invalid_final_checksums, + "CompleteMultipartUpload_incorrect_final_checksums": CompleteMultipartUpload_incorrect_final_checksums, + "CompleteMultipartUpload_should_calculate_the_final_checksum_full_object": CompleteMultipartUpload_should_calculate_the_final_checksum_full_object, + "CompleteMultipartUpload_should_verify_the_final_checksum": CompleteMultipartUpload_should_verify_the_final_checksum, + "CompleteMultipartUpload_checksum_type_mismatch": CompleteMultipartUpload_checksum_type_mismatch, + "CompleteMultipartUpload_should_ignore_the_final_checksum": CompleteMultipartUpload_should_ignore_the_final_checksum, + "CompleteMultipartUpload_success": CompleteMultipartUpload_success, + "CompleteMultipartUpload_racey_success": CompleteMultipartUpload_racey_success, + "PutBucketAcl_non_existing_bucket": PutBucketAcl_non_existing_bucket, + "PutBucketAcl_disabled": PutBucketAcl_disabled, + "PutBucketAcl_none_of_the_options_specified": PutBucketAcl_none_of_the_options_specified, + "PutBucketAcl_invalid_acl_canned_and_acp": PutBucketAcl_invalid_acl_canned_and_acp, + "PutBucketAcl_invalid_acl_canned_and_grants": PutBucketAcl_invalid_acl_canned_and_grants, + "PutBucketAcl_invalid_acl_acp_and_grants": PutBucketAcl_invalid_acl_acp_and_grants, + "PutBucketAcl_invalid_owner": PutBucketAcl_invalid_owner, + "PutBucketAcl_invalid_owner_not_in_body": PutBucketAcl_invalid_owner_not_in_body, + "PutBucketAcl_invalid_empty_owner_id_in_body": PutBucketAcl_invalid_empty_owner_id_in_body, + "PutBucketAcl_invalid_permission_in_body": PutBucketAcl_invalid_permission_in_body, + "PutBucketAcl_invalid_grantee_type_in_body": PutBucketAcl_invalid_grantee_type_in_body, + "PutBucketAcl_empty_grantee_ID_in_body": PutBucketAcl_empty_grantee_ID_in_body, + "PutBucketAcl_success_access_denied": PutBucketAcl_success_access_denied, + "PutBucketAcl_success_grants": PutBucketAcl_success_grants, + "PutBucketAcl_success_canned_acl": PutBucketAcl_success_canned_acl, + "PutBucketAcl_success_acp": PutBucketAcl_success_acp, + "GetBucketAcl_non_existing_bucket": GetBucketAcl_non_existing_bucket, + "GetBucketAcl_translation_canned_public_read": GetBucketAcl_translation_canned_public_read, + "GetBucketAcl_translation_canned_public_read_write": GetBucketAcl_translation_canned_public_read_write, + "GetBucketAcl_translation_canned_private": GetBucketAcl_translation_canned_private, + "GetBucketAcl_access_denied": GetBucketAcl_access_denied, + "GetBucketAcl_success": GetBucketAcl_success, + "PutBucketPolicy_non_existing_bucket": PutBucketPolicy_non_existing_bucket, + "PutBucketPolicy_empty_statement": PutBucketPolicy_empty_statement, + "PutBucketPolicy_invalid_effect": PutBucketPolicy_invalid_effect, + "PutBucketPolicy_empty_actions_string": PutBucketPolicy_empty_actions_string, + "PutBucketPolicy_empty_actions_array": PutBucketPolicy_empty_actions_array, + "PutBucketPolicy_invalid_action": PutBucketPolicy_invalid_action, + "PutBucketPolicy_unsupported_action": PutBucketPolicy_unsupported_action, + "PutBucketPolicy_incorrect_action_wildcard_usage": PutBucketPolicy_incorrect_action_wildcard_usage, + "PutBucketPolicy_empty_principals_string": PutBucketPolicy_empty_principals_string, + "PutBucketPolicy_empty_principals_array": PutBucketPolicy_empty_principals_array, + "PutBucketPolicy_principals_aws_struct_empty_string": PutBucketPolicy_principals_aws_struct_empty_string, + "PutBucketPolicy_principals_aws_struct_empty_string_slice": PutBucketPolicy_principals_aws_struct_empty_string_slice, + "PutBucketPolicy_principals_incorrect_wildcard_usage": PutBucketPolicy_principals_incorrect_wildcard_usage, + "PutBucketPolicy_non_existing_principals": PutBucketPolicy_non_existing_principals, + "PutBucketPolicy_empty_resources_string": PutBucketPolicy_empty_resources_string, + "PutBucketPolicy_empty_resources_array": PutBucketPolicy_empty_resources_array, + "PutBucketPolicy_invalid_resource_prefix": PutBucketPolicy_invalid_resource_prefix, + "PutBucketPolicy_invalid_resource_with_starting_slash": PutBucketPolicy_invalid_resource_with_starting_slash, + "PutBucketPolicy_duplicate_resource": PutBucketPolicy_duplicate_resource, + "PutBucketPolicy_incorrect_bucket_name": PutBucketPolicy_incorrect_bucket_name, + "PutBucketPolicy_object_action_on_bucket_resource": PutBucketPolicy_object_action_on_bucket_resource, + "PutBucketPolicy_bucket_action_on_object_resource": PutBucketPolicy_bucket_action_on_object_resource, + "PutBucketPolicy_success": PutBucketPolicy_success, + "GetBucketPolicy_non_existing_bucket": GetBucketPolicy_non_existing_bucket, + "GetBucketPolicy_not_set": GetBucketPolicy_not_set, + "GetBucketPolicy_success": GetBucketPolicy_success, + "DeleteBucketPolicy_non_existing_bucket": DeleteBucketPolicy_non_existing_bucket, + "DeleteBucketPolicy_remove_before_setting": DeleteBucketPolicy_remove_before_setting, + "DeleteBucketPolicy_success": DeleteBucketPolicy_success, + "PutObjectLockConfiguration_non_existing_bucket": PutObjectLockConfiguration_non_existing_bucket, + "PutObjectLockConfiguration_empty_config": PutObjectLockConfiguration_empty_config, + "PutObjectLockConfiguration_not_enabled_on_bucket_creation": PutObjectLockConfiguration_not_enabled_on_bucket_creation, + "PutObjectLockConfiguration_invalid_status": PutObjectLockConfiguration_invalid_status, + "PutObjectLockConfiguration_invalid_mode": PutObjectLockConfiguration_invalid_mode, + "PutObjectLockConfiguration_both_years_and_days": PutObjectLockConfiguration_both_years_and_days, + "PutObjectLockConfiguration_invalid_years_days": PutObjectLockConfiguration_invalid_years_days, + "PutObjectLockConfiguration_success": PutObjectLockConfiguration_success, + "GetObjectLockConfiguration_non_existing_bucket": GetObjectLockConfiguration_non_existing_bucket, + "GetObjectLockConfiguration_unset_config": GetObjectLockConfiguration_unset_config, + "GetObjectLockConfiguration_success": GetObjectLockConfiguration_success, + "PutObjectRetention_non_existing_bucket": PutObjectRetention_non_existing_bucket, + "PutObjectRetention_non_existing_object": PutObjectRetention_non_existing_object, + "PutObjectRetention_unset_bucket_object_lock_config": PutObjectRetention_unset_bucket_object_lock_config, + "PutObjectRetention_disabled_bucket_object_lock_config": PutObjectRetention_disabled_bucket_object_lock_config, + "PutObjectRetention_expired_retain_until_date": PutObjectRetention_expired_retain_until_date, + "PutObjectRetention_invalid_mode": PutObjectRetention_invalid_mode, + "PutObjectRetention_overwrite_compliance_mode": PutObjectRetention_overwrite_compliance_mode, + "PutObjectRetention_overwrite_governance_without_bypass_specified": PutObjectRetention_overwrite_governance_without_bypass_specified, + "PutObjectRetention_overwrite_governance_with_permission": PutObjectRetention_overwrite_governance_with_permission, + "PutObjectRetention_success": PutObjectRetention_success, + "GetObjectRetention_non_existing_bucket": GetObjectRetention_non_existing_bucket, + "GetObjectRetention_non_existing_object": GetObjectRetention_non_existing_object, + "GetObjectRetention_disabled_lock": GetObjectRetention_disabled_lock, + "GetObjectRetention_unset_config": GetObjectRetention_unset_config, + "GetObjectRetention_success": GetObjectRetention_success, + "PutObjectLegalHold_non_existing_bucket": PutObjectLegalHold_non_existing_bucket, + "PutObjectLegalHold_non_existing_object": PutObjectLegalHold_non_existing_object, + "PutObjectLegalHold_invalid_body": PutObjectLegalHold_invalid_body, + "PutObjectLegalHold_invalid_status": PutObjectLegalHold_invalid_status, + "PutObjectLegalHold_unset_bucket_object_lock_config": PutObjectLegalHold_unset_bucket_object_lock_config, + "PutObjectLegalHold_disabled_bucket_object_lock_config": PutObjectLegalHold_disabled_bucket_object_lock_config, + "PutObjectLegalHold_success": PutObjectLegalHold_success, + "GetObjectLegalHold_non_existing_bucket": GetObjectLegalHold_non_existing_bucket, + "GetObjectLegalHold_non_existing_object": GetObjectLegalHold_non_existing_object, + "GetObjectLegalHold_disabled_lock": GetObjectLegalHold_disabled_lock, + "GetObjectLegalHold_unset_config": GetObjectLegalHold_unset_config, + "GetObjectLegalHold_success": GetObjectLegalHold_success, + "WORMProtection_bucket_object_lock_configuration_compliance_mode": WORMProtection_bucket_object_lock_configuration_compliance_mode, + "WORMProtection_bucket_object_lock_configuration_governance_mode": WORMProtection_bucket_object_lock_configuration_governance_mode, + "WORMProtection_bucket_object_lock_governance_bypass_delete": WORMProtection_bucket_object_lock_governance_bypass_delete, + "WORMProtection_bucket_object_lock_governance_bypass_delete_multiple": WORMProtection_bucket_object_lock_governance_bypass_delete_multiple, + "WORMProtection_object_lock_retention_compliance_locked": WORMProtection_object_lock_retention_compliance_locked, + "WORMProtection_object_lock_retention_governance_locked": WORMProtection_object_lock_retention_governance_locked, + "WORMProtection_object_lock_retention_governance_bypass_overwrite": WORMProtection_object_lock_retention_governance_bypass_overwrite, + "WORMProtection_object_lock_retention_governance_bypass_delete": WORMProtection_object_lock_retention_governance_bypass_delete, + "WORMProtection_object_lock_retention_governance_bypass_delete_mul": WORMProtection_object_lock_retention_governance_bypass_delete_mul, + "WORMProtection_object_lock_legal_hold_locked": WORMProtection_object_lock_legal_hold_locked, + "WORMProtection_root_bypass_governance_retention_delete_object": WORMProtection_root_bypass_governance_retention_delete_object, + "PutObject_overwrite_dir_obj": PutObject_overwrite_dir_obj, + "PutObject_overwrite_file_obj": PutObject_overwrite_file_obj, + "PutObject_overwrite_file_obj_with_nested_obj": PutObject_overwrite_file_obj_with_nested_obj, + "PutObject_dir_obj_with_data": PutObject_dir_obj_with_data, + "CreateMultipartUpload_dir_obj": CreateMultipartUpload_dir_obj, + "IAM_user_access_denied": IAM_user_access_denied, + "IAM_userplus_access_denied": IAM_userplus_access_denied, + "IAM_userplus_CreateBucket": IAM_userplus_CreateBucket, + "IAM_admin_ChangeBucketOwner": IAM_admin_ChangeBucketOwner, + "IAM_ChangeBucketOwner_back_to_root": IAM_ChangeBucketOwner_back_to_root, + "AccessControl_default_ACL_user_access_denied": AccessControl_default_ACL_user_access_denied, + "AccessControl_default_ACL_userplus_access_denied": AccessControl_default_ACL_userplus_access_denied, + "AccessControl_default_ACL_admin_successful_access": AccessControl_default_ACL_admin_successful_access, + "AccessControl_bucket_resource_single_action": AccessControl_bucket_resource_single_action, + "AccessControl_bucket_resource_all_action": AccessControl_bucket_resource_all_action, + "AccessControl_single_object_resource_actions": AccessControl_single_object_resource_actions, + "AccessControl_multi_statement_policy": AccessControl_multi_statement_policy, + "AccessControl_bucket_ownership_to_user": AccessControl_bucket_ownership_to_user, + "AccessControl_root_PutBucketAcl": AccessControl_root_PutBucketAcl, + "AccessControl_user_PutBucketAcl_with_policy_access": AccessControl_user_PutBucketAcl_with_policy_access, + "AccessControl_copy_object_with_starting_slash_for_user": AccessControl_copy_object_with_starting_slash_for_user, + "PutBucketVersioning_non_existing_bucket": PutBucketVersioning_non_existing_bucket, + "PutBucketVersioning_invalid_status": PutBucketVersioning_invalid_status, + "PutBucketVersioning_success_enabled": PutBucketVersioning_success_enabled, + "PutBucketVersioning_success_suspended": PutBucketVersioning_success_suspended, + "GetBucketVersioning_non_existing_bucket": GetBucketVersioning_non_existing_bucket, + "GetBucketVersioning_empty_response": GetBucketVersioning_empty_response, + "GetBucketVersioning_success": GetBucketVersioning_success, + "Versioning_DeleteBucket_not_empty": Versioning_DeleteBucket_not_empty, + "Versioning_PutObject_suspended_null_versionId_obj": Versioning_PutObject_suspended_null_versionId_obj, + "Versioning_PutObject_null_versionId_obj": Versioning_PutObject_null_versionId_obj, + "Versioning_PutObject_overwrite_null_versionId_obj": Versioning_PutObject_overwrite_null_versionId_obj, + "Versioning_PutObject_success": Versioning_PutObject_success, + "Versioning_CopyObject_success": Versioning_CopyObject_success, + "Versioning_CopyObject_non_existing_version_id": Versioning_CopyObject_non_existing_version_id, + "Versioning_CopyObject_from_an_object_version": Versioning_CopyObject_from_an_object_version, + "Versioning_CopyObject_special_chars": Versioning_CopyObject_special_chars, + "Versioning_HeadObject_invalid_versionId": Versioning_HeadObject_invalid_versionId, + "Versioning_HeadObject_invalid_parent": Versioning_HeadObject_invalid_parent, + "Versioning_HeadObject_success": Versioning_HeadObject_success, + "Versioning_HeadObject_without_versionId": Versioning_HeadObject_without_versionId, + "Versioning_HeadObject_delete_marker": Versioning_HeadObject_delete_marker, + "Versioning_GetObject_invalid_versionId": Versioning_GetObject_invalid_versionId, + "Versioning_GetObject_success": Versioning_GetObject_success, + "Versioning_GetObject_delete_marker_without_versionId": Versioning_GetObject_delete_marker_without_versionId, + "Versioning_GetObject_delete_marker": Versioning_GetObject_delete_marker, + "Versioning_GetObject_null_versionId_obj": Versioning_GetObject_null_versionId_obj, + "Versioning_GetObjectAttributes_object_version": Versioning_GetObjectAttributes_object_version, + "Versioning_GetObjectAttributes_delete_marker": Versioning_GetObjectAttributes_delete_marker, + "Versioning_DeleteObject_delete_object_version": Versioning_DeleteObject_delete_object_version, + "Versioning_DeleteObject_non_existing_object": Versioning_DeleteObject_non_existing_object, + "Versioning_DeleteObject_delete_a_delete_marker": Versioning_DeleteObject_delete_a_delete_marker, + "Versioning_Delete_null_versionId_object": Versioning_Delete_null_versionId_object, + "Versioning_DeleteObject_suspended": Versioning_DeleteObject_suspended, + "Versioning_DeleteObjects_success": Versioning_DeleteObjects_success, + "Versioning_DeleteObjects_delete_deleteMarkers": Versioning_DeleteObjects_delete_deleteMarkers, + "ListObjectVersions_non_existing_bucket": ListObjectVersions_non_existing_bucket, + "ListObjectVersions_list_single_object_versions": ListObjectVersions_list_single_object_versions, + "ListObjectVersions_list_multiple_object_versions": ListObjectVersions_list_multiple_object_versions, + "ListObjectVersions_multiple_object_versions_truncated": ListObjectVersions_multiple_object_versions_truncated, + "ListObjectVersions_with_delete_markers": ListObjectVersions_with_delete_markers, + "ListObjectVersions_containing_null_versionId_obj": ListObjectVersions_containing_null_versionId_obj, + "ListObjectVersions_single_null_versionId_object": ListObjectVersions_single_null_versionId_object, + "Versioning_Multipart_Upload_success": Versioning_Multipart_Upload_success, + "Versioning_Multipart_Upload_overwrite_an_object": Versioning_Multipart_Upload_overwrite_an_object, + "Versioning_UploadPartCopy_non_existing_versionId": Versioning_UploadPartCopy_non_existing_versionId, + "Versioning_UploadPartCopy_from_an_object_version": Versioning_UploadPartCopy_from_an_object_version, + "Versioning_Enable_object_lock": Versioning_Enable_object_lock, + "Versioning_status_switch_to_suspended_with_object_lock": Versioning_status_switch_to_suspended_with_object_lock, + "Versioning_PutObjectRetention_invalid_versionId": Versioning_PutObjectRetention_invalid_versionId, + "Versioning_GetObjectRetention_invalid_versionId": Versioning_GetObjectRetention_invalid_versionId, + "Versioning_Put_GetObjectRetention_success": Versioning_Put_GetObjectRetention_success, + "Versioning_PutObjectLegalHold_invalid_versionId": Versioning_PutObjectLegalHold_invalid_versionId, + "Versioning_GetObjectLegalHold_invalid_versionId": Versioning_GetObjectLegalHold_invalid_versionId, + "Versioning_Put_GetObjectLegalHold_success": Versioning_Put_GetObjectLegalHold_success, + "Versioning_WORM_obj_version_locked_with_legal_hold": Versioning_WORM_obj_version_locked_with_legal_hold, + "Versioning_WORM_obj_version_locked_with_governance_retention": Versioning_WORM_obj_version_locked_with_governance_retention, + "Versioning_WORM_obj_version_locked_with_compliance_retention": Versioning_WORM_obj_version_locked_with_compliance_retention, + "Versioning_concurrent_upload_object": Versioning_concurrent_upload_object, } } diff --git a/tests/integration/tests.go b/tests/integration/tests.go index 1946167d..eebaf486 100644 --- a/tests/integration/tests.go +++ b/tests/integration/tests.go @@ -2757,7 +2757,6 @@ func PutObject_special_chars(s *S3Conf) error { if err != nil { return err } - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) res, err := s3client.ListObjectsV2(ctx, &s3.ListObjectsV2Input{ Bucket: &bucket, @@ -2767,8 +2766,8 @@ func PutObject_special_chars(s *S3Conf) error { return err } - if !compareObjects(res.Contents, objs) { - return fmt.Errorf("expected the objects to be %v, instead got %v", + if !compareObjects(objs, res.Contents) { + return fmt.Errorf("expected the objects to be %vß, instead got %v", objStrings(objs), objStrings(res.Contents)) } @@ -3104,6 +3103,12 @@ func PutObject_checksums_success(s *S3Conf) error { obj := "my-obj" for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + if algo == types.ChecksumAlgorithmCrc64nvme { + continue + } res, err := putObjectWithData(int64(i*200), &s3.PutObjectInput{ Bucket: &bucket, Key: &obj, @@ -3113,6 +3118,10 @@ func PutObject_checksums_success(s *S3Conf) error { return err } + if res.res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the object checksum type to be %v, instead got %v", types.ChecksumTypeFullObject, res.res.ChecksumType) + } + switch algo { case types.ChecksumAlgorithmCrc32: if res.res.ChecksumCRC32 == nil { @@ -3507,6 +3516,9 @@ func HeadObject_checksums(s *S3Conf) error { return err } + if res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", el.key, types.ChecksumTypeFullObject, res.ChecksumType) + } if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32)) } @@ -3835,6 +3847,9 @@ func GetObjectAttributes_checksums(s *S3Conf) error { if res.Checksum == nil { return fmt.Errorf("expected non-nil checksum in the response") } + if res.Checksum.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", el.key, types.ChecksumTypeFullObject, res.Checksum.ChecksumType) + } if getString(res.Checksum.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.Checksum.ChecksumCRC32)) } @@ -4010,47 +4025,6 @@ func GetObject_with_meta(s *S3Conf) error { }) } -func GetObject_not_enabled_checksum_mode(s *S3Conf) error { - testName := "GetObject_not_enabled_checksum_mode" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - - _, err := putObjectWithData(350, &s3.PutObjectInput{ - Bucket: &bucket, - Key: &obj, - ChecksumAlgorithm: types.ChecksumAlgorithmSha1, - }, s3client) - if err != nil { - return err - } - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - res, err := s3client.GetObject(ctx, &s3.GetObjectInput{ - Bucket: &bucket, - Key: &obj, - }) - cancel() - if err != nil { - return err - } - - if res.ChecksumCRC32 != nil { - return fmt.Errorf("expected nil crc32 checksum, instead got %v", *res.ChecksumCRC32) - } - if res.ChecksumCRC32C != nil { - return fmt.Errorf("expected nil crc32c checksum, instead got %v", *res.ChecksumCRC32C) - } - if res.ChecksumSHA1 != nil { - return fmt.Errorf("expected nil sha1 checksum, instead got %v", *res.ChecksumSHA1) - } - if res.ChecksumSHA256 != nil { - return fmt.Errorf("expected nil sha256 checksum, instead got %v", *res.ChecksumSHA256) - } - - return nil - }) -} - func GetObject_checksums(s *S3Conf) error { testName := "GetObject_checksums" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -4097,6 +4071,9 @@ func GetObject_checksums(s *S3Conf) error { return err } + if res.ChecksumType != types.ChecksumTypeFullObject { + return fmt.Errorf("expected the %v object checksum type to be %v, instaed got %v", el.key, types.ChecksumTypeFullObject, res.ChecksumType) + } if getString(res.ChecksumCRC32) != getString(out.res.ChecksumCRC32) { return fmt.Errorf("expected crc32 checksum to be %v, instead got %v", getString(out.res.ChecksumCRC32), getString(res.ChecksumCRC32)) } @@ -4654,11 +4631,14 @@ func ListObjects_with_checksum(s *S3Conf) error { testName := "ListObjects_with_checksum" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { contents := []types.Object{} + for i, el := range types.ChecksumAlgorithmCrc32.Values() { + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + if el == types.ChecksumAlgorithmCrc64nvme { + continue + } - checksumAlgos := types.ChecksumAlgorithmCrc32.Values() - checksumAlgos = append(checksumAlgos, "") - - for i, el := range checksumAlgos { key := fmt.Sprintf("obj-%v", i) size := int64(i * 30) out, err := putObjectWithData(size, &s3.PutObjectInput{ @@ -4678,6 +4658,7 @@ func ListObjects_with_checksum(s *S3Conf) error { ChecksumAlgorithm: []types.ChecksumAlgorithm{ el, }, + ChecksumType: out.res.ChecksumType, }) } @@ -4728,7 +4709,7 @@ func ListObjects_list_all_objs(s *S3Conf) error { return fmt.Errorf("expected the Prefix to be nil, instead got %v", *out.Prefix) } - if !compareObjects(out.Contents, contents) { + if !compareObjects(contents, out.Contents) { return fmt.Errorf("expected the contents to be %v, instead got %v", contents, out.Contents) } @@ -5094,7 +5075,7 @@ func ListObjectsV2_list_all_objs(s *S3Conf) error { return fmt.Errorf("expected the Prefix to be nil, instead got %v", *out.Prefix) } - if !compareObjects(out.Contents, contents) { + if !compareObjects(contents, out.Contents) { return fmt.Errorf("expected the contents to be %v, instead got %v", contents, out.Contents) } @@ -5107,10 +5088,14 @@ func ListObjectsV2_with_checksum(s *S3Conf) error { return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { contents := []types.Object{} - checksumAlgos := types.ChecksumAlgorithmCrc32.Values() - checksumAlgos = append(checksumAlgos, "") + for i, el := range types.ChecksumAlgorithmCrc32.Values() { + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + if el == types.ChecksumAlgorithmCrc64nvme { + continue + } - for i, el := range checksumAlgos { key := fmt.Sprintf("obj-%v", i) size := int64(i * 100) out, err := putObjectWithData(size, &s3.PutObjectInput{ @@ -5130,6 +5115,7 @@ func ListObjectsV2_with_checksum(s *S3Conf) error { ChecksumAlgorithm: []types.ChecksumAlgorithm{ el, }, + ChecksumType: out.res.ChecksumType, }) } @@ -6704,6 +6690,32 @@ func CreateMultipartUpload_invalid_checksum_algorithm(s *S3Conf) error { }) } +func CreateMultipartUpload_invalid_checksum_type(s *S3Conf) error { + testName := "CreateMultipartUpload_invalid_checksum_type" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + _, err := createMp(s3client, bucket, "my-mp", withChecksumType(types.ChecksumType("invalid_checksum_type"))) + if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")); err != nil { + return err + } + + return nil + }) +} + +func CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type(s *S3Conf) error { + testName := "CreateMultipartUpload_empty_checksum_algorithm_with_checksum_type" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + for _, el := range types.ChecksumTypeComposite.Values() { + _, err := createMp(s3client, bucket, "my-mp", withChecksumType(el)) + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrChecksumTypeWithAlgo)); err != nil { + return err + } + } + + return nil + }) +} + func CreateMultipartUpload_valid_checksum_algorithm(s *S3Conf) error { testName := "CreateMultipartUpload_valid_checksum_algorithm" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -7089,61 +7101,6 @@ func UploadPart_checksum_algorithm_mistmatch_on_initialization_with_value(s *S3C }) } -func UploadPart_required_checksum(s *S3Conf) error { - testName := "UploadPart_required_checksum" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32c)) - if err != nil { - return err - } - - partNumber := int32(1) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mp.UploadId, - PartNumber: &partNumber, - }) - cancel() - if err := checkApiErr(err, s3err.GetChecksumTypeMismatchErr(types.ChecksumAlgorithmCrc32c, "null")); err != nil { - return err - } - - return nil - }) -} - -func UploadPart_null_checksum(s *S3Conf) error { - testName := "UploadPart_null_checksum" - return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { - obj := "my-obj" - mp, err := createMp(s3client, bucket, obj) - if err != nil { - return err - } - - partNumber := int32(1) - - ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) - _, err = s3client.UploadPart(ctx, &s3.UploadPartInput{ - Bucket: &bucket, - Key: &obj, - UploadId: mp.UploadId, - PartNumber: &partNumber, - ChecksumAlgorithm: types.ChecksumAlgorithmCrc32, - }) - cancel() - if err := checkApiErr(err, s3err.GetChecksumTypeMismatchErr("null", types.ChecksumAlgorithmCrc32)); err != nil { - return err - } - - return nil - }) -} - func UploadPart_incorrect_checksums(s *S3Conf) error { testName := "UploadPart_incorrect_checksums" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { @@ -7209,6 +7166,13 @@ func UploadPart_with_checksums_success(s *S3Conf) error { obj := "my-obj" for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + if algo == types.ChecksumAlgorithmCrc64nvme { + continue + } + mp, err := createMp(s3client, bucket, obj, withChecksum(algo)) if err != nil { return err @@ -8014,7 +7978,7 @@ func ListParts_truncated(s *S3Conf) error { testName := "ListParts_truncated" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - out, err := createMp(s3client, bucket, obj) + out, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) if err != nil { return err } @@ -8047,7 +8011,7 @@ func ListParts_truncated(s *S3Conf) error { if *res.NextPartNumberMarker != fmt.Sprint(*parts[2].PartNumber) { return fmt.Errorf("expected next part number marker to be %v, instead got %v", fmt.Sprint(*parts[2].PartNumber), *res.NextPartNumberMarker) } - if ok := compareParts(res.Parts, parts[:3]); !ok { + if ok := compareParts(parts[:3], res.Parts); !ok { return fmt.Errorf("expected the parts data to be %v, instead got %v", parts[:3], res.Parts) } @@ -8080,6 +8044,12 @@ func ListParts_with_checksums(s *S3Conf) error { obj := "my-obj" for i, algo := range types.ChecksumAlgorithmCrc32.Values() { + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + if algo == types.ChecksumAlgorithmCrc64nvme { + continue + } mp, err := createMp(s3client, bucket, obj, withChecksum(algo)) if err != nil { return err @@ -8114,7 +8084,7 @@ func ListParts_success(s *S3Conf) error { testName := "ListParts_success" return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { obj := "my-obj" - out, err := createMp(s3client, bucket, obj) + out, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) if err != nil { return err } @@ -8324,29 +8294,40 @@ func ListMultipartUploads_with_checksums(s *S3Conf) error { for _, el := range []struct { obj string algo types.ChecksumAlgorithm + t types.ChecksumType }{ { obj: "obj-1", algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeComposite, }, { obj: "obj-2", algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, }, { obj: "obj-3", algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeComposite, }, { obj: "obj-4", algo: types.ChecksumAlgorithmSha256, - }, - { - obj: "obj-5", - }, + t: types.ChecksumTypeComposite, + }, + //FIXME: remove the condition after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + // + // { + // obj: "obj-5", + // algo: types.ChecksumAlgorithmCrc64nvme, + // t: types.ChecksumTypeFullObject, + // }, } { key := el.obj - mp, err := createMp(s3client, bucket, key, withChecksum(el.algo)) + mp, err := createMp(s3client, bucket, key, withChecksum(el.algo), withChecksumType(el.t)) if err != nil { return err } @@ -8356,6 +8337,7 @@ func ListMultipartUploads_with_checksums(s *S3Conf) error { UploadId: mp.UploadId, StorageClass: types.StorageClassStandard, ChecksumAlgorithm: el.algo, + ChecksumType: el.t, }) } @@ -8653,6 +8635,767 @@ func CompleteMultipartUpload_invalid_ETag(s *S3Conf) error { return nil }) } +func CompleteMultipartUpload_invalid_checksum_type(s *S3Conf) error { + testName := "CompleteMultipartUpload_invalid_checksum_type" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 20*1024*1024, 4, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumType("invalid_type"), + }) + cancel() + if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr("x-amz-checksum-type")); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_invalid_checksum_part(s *S3Conf) error { + testName := "CompleteMultipartUpload_invalid_checksum_part" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for i, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + }) + + if i == 0 { + cParts[0].ChecksumCRC32 = getPtr("invalid_checksum") + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeFullObject, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidChecksumPart)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_multiple_checksum_part(s *S3Conf) error { + testName := "CompleteMultipartUpload_multiple_checksum_part" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32), withChecksumType(types.ChecksumTypeComposite)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for i, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + }) + + if i == 0 { + cParts[0].ChecksumSHA1 = getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0=") + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeComposite, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidChecksumPart)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_incorrect_checksum_part(s *S3Conf) error { + testName := "CompleteMultipartUpload_incorrect_checksum_part" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha256), withChecksumType(types.ChecksumTypeComposite)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmSha256)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for i, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumSHA256: el.ChecksumSHA256, + }) + + if i == 0 { + cParts[0].ChecksumSHA256 = getPtr("n2alat9FhKiZXkZO18V2LLcZFM3IT8R7DjSMvK//7WU=") + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeComposite, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrInvalidPart)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_different_checksum_part(s *S3Conf) error { + testName := "CompleteMultipartUpload_different_checksum_part" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32c), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmCrc32c)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for i, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32C: el.ChecksumCRC32C, + }) + + if i == 0 { + cParts[0].ChecksumSHA256 = getPtr("n2alat9FhKiZXkZO18V2LLcZFM3IT8R7DjSMvK//7WU=") + cParts[0].ChecksumCRC32C = nil + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeFullObject, + }) + cancel() + if err := checkApiErr(err, s3err.APIError{ + Code: "BadDigest", + Description: "The sha256 you specified for part 1 did not match what we received.", + HTTPStatusCode: http.StatusBadRequest, + }); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_missing_part_checksum(s *S3Conf) error { + testName := "CompleteMultipartUpload_missing_part_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmSha1), withChecksumType(types.ChecksumTypeComposite)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmSha1)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for i, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumSHA1: el.ChecksumSHA1, + }) + + if i == 0 { + cParts[0].ChecksumSHA1 = nil + } + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeComposite, + }) + cancel() + if err := checkApiErr(err, s3err.APIError{ + Code: "InvalidRequest", + Description: "The upload was created using a sha1 checksum. The complete request must include the checksum for each part. It was missing for part 1 in the request.", + HTTPStatusCode: http.StatusBadRequest, + }); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_multiple_final_checksums(s *S3Conf) error { + testName := "CompleteMultipartUpload_multiple_final_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(types.ChecksumAlgorithmCrc32), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32C, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumCRC32: getPtr("sGc9Hg=="), + ChecksumCRC32C: getPtr("/2NsFg=="), + ChecksumType: types.ChecksumTypeFullObject, + }) + cancel() + if err := checkApiErr(err, s3err.GetAPIError(s3err.ErrMultipleChecksumHeaders)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_invalid_final_checksums(s *S3Conf) error { + testName := "CompleteMultipartUpload_invalid_final_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + for _, el := range []struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + }{ + { + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + { + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmSha256, + t: types.ChecksumTypeComposite, + }, + //FIXME: uncomment the object after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + // + // { + // algo: types.ChecksumAlgorithmCrc64nvme, + // t: types.ChecksumTypeFullObject, + // }, + } { + + mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo), withChecksumType(el.t)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(el.algo)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32C, + ChecksumCRC32C: el.ChecksumCRC32C, + ChecksumSHA1: el.ChecksumSHA1, + ChecksumSHA256: el.ChecksumSHA256, + }) + } + + mpInput := &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: el.t, + } + + switch el.algo { + case types.ChecksumAlgorithmCrc32: + mpInput.ChecksumCRC32 = getPtr("invalid_crc32") + case types.ChecksumAlgorithmCrc32c: + mpInput.ChecksumCRC32C = getPtr("invalid_crc32c") + case types.ChecksumAlgorithmSha1: + mpInput.ChecksumSHA1 = getPtr("invalid_sha1") + case types.ChecksumAlgorithmSha256: + mpInput.ChecksumSHA256 = getPtr("invalid_sha256") + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, mpInput) + cancel() + if err := checkApiErr(err, s3err.GetInvalidChecksumHeaderErr(fmt.Sprintf("x-amz-checksum-%v", strings.ToLower(string(el.algo))))); err != nil { + return err + } + } + + return nil + }) +} + +func CompleteMultipartUpload_incorrect_final_checksums(s *S3Conf) error { + testName := "CompleteMultipartUpload_incorrect_final_checksums" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + for _, el := range []struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + }{ + { + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + { + algo: types.ChecksumAlgorithmSha1, + t: types.ChecksumTypeComposite, + }, + { + algo: types.ChecksumAlgorithmSha256, + t: types.ChecksumTypeComposite, + }, + //FIXME: uncomment the object after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + // + // { + // algo: types.ChecksumAlgorithmCrc64nvme, + // t: types.ChecksumTypeFullObject, + // }, + } { + mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo), withChecksumType(el.t)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 5*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(el.algo)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + ChecksumCRC32C: el.ChecksumCRC32C, + ChecksumSHA1: el.ChecksumSHA1, + ChecksumSHA256: el.ChecksumSHA256, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + // Provide one of the parts checksum. In any case + // the final checksum will differ from one of the parts checksum + ChecksumCRC32: cParts[0].ChecksumCRC32, + ChecksumCRC32C: cParts[0].ChecksumCRC32C, + ChecksumSHA1: cParts[0].ChecksumSHA1, + ChecksumSHA256: cParts[0].ChecksumSHA256, + ChecksumType: el.t, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumBadDigestErr(el.algo)); err != nil { + return err + } + } + + return nil + }) +} + +func CompleteMultipartUpload_should_calculate_the_final_checksum_full_object(s *S3Conf) error { + testName := "CompleteMultipartUpload_should_calculate_the_final_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + for _, el := range []struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + }{ + { + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeFullObject, + }, + { + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + //FIXME: uncomment the object after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + // + // { + // algo: types.ChecksumAlgorithmCrc64nvme, + // t: types.ChecksumTypeFullObject, + // }, + } { + mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo), withChecksumType(el.t)) + if err != nil { + return err + } + + parts, csum, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(el.algo)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + ChecksumCRC32C: el.ChecksumCRC32C, + ChecksumSHA1: el.ChecksumSHA1, + ChecksumSHA256: el.ChecksumSHA256, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: el.t, + }) + cancel() + if err != nil { + return err + } + + switch el.algo { + case types.ChecksumAlgorithmCrc32: + if getString(res.ChecksumCRC32) != csum { + return fmt.Errorf("expected the final crc32 checksum to be %v, instead got %v", csum, getString(res.ChecksumCRC32)) + } + case types.ChecksumAlgorithmCrc32c: + if getString(res.ChecksumCRC32C) != csum { + return fmt.Errorf("expected the final crc32c checksum to be %v, instead got %v", csum, getString(res.ChecksumCRC32C)) + } + } + } + + return nil + }) +} + +func CompleteMultipartUpload_should_verify_the_final_checksum(s *S3Conf) error { + testName := "CompleteMultipartUpload_should_verify_the_final_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + for _, el := range []struct { + algo types.ChecksumAlgorithm + t types.ChecksumType + }{ + { + algo: types.ChecksumAlgorithmCrc32, + t: types.ChecksumTypeFullObject, + }, + { + algo: types.ChecksumAlgorithmCrc32c, + t: types.ChecksumTypeFullObject, + }, + //FIXME: uncomment the object after the crc64nvme + // checksum support is added in the aws sdk + // https://github.com/aws/aws-sdk-go-v2/issues/2985 + // + // { + // algo: types.ChecksumAlgorithmCrc64nvme, + // t: types.ChecksumTypeFullObject, + // }, + } { + mp, err := createMp(s3client, bucket, obj, withChecksum(el.algo), withChecksumType(el.t)) + if err != nil { + return err + } + + parts, csum, err := uploadParts(s3client, 15*1024*1024, 3, bucket, obj, *mp.UploadId, withChecksum(el.algo)) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + ChecksumCRC32C: el.ChecksumCRC32C, + ChecksumSHA1: el.ChecksumSHA1, + ChecksumSHA256: el.ChecksumSHA256, + }) + } + + mpInput := &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: el.t, + } + + switch el.algo { + case types.ChecksumAlgorithmCrc32: + mpInput.ChecksumCRC32 = &csum + case types.ChecksumAlgorithmCrc32c: + mpInput.ChecksumCRC32C = &csum + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CompleteMultipartUpload(ctx, mpInput) + cancel() + if err != nil { + return err + } + + switch el.algo { + case types.ChecksumAlgorithmCrc32: + if getString(res.ChecksumCRC32) != csum { + return fmt.Errorf("expected the final crc32 checksum to be %v, instead got %v", csum, getString(res.ChecksumCRC32)) + } + case types.ChecksumAlgorithmCrc32c: + if getString(res.ChecksumCRC32C) != csum { + return fmt.Errorf("expected the final crc32c checksum to be %v, instead got %v", csum, getString(res.ChecksumCRC32C)) + } + } + } + + return nil + }) +} + +func CompleteMultipartUpload_checksum_type_mismatch(s *S3Conf) error { + testName := "CompleteMultipartUpload_checksum_type_mismatch" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj, withChecksum(types.ChecksumAlgorithmCrc32), withChecksumType(types.ChecksumTypeFullObject)) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 20*1024*1024, 4, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + ChecksumCRC32: el.ChecksumCRC32, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + _, err = s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumType: types.ChecksumTypeComposite, + }) + cancel() + if err := checkApiErr(err, s3err.GetChecksumTypeMismatchOnMpErr(types.ChecksumTypeFullObject)); err != nil { + return err + } + + return nil + }) +} + +func CompleteMultipartUpload_should_ignore_the_final_checksum(s *S3Conf) error { + testName := "CompleteMultipartUpload_should_ignore_the_final_checksum" + return actionHandler(s, testName, func(s3client *s3.Client, bucket string) error { + obj := "my-obj" + mp, err := createMp(s3client, bucket, obj) + if err != nil { + return err + } + + parts, _, err := uploadParts(s3client, 20*1024*1024, 4, bucket, obj, *mp.UploadId) + if err != nil { + return err + } + + cParts := []types.CompletedPart{} + for _, el := range parts { + cParts = append(cParts, types.CompletedPart{ + ETag: el.ETag, + PartNumber: el.PartNumber, + }) + } + + ctx, cancel := context.WithTimeout(context.Background(), shortTimeout) + res, err := s3client.CompleteMultipartUpload(ctx, &s3.CompleteMultipartUploadInput{ + Bucket: &bucket, + Key: &obj, + UploadId: mp.UploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: cParts, + }, + ChecksumSHA1: getPtr("Kq5sNclPz7QV2+lfQIuc6R7oRu0="), // should ignore this + }) + cancel() + if err != nil { + return err + } + + if res.ChecksumCRC32 != nil { + return fmt.Errorf("expected nil crc32 checksum, insted got %v", *res.ChecksumCRC32) + } + if res.ChecksumCRC32C != nil { + return fmt.Errorf("expected nil crc32c checksum, insted got %v", *res.ChecksumCRC32C) + } + if res.ChecksumSHA1 != nil { + return fmt.Errorf("expected nil sha1 checksum, insted got %v", *res.ChecksumSHA1) + } + if res.ChecksumSHA256 != nil { + return fmt.Errorf("expected nil sha256 checksum, insted got %v", *res.ChecksumSHA256) + } + + return nil + }) +} func CompleteMultipartUpload_small_upload_size(s *S3Conf) error { testName := "CompleteMultipartUpload_small_upload_size" diff --git a/tests/integration/utils.go b/tests/integration/utils.go index 5452f6af..a76a6dd6 100644 --- a/tests/integration/utils.go +++ b/tests/integration/utils.go @@ -481,6 +481,7 @@ func putObjectWithData(lgth int64, input *s3.PutObjectInput, client *s3.Client) type mpCfg struct { checksumAlgorithm types.ChecksumAlgorithm + checksumType types.ChecksumType } type mpOpt func(*mpCfg) @@ -488,6 +489,9 @@ type mpOpt func(*mpCfg) func withChecksum(algo types.ChecksumAlgorithm) mpOpt { return func(mc *mpCfg) { mc.checksumAlgorithm = algo } } +func withChecksumType(t types.ChecksumType) mpOpt { + return func(mc *mpCfg) { mc.checksumType = t } +} func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.CreateMultipartUploadOutput, error) { cfg := new(mpCfg) @@ -499,6 +503,7 @@ func createMp(s3client *s3.Client, bucket, key string, opts ...mpOpt) (*s3.Creat Bucket: &bucket, Key: &key, ChecksumAlgorithm: cfg.checksumAlgorithm, + ChecksumType: cfg.checksumType, }) cancel() return out, err @@ -535,6 +540,9 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool { if item.ChecksumAlgorithm != list2[i].ChecksumAlgorithm { return false } + if item.ChecksumType != list2[i].ChecksumType { + return false + } } return true @@ -542,30 +550,52 @@ func compareMultipartUploads(list1, list2 []types.MultipartUpload) bool { func compareParts(parts1, parts2 []types.Part) bool { if len(parts1) != len(parts2) { + fmt.Printf("list length are not equal: %v != %v\n", len(parts1), len(parts2)) return false } for i, prt := range parts1 { if *prt.PartNumber != *parts2[i].PartNumber { + fmt.Printf("partNumbers are not equal, %v != %v\n", *prt.PartNumber, *parts2[i].PartNumber) return false } if *prt.ETag != *parts2[i].ETag { + fmt.Printf("etags are not equal, %v != %v\n", *prt.ETag, *parts2[i].ETag) return false } if *prt.Size != *parts2[i].Size { + fmt.Printf("sizes are not equal, %v != %v\n", *prt.Size, *parts2[i].Size) return false } - if getString(prt.ChecksumCRC32) != getString(parts2[i].ChecksumCRC32) { - return false + if prt.ChecksumCRC32 != nil { + if *prt.ChecksumCRC32 != getString(parts2[i].ChecksumCRC32) { + fmt.Printf("crc32 checksums are not equal, %v != %v\n", *prt.ChecksumCRC32, *parts2[i].ChecksumCRC32) + return false + } } - if getString(prt.ChecksumCRC32C) != getString(parts2[i].ChecksumCRC32C) { - return false + if prt.ChecksumCRC32C != nil { + if *prt.ChecksumCRC32C != getString(parts2[i].ChecksumCRC32C) { + fmt.Printf("crc32c checksums are not equal, %v != %v\n", *prt.ChecksumCRC32C, *parts2[i].ChecksumCRC32C) + return false + } } - if getString(prt.ChecksumSHA1) != getString(parts2[i].ChecksumSHA1) { - return false + if prt.ChecksumSHA1 != nil { + if *prt.ChecksumSHA1 != getString(parts2[i].ChecksumSHA1) { + fmt.Printf("sha1 checksums are not equal, %v != %v\n", *prt.ChecksumSHA1, *parts2[i].ChecksumSHA1) + return false + } } - if getString(prt.ChecksumSHA256) != getString(parts2[i].ChecksumSHA256) { - return false + if prt.ChecksumSHA256 != nil { + if *prt.ChecksumSHA256 != getString(parts2[i].ChecksumSHA256) { + fmt.Printf("sha256 checksums are not equal, %v != %v\n", *prt.ChecksumSHA256, *parts2[i].ChecksumSHA256) + return false + } + } + if prt.ChecksumCRC64NVME != nil { + if *prt.ChecksumCRC64NVME != getString(parts2[i].ChecksumCRC64NVME) { + fmt.Printf("crc64nvme checksums are not equal, %v != %v\n", *prt.ChecksumCRC64NVME, *parts2[i].ChecksumCRC64NVME) + return false + } } } return true @@ -691,6 +721,13 @@ func compareObjects(list1, list2 []types.Object) bool { return false } } + if obj.ChecksumType != "" { + if obj.ChecksumType[0] != list2[i].ChecksumType[0] { + fmt.Printf("checksum types are not equal: (%q %q) %v != %v\n", + *obj.Key, *list2[i].Key, obj.ChecksumType[0], list2[i].ChecksumType[0]) + return false + } + } } return true