Skip to content

Commit

Permalink
feat: Adds the CRC64NVME checksum support in the gateway. Adds checks…
Browse files Browse the repository at this point in the history 
…um-type support for the checksum implementation
  • Loading branch information
@niksis02
niksis02 committed Feb 14, 2025
1 parent 6956757 commit 29160ab
Show file tree
Hide file tree
Showing 11 changed files with 2,161 additions and 709 deletions.
326 changes: 225 additions & 101 deletions backend/posix/posix.go
View file

Large diffs are not rendered by default.

56 changes: 39 additions & 17 deletions backend/s3proxy/s3.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -256,8 +256,10 @@ func (s *S3Proxy) ListMultipartUploads(ctx context.Context, input *s3.ListMultip
ID: *u.Owner.ID,
DisplayName: *u.Owner.DisplayName,
},
StorageClass: u.StorageClass,
Initiated: *u.Initiated,
StorageClass: u.StorageClass,
Initiated: *u.Initiated,
ChecksumAlgorithm: u.ChecksumAlgorithm,
ChecksumType: &u.ChecksumType,
})
}

Expand Down Expand Up @@ -293,10 +295,15 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
var parts []s3response.Part
for _, p := range output.Parts {
parts = append(parts, s3response.Part{
PartNumber: int(*p.PartNumber),
LastModified: *p.LastModified,
ETag: *p.ETag,
Size: *p.Size,
PartNumber: int(*p.PartNumber),
LastModified: *p.LastModified,
ETag: *p.ETag,
Size: *p.Size,
ChecksumCRC32: p.ChecksumCRC32,
ChecksumCRC32C: p.ChecksumCRC32C,
ChecksumCRC64NVME: p.ChecksumCRC64NVME,
ChecksumSHA1: p.ChecksumSHA1,
ChecksumSHA256: p.ChecksumSHA256,
})
}
pnm, err := strconv.Atoi(*output.PartNumberMarker)
Expand Down Expand Up @@ -329,6 +336,8 @@ func (s *S3Proxy) ListParts(ctx context.Context, input *s3.ListPartsInput) (s3re
MaxParts: int(*output.MaxParts),
IsTruncated: *output.IsTruncated,
Parts: parts,
ChecksumAlgorithm: output.ChecksumAlgorithm,
ChecksumType: &output.ChecksumType,
}, nil
}

Expand All @@ -348,8 +357,13 @@ func (s *S3Proxy) UploadPartCopy(ctx context.Context, input *s3.UploadPartCopyIn
}

return s3response.CopyPartResult{
LastModified: *output.CopyPartResult.LastModified,
ETag: output.CopyPartResult.ETag,
LastModified: *output.CopyPartResult.LastModified,
ETag: output.CopyPartResult.ETag,
ChecksumCRC32: output.CopyPartResult.ChecksumCRC32,
ChecksumCRC32C: output.CopyPartResult.ChecksumCRC32C,
ChecksumCRC64NVME: output.CopyPartResult.ChecksumCRC64NVME,
ChecksumSHA1: output.CopyPartResult.ChecksumSHA1,
ChecksumSHA256: output.CopyPartResult.ChecksumSHA256,
}, nil
}

Expand All @@ -369,8 +383,13 @@ func (s *S3Proxy) PutObject(ctx context.Context, input *s3.PutObjectInput) (s3re
}

return s3response.PutObjectOutput{
ETag: *output.ETag,
VersionID: versionID,
ETag: *output.ETag,
VersionID: versionID,
ChecksumCRC32: output.ChecksumCRC32,
ChecksumCRC32C: output.ChecksumCRC32C,
ChecksumCRC64NVME: output.ChecksumCRC64NVME,
ChecksumSHA1: output.ChecksumSHA1,
ChecksumSHA256: output.ChecksumSHA256,
}, nil
}

Expand Down Expand Up @@ -421,6 +440,7 @@ func (s *S3Proxy) GetObjectAttributes(ctx context.Context, input *s3.GetObjectAt
ObjectSize: out.ObjectSize,
StorageClass: out.StorageClass,
ObjectParts: &parts,
Checksum: out.Checksum,
}, handleError(err)
}

Expand Down Expand Up @@ -833,13 +853,15 @@ func convertObjects(objs []types.Object) []s3response.Object {

for _, obj := range objs {
result = append(result, s3response.Object{
ETag: obj.ETag,
Key: obj.Key,
LastModified: obj.LastModified,
Owner: obj.Owner,
Size: obj.Size,
RestoreStatus: obj.RestoreStatus,
StorageClass: obj.StorageClass,
ETag: obj.ETag,
Key: obj.Key,
LastModified: obj.LastModified,
Owner: obj.Owner,
Size: obj.Size,
RestoreStatus: obj.RestoreStatus,
StorageClass: obj.StorageClass,
ChecksumAlgorithm: obj.ChecksumAlgorithm,
ChecksumType: &obj.ChecksumType,
})
}

Expand Down
75 changes: 68 additions & 7 deletions s3api/controllers/base.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -607,6 +607,18 @@ func (c S3ApiController) GetActions(ctx *fiber.Ctx) error {
Value: *res.ChecksumSHA256,
})
}
if res.ChecksumCRC64NVME != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
}
if res.ChecksumType != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-type",
Value: string(res.ChecksumType),
})
}

// Set x-amz-meta-... headers
utils.SetMetaHeaders(ctx, res.Metadata)
Expand Down Expand Up @@ -2079,6 +2091,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
})
if err == nil {
headers := []utils.CustomHeader{}
Expand Down Expand Up @@ -2112,6 +2125,12 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Value: *res.ChecksumSHA256,
})
}
if res.ChecksumCRC64NVME != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
}

utils.SetResponseHeaders(ctx, headers)
}
Expand Down Expand Up @@ -2511,6 +2530,7 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
})
if err != nil {
return SendResponse(ctx, err,
Expand Down Expand Up @@ -2562,6 +2582,18 @@ func (c S3ApiController) PutActions(ctx *fiber.Ctx) error {
Value: getstring(res.ChecksumSHA256),
})
}
if getstring(res.ChecksumCRC64NVME) != "" {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: getstring(res.ChecksumCRC64NVME),
})
}
if res.ChecksumType != nil {
hdrs = append(hdrs, utils.CustomHeader{
Key: "x-amz-checksum-type",
Value: string(*res.ChecksumType),
})
}

utils.SetResponseHeaders(ctx, hdrs)

Expand Down Expand Up @@ -3206,6 +3238,18 @@ func (c S3ApiController) HeadObject(ctx *fiber.Ctx) error {
Value: *res.ChecksumSHA256,
})
}
if res.ChecksumCRC64NVME != nil {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-crc64nvme",
Value: *res.ChecksumCRC64NVME,
})
}
if res.ChecksumType != "" {
headers = append(headers, utils.CustomHeader{
Key: "x-amz-checksum-type",
Value: string(res.ChecksumType),
})
}

contentType := getstring(res.ContentType)
if contentType == "" {
Expand Down Expand Up @@ -3430,6 +3474,21 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
})
}

checksumType := types.ChecksumType(ctx.Get("x-amz-checksum-type"))
err = utils.IsChecksumTypeValid(checksumType)
if err != nil {
if c.debug {
log.Printf("invalid checksum type: %v", err)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Logger: c.logger,
MetricsMng: c.mm,
Action: metrics.ActionCompleteMultipartUpload,
BucketOwner: parsedAcl.Owner,
})
}

res, err := c.be.CompleteMultipartUpload(ctx.Context(),
&s3.CompleteMultipartUploadInput{
Bucket: &bucket,
Expand All @@ -3438,10 +3497,12 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
MultipartUpload: &types.CompletedMultipartUpload{
Parts: data.Parts,
},
ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC32: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32]),
ChecksumCRC32C: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc32c]),
ChecksumSHA1: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha1]),
ChecksumSHA256: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmSha256]),
ChecksumCRC64NVME: backend.GetPtrFromString(checksums[types.ChecksumAlgorithmCrc64nvme]),
ChecksumType: checksumType,
})
if err == nil {
if getstring(res.VersionId) != "" {
Expand Down Expand Up @@ -3507,11 +3568,10 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {

metadata := utils.GetUserMetaData(&ctx.Request().Header)

checksumAlgorithm := types.ChecksumAlgorithm(ctx.Get("x-amz-checksum-algorithm"))
err = utils.IsChecksumAlgorithmValid(checksumAlgorithm)
checksumAlgorithm, checksumType, err := utils.ParseCreateMpChecksumHeaders(ctx)
if err != nil {
if c.debug {
log.Printf("invalid checksum algorithm: %v", checksumAlgorithm)
log.Printf("err parsing checksum headers: %v", err)
}
return SendXMLResponse(ctx, nil, err,
&MetaOpts{
Expand All @@ -3534,6 +3594,7 @@ func (c S3ApiController) CreateActions(ctx *fiber.Ctx) error {
ObjectLockLegalHoldStatus: objLockState.LegalHoldStatus,
Metadata: metadata,
ChecksumAlgorithm: checksumAlgorithm,
ChecksumType: checksumType,
})
if err == nil {
if checksumAlgorithm != "" {
Expand Down
70 changes: 70 additions & 0 deletions s3api/utils/csum-reader.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,9 +21,12 @@ import (
"encoding/base64"
"encoding/hex"
"errors"
"fmt"
"hash"
"hash/crc32"
"hash/crc64"
"io"
"math/bits"

"github.com/aws/aws-sdk-go-v2/service/s3/types"
"github.com/versity/versitygw/s3err"
Expand All @@ -45,6 +48,8 @@ const (
HashTypeCRC32 HashType = "crc32"
// HashTypeCRC32C generates CRC32C Base64-Encoded checksum for the data stream
HashTypeCRC32C HashType = "crc32c"
// HashTypeCRC64NVME generates CRC64NVME Base64-Encoded checksum for the data stream
HashTypeCRC64NVME HashType = "crc64nvme"
// HashTypeNone is a no-op checksum for the data stream
HashTypeNone HashType = "none"
)
Expand Down Expand Up @@ -83,6 +88,8 @@ func NewHashReader(r io.Reader, expectedSum string, ht HashType) (*HashReader, e
hash = crc32.NewIEEE()
case HashTypeCRC32C:
hash = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeCRC64NVME:
hash = crc64.New(crc64.MakeTable(bits.Reverse64(0xad93d23594c93659)))
case HashTypeNone:
hash = noop{}
default:
Expand Down Expand Up @@ -136,6 +143,11 @@ func (hr *HashReader) Read(p []byte) (int, error) {
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmSha256)
}
case HashTypeCRC64NVME:
sum := hr.Sum()
if sum != hr.sum {
return n, s3err.GetChecksumBadDigestErr(types.ChecksumAlgorithmCrc64nvme)
}
default:
return n, errInvalidHashType
}
Expand All @@ -162,6 +174,8 @@ func (hr *HashReader) Sum() string {
return Base64SumString(hr.hash.Sum(nil))
case HashTypeSha256:
return Base64SumString(hr.hash.Sum(nil))
case HashTypeCRC64NVME:
return Base64SumString(hr.hash.Sum(nil))
default:
return ""
}
Expand All @@ -183,3 +197,59 @@ func (n noop) Sum(b []byte) []byte { return []byte{} }
func (n noop) Reset() {}
func (n noop) Size() int { return 0 }
func (n noop) BlockSize() int { return 1 }

// NewCompositeChecksumReader initializes a composite checksum
// processor, which decodes and validates the provided
// checksums and returns the final checksum based on
// the previous processings.
//
// The supported checksum types are:
// - CRC32
// - CRC32C
// - SHA1
// - SHA256
func NewCompositeChecksumReader(ht HashType) (*CompositeChecksumReader, error) {
var hasher hash.Hash
switch ht {
case HashTypeSha256:
hasher = sha256.New()
case HashTypeSha1:
hasher = sha1.New()
case HashTypeCRC32:
hasher = crc32.NewIEEE()
case HashTypeCRC32C:
hasher = crc32.New(crc32.MakeTable(crc32.Castagnoli))
case HashTypeNone:
hasher = noop{}
default:
return nil, errInvalidHashType
}

return &CompositeChecksumReader{
hasher: hasher,
}, nil
}

type CompositeChecksumReader struct {
hasher hash.Hash
}

// Decodes and writes the checksum in the hasher
func (ccr *CompositeChecksumReader) Process(checksum string) error {
data, err := base64.StdEncoding.DecodeString(checksum)
if err != nil {
return fmt.Errorf("base64 decode: %w", err)
}

_, err = ccr.hasher.Write(data)
if err != nil {
return fmt.Errorf("hash write: %w", err)
}

return nil
}

// Returns the base64 encoded composite checksum
func (ccr *CompositeChecksumReader) Sum() string {
return Base64SumString(ccr.hasher.Sum(nil))
}
Loading

0 comments on commit 29160ab

Please sign in to comment.