From bf38a03af97e9986f0d2b571acb4acd792aeea9d Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Mon, 8 Jan 2024 13:11:40 -0800 Subject: [PATCH 1/2] chore: remove azure bug comment This comment references a bug that was fixed in the v1.2.1 sdk update: https://github.com/Azure/azure-sdk-for-go/issues/22171 --- backend/azure/azure.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 021ce4c3..762abc3a 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -395,8 +395,6 @@ func (az *Azure) DeleteObjectTagging(ctx context.Context, bucket, object string) return err } - //TODO: SDK has a bug here: it recommends to use the method to remove tags by passing an empty map, - // but the method panics because of incorrect implementation _, err = client.SetTags(ctx, map[string]string{}, nil) if err != nil { return azureErrToS3Err(err) From a86a8cbce5d130f7febdbd36605abb9179c3e54b Mon Sep 17 00:00:00 2001 From: Ben McClelland Date: Mon, 8 Jan 2024 13:40:18 -0800 Subject: [PATCH 2/2] fix: add azure CreateMultipartUpload to allow clients to work as expected The azure sdk doesnt use a separate function to initialize a multipart upload, so CreateMultipartUpload becomes a no-op. But we still need to have it return success so that clients wont get an unexpected error. --- backend/azure/azure.go | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/backend/azure/azure.go b/backend/azure/azure.go index 762abc3a..fe563044 100644 --- a/backend/azure/azure.go +++ b/backend/azure/azure.go @@ -403,7 +403,23 @@ func (az *Azure) DeleteObjectTagging(ctx context.Context, bucket, object string) return nil } -// Multipart upload starts with UploadPart action. +func (az *Azure) CreateMultipartUpload(ctx context.Context, input *s3.CreateMultipartUploadInput) (*s3.CreateMultipartUploadOutput, error) { + // Multipart upload starts with UploadPart action so there is no + // correlating function for creating mutlipart uploads. + // TODO: since azure only allows for a single multipart upload + // for an object name at a time, we need to send an error back to + // the client if there is already an outstanding upload in progress + // for this object. + // Alternatively, is there something we can do with upload ids to + // keep concurrent uploads unique still? I haven't found an efficient + // way to rename final objects. + return &s3.CreateMultipartUploadOutput{ + Bucket: input.Bucket, + Key: input.Key, + UploadId: input.Key, + }, nil +} + // Each part is translated into an uncommitted block in a newly created blob in staging area func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (etag string, err error) { client, err := az.getBlockBlobClient(*input.Bucket, *input.Key) @@ -411,6 +427,10 @@ func (az *Azure) UploadPart(ctx context.Context, input *s3.UploadPartInput) (eta return "", err } + // TODO: request streamable version of StageBlock() + // (*blockblob.Client).StageBlock does not have a streamable + // version of this function at this time, so we need to cache + // the body in memory to create an io.ReadSeekCloser rdr, err := getReadSeekCloser(input.Body) if err != nil { return "", err @@ -558,6 +578,7 @@ func (az *Azure) ListMultipartUploads(ctx context.Context, input *s3.ListMultipa // Deletes the block blob with committed/uncommitted blocks func (az *Azure) AbortMultipartUpload(ctx context.Context, input *s3.AbortMultipartUploadInput) error { + // TODO: need to verify this blob has uncommitted blocks? _, err := az.client.DeleteBlob(ctx, *input.Bucket, *input.Key, nil) if err != nil { return azureErrToS3Err(err) @@ -712,7 +733,6 @@ func azureErrToS3Err(apiErr error) error { Description: azErr.RawResponse.Status, HTTPStatusCode: azErr.StatusCode, } - fmt.Println(resp) return resp }