update e2e tests

.github/workflows/e2e-test.yml

@@ -30,15 +30,15 @@ jobs:
           echo AWS_S3_BUCKET='test-bucket' >> .env
           echo S3API_SERVICE_PORT='5005' >> .env
           echo AUTH_LEVEL=0 >> .env
-
-      - name: Substitute secret variables in JSON
-        env:
-          auth_password: ${{ secrets.AUTH_PASSWORD }}
-          KEYCLOAK_SECRET: ${{ secrets.KEYCLOAK_SECRET }}
-        run: |
-          echo "Is auth_password set: $(if [ -z "$auth_password" ]; then echo "No"; else echo "Yes"; fi)"
-          echo "Is KEYCLOAK_SECRET set: $(if [ -z "$KEYCLOAK_SECRET" ]; then echo "No"; else echo "Yes"; fi)"
-          envsubst < e2e-test/e2eEnv.template.json > e2e-test/e2eEnv.json
+          echo INIT_AUTH=0 >> .env
+      # - name: Substitute secret variables in JSON
+      #   env:
+      #     auth_password: ${{ secrets.AUTH_PASSWORD }}
+      #     KEYCLOAK_SECRET: ${{ secrets.KEYCLOAK_SECRET }}
+      #   run: |
+      #     echo "Is auth_password set: $(if [ -z "$auth_password" ]; then echo "No"; else echo "Yes"; fi)"
+      #     echo "Is KEYCLOAK_SECRET set: $(if [ -z "$KEYCLOAK_SECRET" ]; then echo "No"; else echo "Yes"; fi)"
+      #     envsubst < e2e-test/e2eEnv.template.json > e2e-test/e2eEnv.json
       - name: Build the docker-compose stack
         run: docker-compose build -d
 

blobstore/blobstore.go

@@ -84,8 +84,7 @@ func (s3Ctrl *S3Controller) checkAndAdjustPrefix(bucket, prefix string) (string,
 	if prefix != "" && prefix != "./" && prefix != "/" {
 		isObject, err := s3Ctrl.KeyExists(bucket, prefix)
 		if err != nil {
-			fmt.Println(err)
-			return "", configberry.HandleAWSError(err, "error checking if object exists")
+			return "", configberry.HandleAWSError(err, "error checking if prefix is an object")
 		}
 		if isObject {
 			objMeta, err := s3Ctrl.GetMetaData(bucket, prefix)

blobstore/creds.go

@@ -86,6 +86,39 @@ func (mc MinioConfig) minIOSessionManager() (*s3.S3, *session.Session, error) {
 		log.Info("Bucket already exists")
 	}
 
+	// Create the policy as a byte array
+	policyBytes := []byte(`{
+    "Version": "2012-10-17",
+    "Statement": [
+      {
+        "Effect": "Allow",
+        "Principal": {
+          "AWS": [
+            "*"
+        ]
+      },
+      "Action": [
+        "s3:GetObject",
+        "s3:PutObject"
+      ],
+      "Resource": [
+        "arn:aws:s3:::test-bucket/*"
+      ]
+    }
+  ]}`)
+
+	// Set the policy on the bucket
+	_, err = s3SVC.PutBucketPolicy(&s3.PutBucketPolicyInput{
+		Bucket: aws.String(mc.Bucket),
+		Policy: aws.String(string(policyBytes)),
+	})
+	if err != nil {
+		log.Errorf("error setting bucket policy: %s", err.Error())
+		// Handle error appropriately
+	} else {
+		log.Info("Bucket policy set successfully")
+	}
+
 	return s3SVC, sess, nil
 }
 

blobstore/delete.go

@@ -12,6 +12,25 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
+func (s3Ctrl *S3Controller) DeleteObjectIfExists(bucket, key string) error {
+	// Check if the object exists
+	if _, err := s3Ctrl.GetMetaData(bucket, key); err != nil {
+		return err
+	}
+
+	// Delete the object
+	deleteInput := &s3.DeleteObjectInput{
+		Bucket: aws.String(bucket),
+		Key:    aws.String(key),
+	}
+	_, err := s3Ctrl.S3Svc.DeleteObject(deleteInput)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (s3Ctrl *S3Controller) DeleteList(page *s3.ListObjectsV2Output, bucket string) error {
 	if len(page.Contents) == 0 {
 		return nil // No objects to delete in this page
@@ -81,28 +100,9 @@ func (bh *BlobHandler) HandleDeleteObject(c echo.Context) error {
 		return configberry.HandleErrorResponse(c, appErr)
 	}
 
-	// If the key is not a folder, proceed with deleting a single object
-	keyExist, err := s3Ctrl.KeyExists(bucket, key)
-	if err != nil {
-		appErr := configberry.HandleAWSError(err, "error checking if object exists")
-		log.Error(configberry.LogErrorFormatter(appErr, true))
-		return configberry.HandleErrorResponse(c, appErr)
-	}
-
-	if !keyExist {
-		appErr := configberry.NewAppError(configberry.NotFoundError, fmt.Sprintf("object %s not found", key), nil)
-		log.Error(configberry.LogErrorFormatter(appErr, true))
-		return configberry.HandleErrorResponse(c, appErr)
-	}
-
-	deleteInput := &s3.DeleteObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(key),
-	}
-
-	_, err = s3Ctrl.S3Svc.DeleteObject(deleteInput)
+	err = s3Ctrl.DeleteObjectIfExists(bucket, key)
 	if err != nil {
-		appErr := configberry.NewAppError(configberry.InternalServerError, fmt.Sprintf("error deleting object. %s", err.Error()), nil)
+		appErr := configberry.HandleAWSError(err, fmt.Sprintf("error deleting object %s", key))
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}

blobstore/list.go

@@ -153,7 +153,7 @@ func (bh *BlobHandler) HandleListByPrefix(c echo.Context) error {
 
 	err = s3Ctrl.GetListWithCallBack(bucket, prefix, delimiter, processPage)
 	if err != nil {
-		appErr := configberry.NewAppError(configberry.InternalServerError, "error processing objects", err)
+		appErr := configberry.HandleAWSError(err, "error processing objects")
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}

blobstore/metadata.go

@@ -22,7 +22,6 @@ func (s3Ctrl *S3Controller) GetMetaData(bucket, key string) (*s3.HeadObjectOutpu
 	if err != nil {
 		return nil, err
 	}
-
 	return result, nil
 }
 
@@ -61,29 +60,20 @@ func (bh *BlobHandler) HandleGetSize(c echo.Context) error {
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}
-
-	permissions, fullAccess, appErr := bh.getS3ReadPermissions(c, bucket)
+	adjustedPrefix, appErr := s3Ctrl.checkAndAdjustPrefix(bucket, prefix)
 	if appErr != nil {
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}
-
-	if !fullAccess && !isPermittedPrefix(bucket, prefix, permissions) {
-		appErr := configberry.NewAppError(configberry.ForbiddenError, fmt.Sprintf("user does not have permission to read the %s prefix", prefix), err)
-		log.Error(configberry.LogErrorFormatter(appErr, true))
-		return configberry.HandleErrorResponse(c, appErr)
-	}
-
-	// Check if the prefix points directly to an object
-	isObject, err := s3Ctrl.KeyExists(bucket, prefix)
-	if err != nil {
-		appErr := configberry.NewAppError(configberry.InternalServerError, "error checking if prefix is an object", err)
+	prefix = adjustedPrefix
+	permissions, fullAccess, appErr := bh.getS3ReadPermissions(c, bucket)
+	if appErr != nil {
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}
 
-	if isObject {
-		appErr := configberry.NewAppError(configberry.TeapotError, fmt.Sprintf("the provided prefix %s points to a single object rather than a collection", prefix), err)
+	if !fullAccess && !isPermittedPrefix(bucket, prefix, permissions) {
+		appErr := configberry.NewAppError(configberry.ForbiddenError, fmt.Sprintf("user does not have permission to read the %s prefix", prefix), err)
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}
@@ -149,7 +139,7 @@ func (bh *BlobHandler) HandleGetMetaData(c echo.Context) error {
 
 	result, err := s3Ctrl.GetMetaData(bucket, key)
 	if err != nil {
-		appErr := configberry.HandleAWSError(err, "error getting metadata")
+		appErr := configberry.HandleAWSError(err, fmt.Sprintf("error getting metadata for %s", key))
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}

blobstore/move.go

@@ -64,15 +64,7 @@ func (s3Ctrl *S3Controller) CopyObject(bucket, srcObjectKey, destObjectKey strin
 	if srcObjectKey == destObjectKey {
 		return fmt.Errorf("source `%s` and destination `%s` keys are identical; no action taken", srcObjectKey, destObjectKey)
 	}
-	// Check if the old key exists in the bucket
-	oldKeyExists, err := s3Ctrl.KeyExists(bucket, srcObjectKey)
-	if err != nil {
-		return err
-	}
 
-	if !oldKeyExists {
-		return fmt.Errorf("`srcObjectKey` " + srcObjectKey + " does not exist")
-	}
 	// Check if the new key already exists in the bucket
 	newKeyExists, err := s3Ctrl.KeyExists(bucket, destObjectKey)
 	if err != nil {
@@ -150,6 +142,11 @@ func (bh *BlobHandler) HandleMoveObject(c echo.Context) error {
 		log.Error(configberry.LogErrorFormatter(appErr, false))
 		return configberry.HandleErrorResponse(c, appErr)
 	}
+	if params["srcObjectKey"] == params["destObjectKey"] {
+		appErr := configberry.NewAppError(configberry.ValidationError, fmt.Sprintf("source `%s` and destination `%s` keys are identical; no action taken", params["srcObjectKey"], params["destObjectKey"]), nil)
+		log.Error(configberry.LogErrorFormatter(appErr, true))
+		return configberry.HandleErrorResponse(c, appErr)
+	}
 
 	bucket := c.QueryParam("bucket")
 	s3Ctrl, err := bh.GetController(bucket)
@@ -159,6 +156,18 @@ func (bh *BlobHandler) HandleMoveObject(c echo.Context) error {
 		return configberry.HandleErrorResponse(c, appErr)
 	}
 
+	newKeyExists, err := s3Ctrl.KeyExists(bucket, params["destObjectKey"])
+	if err != nil {
+		return err
+	}
+
+	if newKeyExists {
+		appErr := configberry.NewAppError(configberry.ConflictError, fmt.Sprintf("%s already exists in the bucket; duplication will cause an overwrite. Please rename dest_key to a different name", params["destObjectKey"]), err)
+		log.Error(configberry.LogErrorFormatter(appErr, true))
+		return configberry.HandleErrorResponse(c, appErr)
+
+	}
+
 	err = s3Ctrl.CopyObject(bucket, params["srcObjectKey"], params["destObjectKey"])
 	if err != nil {
 		appErr := configberry.HandleAWSError(err, "error copying prefix")

blobstore/object_content.go

@@ -12,13 +12,6 @@ import (
 )
 
 func (s3Ctrl *S3Controller) FetchObjectContent(bucket string, key string) (io.ReadCloser, error) {
-	keyExist, err := s3Ctrl.KeyExists(bucket, key)
-	if err != nil {
-		return nil, err
-	}
-	if !keyExist {
-		return nil, fmt.Errorf("object %s not found", key)
-	}
 	input := &s3.GetObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
@@ -67,7 +60,7 @@ func (bh *BlobHandler) HandleObjectContents(c echo.Context) error {
 	}
 	body, err := io.ReadAll(outPutBody)
 	if err != nil {
-		appErr := configberry.NewAppError(configberry.ForbiddenError, "error reading objects body", err)
+		appErr := configberry.NewAppError(configberry.InternalServerError, "error reading objects body", err)
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}

blobstore/presigned_url.go

@@ -18,6 +18,9 @@ import (
 
 func (s3Ctrl *S3Controller) GetDownloadPresignedURL(bucket, key string, expDays int) (string, error) {
 	duration := time.Duration(expDays) * 24 * time.Hour
+	if _, err := s3Ctrl.GetMetaData(bucket, key); err != nil { //this is to check if the object exists or not, it will return an AWS error
+		return "", err
+	}
 	req, _ := s3Ctrl.S3Svc.GetObjectRequest(&s3.GetObjectInput{
 		Bucket: aws.String(bucket),
 		Key:    aws.String(key),
@@ -53,21 +56,9 @@ func (bh *BlobHandler) HandleGetPresignedDownloadURL(c echo.Context) error {
 		return configberry.HandleErrorResponse(c, appErr)
 	}
 
-	keyExist, err := s3Ctrl.KeyExists(bucket, key)
-	if err != nil {
-		appErr := configberry.NewAppError(configberry.InternalServerError, "error checking if object exists", err)
-		log.Error(configberry.LogErrorFormatter(appErr, true))
-		return configberry.HandleErrorResponse(c, appErr)
-	}
-	if !keyExist {
-		appErr := configberry.NewAppError(configberry.NotFoundError, fmt.Sprintf("object %s not found", key), err)
-		log.Error(configberry.LogErrorFormatter(appErr, true))
-		return configberry.HandleErrorResponse(c, appErr)
-	}
-
 	url, err := s3Ctrl.GetDownloadPresignedURL(bucket, key, bh.Config.DefaultDownloadPresignedUrlExpiration)
 	if err != nil {
-		appErr := configberry.HandleAWSError(err, "error getting presigned URL")
+		appErr := configberry.HandleAWSError(err, fmt.Sprintf("error getting presigned download URL for object %s", key))
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}

blobstore/upload.go

@@ -125,14 +125,41 @@ func (s3Ctrl *S3Controller) UploadS3Obj(bucket string, key string, body io.ReadC
 // function to retrieve presigned url for a normal one time upload. You can only upload 5GB files at a time.
 func (s3Ctrl *S3Controller) GetUploadPresignedURL(bucket string, key string, expMin int) (string, error) {
 	duration := time.Duration(expMin) * time.Minute
-	req, _ := s3Ctrl.S3Svc.PutObjectRequest(&s3.PutObjectInput{
-		Bucket: aws.String(bucket),
-		Key:    aws.String(key),
-	})
+	var urlStr string
+	var err error
+	if s3Ctrl.S3Mock {
+		// Create a temporary S3 client with the modified endpoint
+		//this is done  so that the presigned url starts with localhost:9000 instead of
+		//minio:9000 which would cause an error due to cors origin policy
+		tempS3Svc, err := session.NewSession(&aws.Config{
+			Endpoint:         aws.String("http://localhost:9000"),
+			Region:           s3Ctrl.S3Svc.Config.Region,
+			Credentials:      s3Ctrl.S3Svc.Config.Credentials,
+			S3ForcePathStyle: aws.Bool(true),
+		})
+		if err != nil {
+			return "", fmt.Errorf("error creating temporary s3 session: %s", err.Error())
+		}
 
-	urlStr, err := req.Presign(duration)
-	if err != nil {
-		return "", err
+		// Generate the request using the temporary client
+		req, _ := s3.New(tempS3Svc).PutObjectRequest(&s3.PutObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(key),
+		})
+		urlStr, err = req.Presign(duration)
+		if err != nil {
+			return "", err
+		}
+	} else {
+		// Generate the request using the original client
+		req, _ := s3Ctrl.S3Svc.PutObjectRequest(&s3.PutObjectInput{
+			Bucket: aws.String(bucket),
+			Key:    aws.String(key),
+		})
+		urlStr, err = req.Presign(duration)
+		if err != nil {
+			return "", err
+		}
 	}
 
 	return urlStr, nil
@@ -297,7 +324,7 @@ func (bh *BlobHandler) HandleMultipartUpload(c echo.Context) error {
 
 	err = s3Ctrl.UploadS3Obj(bucket, key, body)
 	if err != nil {
-		appErr := configberry.NewAppError(configberry.InternalServerError, "error uploading S3 object", err)
+		appErr := configberry.HandleAWSError(err, "error uploading S3 object")
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 
@@ -354,10 +381,11 @@ func (bh *BlobHandler) HandleGetPresignedUploadURL(c echo.Context) error {
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}
+
 	//if the user did not provided both upload_id and part_number then we returned normal presigned URL
 	presignedURL, err := s3Ctrl.GetUploadPresignedURL(bucket, key, bh.Config.DefaultUploadPresignedUrlExpiration)
 	if err != nil {
-		appErr := configberry.NewAppError(configberry.InternalServerError, "error generating presigned URL", err)
+		appErr := configberry.HandleAWSError(err, "error generating presigned URL")
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}
@@ -391,7 +419,7 @@ func (bh *BlobHandler) HandleGetMultipartUploadID(c echo.Context) error {
 
 	uploadID, err := s3Ctrl.GetMultiPartUploadID(bucket, key)
 	if err != nil {
-		appErr := configberry.NewAppError(configberry.InternalServerError, "error retrieving multipart Upload ID", err)
+		appErr := configberry.HandleAWSError(err, "error retrieving multipart Upload ID")
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}
@@ -440,7 +468,7 @@ func (bh *BlobHandler) HandleCompleteMultipartUpload(c echo.Context) error {
 
 	_, err = s3Ctrl.CompleteMultipartUpload(bucket, key, req.UploadID, s3Parts)
 	if err != nil {
-		appErr := configberry.NewAppError(configberry.InternalServerError, fmt.Sprintf("error completing the multipart Upload for key %s", key), err)
+		appErr := configberry.HandleAWSError(err, fmt.Sprintf("error completing the multipart Upload for key %s", key))
 		log.Error(configberry.LogErrorFormatter(appErr, true))
 		return configberry.HandleErrorResponse(c, appErr)
 	}