Skip to content

Commit 47ab4ad

Browse files
ShaneMPutnamShaneMPutnam
authored
Merge pull request #37 from Dewberry/bug-list-size-overload
Bug list size overload
2 parents 8309dd2 + 0c976c9 commit 47ab4ad

15 files changed

Lines changed: 970 additions & 453 deletions

‎.github/workflows/e2e-test.yml‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -30,11 +30,6 @@ jobs:
3030
echo AWS_S3_BUCKET='test-bucket' >> .env
3131
echo S3API_SERVICE_PORT='5005' >> .env
3232
echo AUTH_LEVEL=0 >> .env
33-
echo POSTGRES_CONN_STRING='postgres://user:password@postgres:5432/db?sslmode=disable' >> .env
34-
echo POSTGRES_PASSWORD='password' >> .env
35-
echo POSTGRES_USER='user' >> .env
36-
echo POSTGRES_DB='db' >> .env
37-
echo PG_LOG_CHECKPOINTS='off' >> .env
3833
3934
- name: Substitute secret variables in JSON
4035
env:

‎blobstore/blobhandler.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ type S3Controller struct {
2121
Sess *session.Session
2222
S3Svc *s3.S3
2323
Buckets []string
24+
S3Mock bool
2425
}
2526

2627
// Config holds the configuration settings for the REST API server.
@@ -92,13 +93,12 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
9293
}
9394

9495
// Configure the BlobHandler with MinIO session and bucket information
95-
config.S3Controllers = []S3Controller{{Sess: sess, S3Svc: s3SVC, Buckets: []string{creds.Bucket}}}
96+
config.S3Controllers = []S3Controller{{Sess: sess, S3Svc: s3SVC, Buckets: []string{creds.Bucket}, S3Mock: true}}
9697
// Return the configured BlobHandler
9798
return &config, nil
9899
}
99100

100101
// Using AWS S3
101-
102102
// Load AWS credentials from the provided .env.json file
103103
log.Debug("looking for .env.json")
104104
awsConfig, err := newAWSConfig(envJson)
@@ -153,7 +153,7 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
153153
}
154154

155155
if len(bucketNames) > 0 {
156-
config.S3Controllers = append(config.S3Controllers, S3Controller{Sess: sess, S3Svc: s3SVC, Buckets: bucketNames})
156+
config.S3Controllers = append(config.S3Controllers, S3Controller{Sess: sess, S3Svc: s3SVC, Buckets: bucketNames, S3Mock: false})
157157
}
158158
}
159159

‎blobstore/buckets.go‎

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -112,17 +112,15 @@ func (bh *BlobHandler) HandleListBuckets(c echo.Context) error {
112112
}
113113
}
114114
bh.Mu.Unlock()
115-
116-
log.Info("HandleListBuckets: Successfully retrieved list of buckets")
117-
115+
log.Info("Successfully retrieved list of buckets")
118116
return c.JSON(http.StatusOK, allBuckets)
119117
}
120118

121119
// func (bh *BlobHandler) HandleCreateBucket(c echo.Context) error {
122120
// bucketName := c.QueryParam("name")
123121

124122
// if bucketName == "" {
125-
// err := errors.New("request must include a `name` parameter")
123+
// err := fmt.Errorf("request must include a `name` parameter")
126124
// log.Info("HandleCreateBucket: " + err.Error())
127125
// return c.JSON(http.StatusBadRequest, err.Error())
128126
// }
@@ -157,7 +155,7 @@ func (bh *BlobHandler) HandleListBuckets(c echo.Context) error {
157155
// bucketName := c.QueryParam("name")
158156

159157
// if bucketName == "" {
160-
// err := errors.New("request must include a `name` parameter")
158+
// err := fmt.Errorf("request must include a `name` parameter")
161159
// log.Info("HandleDeleteBucket: " + err.Error())
162160
// return c.JSON(http.StatusBadRequest, err.Error())
163161
// }
@@ -177,7 +175,7 @@ func (bh *BlobHandler) HandleListBuckets(c echo.Context) error {
177175
// bucketName := c.QueryParam("name")
178176

179177
// if bucketName == "" {
180-
// err := errors.New("request must include a `name` parameter")
178+
// err := fmt.Errorf("request must include a `name` parameter")
181179
// log.Info("HandleGetBucketACL: " + err.Error())
182180
// return c.JSON(http.StatusBadRequest, err.Error())
183181
// }

‎blobstore/delete.go‎

Lines changed: 78 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package blobstore
22

33
import (
4-
"errors"
54
"fmt"
65
"net/http"
76
"strings"
@@ -12,39 +11,45 @@ import (
1211
log "github.com/sirupsen/logrus"
1312
)
1413

15-
func (s3Ctrl *S3Controller) RecursivelyDeleteObjects(bucket, prefix string) error {
16-
prefixPath := strings.Trim(prefix, "/") + "/"
17-
query := &s3.ListObjectsV2Input{
18-
Bucket: aws.String(bucket),
19-
Prefix: aws.String(prefixPath),
14+
func (s3Ctrl *S3Controller) DeleteList(page *s3.ListObjectsV2Output, bucket string) error {
15+
if len(page.Contents) == 0 {
16+
return nil // No objects to delete in this page
2017
}
21-
resp, err := s3Ctrl.S3Svc.ListObjectsV2(query)
18+
19+
var objectsToDelete []*s3.ObjectIdentifier
20+
for _, obj := range page.Contents {
21+
objectsToDelete = append(objectsToDelete, &s3.ObjectIdentifier{Key: obj.Key})
22+
}
23+
24+
// Perform the delete operation for the current page
25+
_, err := s3Ctrl.S3Svc.DeleteObjects(&s3.DeleteObjectsInput{
26+
Bucket: aws.String(bucket),
27+
Delete: &s3.Delete{
28+
Objects: objectsToDelete,
29+
Quiet: aws.Bool(true),
30+
},
31+
})
2232
if err != nil {
23-
return fmt.Errorf("recursivelyDeleteObjects: error listing objects: %s", err)
33+
return fmt.Errorf("error deleting objects: %v", err)
2434
}
25-
if len(resp.Contents) > 0 {
26-
var objectsToDelete []*s3.ObjectIdentifier
2735

28-
for _, obj := range resp.Contents {
29-
objectsToDelete = append(objectsToDelete, &s3.ObjectIdentifier{
30-
Key: obj.Key,
31-
})
32-
}
36+
return nil
37+
}
3338

34-
if len(objectsToDelete) > 0 {
35-
_, err = s3Ctrl.S3Svc.DeleteObjects(&s3.DeleteObjectsInput{
36-
Bucket: aws.String(bucket),
37-
Delete: &s3.Delete{
38-
Objects: objectsToDelete,
39-
},
40-
})
41-
42-
if err != nil {
43-
return fmt.Errorf("recursivelyDeleteObjects: error Deleting objects %v: %s", objectsToDelete, err)
44-
}
39+
func (s3Ctrl *S3Controller) RecursivelyDeleteObjects(bucket, prefix string) error {
40+
var objectsFound bool
41+
err := s3Ctrl.GetListWithCallBack(bucket, prefix, false, func(page *s3.ListObjectsV2Output) error {
42+
if len(page.Contents) > 0 {
43+
objectsFound = true
4544
}
46-
} else {
47-
return fmt.Errorf("recursivelyDeleteObjects: object %s not found and no objects were deleted", prefixPath)
45+
return s3Ctrl.DeleteList(page, bucket)
46+
})
47+
if err != nil {
48+
return fmt.Errorf("error processing objects for deletion: %v", err)
49+
}
50+
51+
if !objectsFound {
52+
return fmt.Errorf("prefix not found")
4853
}
4954
return nil
5055
}
@@ -56,28 +61,29 @@ func (bh *BlobHandler) HandleDeleteObject(c echo.Context) error {
5661
bucket := c.QueryParam("bucket")
5762
s3Ctrl, err := bh.GetController(bucket)
5863
if err != nil {
59-
errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error())
64+
errMsg := fmt.Errorf("parameter `bucket` %s is not available, %s", bucket, err.Error())
6065
log.Error(errMsg.Error())
6166
return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
6267
}
6368

6469
key := c.QueryParam("key")
6570
if key == "" {
66-
err := errors.New("parameter 'key' is required")
67-
log.Errorf("HandleDeleteObjects: %s", err.Error())
68-
return c.JSON(http.StatusUnprocessableEntity, err.Error())
71+
errMsg := fmt.Errorf("parameter `key` is required")
72+
log.Error(errMsg.Error())
73+
return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
6974
}
7075

7176
// If the key is not a folder, proceed with deleting a single object
7277
keyExist, err := s3Ctrl.KeyExists(bucket, key)
7378
if err != nil {
74-
log.Errorf("HandleDeleteObjects: Error checking if key exists: %s", err.Error())
75-
return c.JSON(http.StatusInternalServerError, err)
79+
errMsg := fmt.Errorf("error checking if object exists: %s", err.Error())
80+
log.Error(errMsg.Error())
81+
return c.JSON(http.StatusInternalServerError, errMsg.Error())
7682
}
7783
if !keyExist {
78-
err := fmt.Errorf("object %s not found", key)
79-
log.Errorf("HandleDeleteObjects: %s", err.Error())
80-
return c.JSON(http.StatusNotFound, err.Error())
84+
errMsg := fmt.Errorf("object %s not found", key)
85+
log.Error(errMsg.Error())
86+
return c.JSON(http.StatusNotFound, errMsg.Error())
8187
}
8288

8389
deleteInput := &s3.DeleteObjectInput{
@@ -87,54 +93,45 @@ func (bh *BlobHandler) HandleDeleteObject(c echo.Context) error {
8793

8894
_, err = s3Ctrl.S3Svc.DeleteObject(deleteInput)
8995
if err != nil {
90-
msg := fmt.Sprintf("error deleting object. %s", err.Error())
91-
log.Errorf("HandleDeleteObjects: %s", err.Error())
92-
return c.JSON(http.StatusInternalServerError, msg)
96+
errMsg := fmt.Errorf("error deleting object. %s", err.Error())
97+
log.Error(errMsg.Error())
98+
return c.JSON(http.StatusInternalServerError, errMsg.Error())
9399
}
94100

95-
log.Info("HandleDeleteObjects: Successfully deleted file with key:", key)
101+
log.Infof("successfully deleted file with key: %s", key)
96102
return c.JSON(http.StatusOK, fmt.Sprintf("Successfully deleted object: %s", key))
97103
}
98104

99105
func (bh *BlobHandler) HandleDeletePrefix(c echo.Context) error {
100106
bucket := c.QueryParam("bucket")
101107
s3Ctrl, err := bh.GetController(bucket)
102108
if err != nil {
103-
errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error())
109+
errMsg := fmt.Errorf("parameter `bucket` %s is not available, %s", bucket, err.Error())
104110
log.Error(errMsg.Error())
105111
return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
106112
}
107-
108113
prefix := c.QueryParam("prefix")
109114
if prefix == "" {
110-
err = errors.New("parameter 'prefix' is required")
111-
log.Errorf("HandleDeleteObjects: %s", err.Error())
112-
return c.JSON(http.StatusUnprocessableEntity, err.Error())
115+
errMsg := fmt.Errorf("parameter `prefix` is required")
116+
log.Error(errMsg.Error())
117+
return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
113118
}
114119
if !strings.HasSuffix(prefix, "/") {
115120
prefix = prefix + "/"
116121
}
117-
response, err := s3Ctrl.GetList(bucket, prefix, false)
118-
if err != nil {
119-
log.Errorf("HandleDeleteObjects: Error getting list: %s", err.Error())
120-
return c.JSON(http.StatusInternalServerError, err)
121-
}
122-
if *response.KeyCount == 0 {
123-
err := fmt.Errorf("the specified prefix %s does not exist in S3", prefix)
124-
log.Errorf("HandleDeleteObjects: %s", err.Error())
125-
return c.JSON(http.StatusNotFound, err.Error())
126-
}
127-
// This will recursively delete all objects with the specified prefix
128122
err = s3Ctrl.RecursivelyDeleteObjects(bucket, prefix)
129123
if err != nil {
130-
msg := fmt.Sprintf("error deleting objects. %s", err.Error())
131-
log.Errorf("HandleDeleteObjects: %s", msg)
132-
return c.JSON(http.StatusInternalServerError, msg)
124+
if strings.Contains(err.Error(), "prefix not found") {
125+
errMsg := fmt.Errorf("no objects found with prefix: %s", prefix)
126+
log.Error(errMsg.Error())
127+
return c.JSON(http.StatusNotFound, errMsg.Error())
128+
}
129+
errMsg := fmt.Errorf("error deleting objects: %s", err.Error())
130+
log.Error(errMsg.Error())
131+
return c.JSON(http.StatusInternalServerError, errMsg.Error())
133132
}
134-
135-
log.Info("HandleDeleteObjects: Successfully deleted prefix and its contents for prefix:", prefix)
133+
log.Info("Successfully deleted prefix and its contents for prefix:", prefix)
136134
return c.JSON(http.StatusOK, "Successfully deleted prefix and its contents")
137-
138135
}
139136

140137
func (s3Ctrl *S3Controller) DeleteKeys(bucket string, key []string) error {
@@ -157,7 +154,7 @@ func (s3Ctrl *S3Controller) DeleteKeys(bucket string, key []string) error {
157154

158155
_, err := s3Ctrl.S3Svc.DeleteObjects(input)
159156
if err != nil {
160-
return fmt.Errorf("deleteKeys: error Deleting objects: %s", err.Error())
157+
return fmt.Errorf("error deleting objects: %s", err.Error())
161158
}
162159
return nil
163160
}
@@ -169,21 +166,22 @@ func (bh *BlobHandler) HandleDeleteObjectsByList(c echo.Context) error {
169166
}
170167
var deleteRequest DeleteRequest
171168
if err := c.Bind(&deleteRequest); err != nil {
172-
log.Errorf("HandleDeleteObjectsByList: Error parsing request body: %s" + err.Error())
173-
return c.JSON(http.StatusBadRequest, "Invalid request body")
169+
errMsg := fmt.Errorf("error parsing request body: %s" + err.Error())
170+
log.Error(errMsg.Error())
171+
return c.JSON(http.StatusBadRequest, errMsg.Error())
174172
}
175173

176174
// Ensure there are keys to delete
177175
if len(deleteRequest.Keys) == 0 {
178-
errMsg := "No keys to delete. Please provide 'keys' in the request body."
179-
log.Errorf("HandleDeleteObjectsByList: %s", errMsg)
180-
return c.JSON(http.StatusUnprocessableEntity, errMsg)
176+
errMsg := fmt.Errorf("no keys to delete. Please provide 'keys' in the request body")
177+
log.Error(errMsg.Error())
178+
return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
181179
}
182180

183181
bucket := c.QueryParam("bucket")
184182
s3Ctrl, err := bh.GetController(bucket)
185183
if err != nil {
186-
errMsg := fmt.Errorf("bucket %s is not available, %s", bucket, err.Error())
184+
errMsg := fmt.Errorf("`bucket` %s is not available, %s", bucket, err.Error())
187185
log.Error(errMsg.Error())
188186
return c.JSON(http.StatusUnprocessableEntity, errMsg.Error())
189187
}
@@ -197,14 +195,14 @@ func (bh *BlobHandler) HandleDeleteObjectsByList(c echo.Context) error {
197195
// Check if the key exists before appending it to the keys list
198196
keyExists, err := s3Ctrl.KeyExists(bucket, s3Path)
199197
if err != nil {
200-
msg := fmt.Errorf("error checking if key exists. %s", err.Error())
201-
log.Errorf("HandleDeleteObjectsByList: %s", msg)
202-
return c.JSON(http.StatusInternalServerError, msg)
198+
errMsg := fmt.Errorf("error checking if object exists. %s", err.Error())
199+
log.Error(errMsg.Error())
200+
return c.JSON(http.StatusInternalServerError, errMsg)
203201
}
204202
if !keyExists {
205-
errMsg := fmt.Sprintf("object %s not found", s3Path)
206-
log.Errorf("HandleDeleteObjectsByList: %s", errMsg)
207-
return c.JSON(http.StatusNotFound, errMsg)
203+
errMsg := fmt.Errorf("object %s not found", s3Path)
204+
log.Error(errMsg.Error())
205+
return c.JSON(http.StatusNotFound, errMsg.Error())
208206
}
209207

210208
keys = append(keys, *key)
@@ -213,11 +211,11 @@ func (bh *BlobHandler) HandleDeleteObjectsByList(c echo.Context) error {
213211
// Delete the objects using the deleteKeys function
214212
err = s3Ctrl.DeleteKeys(bucket, keys)
215213
if err != nil {
216-
msg := fmt.Sprintf("error deleting objects. %s", err.Error())
217-
log.Errorf("HandleDeleteObjectsByList: %s", msg)
218-
return c.JSON(http.StatusInternalServerError, msg)
214+
errMsg := fmt.Errorf("error deleting objects. %s", err.Error())
215+
log.Error(errMsg.Error())
216+
return c.JSON(http.StatusInternalServerError, errMsg)
219217
}
220218

221-
log.Info("HandleDeleteObjectsByList: Successfully deleted objects:", deleteRequest.Keys)
219+
log.Info("Successfully deleted objects:", deleteRequest.Keys)
222220
return c.JSON(http.StatusOK, "Successfully deleted objects")
223221
}

0 commit comments

Comments
 (0)