Support BucketLinks on admin command (Fix #1081)

backend/posix/posix.go

@@ -249,35 +249,15 @@ func (p *Posix) doesBucketAndObjectExist(bucket, object string) error {
 }
 
 func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) {
-	entries, err := os.ReadDir(".")
+	fis, err := listBucketFileInfos(p.bucketlinks)
 	if err != nil {
-		return s3response.ListAllMyBucketsResult{},
-			fmt.Errorf("readdir buckets: %w", err)
+		return s3response.ListAllMyBucketsResult{}, fmt.Errorf("listBucketFileInfos : %w", err)
 	}
 
 	var cToken string
 
 	var buckets []s3response.ListAllMyBucketsEntry
-	for _, entry := range entries {
-		fi, err := entry.Info()
-		if err != nil {
-			// skip entries returning errors
-			continue
-		}
-
-		if p.bucketlinks && entry.Type() == fs.ModeSymlink {
-			fi, err = os.Stat(entry.Name())
-			if err != nil {
-				// skip entries returning errors
-				continue
-			}
-		}
-
-		if !fi.IsDir() {
-			// buckets must be a directory
-			continue
-		}
-
+	for _, fi := range fis {
 		if !strings.HasPrefix(fi.Name(), input.Prefix) {
 			continue
 		}
@@ -294,13 +274,13 @@ func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput
 		// return all the buckets for admin users
 		if input.IsAdmin {
 			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
-				Name:         entry.Name(),
+				Name:         fi.Name(),
 				CreationDate: fi.ModTime(),
 			})
 			continue
 		}
 
-		aclTag, err := p.meta.RetrieveAttribute(nil, entry.Name(), "", aclkey)
+		aclTag, err := p.meta.RetrieveAttribute(nil, fi.Name(), "", aclkey)
 		if errors.Is(err, meta.ErrNoSuchKey) {
 			// skip buckets without acl tag
 			continue
@@ -317,7 +297,7 @@ func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput
 
 		if acl.Owner == input.Owner {
 			buckets = append(buckets, s3response.ListAllMyBucketsEntry{
-				Name:         entry.Name(),
+				Name:         fi.Name(),
 				CreationDate: fi.ModTime(),
 			})
 		}
@@ -4742,23 +4722,47 @@ func (p *Posix) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte
 	return p.PutBucketAcl(ctx, bucket, acl)
 }
 
-func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) {
+func listBucketFileInfos(bucketlinks bool) ([]fs.FileInfo, error) {
 	entries, err := os.ReadDir(".")
 	if err != nil {
-		return buckets, fmt.Errorf("readdir buckets: %w", err)
+		return nil, fmt.Errorf("readdir buckets: %w", err)
 	}
 
+	var fis []fs.FileInfo
 	for _, entry := range entries {
-		if !entry.IsDir() {
+		fi, err := entry.Info()
+		if err != nil {
 			continue
 		}
 
-		fi, err := entry.Info()
-		if err != nil {
+		if bucketlinks && entry.Type() == fs.ModeSymlink {
+			fi, err = os.Stat(entry.Name())
+			if err != nil {
+				// skip entries returning errors
+				continue
+			}
+		}
+
+		if !fi.IsDir() {
+			// buckets must be a directory
 			continue
 		}
 
-		aclTag, err := p.meta.RetrieveAttribute(nil, entry.Name(), "", aclkey)
+		fis = append(fis, fi)
+	}
+	return fis, nil
+
+}
+
+func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) {
+	fis, err := listBucketFileInfos(p.bucketlinks)
+	if err != nil {
+		return buckets, fmt.Errorf("listBucketFileInfos: %w", err)
+	}
+
+	for _, fi := range fis {
+
+		aclTag, err := p.meta.RetrieveAttribute(nil, fi.Name(), "", aclkey)
 		if err != nil && !errors.Is(err, meta.ErrNoSuchKey) {
 			return buckets, fmt.Errorf("get acl tag: %w", err)
 		}

backend/posix/posix_test.go

@@ -0,0 +1,97 @@
+// Copyright 2023 Versity Software
+// This file is licensed under the Apache License, Version 2.0
+// (the "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package posix
+
+import (
+	"context"
+	"os"
+	"sort"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/versity/versitygw/backend/meta"
+)
+
+func TestListBucketsAndOwnersBucketLinks(t *testing.T) {
+	a := assert.New(t)
+	type link struct {
+		from string
+		to   string
+	}
+	testCases := []struct {
+		name     string
+		dirs     []string
+		files    []string
+		links    []link
+		expected []string
+	}{
+		{"empty", []string{}, []string{}, nil, []string{}},
+		{"single", []string{"abucket"}, []string{}, nil, []string{"abucket"}},
+		{"basic three", []string{"ccc", "bbb", "aaa"}, []string{}, nil, []string{"aaa", "bbb", "ccc"}},
+		{"case sensitive", []string{"Ccc", "bBb", "aaA"}, []string{}, nil, []string{"Ccc", "aaA", "bBb"}},
+		{"link to single dir", []string{"frombucket"}, []string{}, []link{{"frombucket", "tobucket"}}, []string{"frombucket", "tobucket"}},
+		{"link to single file", []string{}, []string{"fromfile"}, []link{{"fromfile", "tofile"}}, []string{}},
+		{"link to non-existent", []string{}, []string{}, []link{{"doesnotexist", "tofile"}}, []string{}},
+	}
+
+	ctx := context.Background()
+	var err error
+
+	for _, tc := range testCases {
+		gwDir := t.TempDir()
+
+		t.Logf("%s: working in gw dir [%s]", tc.name, gwDir)
+		os.Chdir(gwDir)
+		for _, dir := range tc.dirs {
+			err = os.Mkdir(dir, 0755)
+			if err != nil {
+				t.Fatalf("Failed to setup test: Mkdir err %s", err)
+			}
+		}
+		for _, file := range tc.files {
+			f, err := os.Create(file)
+			if err != nil {
+				t.Fatalf("Failed to setup test: Mkdir err %s", err)
+			}
+			f.Close()
+		}
+		for _, link := range tc.links {
+			err = os.Symlink(link.from, link.to)
+			if err != nil {
+				t.Fatalf("Failed to setup test: Link err %s", err)
+			}
+		}
+
+		meta := meta.XattrMeta{}
+		var opts PosixOpts
+
+		opts.BucketLinks = true
+
+		p, err := New(gwDir, meta, opts)
+		if err != nil {
+			t.Errorf("Can't create posix backend")
+		}
+		resp, err := p.ListBucketsAndOwners(ctx)
+		if err != nil {
+			t.Fatalf("ListBucketsAndOwners failed: %s", err)
+		}
+		got := make([]string, 0)
+		for _, bucket := range resp {
+			got = append(got, bucket.Name)
+		}
+		sort.Strings(got)
+		a.Equal(tc.expected, got)
+	}
+}

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/pkg/xattr v0.4.10
 	github.com/segmentio/kafka-go v0.4.47
 	github.com/smira/go-statsd v1.3.4
+	github.com/stretchr/testify v1.10.0
 	github.com/urfave/cli/v2 v2.27.5
 	github.com/valyala/fasthttp v1.59.0
 	github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44
@@ -39,6 +40,7 @@ require (
 	github.com/aws/aws-sdk-go-v2/service/sso v1.24.16 // indirect
 	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.15 // indirect
 	github.com/aws/aws-sdk-go-v2/service/sts v1.33.15 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
@@ -51,11 +53,13 @@ require (
 	github.com/nats-io/nuid v1.0.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
 	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	golang.org/x/crypto v0.35.0 // indirect
 	golang.org/x/net v0.35.0 // indirect
 	golang.org/x/text v0.22.0 // indirect
 	golang.org/x/time v0.10.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )
 
 require (

go.sum

@@ -122,6 +122,10 @@ github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeW
 github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
 github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE=
@@ -156,6 +160,8 @@ github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93Ge
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
 github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
+github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
 github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
 github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk=
@@ -167,8 +173,9 @@ github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c
 github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
-github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
@@ -296,6 +303,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=