diff --git a/backend/posix/posix.go b/backend/posix/posix.go index 52a5c152..44cb35cc 100644 --- a/backend/posix/posix.go +++ b/backend/posix/posix.go @@ -249,35 +249,15 @@ func (p *Posix) doesBucketAndObjectExist(bucket, object string) error { } func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput) (s3response.ListAllMyBucketsResult, error) { - entries, err := os.ReadDir(".") + fis, err := listBucketFileInfos(p.bucketlinks) if err != nil { - return s3response.ListAllMyBucketsResult{}, - fmt.Errorf("readdir buckets: %w", err) + return s3response.ListAllMyBucketsResult{}, fmt.Errorf("listBucketFileInfos : %w", err) } var cToken string var buckets []s3response.ListAllMyBucketsEntry - for _, entry := range entries { - fi, err := entry.Info() - if err != nil { - // skip entries returning errors - continue - } - - if p.bucketlinks && entry.Type() == fs.ModeSymlink { - fi, err = os.Stat(entry.Name()) - if err != nil { - // skip entries returning errors - continue - } - } - - if !fi.IsDir() { - // buckets must be a directory - continue - } - + for _, fi := range fis { if !strings.HasPrefix(fi.Name(), input.Prefix) { continue } @@ -294,13 +274,13 @@ func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput // return all the buckets for admin users if input.IsAdmin { buckets = append(buckets, s3response.ListAllMyBucketsEntry{ - Name: entry.Name(), + Name: fi.Name(), CreationDate: fi.ModTime(), }) continue } - aclTag, err := p.meta.RetrieveAttribute(nil, entry.Name(), "", aclkey) + aclTag, err := p.meta.RetrieveAttribute(nil, fi.Name(), "", aclkey) if errors.Is(err, meta.ErrNoSuchKey) { // skip buckets without acl tag continue @@ -317,7 +297,7 @@ func (p *Posix) ListBuckets(_ context.Context, input s3response.ListBucketsInput if acl.Owner == input.Owner { buckets = append(buckets, s3response.ListAllMyBucketsEntry{ - Name: entry.Name(), + Name: fi.Name(), CreationDate: fi.ModTime(), }) } @@ -4742,23 +4722,47 @@ func (p *Posix) ChangeBucketOwner(ctx context.Context, bucket string, acl []byte return p.PutBucketAcl(ctx, bucket, acl) } -func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) { +func listBucketFileInfos(bucketlinks bool) ([]fs.FileInfo, error) { entries, err := os.ReadDir(".") if err != nil { - return buckets, fmt.Errorf("readdir buckets: %w", err) + return nil, fmt.Errorf("readdir buckets: %w", err) } + var fis []fs.FileInfo for _, entry := range entries { - if !entry.IsDir() { + fi, err := entry.Info() + if err != nil { continue } - fi, err := entry.Info() - if err != nil { + if bucketlinks && entry.Type() == fs.ModeSymlink { + fi, err = os.Stat(entry.Name()) + if err != nil { + // skip entries returning errors + continue + } + } + + if !fi.IsDir() { + // buckets must be a directory continue } - aclTag, err := p.meta.RetrieveAttribute(nil, entry.Name(), "", aclkey) + fis = append(fis, fi) + } + return fis, nil + +} + +func (p *Posix) ListBucketsAndOwners(ctx context.Context) (buckets []s3response.Bucket, err error) { + fis, err := listBucketFileInfos(p.bucketlinks) + if err != nil { + return buckets, fmt.Errorf("listBucketFileInfos: %w", err) + } + + for _, fi := range fis { + + aclTag, err := p.meta.RetrieveAttribute(nil, fi.Name(), "", aclkey) if err != nil && !errors.Is(err, meta.ErrNoSuchKey) { return buckets, fmt.Errorf("get acl tag: %w", err) } diff --git a/backend/posix/posix_test.go b/backend/posix/posix_test.go new file mode 100644 index 00000000..b342e54a --- /dev/null +++ b/backend/posix/posix_test.go @@ -0,0 +1,97 @@ +// Copyright 2023 Versity Software +// This file is licensed under the Apache License, Version 2.0 +// (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package posix + +import ( + "context" + "os" + "sort" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/versity/versitygw/backend/meta" +) + +func TestListBucketsAndOwnersBucketLinks(t *testing.T) { + a := assert.New(t) + type link struct { + from string + to string + } + testCases := []struct { + name string + dirs []string + files []string + links []link + expected []string + }{ + {"empty", []string{}, []string{}, nil, []string{}}, + {"single", []string{"abucket"}, []string{}, nil, []string{"abucket"}}, + {"basic three", []string{"ccc", "bbb", "aaa"}, []string{}, nil, []string{"aaa", "bbb", "ccc"}}, + {"case sensitive", []string{"Ccc", "bBb", "aaA"}, []string{}, nil, []string{"Ccc", "aaA", "bBb"}}, + {"link to single dir", []string{"frombucket"}, []string{}, []link{{"frombucket", "tobucket"}}, []string{"frombucket", "tobucket"}}, + {"link to single file", []string{}, []string{"fromfile"}, []link{{"fromfile", "tofile"}}, []string{}}, + {"link to non-existent", []string{}, []string{}, []link{{"doesnotexist", "tofile"}}, []string{}}, + } + + ctx := context.Background() + var err error + + for _, tc := range testCases { + gwDir := t.TempDir() + + t.Logf("%s: working in gw dir [%s]", tc.name, gwDir) + os.Chdir(gwDir) + for _, dir := range tc.dirs { + err = os.Mkdir(dir, 0755) + if err != nil { + t.Fatalf("Failed to setup test: Mkdir err %s", err) + } + } + for _, file := range tc.files { + f, err := os.Create(file) + if err != nil { + t.Fatalf("Failed to setup test: Mkdir err %s", err) + } + f.Close() + } + for _, link := range tc.links { + err = os.Symlink(link.from, link.to) + if err != nil { + t.Fatalf("Failed to setup test: Link err %s", err) + } + } + + meta := meta.XattrMeta{} + var opts PosixOpts + + opts.BucketLinks = true + + p, err := New(gwDir, meta, opts) + if err != nil { + t.Errorf("Can't create posix backend") + } + resp, err := p.ListBucketsAndOwners(ctx) + if err != nil { + t.Fatalf("ListBucketsAndOwners failed: %s", err) + } + got := make([]string, 0) + for _, bucket := range resp { + got = append(got, bucket.Name) + } + sort.Strings(got) + a.Equal(tc.expected, got) + } +} diff --git a/go.mod b/go.mod index 9542136b..687fe7a1 100644 --- a/go.mod +++ b/go.mod @@ -22,6 +22,7 @@ require ( github.com/pkg/xattr v0.4.10 github.com/segmentio/kafka-go v0.4.47 github.com/smira/go-statsd v1.3.4 + github.com/stretchr/testify v1.10.0 github.com/urfave/cli/v2 v2.27.5 github.com/valyala/fasthttp v1.59.0 github.com/versity/scoutfs-go v0.0.0-20240325223134-38eb2f5f7d44 @@ -39,6 +40,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sso v1.24.16 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.15 // indirect github.com/aws/aws-sdk-go-v2/service/sts v1.33.15 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/go-asn1-ber/asn1-ber v1.5.7 // indirect github.com/golang-jwt/jwt/v5 v5.2.1 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect @@ -51,11 +53,13 @@ require ( github.com/nats-io/nuid v1.0.1 // indirect github.com/pierrec/lz4/v4 v4.1.22 // indirect github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/ryanuber/go-glob v1.0.0 // indirect golang.org/x/crypto v0.35.0 // indirect golang.org/x/net v0.35.0 // indirect golang.org/x/text v0.22.0 // indirect golang.org/x/time v0.10.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/go.sum b/go.sum index c2e63b0a..36ea531e 100644 --- a/go.sum +++ b/go.sum @@ -122,6 +122,10 @@ github.com/keybase/go-keychain v0.0.0-20231219164618-57a3676c3af6/go.mod h1:3VeW github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= @@ -156,6 +160,8 @@ github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93Ge github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/go-glob v1.0.0 h1:iQh3xXAumdQ+4Ufa5b25cRpC5TYKlno6hsv6Cb3pkBk= @@ -167,8 +173,9 @@ github.com/smira/go-statsd v1.3.4 h1:kBYWcLSGT+qC6JVbvfz48kX7mQys32fjDOPrfmsSx2c github.com/smira/go-statsd v1.3.4/go.mod h1:RjdsESPgDODtg1VpVVf9MJrEW2Hw0wtRNbmB1CAhu6A= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -296,6 +303,8 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=