diff --git a/go.mod b/go.mod index c7109ac..b1e25cd 100644 --- a/go.mod +++ b/go.mod @@ -3,14 +3,9 @@ module github.com/seantis/roots require ( cloud.google.com/go v0.34.0 // indirect github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f - github.com/codeclysm/extract v2.0.0+incompatible github.com/dankinder/httpmock v0.0.0-20181129004041-3d90f378770c github.com/davecgh/go-spew v1.1.1 // indirect github.com/jawher/mow.cli v1.0.4 - github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect - github.com/juju/loggo v0.0.0-20180524022052-584905176618 // indirect - github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 // indirect - github.com/kr/pretty v0.1.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/stretchr/objx v0.1.1 // indirect github.com/stretchr/testify v1.2.2 @@ -18,8 +13,4 @@ require ( golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect google.golang.org/appengine v1.3.0 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect - gopkg.in/h2non/filetype.v1 v1.0.5 // indirect - gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect - gopkg.in/yaml.v2 v2.2.2 // indirect ) diff --git a/go.sum b/go.sum index 196d25f..741ce74 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,6 @@ cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f h1:tbgFqBK8r77y+mT2RKkQ8ukhk/uvPtPZvr3a3166YNw= github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0= -github.com/codeclysm/extract v2.0.0+incompatible h1:+b4WsD7YuZ5u3iW5T5TWbO764zUyEpQZSH5tZbjAxXQ= -github.com/codeclysm/extract v2.0.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks= github.com/dankinder/httpmock v0.0.0-20181129004041-3d90f378770c h1:hYKEPoKfwYM16J6rED0x31S/cba6TjJrVYhUmdjbIBU= github.com/dankinder/httpmock v0.0.0-20181129004041-3d90f378770c/go.mod h1:MzyaGzqQVRblPUxLf82eHwghskLVzikyebSiOZX7U6Y= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -12,17 +10,6 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/jawher/mow.cli v1.0.4 h1:hKjm95J7foZ2ngT8tGb15Aq9rj751R7IUDjG+5e3cGA= github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok= -github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q= -github.com/juju/loggo v0.0.0-20180524022052-584905176618 h1:MK144iBQF9hTSwBW/9eJm034bVoG30IshVm688T2hi8= -github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 h1:WQM1NildKThwdP7qWrNAFGzp4ijNLw8RlgENkaI4MJs= -github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= @@ -39,12 +26,3 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y= -gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= -gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/pkg/image/store.go b/pkg/image/store.go index 19f3654..347c872 100644 --- a/pkg/image/store.go +++ b/pkg/image/store.go @@ -174,6 +174,8 @@ func (s *Store) Extract(ctx context.Context, r *Remote, dst string) error { // process the layers in order digests := make([]string, len(results)) + dirmodes := make(map[string]os.FileMode) + for i := range results { result := <-results[i] @@ -181,7 +183,7 @@ func (s *Store) Extract(ctx context.Context, r *Remote, dst string) error { return fmt.Errorf("error downloading %s: %v", result.Digest, result.Error) } - err := untarLayer(ctx, result.Path, dst) + err := untarLayer(ctx, result.Path, dst, dirmodes) if err != nil { return fmt.Errorf("error extracting %s: %v", result.Path, err) @@ -190,6 +192,11 @@ func (s *Store) Extract(ctx context.Context, r *Remote, dst string) error { digests[i] = result.Digest } + // set the correct permissions for all directories + if err := setDirectoryPermissions(dirmodes); err != nil { + return fmt.Errorf("error setting directory permissions: %v", err) + } + // record the destination in the cache return s.saveLink(dst, digests) } diff --git a/pkg/image/untar.go b/pkg/image/untar.go index b506de3..e7217c2 100644 --- a/pkg/image/untar.go +++ b/pkg/image/untar.go @@ -11,9 +11,8 @@ import ( "path" "path/filepath" "regexp" + "sort" "strings" - - "github.com/codeclysm/extract" ) // detect relative paths that try to escape the destination directory @@ -25,7 +24,7 @@ type walkHandler func(*tar.Header, *tar.Reader) error // untarLayer takes an OCI layer and extracts it into a directory, observing // any whiteouts that might be specified in the layer. // See: https://github.com/opencontainers/image-spec/blob/master/layer.md -func untarLayer(ctx context.Context, archive, dst string) error { +func untarLayer(ctx context.Context, archive, dst string, dirmodes map[string]os.FileMode) error { r, err := os.Open(archive) if err == nil { defer r.Close() @@ -40,6 +39,11 @@ func untarLayer(ctx context.Context, archive, dst string) error { return err } + reset := func() { + r.Seek(0, 0) + gzr.Reset(r) + } + // pre-process the archive err = walkTar(ctx, gzr, func(h *tar.Header, r *tar.Reader) error { @@ -55,6 +59,18 @@ func untarLayer(ctx context.Context, archive, dst string) error { return fmt.Errorf("refusing to extract unsafe path: %s", h.Name) } + // create directory structure + if h.Typeflag == tar.TypeDir { + file := filepath.Join(dst, h.Name) + + if err := os.MkdirAll(file, 0755); err != nil { + return fmt.Errorf("error creating directory %s: %v", file, err) + } + + // store actual file mode of directories to set them later + dirmodes[file] = os.FileMode(h.Mode) + } + return nil }) @@ -62,25 +78,88 @@ func untarLayer(ctx context.Context, archive, dst string) error { return err } - // then extract all non-whiteout files - r.Seek(0, 0) - gzr.Reset(r) + reset() - err = extract.Tar(ctx, gzr, dst, func(name string) string { + // create all regular files + err = walkTar(ctx, gzr, func(h *tar.Header, r *tar.Reader) error { + + // skip anything but regular files + if h.Typeflag != tar.TypeReg { + return nil + } // skip whiteout files - if isWhiteoutPath(name) { - return "" + if isWhiteoutPath(h.Name) { + return nil } - return name + // remove the file if it exists + file := filepath.Join(dst, h.Name) + + if info, err := os.Stat(file); err == nil && !info.IsDir() { + if err := os.Remove(file); err != nil { + return fmt.Errorf("error replacing %s: %v", file, err) + } + } + + // copy the file + f, err := os.OpenFile(file, os.O_CREATE|os.O_RDWR, os.FileMode(h.Mode)) + if err != nil { + return fmt.Errorf("error creating %s: %v", file, err) + } + + if _, err := io.Copy(f, r); err != nil { + return fmt.Errorf("error copying %s: %v", file, err) + } + + return f.Close() }) if err != nil { return err } - return nil + reset() + + // create links + return walkTar(ctx, gzr, func(h *tar.Header, r *tar.Reader) error { + + // skip anything that isn't a link + if h.Typeflag != tar.TypeLink && h.Typeflag != tar.TypeSymlink { + return nil + } + + new := filepath.Join(dst, h.Name) + + var old string + if h.Linkname[0] == '.' || !strings.Contains(h.Linkname, "/") { + old = filepath.Join(filepath.Dir(new), h.Linkname) + } else { + old = filepath.Join(dst, h.Linkname) + } + + // remove the link if it exists + if info, err := os.Lstat(new); err == nil && !info.IsDir() { + if err := os.Remove(new); err != nil { + return fmt.Errorf("error replacing %s: %v", new, err) + } + } + + // create hard links + if h.Typeflag == tar.TypeLink { + if err := os.Link(old, new); err != nil { + return fmt.Errorf("error creating hard link %s->%s: %v", new, old, err) + } + return nil + } + + // create symbolic links + if err := os.Symlink(h.Linkname, new); err != nil { + return fmt.Errorf("error creating symbolic link %s->%s: %v", new, old, err) + } + + return nil + }) } // walkTar takes a gzip.Reader and calls a handler function @@ -110,6 +189,41 @@ func walkTar(ctx context.Context, gzr *gzip.Reader, handler walkHandler) error { } } +// setDirectoryPermissions takes a list of directories with file permissions +// and applies the permissions to those files +func setDirectoryPermissions(dirmodes map[string]os.FileMode) error { + + // process directories with longer paths first, to set the permissions + // of children before setting the permissions of parents + order := make([]string, 0, len(dirmodes)) + for path := range dirmodes { + + // it's possible that certain paths do not exist anymore, if a + // whiteout was applied in the process + if info, err := os.Stat(path); os.IsNotExist(err) { + continue + } else if err != nil { + return fmt.Errorf("error accessing %s: %v", path, err) + } else if !info.IsDir() { + return fmt.Errorf("not a directory: %s", path) + } + + order = append(order, path) + } + + sort.Slice(order, func(j, k int) bool { + return len(order[j]) > len(order[k]) + }) + + for _, path := range order { + if err := os.Chmod(path, dirmodes[path]); err != nil { + return fmt.Errorf("error setting %04o on %s: %v", dirmodes[path], path, err) + } + } + + return nil +} + // applyWhiteout takes a destination and a relative whiteout path and applies it func applyWhiteout(dst, whiteout string) error { if strings.HasSuffix(whiteout, ".wh..wh..opq") {