Fixes pull failing to create some files/folders

go.mod

@@ -3,23 +3,14 @@ module github.com/seantis/roots
 require (
 	cloud.google.com/go v0.34.0 // indirect
 	github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f
-	github.com/codeclysm/extract v2.0.0+incompatible
 	github.com/dankinder/httpmock v0.0.0-20181129004041-3d90f378770c
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/jawher/mow.cli v1.0.4
-	github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 // indirect
-	github.com/juju/loggo v0.0.0-20180524022052-584905176618 // indirect
-	github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 // indirect
-	github.com/kr/pretty v0.1.0 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/stretchr/objx v0.1.1 // indirect
 	github.com/stretchr/testify v1.2.2
 	golang.org/x/net v0.0.0-20181207154023-610586996380 // indirect
 	golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890
 	golang.org/x/sync v0.0.0-20181108010431-42b317875d0f // indirect
 	google.golang.org/appengine v1.3.0 // indirect
-	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
-	gopkg.in/h2non/filetype.v1 v1.0.5 // indirect
-	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
-	gopkg.in/yaml.v2 v2.2.2 // indirect
 )

go.sum

@@ -2,8 +2,6 @@ cloud.google.com/go v0.34.0 h1:eOI3/cP2VTU6uZLDYAoic+eyzzB9YyGmJ7eIjl8rOPg=
 cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f h1:tbgFqBK8r77y+mT2RKkQ8ukhk/uvPtPZvr3a3166YNw=
 github.com/alexflint/go-filemutex v0.0.0-20171028004239-d358565f3c3f/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
-github.com/codeclysm/extract v2.0.0+incompatible h1:+b4WsD7YuZ5u3iW5T5TWbO764zUyEpQZSH5tZbjAxXQ=
-github.com/codeclysm/extract v2.0.0+incompatible/go.mod h1:2nhFMPHiU9At61hz+12bfrlpXSUrOnK+wR+KlGO4Uks=
 github.com/dankinder/httpmock v0.0.0-20181129004041-3d90f378770c h1:hYKEPoKfwYM16J6rED0x31S/cba6TjJrVYhUmdjbIBU=
 github.com/dankinder/httpmock v0.0.0-20181129004041-3d90f378770c/go.mod h1:MzyaGzqQVRblPUxLf82eHwghskLVzikyebSiOZX7U6Y=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -12,17 +10,6 @@ github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/jawher/mow.cli v1.0.4 h1:hKjm95J7foZ2ngT8tGb15Aq9rj751R7IUDjG+5e3cGA=
 github.com/jawher/mow.cli v1.0.4/go.mod h1:5hQj2V8g+qYmLUVWqu4Wuja1pI57M83EChYLVZ0sMKk=
-github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5 h1:rhqTjzJlm7EbkELJDKMTU7udov+Se0xZkWmugr6zGok=
-github.com/juju/errors v0.0.0-20181118221551-089d3ea4e4d5/go.mod h1:W54LbzXuIE0boCoNJfwqpmkKJ1O4TCTZMetAt6jGk7Q=
-github.com/juju/loggo v0.0.0-20180524022052-584905176618 h1:MK144iBQF9hTSwBW/9eJm034bVoG30IshVm688T2hi8=
-github.com/juju/loggo v0.0.0-20180524022052-584905176618/go.mod h1:vgyd7OREkbtVEN/8IXZe5Ooef3LQePvuBm9UWj6ZL8U=
-github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073 h1:WQM1NildKThwdP7qWrNAFGzp4ijNLw8RlgENkaI4MJs=
-github.com/juju/testing v0.0.0-20180920084828-472a3e8b2073/go.mod h1:63prj8cnj0tU0S9OHjGJn+b1h0ZghCndfnbQolrYTwA=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A=
@@ -39,12 +26,3 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk=
 google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/h2non/filetype.v1 v1.0.5 h1:CC1jjJjoEhNVbMhXYalmGBhOBK2V70Q1N850wt/98/Y=
-gopkg.in/h2non/filetype.v1 v1.0.5/go.mod h1:M0yem4rwSX5lLVrkEuRRp2/NinFMD5vgJ4DlAhZcfNo=
-gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU=
-gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
-gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

pkg/image/store.go

@@ -174,14 +174,16 @@ func (s *Store) Extract(ctx context.Context, r *Remote, dst string) error {
 
 	// process the layers in order
 	digests := make([]string, len(results))
+	dirmodes := make(map[string]os.FileMode)
+
 	for i := range results {
 		result := <-results[i]
 
 		if result.Error != nil {
 			return fmt.Errorf("error downloading %s: %v", result.Digest, result.Error)
 		}
 
-		err := untarLayer(ctx, result.Path, dst)
+		err := untarLayer(ctx, result.Path, dst, dirmodes)
 
 		if err != nil {
 			return fmt.Errorf("error extracting %s: %v", result.Path, err)
@@ -190,6 +192,11 @@ func (s *Store) Extract(ctx context.Context, r *Remote, dst string) error {
 		digests[i] = result.Digest
 	}
 
+	// set the correct permissions for all directories
+	if err := setDirectoryPermissions(dirmodes); err != nil {
+		return fmt.Errorf("error setting directory permissions: %v", err)
+	}
+
 	// record the destination in the cache
 	return s.saveLink(dst, digests)
 }

pkg/image/untar.go

@@ -11,9 +11,8 @@ import (
 	"path"
 	"path/filepath"
 	"regexp"
+	"sort"
 	"strings"
-
-	"github.com/codeclysm/extract"
 )
 
 // detect relative paths that try to escape the destination directory
@@ -25,7 +24,7 @@ type walkHandler func(*tar.Header, *tar.Reader) error
 // untarLayer takes an OCI layer and extracts it into a directory, observing
 // any whiteouts that might be specified in the layer.
 // See: https://github.com/opencontainers/image-spec/blob/master/layer.md
-func untarLayer(ctx context.Context, archive, dst string) error {
+func untarLayer(ctx context.Context, archive, dst string, dirmodes map[string]os.FileMode) error {
 	r, err := os.Open(archive)
 	if err == nil {
 		defer r.Close()
@@ -40,6 +39,11 @@ func untarLayer(ctx context.Context, archive, dst string) error {
 		return err
 	}
 
+	reset := func() {
+		r.Seek(0, 0)
+		gzr.Reset(r)
+	}
+
 	// pre-process the archive
 	err = walkTar(ctx, gzr, func(h *tar.Header, r *tar.Reader) error {
 
@@ -55,32 +59,107 @@ func untarLayer(ctx context.Context, archive, dst string) error {
 			return fmt.Errorf("refusing to extract unsafe path: %s", h.Name)
 		}
 
+		// create directory structure
+		if h.Typeflag == tar.TypeDir {
+			file := filepath.Join(dst, h.Name)
+
+			if err := os.MkdirAll(file, 0755); err != nil {
+				return fmt.Errorf("error creating directory %s: %v", file, err)
+			}
+
+			// store actual file mode of directories to set them later
+			dirmodes[file] = os.FileMode(h.Mode)
+		}
+
 		return nil
 	})
 
 	if err != nil {
 		return err
 	}
 
-	// then extract all non-whiteout files
-	r.Seek(0, 0)
-	gzr.Reset(r)
+	reset()
 
-	err = extract.Tar(ctx, gzr, dst, func(name string) string {
+	// create all regular files
+	err = walkTar(ctx, gzr, func(h *tar.Header, r *tar.Reader) error {
+
+		// skip anything but regular files
+		if h.Typeflag != tar.TypeReg {
+			return nil
+		}
 
 		// skip whiteout files
-		if isWhiteoutPath(name) {
-			return ""
+		if isWhiteoutPath(h.Name) {
+			return nil
 		}
 
-		return name
+		// remove the file if it exists
+		file := filepath.Join(dst, h.Name)
+
+		if info, err := os.Stat(file); err == nil && !info.IsDir() {
+			if err := os.Remove(file); err != nil {
+				return fmt.Errorf("error replacing %s: %v", file, err)
+			}
+		}
+
+		// copy the file
+		f, err := os.OpenFile(file, os.O_CREATE|os.O_RDWR, os.FileMode(h.Mode))
+		if err != nil {
+			return fmt.Errorf("error creating %s: %v", file, err)
+		}
+
+		if _, err := io.Copy(f, r); err != nil {
+			return fmt.Errorf("error copying %s: %v", file, err)
+		}
+
+		return f.Close()
 	})
 
 	if err != nil {
 		return err
 	}
 
-	return nil
+	reset()
+
+	// create links
+	return walkTar(ctx, gzr, func(h *tar.Header, r *tar.Reader) error {
+
+		// skip anything that isn't a link
+		if h.Typeflag != tar.TypeLink && h.Typeflag != tar.TypeSymlink {
+			return nil
+		}
+
+		new := filepath.Join(dst, h.Name)
+
+		var old string
+		if h.Linkname[0] == '.' || !strings.Contains(h.Linkname, "/") {
+			old = filepath.Join(filepath.Dir(new), h.Linkname)
+		} else {
+			old = filepath.Join(dst, h.Linkname)
+		}
+
+		// remove the link if it exists
+		if info, err := os.Lstat(new); err == nil && !info.IsDir() {
+			if err := os.Remove(new); err != nil {
+				return fmt.Errorf("error replacing %s: %v", new, err)
+			}
+		}
+
+		// create hard links
+		if h.Typeflag == tar.TypeLink {
+			if err := os.Link(old, new); err != nil {
+				return fmt.Errorf("error creating hard link %s->%s: %v", new, old, err)
+			}
+			return nil
+		}
+
+		// create symbolic links
+		if err := os.Symlink(h.Linkname, new); err != nil {
+			return fmt.Errorf("error creating symbolic link %s->%s: %v", new, old, err)
+		}
+
+		return nil
+	})
 }
 
 // walkTar takes a gzip.Reader and calls a handler function
@@ -110,6 +189,41 @@ func walkTar(ctx context.Context, gzr *gzip.Reader, handler walkHandler) error {
 	}
 }
 
+// setDirectoryPermissions takes a list of directories with file permissions
+// and applies the permissions to those files
+func setDirectoryPermissions(dirmodes map[string]os.FileMode) error {
+
+	// process directories with longer paths first, to set the permissions
+	// of children before setting the permissions of parents
+	order := make([]string, 0, len(dirmodes))
+	for path := range dirmodes {
+
+		// it's possible that certain paths do not exist anymore, if a
+		// whiteout was applied in the process
+		if info, err := os.Stat(path); os.IsNotExist(err) {
+			continue
+		} else if err != nil {
+			return fmt.Errorf("error accessing %s: %v", path, err)
+		} else if !info.IsDir() {
+			return fmt.Errorf("not a directory: %s", path)
+		}
+
+		order = append(order, path)
+	}
+
+	sort.Slice(order, func(j, k int) bool {
+		return len(order[j]) > len(order[k])
+	})
+
+	for _, path := range order {
+		if err := os.Chmod(path, dirmodes[path]); err != nil {
+			return fmt.Errorf("error setting %04o on %s: %v", dirmodes[path], path, err)
+		}
+	}
+
+	return nil
+}
+
 // applyWhiteout takes a destination and a relative whiteout path and applies it
 func applyWhiteout(dst, whiteout string) error {
 	if strings.HasSuffix(whiteout, ".wh..wh..opq") {