feat(erofs): initial commit for erofs support

Fixes opencontainers/image-spec#1190

Signed-off-by: Ramkumar Chinchani <[email protected]>

Author: rchincha

pkg/erofs/erofs.go

@@ -0,0 +1,180 @@
+package erofs
+
+import (
+	"bytes"
+	"io"
+	"os"
+	"os/exec"
+	"path"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+)
+
+var checkZstdSupported sync.Once
+var zstdIsSuspported bool
+
+// ExcludePaths represents a list of paths to exclude in a erofs listing.
+// Users should do something like filepath.Walk() over the whole filesystem,
+// calling AddExclude() or AddInclude() based on whether they want to include
+// or exclude a particular file. Note that if e.g. /usr is excluded, then
+// everyting underneath is also implicitly excluded. The
+// AddExclude()/AddInclude() methods do the math to figure out what is the
+// correct set of things to exclude or include based on what paths have been
+// previously included or excluded.
+type ExcludePaths struct {
+	exclude map[string]bool
+	include []string
+}
+
+func NewExcludePaths() *ExcludePaths {
+	return &ExcludePaths{
+		exclude: map[string]bool{},
+		include: []string{},
+	}
+}
+
+func (eps *ExcludePaths) AddExclude(p string) {
+	for _, inc := range eps.include {
+		// If /usr/bin/ls has changed but /usr hasn't, we don't want to list
+		// /usr in the include paths any more, so let's be sure to only
+		// add things which aren't prefixes.
+		if strings.HasPrefix(inc, p) {
+			return
+		}
+	}
+	eps.exclude[p] = true
+}
+
+func (eps *ExcludePaths) AddInclude(orig string, isDir bool) {
+	// First, remove this thing and all its parents from exclude.
+	p := orig
+
+	// normalize to the first dir
+	if !isDir {
+		p = path.Dir(p)
+	}
+	for {
+		// our paths are all absolute, so this is a base case
+		if p == "/" {
+			break
+		}
+
+		delete(eps.exclude, p)
+		p = filepath.Dir(p)
+	}
+
+	// now add it to the list of includes, so we don't accidentally re-add
+	// anything above.
+	eps.include = append(eps.include, orig)
+}
+
+func (eps *ExcludePaths) String() (string, error) {
+	var buf bytes.Buffer
+	for p := range eps.exclude {
+		_, err := buf.WriteString(p)
+		if err != nil {
+			return "", err
+		}
+		_, err = buf.WriteString("\n")
+		if err != nil {
+			return "", err
+		}
+	}
+
+	_, err := buf.WriteString("\n")
+	if err != nil {
+		return "", err
+	}
+
+	return buf.String(), nil
+}
+
+func MakeErofs(tempdir string, rootfs string, eps *ExcludePaths, verity VerityMetadata) (io.ReadCloser, string, string, error) {
+	var excludesFile string
+	var err error
+	var toExclude string
+	var rootHash string
+
+	if eps != nil {
+		toExclude, err = eps.String()
+		if err != nil {
+			return nil, "", rootHash, errors.Wrapf(err, "couldn't create exclude path list")
+		}
+	}
+
+	if len(toExclude) != 0 {
+		excludes, err := os.CreateTemp(tempdir, "stacker-erofs-exclude-")
+		if err != nil {
+			return nil, "", rootHash, err
+		}
+		defer os.Remove(excludes.Name())
+
+		excludesFile = excludes.Name()
+		_, err = excludes.WriteString(toExclude)
+		excludes.Close()
+		if err != nil {
+			return nil, "", rootHash, err
+		}
+	}
+
+	tmpErofs, err := os.CreateTemp(tempdir, "stacker-erofs-img-")
+	if err != nil {
+		return nil, "", rootHash, err
+	}
+	tmpErofs.Close()
+	os.Remove(tmpErofs.Name())
+	defer os.Remove(tmpErofs.Name())
+	args := []string{rootfs, tmpErofs.Name()}
+	compression := GzipCompression
+	if mkerofsSupportsZstd() {
+		args = append(args, "-z", "zstd")
+		compression = ZstdCompression
+	}
+	if len(toExclude) != 0 {
+		args = append(args, "--exclude-path", excludesFile)
+	}
+	cmd := exec.Command("mkfs.erofs", args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err = cmd.Run(); err != nil {
+		return nil, "", rootHash, errors.Wrap(err, "couldn't build erofs")
+	}
+
+	if verity {
+		rootHash, err = appendVerityData(tmpErofs.Name())
+		if err != nil {
+			return nil, "", rootHash, err
+		}
+	}
+
+	blob, err := os.Open(tmpErofs.Name())
+	if err != nil {
+		return nil, "", rootHash, errors.WithStack(err)
+	}
+
+	return blob, GenerateErofsMediaType(compression, verity), rootHash, nil
+}
+
+func mkerofsSupportsZstd() bool {
+	checkZstdSupported.Do(func() {
+		var stdoutBuffer strings.Builder
+		var stderrBuffer strings.Builder
+
+		cmd := exec.Command("mkfs.erofs", "--help")
+		cmd.Stdout = &stdoutBuffer
+		cmd.Stderr = &stderrBuffer
+
+		// Ignore errs here as `mkerofs --help` exit status code is 1
+		_ = cmd.Run()
+
+		if strings.Contains(stdoutBuffer.String(), "zstd") ||
+			strings.Contains(stderrBuffer.String(), "zstd") {
+			zstdIsSuspported = true
+		}
+	})
+
+	return zstdIsSuspported
+}

pkg/erofs/mediatype.go

@@ -0,0 +1,37 @@
+package erofs
+
+import (
+	"fmt"
+	"strings"
+)
+
+type ErofsCompression string
+type VerityMetadata bool
+
+const (
+	BaseMediaTypeLayerErofs = "application/vnd.stacker.image.layer.erofs"
+
+	GzipCompression ErofsCompression = "gzip"
+	ZstdCompression ErofsCompression = "zstd"
+
+	veritySuffix = "verity"
+
+	VerityMetadataPresent VerityMetadata = true
+	VerityMetadataMissing VerityMetadata = false
+)
+
+func IsErofsMediaType(mediaType string) bool {
+	return strings.HasPrefix(mediaType, BaseMediaTypeLayerErofs)
+}
+
+func GenerateErofsMediaType(comp ErofsCompression, verity VerityMetadata) string {
+	verityString := ""
+	if verity {
+		verityString = fmt.Sprintf("+%s", veritySuffix)
+	}
+	return fmt.Sprintf("%s+%s%s", BaseMediaTypeLayerErofs, comp, verityString)
+}
+
+func HasVerityMetadata(mediaType string) VerityMetadata {
+	return VerityMetadata(strings.HasSuffix(mediaType, veritySuffix))
+}