Skip to content

Commit ed7a259

Browse files
rchincharchincha
committed
feat: add support for erofs layers
atomfs has added support for additional filesystem types such as erofs. Signed-off-by: Ramkumar Chinchani <[email protected]> Signed-off-by: Ramkumar Chinchani <[email protected]>
1 parent c34ba3a commit ed7a259

16 files changed

+230
-44
lines changed

‎.github/workflows/build.yaml

+1
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,7 @@ jobs:
6767
- name: install dependencies
6868
run: |
6969
./install-build-deps.sh
70+
sudo apt-get install -y linux-modules-extra-$(uname -r)
7071
echo "running kernel is: $(uname -a)"
7172
- name: docker-clone
7273
run: |

‎.github/workflows/coverage.yaml

+2-1
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ on:
3232

3333
jobs:
3434
build:
35-
runs-on: ubuntu-22.04
35+
runs-on: ubuntu-24.04
3636
services:
3737
registry:
3838
image: ghcr.io/project-stacker/registry:2
@@ -68,6 +68,7 @@ jobs:
6868
- name: install dependencies
6969
run: |
7070
./install-build-deps.sh
71+
sudo apt-get install -y linux-modules-extra-$(uname -r)
7172
echo "running kernel is: $(uname -a)"
7273
- name: docker-clone
7374
run: |

‎cmd/stacker/build.go

+5-5
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"fmt"
55

66
cli "github.com/urfave/cli/v2"
7-
"machinerun.io/atomfs/squashfs"
7+
"machinerun.io/atomfs/pkg/verity"
88
"stackerbuild.io/stacker/pkg/stacker"
99
"stackerbuild.io/stacker/pkg/types"
1010
)
@@ -52,12 +52,12 @@ func initCommonBuildFlags() []cli.Flag {
5252
},
5353
&cli.StringSliceFlag{
5454
Name: "layer-type",
55-
Usage: "set the output layer type (supported values: tar, squashfs); can be supplied multiple times",
55+
Usage: "set the output layer type (supported values: tar, squashfs, erofs); can be supplied multiple times",
5656
Value: cli.NewStringSlice("tar"),
5757
},
5858
&cli.BoolFlag{
59-
Name: "no-squashfs-verity",
60-
Usage: "do not append dm-verity data to squashfs archives",
59+
Name: "no-verity",
60+
Usage: "do not append dm-verity data to fs archives",
6161
},
6262
&cli.BoolFlag{
6363
Name: "require-hash",
@@ -103,7 +103,7 @@ func newBuildArgs(ctx *cli.Context) (stacker.BuildArgs, error) {
103103
AnnotationsNamespace: ctx.String("annotations-namespace"),
104104
}
105105
var err error
106-
verity := squashfs.VerityMetadata(!ctx.Bool("no-squashfs-verity"))
106+
verity := verity.VerityMetadata(!ctx.Bool("no-verity"))
107107
args.LayerTypes, err = types.NewLayerTypes(ctx.StringSlice("layer-type"), verity)
108108
return args, err
109109
}

‎cmd/stacker/inspect.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ import (
1111
"github.com/opencontainers/umoci/oci/casext"
1212
"github.com/pkg/errors"
1313
cli "github.com/urfave/cli/v2"
14-
stackeroci "machinerun.io/atomfs/oci"
14+
stackeroci "machinerun.io/atomfs/pkg/oci"
1515
)
1616

1717
var inspectCmd = cli.Command{

‎cmd/stacker/internal_go.go

+4-4
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ import (
99
"github.com/pkg/errors"
1010
cli "github.com/urfave/cli/v2"
1111
"golang.org/x/sys/unix"
12-
"machinerun.io/atomfs"
12+
"machinerun.io/atomfs/pkg/molecule"
1313
"stackerbuild.io/stacker/pkg/lib"
1414
"stackerbuild.io/stacker/pkg/log"
1515
"stackerbuild.io/stacker/pkg/overlay"
@@ -176,14 +176,14 @@ func doAtomfsMount(ctx *cli.Context) error {
176176
tag := ctx.Args().Get(0)
177177
mountpoint := ctx.Args().Get(1)
178178

179-
opts := atomfs.MountOCIOpts{
179+
opts := molecule.MountOCIOpts{
180180
OCIDir: config.OCIDir,
181181
Tag: tag,
182182
Target: mountpoint,
183183
AllowMissingVerityData: true,
184184
}
185185

186-
mol, err := atomfs.BuildMoleculeFromOCI(opts)
186+
mol, err := molecule.BuildMoleculeFromOCI(opts)
187187
if err != nil {
188188
return err
189189
}
@@ -199,5 +199,5 @@ func doAtomfsUmount(ctx *cli.Context) error {
199199
}
200200

201201
mountpoint := ctx.Args().Get(0)
202-
return atomfs.Umount(mountpoint)
202+
return molecule.Umount(mountpoint)
203203
}

‎cmd/stacker/publish.go

+3-3
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@ package main
33
import (
44
"github.com/pkg/errors"
55
cli "github.com/urfave/cli/v2"
6-
"machinerun.io/atomfs/squashfs"
6+
"machinerun.io/atomfs/pkg/verity"
77
"stackerbuild.io/stacker/pkg/lib"
88
"stackerbuild.io/stacker/pkg/stacker"
99
"stackerbuild.io/stacker/pkg/types"
@@ -69,7 +69,7 @@ var publishCmd = cli.Command{
6969
},
7070
&cli.StringSliceFlag{
7171
Name: "layer-type",
72-
Usage: "set the output layer type (supported values: tar, squashfs); can be supplied multiple times",
72+
Usage: "set the output layer type (supported values: tar, squashfs, erofs); can be supplied multiple times",
7373
Value: cli.NewStringSlice("tar"),
7474
},
7575
&cli.StringSliceFlag{
@@ -108,7 +108,7 @@ func beforePublish(ctx *cli.Context) error {
108108
}
109109

110110
func doPublish(ctx *cli.Context) error {
111-
verity := squashfs.VerityMetadata(!ctx.Bool("no-squashfs-verity"))
111+
verity := verity.VerityMetadata(!ctx.Bool("no-verity"))
112112
layerTypes, err := types.NewLayerTypes(ctx.StringSlice("layer-type"), verity)
113113
if err != nil {
114114
return err

‎cmd/stacker/validate.go

+2
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,8 @@ func validateLayerTypeFlags(ctx *cli.Context) error {
5050
break
5151
case "squashfs":
5252
break
53+
case "erofs":
54+
break
5355
default:
5456
return errors.Errorf("unknown layer type: %s", layerType)
5557
}

‎go.mod

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ require (
1919
github.com/mitchellh/hashstructure v1.1.0
2020
github.com/moby/buildkit v0.11.4
2121
github.com/opencontainers/go-digest v1.0.0
22-
github.com/opencontainers/image-spec v1.1.0-rc4
22+
github.com/opencontainers/image-spec v1.1.0
2323
github.com/opencontainers/umoci v0.4.8-0.20220412065115-12453f247749
2424
github.com/pkg/errors v0.9.1
2525
github.com/pkg/xattr v0.4.9
@@ -290,5 +290,6 @@ require (
290290

291291
replace (
292292
github.com/opencontainers/umoci => github.com/project-stacker/umoci v0.0.0-20240906174318-e9397ba4ced0
293+
machinerun.io/atomfs => github.com/rchincha/atomfs v0.0.0-20250216002037-e582f37114f4
293294
stackerbuild.io/stacker-bom => github.com/project-stacker/stacker-bom v0.0.0-20240509203427-4d685e046780
294295
)

‎go.sum

+4-4
Original file line numberDiff line numberDiff line change
@@ -754,8 +754,8 @@ github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3ev
754754
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
755755
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
756756
github.com/opencontainers/image-spec v1.0.2/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
757-
github.com/opencontainers/image-spec v1.1.0-rc4 h1:oOxKUJWnFC4YGHCCMNql1x4YaDfYBTS5Y4x/Cgeo1E0=
758-
github.com/opencontainers/image-spec v1.1.0-rc4/go.mod h1:X4pATf0uXsnn3g5aiGIsVnJBR4mxhKzfwmvK/B2NTm8=
757+
github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
758+
github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
759759
github.com/opencontainers/runc v1.2.3 h1:fxE7amCzfZflJO2lHXf4y/y8M1BoAqp+FVmG19oYB80=
760760
github.com/opencontainers/runc v1.2.3/go.mod h1:nSxcWUydXrsBZVYNSkTjoQ/N6rcyTtn+1SD5D4+kRIM=
761761
github.com/opencontainers/runtime-spec v1.2.0 h1:z97+pHb3uELt/yiAWD691HNHQIF07bE7dzrbT927iTk=
@@ -821,6 +821,8 @@ github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+Gx
821821
github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA=
822822
github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
823823
github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
824+
github.com/rchincha/atomfs v0.0.0-20250216002037-e582f37114f4 h1:3V1d2yzgSHqMwe5CQnFV/N6Zej1qpQV1OrQpnmTsmv4=
825+
github.com/rchincha/atomfs v0.0.0-20250216002037-e582f37114f4/go.mod h1:jrGbuGXiCPi4LFoMvcPRnqPGlxe3pW8UtLEnhUcGRmI=
824826
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
825827
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
826828
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -1598,8 +1600,6 @@ honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWh
15981600
honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
15991601
honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
16001602
honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
1601-
machinerun.io/atomfs v1.1.3 h1:oV1SH7VI2MqAks7FlirhLLKvyVcJkMB0NFevXF8EJaU=
1602-
machinerun.io/atomfs v1.1.3/go.mod h1:qXz4epm3/7vEpEyf9YaTCafp3CwbUeDa1XrYyx7qbPc=
16031603
modernc.org/libc v1.37.6 h1:orZH3c5wmhIQFTXF+Nt+eeauyd+ZIt2BX6ARe+kD+aw=
16041604
modernc.org/libc v1.37.6/go.mod h1:YAXkAZ8ktnkCKaN9sw/UDeUVkGYJ/YquGO4FTi5nmHE=
16051605
modernc.org/mathutil v1.6.0 h1:fRe9+AmYlaej+64JsEEhoWuAYBkOtQiMEU7n/XgfYi4=

‎install-build-deps.sh

+2
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,8 @@ installdeps_ubuntu() {
4444
squashfuse
4545
libarchive-tools
4646
shellcheck
47+
erofs-utils
48+
erofsfuse
4749
)
4850

4951
case "$VERSION_ID" in

‎pkg/lib/image_test.go

+3-2
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,8 @@ import (
1313
"github.com/opencontainers/umoci/mutate"
1414
"github.com/opencontainers/umoci/oci/casext"
1515
"github.com/stretchr/testify/assert"
16-
"machinerun.io/atomfs/squashfs"
16+
"machinerun.io/atomfs/pkg/squashfs"
17+
"machinerun.io/atomfs/pkg/verity"
1718
)
1819

1920
func createImage(dir string, tag string) error {
@@ -48,7 +49,7 @@ func createImage(dir string, tag string) error {
4849

4950
// need *something* in the layer, why not just recursively include the
5051
// OCI image for maximum confusion :)
51-
layer, mediaType, _, err := squashfs.MakeSquashfs(dir, path.Join(dir, "oci"), nil, squashfs.VerityMetadataMissing)
52+
layer, mediaType, _, err := squashfs.MakeSquashfs(dir, path.Join(dir, "oci"), nil, verity.VerityMetadataMissing)
5253
if err != nil {
5354
return err
5455
}

‎pkg/overlay/metadata.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ import (
1010
ispec "github.com/opencontainers/image-spec/specs-go/v1"
1111
"github.com/opencontainers/umoci/oci/casext"
1212
"github.com/pkg/errors"
13-
stackeroci "machinerun.io/atomfs/oci"
13+
stackeroci "machinerun.io/atomfs/pkg/oci"
1414
"stackerbuild.io/stacker/pkg/log"
1515
"stackerbuild.io/stacker/pkg/types"
1616
)

‎pkg/overlay/pack.go

+10-7
Original file line numberDiff line numberDiff line change
@@ -22,8 +22,9 @@ import (
2222
"github.com/opencontainers/umoci/oci/layer"
2323
"github.com/pkg/errors"
2424
"github.com/pkg/xattr"
25-
stackeroci "machinerun.io/atomfs/oci"
26-
"machinerun.io/atomfs/squashfs"
25+
stackerfs "machinerun.io/atomfs/pkg/fs"
26+
stackeroci "machinerun.io/atomfs/pkg/oci"
27+
"machinerun.io/atomfs/pkg/verity"
2728
"stackerbuild.io/stacker/pkg/lib"
2829
"stackerbuild.io/stacker/pkg/log"
2930
"stackerbuild.io/stacker/pkg/storage"
@@ -280,7 +281,8 @@ func generateBlob(layerType types.LayerType, contents string, ociDir string, low
280281
blob = layer.GenerateInsertLayer(contents, "/", false, &packOptions)
281282
mediaType = ispec.MediaTypeImageLayer
282283
} else {
283-
blob, mediaType, rootHash, err = squashfs.MakeSquashfs(ociDir, contents, nil, layerType.Verity)
284+
fsi := stackerfs.New(stackerfs.FilesystemType(layerType.Type))
285+
blob, mediaType, rootHash, err = fsi.Make(ociDir, contents, nil, layerType.Verity)
284286
if err != nil {
285287
return nil, "", "", err
286288
}
@@ -303,7 +305,7 @@ func ociPutBlob(blob io.ReadCloser, config types.StackerConfig, layerMediaType s
303305

304306
annotations := map[string]string{}
305307
if rootHash != "" {
306-
annotations[squashfs.VerityRootHashAnnotation] = rootHash
308+
annotations[verity.VerityRootHashAnnotation] = rootHash
307309
}
308310

309311
desc := ispec.Descriptor{
@@ -443,7 +445,7 @@ func generateLayer(config types.StackerConfig, _ casext.Engine, mutators []*muta
443445
} else {
444446
annotations := map[string]string{}
445447
if rootHash != "" {
446-
annotations[squashfs.VerityRootHashAnnotation] = rootHash
448+
annotations[verity.VerityRootHashAnnotation] = rootHash
447449
}
448450
desc, err = mutator.Add(context.Background(), mediaType, blob, history, mutate.NoopCompressor, annotations)
449451
if err != nil {
@@ -693,10 +695,11 @@ func unpackOne(l ispec.Descriptor, ociDir string, extractDir string) error {
693695
return nil
694696
}
695697

696-
if squashfs.IsSquashfsMediaType(l.MediaType) {
697-
return squashfs.ExtractSingleSquash(
698+
if fsi := stackerfs.NewFromMediaType(l.MediaType); fsi != nil {
699+
return fsi.ExtractSingle(
698700
path.Join(ociDir, "blobs", "sha256", l.Digest.Encoded()), extractDir)
699701
}
702+
700703
switch l.MediaType {
701704
case ispec.MediaTypeImageLayer, ispec.MediaTypeImageLayerGzip:
702705
tarEx.Lock()

‎pkg/types/layer_type.go

+24-13
Original file line numberDiff line numberDiff line change
@@ -7,14 +7,16 @@ import (
77

88
ispec "github.com/opencontainers/image-spec/specs-go/v1"
99
"github.com/pkg/errors"
10-
"machinerun.io/atomfs/squashfs"
10+
"machinerun.io/atomfs/pkg/erofs"
11+
"machinerun.io/atomfs/pkg/squashfs"
12+
"machinerun.io/atomfs/pkg/verity"
1113
)
1214

1315
var ErrEmptyLayers = errors.New("empty layers")
1416

1517
type LayerType struct {
1618
Type string
17-
Verity squashfs.VerityMetadata
19+
Verity verity.VerityMetadata
1820
}
1921

2022
func (lt LayerType) String() string {
@@ -44,15 +46,17 @@ func (lt *LayerType) UnmarshalText(text []byte) error {
4446
return errors.Wrapf(err, "bad verity bool: %s", fields[1])
4547
}
4648

47-
lt.Verity = squashfs.VerityMetadata(result)
49+
lt.Verity = verity.VerityMetadata(result)
4850

4951
return nil
5052
}
5153

52-
func NewLayerType(lt string, verity squashfs.VerityMetadata) (LayerType, error) {
54+
func NewLayerType(lt string, verity verity.VerityMetadata) (LayerType, error) {
5355
switch lt {
5456
case "squashfs":
5557
return LayerType{Type: lt, Verity: verity}, nil
58+
case "erofs":
59+
return LayerType{Type: lt, Verity: verity}, nil
5660
case "tar":
5761
return LayerType{Type: lt}, nil
5862
default:
@@ -62,31 +66,38 @@ func NewLayerType(lt string, verity squashfs.VerityMetadata) (LayerType, error)
6266

6367
func NewLayerTypeManifest(manifest ispec.Manifest) (LayerType, error) {
6468
if len(manifest.Layers) == 0 {
65-
return NewLayerType("tar", squashfs.VerityMetadataMissing)
69+
return NewLayerType("tar", verity.VerityMetadataMissing)
6670
}
6771

72+
_, verityMetadataPresent := manifest.Layers[0].Annotations[verity.VerityRootHashAnnotation]
73+
6874
switch manifest.Layers[0].MediaType {
6975
case squashfs.BaseMediaTypeLayerSquashfs:
7076
// older stackers generated media types without compression information
7177
fallthrough
72-
case squashfs.GenerateSquashfsMediaType(squashfs.GzipCompression, squashfs.VerityMetadataMissing):
78+
case squashfs.GenerateSquashfsMediaType(squashfs.GzipCompression):
79+
fallthrough
80+
case squashfs.GenerateSquashfsMediaType(squashfs.ZstdCompression):
81+
return NewLayerType("squashfs", verity.VerityMetadata(verityMetadataPresent))
82+
case erofs.BaseMediaTypeLayerErofs:
83+
// older stackers generated media types without compression information
84+
fallthrough
85+
case erofs.GenerateErofsMediaType(erofs.LZ4HCCompression):
7386
fallthrough
74-
case squashfs.GenerateSquashfsMediaType(squashfs.ZstdCompression, squashfs.VerityMetadataMissing):
75-
return NewLayerType("squashfs", squashfs.VerityMetadataMissing)
76-
case squashfs.GenerateSquashfsMediaType(squashfs.GzipCompression, squashfs.VerityMetadataPresent):
87+
case erofs.GenerateErofsMediaType(erofs.LZ4Compression):
7788
fallthrough
78-
case squashfs.GenerateSquashfsMediaType(squashfs.ZstdCompression, squashfs.VerityMetadataPresent):
79-
return NewLayerType("squashfs", squashfs.VerityMetadataPresent)
89+
case erofs.GenerateErofsMediaType(erofs.ZstdCompression):
90+
return NewLayerType("erofs", verity.VerityMetadata(verityMetadataPresent))
8091
case ispec.MediaTypeImageLayerGzip:
8192
fallthrough
8293
case ispec.MediaTypeImageLayer:
83-
return NewLayerType("tar", squashfs.VerityMetadataMissing)
94+
return NewLayerType("tar", verity.VerityMetadataMissing)
8495
default:
8596
return LayerType{}, errors.Errorf("invalid layer type %s", manifest.Layers[0].MediaType)
8697
}
8798
}
8899

89-
func NewLayerTypes(lts []string, verity squashfs.VerityMetadata) ([]LayerType, error) {
100+
func NewLayerTypes(lts []string, verity verity.VerityMetadata) ([]LayerType, error) {
90101
ret := []LayerType{}
91102
for _, lt := range lts {
92103
hoisted, err := NewLayerType(lt, verity)

0 commit comments

Comments
 (0)