Skip to content

Commit 4de29a0

Browse files
mangelajomangelajo
committed
Support overlayfs whiteouts on checkout
Introduces an intermediate format for overlayfs storage, where .wh-ostree. prefixed files will be converted into char 0:0 whiteout devices used by overlayfs to mark deletions across layers. The CI scripts now uses a volume for the scratch directories previously in /var/tmp otherwise we cannot create whiteout devices into an overlayfs mounted filesystem. Related-Issue: #2712
1 parent c6c3c5a commit 4de29a0

File tree

13 files changed

+283
-9
lines changed

13 files changed

+283
-9
lines changed

‎.github/workflows/tests.yml

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -172,8 +172,14 @@ jobs:
172172
# An empty string isn't valid, so a dummy --label option is always
173173
# added.
174174
options: --label ostree ${{ matrix.container-options }}
175+
# make sure tests are performed on a non-overlayfs filesystem
176+
volumes:
177+
- tmp_dir:/test-tmp
178+
env:
179+
TEST_TMPDIR: /test-tmp
175180

176181
steps:
182+
177183
- name: Pre-checkout setup
178184
run: ${{ matrix.pre-checkout-setup }}
179185
if: ${{ matrix.pre-checkout-setup }}
@@ -187,7 +193,7 @@ jobs:
187193
run: ./ci/gh-install.sh ${{ matrix.extra-packages }}
188194

189195
- name: Add non-root user
190-
run: "useradd builder && chown -R -h builder: ."
196+
run: "useradd builder && chown -R -h builder: . $TEST_TMPDIR"
191197

192198
- name: Build and test
193199
run: runuser -u builder -- ./ci/gh-build.sh ${{ matrix.configure-options }}

‎Makefile-tests.am

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,7 @@ _installed_or_uninstalled_test_scripts = \
107107
tests/test-admin-deploy-nomerge.sh \
108108
tests/test-admin-deploy-none.sh \
109109
tests/test-admin-deploy-bootid-gc.sh \
110+
tests/test-admin-deploy-whiteouts.sh \
110111
tests/test-osupdate-dtb.sh \
111112
tests/test-admin-instutil-set-kargs.sh \
112113
tests/test-admin-upgrade-not-backwards.sh \

‎bash/ostree

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -249,6 +249,7 @@ _ostree_checkout() {
249249
--union-identical
250250
--user-mode -U
251251
--whiteouts
252+
--process-passthrough-whiteouts
252253
"
253254

254255
local options_with_args="

‎man/ostree-checkout.xml

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,17 @@ License along with this library. If not, see <https://www.gnu.org/licenses/>.
114114
</para></listitem>
115115
</varlistentry>
116116

117+
<varlistentry>
118+
<term><option>--process-passthrough-whiteouts</option></term>
119+
120+
<listitem><para>
121+
Enable overlayfs whiteout extraction into 0:0 character devices.
122+
Overlayfs whiteouts are encoded inside ostree as <literal>.ostree-wh.filename</literal>
123+
and extracted as 0:0 character devices. This is useful to carry
124+
container storage embedded into ostree.
125+
</para></listitem>
126+
</varlistentry>
127+
117128
<varlistentry>
118129
<term><option>--allow-noent</option></term>
119130

‎src/libostree/ostree-repo-checkout.c

Lines changed: 119 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@
3535
#define WHITEOUT_PREFIX ".wh."
3636
#define OPAQUE_WHITEOUT_NAME ".wh..wh..opq"
3737

38+
#define OVERLAYFS_WHITEOUT_PREFIX ".ostree-wh."
39+
3840
/* Per-checkout call state/caching */
3941
typedef struct {
4042
GString *path_buf; /* buffer for real path if filtering enabled */
@@ -582,6 +584,108 @@ checkout_file_hardlink (OstreeRepo *self,
582584
return TRUE;
583585
}
584586

587+
static gboolean
588+
_checkout_overlayfs_whiteout_at_no_overwrite (OstreeRepoCheckoutAtOptions *options,
589+
int destination_dfd,
590+
const char *destination_name,
591+
GFileInfo *file_info,
592+
GVariant *xattrs,
593+
GCancellable *cancellable,
594+
GError **error)
595+
{
596+
guint32 file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
597+
if (mknodat(destination_dfd, destination_name, (file_mode & ~S_IFMT) | S_IFCHR, (dev_t)0) < 0)
598+
return glnx_throw_errno_prefix (error, "Creating whiteout char device");
599+
if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER)
600+
{
601+
if (xattrs != NULL &&
602+
!glnx_dfd_name_set_all_xattrs(destination_dfd, destination_name, xattrs,
603+
cancellable, error))
604+
return glnx_throw_errno_prefix (error, "Setting xattrs for whiteout char device");
605+
606+
if (TEMP_FAILURE_RETRY(fchownat(destination_dfd, destination_name,
607+
g_file_info_get_attribute_uint32 (file_info, "unix::uid"),
608+
g_file_info_get_attribute_uint32 (file_info, "unix::gid"),
609+
AT_SYMLINK_NOFOLLOW) < 0))
610+
return glnx_throw_errno_prefix (error, "fchownat");
611+
if (TEMP_FAILURE_RETRY (fchmodat (destination_dfd, destination_name, file_mode & ~S_IFMT, 0)) < 0)
612+
return glnx_throw_errno_prefix (error, "fchmodat %s to 0%o", destination_name, file_mode & ~S_IFMT);
613+
}
614+
615+
return TRUE;
616+
}
617+
618+
static gboolean
619+
_checkout_overlayfs_whiteout_at (OstreeRepo *repo,
620+
OstreeRepoCheckoutAtOptions *options,
621+
int destination_dfd,
622+
const char *destination_name,
623+
GFileInfo *file_info,
624+
GVariant *xattrs,
625+
GCancellable *cancellable,
626+
GError **error)
627+
{
628+
if (_checkout_overlayfs_whiteout_at_no_overwrite(options, destination_dfd, destination_name,
629+
file_info, xattrs, cancellable, error))
630+
return TRUE;
631+
632+
if (!error || !g_error_matches(*error, G_IO_ERROR, G_IO_ERROR_EXISTS))
633+
return FALSE;
634+
635+
guint32 uid = g_file_info_get_attribute_uint32 (file_info, "unix::uid");
636+
guint32 gid = g_file_info_get_attribute_uint32 (file_info, "unix::gid");
637+
guint32 file_mode = g_file_info_get_attribute_uint32 (file_info, "unix::mode");
638+
639+
struct stat dest_stbuf;
640+
641+
switch(options->overwrite_mode)
642+
{
643+
case OSTREE_REPO_CHECKOUT_OVERWRITE_NONE:
644+
return FALSE;
645+
case OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_FILES:
646+
g_clear_error(error);
647+
if (!ot_ensure_unlinked_at (destination_dfd, destination_name, error))
648+
return FALSE;
649+
return _checkout_overlayfs_whiteout_at_no_overwrite(options, destination_dfd, destination_name,
650+
file_info, xattrs, cancellable, error);
651+
case OSTREE_REPO_CHECKOUT_OVERWRITE_ADD_FILES:
652+
g_clear_error(error);
653+
return TRUE;
654+
655+
case OSTREE_REPO_CHECKOUT_OVERWRITE_UNION_IDENTICAL:
656+
g_clear_error(error);
657+
if (!glnx_fstatat(destination_dfd, destination_name, &dest_stbuf, AT_SYMLINK_NOFOLLOW,
658+
error))
659+
return FALSE;
660+
if (!(repo->disable_xattrs || repo->mode == OSTREE_REPO_MODE_BARE_USER_ONLY))
661+
{
662+
g_autoptr(GVariant) fs_xattrs;
663+
if (!glnx_dfd_name_get_all_xattrs (destination_dfd, destination_name,
664+
&fs_xattrs, cancellable, error))
665+
return FALSE;
666+
if (!g_variant_equal(fs_xattrs, xattrs))
667+
return glnx_throw(error, "existing destination file %s xattrs don't match",
668+
destination_name);
669+
}
670+
if (options->mode != OSTREE_REPO_CHECKOUT_MODE_USER)
671+
{
672+
if (gid != dest_stbuf.st_gid)
673+
return glnx_throw(error, "existing destination file %s does not match gid %d",
674+
destination_name, gid);
675+
676+
if (uid != dest_stbuf.st_uid)
677+
return glnx_throw(error, "existing destination file %s does not match uid %d",
678+
destination_name, gid);
679+
680+
if ((file_mode & ALLPERMS) != (dest_stbuf.st_mode & ALLPERMS))
681+
return glnx_throw(error, "existing destination file %s does not match mode %o",
682+
destination_name, file_mode);
683+
}
684+
break;
685+
}
686+
return TRUE;
687+
}
688+
585689
static gboolean
586690
checkout_one_file_at (OstreeRepo *repo,
587691
OstreeRepoCheckoutAtOptions *options,
@@ -603,7 +707,8 @@ checkout_one_file_at (OstreeRepo *repo,
603707

604708
/* FIXME - avoid the GFileInfo here */
605709
g_autoptr(GFileInfo) source_info = NULL;
606-
if (!ostree_repo_load_file (repo, checksum, NULL, &source_info, NULL,
710+
g_autoptr(GVariant) source_xattrs = NULL;
711+
if (!ostree_repo_load_file (repo, checksum, NULL, &source_info, &source_xattrs,
607712
cancellable, error))
608713
return FALSE;
609714

@@ -623,6 +728,7 @@ checkout_one_file_at (OstreeRepo *repo,
623728
const gboolean is_unreadable = (!is_symlink && (source_mode & S_IRUSR) == 0);
624729
const gboolean is_whiteout = (!is_symlink && options->process_whiteouts &&
625730
g_str_has_prefix (destination_name, WHITEOUT_PREFIX));
731+
const gboolean is_overlayfs_whiteout = (!is_symlink && g_str_has_prefix (destination_name, OVERLAYFS_WHITEOUT_PREFIX));
626732
const gboolean is_reg_zerosized = (!is_symlink && g_file_info_get_size (source_info) == 0);
627733
const gboolean override_user_unreadable = (options->mode == OSTREE_REPO_CHECKOUT_MODE_USER && is_unreadable);
628734

@@ -643,6 +749,18 @@ checkout_one_file_at (OstreeRepo *repo,
643749

644750
need_copy = FALSE;
645751
}
752+
else if (is_overlayfs_whiteout && options->process_passthrough_whiteouts)
753+
{
754+
const char *name = destination_name + (sizeof (OVERLAYFS_WHITEOUT_PREFIX) - 1);
755+
756+
if (!name[0])
757+
return glnx_throw (error, "Invalid empty overlayfs whiteout '%s'", name);
758+
759+
g_assert (name[0] != '/'); /* Sanity */
760+
761+
return _checkout_overlayfs_whiteout_at(repo, options, destination_dfd, name,
762+
source_info, source_xattrs, cancellable, error);
763+
}
646764
else if (is_reg_zerosized || override_user_unreadable)
647765
{
648766
/* In https://github.com/ostreedev/ostree/commit/673cacd633f9d6b653cdea530657d3e780a41bbd we

‎src/libostree/ostree-repo.h

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -989,8 +989,9 @@ typedef struct {
989989
gboolean force_copy; /* Since: 2017.6 */
990990
gboolean bareuseronly_dirs; /* Since: 2017.7 */
991991
gboolean force_copy_zerosized; /* Since: 2018.9 */
992-
gboolean unused_bools[4];
993-
/* 4 byte hole on 64 bit */
992+
gboolean process_passthrough_whiteouts;
993+
gboolean unused_bools[3];
994+
/* 3 byte hole on 64 bit */
994995

995996
const char *subpath;
996997

‎src/libostree/ostree-sysroot-deploy.c

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -641,7 +641,7 @@ checkout_deployment_tree (OstreeSysroot *sysroot,
641641
return FALSE;
642642

643643
/* Generate hardlink farm, then opendir it */
644-
OstreeRepoCheckoutAtOptions checkout_opts = { 0, };
644+
OstreeRepoCheckoutAtOptions checkout_opts = { .process_passthrough_whiteouts = TRUE };
645645
if (!ostree_repo_checkout_at (repo, &checkout_opts, osdeploy_dfd,
646646
checkout_target_name, csum,
647647
cancellable, error))

‎src/ostree/ot-builtin-checkout.c

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ static gboolean opt_union;
3737
static gboolean opt_union_add;
3838
static gboolean opt_union_identical;
3939
static gboolean opt_whiteouts;
40+
static gboolean opt_process_passthrough_whiteouts;
4041
static gboolean opt_from_stdin;
4142
static char *opt_from_file;
4243
static gboolean opt_disable_fsync;
@@ -77,6 +78,7 @@ static GOptionEntry options[] = {
7778
{ "union-add", 0, 0, G_OPTION_ARG_NONE, &opt_union_add, "Keep existing files/directories, only add new", NULL },
7879
{ "union-identical", 0, 0, G_OPTION_ARG_NONE, &opt_union_identical, "When layering checkouts, error out if a file would be replaced with a different version, but add new files and directories", NULL },
7980
{ "whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_whiteouts, "Process 'whiteout' (Docker style) entries", NULL },
81+
{ "process-passthrough-whiteouts", 0, 0, G_OPTION_ARG_NONE, &opt_process_passthrough_whiteouts, "Enable overlayfs whiteout extraction into char 0:0 devices", NULL },
8082
{ "allow-noent", 0, 0, G_OPTION_ARG_NONE, &opt_allow_noent, "Do nothing if specified path does not exist", NULL },
8183
{ "from-stdin", 0, 0, G_OPTION_ARG_NONE, &opt_from_stdin, "Process many checkouts from standard input", NULL },
8284
{ "from-file", 0, 0, G_OPTION_ARG_STRING, &opt_from_file, "Process many checkouts from input file", "FILE" },
@@ -129,7 +131,8 @@ process_one_checkout (OstreeRepo *repo,
129131
if (opt_disable_cache || opt_whiteouts || opt_require_hardlinks ||
130132
opt_union_add || opt_force_copy || opt_force_copy_zerosized ||
131133
opt_bareuseronly_dirs || opt_union_identical ||
132-
opt_skiplist_file || opt_selinux_policy || opt_selinux_prefix)
134+
opt_skiplist_file || opt_selinux_policy || opt_selinux_prefix ||
135+
opt_process_passthrough_whiteouts)
133136
{
134137
OstreeRepoCheckoutAtOptions checkout_options = { 0, };
135138

@@ -162,6 +165,8 @@ process_one_checkout (OstreeRepo *repo,
162165
}
163166
if (opt_whiteouts)
164167
checkout_options.process_whiteouts = TRUE;
168+
if (opt_process_passthrough_whiteouts)
169+
checkout_options.process_passthrough_whiteouts = TRUE;
165170
if (subpath)
166171
checkout_options.subpath = subpath;
167172

‎tests/archive-test.sh

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,11 @@ mkdir -p test-overlays
7171
date > test-overlays/overlaid-file
7272
$OSTREE commit ${COMMIT_ARGS} -b test-base --base test2 --owner-uid 42 --owner-gid 42 test-overlays/
7373
$OSTREE ls -R test-base > ls.txt
74-
assert_streq "$(wc -l < ls.txt)" 14
74+
if can_create_whiteout_devices; then
75+
assert_streq "$(wc -l < ls.txt)" 17
76+
else
77+
assert_streq "$(wc -l < ls.txt)" 14
78+
fi
79+
7580
assert_streq "$(grep '42.*42' ls.txt | wc -l)" 2
7681
echo "ok commit overlay base"

‎tests/basic-test.sh

Lines changed: 28 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@
1919

2020
set -euo pipefail
2121

22-
echo "1..$((88 + ${extra_basic_tests:-0}))"
22+
echo "1..$((90 + ${extra_basic_tests:-0}))"
2323

2424
CHECKOUT_U_ARG=""
2525
CHECKOUT_H_ARGS="-H"
@@ -1203,3 +1203,30 @@ if test "$(id -u)" != "0"; then
12031203
else
12041204
echo "ok # SKIP not run when root"
12051205
fi
1206+
1207+
if ! skip_one_without_whiteouts_devices; then
1208+
cd ${test_tmpdir}
1209+
rm checkout-test2 -rf
1210+
$OSTREE checkout test2 checkout-test2
1211+
1212+
assert_not_has_file checkout-test2/whiteouts/whiteout
1213+
assert_not_has_file checkout-test2/whiteouts/whiteout2
1214+
assert_has_file checkout-test2/whiteouts/.ostree-wh.whiteout
1215+
assert_has_file checkout-test2/whiteouts/.ostree-wh.whiteout2
1216+
1217+
echo "ok checkout: no whiteout passthrough by default"
1218+
fi
1219+
1220+
if ! skip_one_without_whiteouts_devices; then
1221+
cd ${test_tmpdir}
1222+
rm checkout-test2 -rf
1223+
$OSTREE checkout --process-passthrough-whiteouts test2 checkout-test2
1224+
1225+
assert_not_has_file checkout-test2/whiteouts/.ostree-wh.whiteout
1226+
assert_not_has_file checkout-test2/whiteouts/.ostree-wh.whiteout2
1227+
1228+
assert_is_whiteout_device checkout-test2/whiteouts/whiteout
1229+
assert_is_whiteout_device checkout-test2/whiteouts/whiteout2
1230+
1231+
echo "ok checkout: whiteout with overlayfs passthrough processing"
1232+
fi

‎tests/kolainst/data-shared/libtest-core.sh

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,13 @@ assert_file_has_mode () {
163163
fi
164164
}
165165

166+
assert_is_whiteout_device () {
167+
device_details="$(stat -c '%F %t:%T' $1)"
168+
if [ "$device_details" != "character special file 0:0" ]; then
169+
fatal "File '$1' is not a whiteout character device 0:0"
170+
fi
171+
}
172+
166173
assert_symlink_has_content () {
167174
if ! test -L "$1"; then
168175
fatal "File '$1' is not a symbolic link"

0 commit comments

Comments
 (0)