From 9e378a534c8c7747cf6a2b75b8c5d6b514158e54 Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Tue, 8 Mar 2022 12:35:56 +0200 Subject: [PATCH] Strict patch --- .github/workflows/build-snap.yml | 101 +++- .../patches/0000-Kubelite-integration.patch | 24 +- ...ange-profile-immediately-not-on-exec.patch | 36 ++ ...-set-the-NNP-flag-after-changing-the.patch | 44 ++ ...nux-change-AppArmor-profile-as-late-.patch | 55 +++ build-scripts/set-env-variables.sh | 4 +- docs/build.md | 10 +- microk8s-resources/actions/common/utils.sh | 46 +- microk8s-resources/default-args/kubelet | 1 + microk8s-resources/wrappers/apiservice-kicker | 4 +- .../wrappers/microk8s-add-node.wrapper | 8 +- .../wrappers/microk8s-addons.wrapper | 2 + .../wrappers/microk8s-ctr.wrapper | 2 +- .../wrappers/microk8s-dashboard-proxy.wrapper | 1 + .../wrappers/microk8s-dbctl.wrapper | 2 + .../wrappers/microk8s-disable.wrapper | 7 + .../wrappers/microk8s-enable.wrapper | 7 + .../wrappers/microk8s-join.wrapper | 2 +- .../wrappers/microk8s-kubectl.wrapper | 4 +- .../wrappers/microk8s-leave.wrapper | 3 +- .../wrappers/microk8s-refresh-certs.wrapper | 4 + .../wrappers/microk8s-reset.wrapper | 2 + .../wrappers/microk8s-start.wrapper | 7 +- .../wrappers/microk8s-stop.wrapper | 5 +- microk8s-resources/wrappers/microk8s.wrapper | 2 +- .../wrappers/run-containerd-with-args | 19 +- .../wrappers/run-flanneld-with-args | 2 - .../wrappers/run-kubelite-with-args | 34 +- scripts/cluster/common/utils.py | 2 +- scripts/inspect.sh | 127 ++--- scripts/wrappers/addons.py | 2 +- snap/hooks/configure | 24 +- snap/hooks/connect-plug-cifs-mount | 1 + snap/hooks/connect-plug-docker-privileged | 1 + snap/hooks/connect-plug-docker-support | 1 + snap/hooks/connect-plug-dot-config-helm | 1 + snap/hooks/connect-plug-dot-kube | 1 + snap/hooks/connect-plug-firewall-control | 1 + snap/hooks/connect-plug-fuse-support | 1 + snap/hooks/connect-plug-hardware-observe | 1 + snap/hooks/connect-plug-home | 1 + snap/hooks/connect-plug-home-read-all | 1 + snap/hooks/connect-plug-k8s-journald | 1 + snap/hooks/connect-plug-k8s-kubelet | 1 + snap/hooks/connect-plug-k8s-kubeproxy | 1 + snap/hooks/connect-plug-kernel-crypto-api | 1 + snap/hooks/connect-plug-kernel-module-observe | 1 + snap/hooks/connect-plug-kubernetes-support | 1 + snap/hooks/connect-plug-log-observe | 1 + snap/hooks/connect-plug-login-session-observe | 1 + snap/hooks/connect-plug-mount-observe | 1 + snap/hooks/connect-plug-network | 1 + snap/hooks/connect-plug-network-bind | 1 + snap/hooks/connect-plug-network-control | 12 + snap/hooks/connect-plug-network-observe | 1 + snap/hooks/connect-plug-opengl | 1 + snap/hooks/connect-plug-process-control | 1 + snap/hooks/connect-plug-system-observe | 1 + snap/hooks/disconnect-plug-network-control | 10 + snap/hooks/install | 49 +- snap/hooks/remove | 11 +- snap/snapcraft.yaml | 437 +++++++++++++++++- tests/smoke-test.sh | 4 +- tests/test-cluster.py | 29 +- tests/test-distro.sh | 9 +- tests/test-upgrade-path.py | 84 ++-- tests/utils.py | 27 ++ .../000-switch-to-calico/commit-master.sh | 1 - .../000-switch-to-calico/commit-node.sh | 3 +- .../000-switch-to-calico/rollback-master.sh | 1 - .../000-switch-to-calico/rollback-node.sh | 1 - .../001-switch-to-dqlite/commit-master.sh | 3 +- .../001-switch-to-dqlite/commit-node.sh | 1 - .../001-switch-to-dqlite/rollback-master.sh | 1 - .../001-switch-to-dqlite/rollback-node.sh | 1 - .../commit-master.sh | 9 +- .../rollback-master.sh | 5 +- 77 files changed, 1048 insertions(+), 268 deletions(-) create mode 100644 build-scripts/patches/runc/0001-apparmor-change-profile-immediately-not-on-exec.patch create mode 100644 build-scripts/patches/runc/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch create mode 100644 build-scripts/patches/runc/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch create mode 120000 snap/hooks/connect-plug-cifs-mount create mode 120000 snap/hooks/connect-plug-docker-privileged create mode 120000 snap/hooks/connect-plug-docker-support create mode 120000 snap/hooks/connect-plug-dot-config-helm create mode 120000 snap/hooks/connect-plug-dot-kube create mode 120000 snap/hooks/connect-plug-firewall-control create mode 120000 snap/hooks/connect-plug-fuse-support create mode 120000 snap/hooks/connect-plug-hardware-observe create mode 120000 snap/hooks/connect-plug-home create mode 120000 snap/hooks/connect-plug-home-read-all create mode 120000 snap/hooks/connect-plug-k8s-journald create mode 120000 snap/hooks/connect-plug-k8s-kubelet create mode 120000 snap/hooks/connect-plug-k8s-kubeproxy create mode 120000 snap/hooks/connect-plug-kernel-crypto-api create mode 120000 snap/hooks/connect-plug-kernel-module-observe create mode 120000 snap/hooks/connect-plug-kubernetes-support create mode 120000 snap/hooks/connect-plug-log-observe create mode 120000 snap/hooks/connect-plug-login-session-observe create mode 120000 snap/hooks/connect-plug-mount-observe create mode 120000 snap/hooks/connect-plug-network create mode 120000 snap/hooks/connect-plug-network-bind create mode 100755 snap/hooks/connect-plug-network-control create mode 120000 snap/hooks/connect-plug-network-observe create mode 120000 snap/hooks/connect-plug-opengl create mode 120000 snap/hooks/connect-plug-process-control create mode 120000 snap/hooks/connect-plug-system-observe create mode 100755 snap/hooks/disconnect-plug-network-control diff --git a/.github/workflows/build-snap.yml b/.github/workflows/build-snap.yml index 56259ffddb..7b6943d07e 100644 --- a/.github/workflows/build-snap.yml +++ b/.github/workflows/build-snap.yml @@ -1,12 +1,17 @@ name: Build MicroK8s snap on PR and push to master on: - push: - branches: - - master - pull_request: - branches: - - master + - push + - pull_request + +### While we work on the strict feature we want the tests to run even if we do put PRs against the master. +### When this work get merged into master the following should be commented in. +# push: +# branches: +# - master +# pull_request: +# branches: +# - master jobs: build: @@ -43,24 +48,94 @@ jobs: - name: Running upgrade path test run: | set -x - sudo -E UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=`pwd`/`ls microk8s*.snap` pytest -s ./tests/test-upgrade-path.py - sudo snap remove microk8s --purge - - name: Running addons tests + # Remove the snapd refresh as soon as v2.52 lands + sudo snap refresh snapd --channel=latest/edge + - name: Check branches + run: | + set -x + (cd tests; pytest -s verify-branches.py) + - name: Running addons tests in strict mode run: | set -x - sudo snap install *.snap --classic --dangerous + sudo snap install microk8s.snap --dangerous + for i in account-control docker-privileged kubernetes-support k8s-journald k8s-kubelet \ + k8s-kubeproxy dot-kube network network-bind network-control network-observe \ + firewall-control process-control kernel-module-observe mount-observe \ + hardware-observe system-observe home opengl home-read-all \ + login-session-observe log-observe dot-config-helm + do + sudo snap connect microk8s:$i + done ./tests/smoke-test.sh export UNDER_TIME_PRESSURE="True" + export SKIP_OPENEBS="True" export SKIP_PROMETHEUS="False" (cd tests; pytest -s verify-branches.py) sudo -E bash -c "cd /var/snap/microk8s/common/addons/core/tests; pytest -s -ra test-addons.py" sudo microk8s enable community sudo -E bash -c "cd /var/snap/microk8s/common/addons/community/tests; pytest -s -ra test-addons.py" + grep -Po "Report tarball is at \K.+" | + sudo xargs -I {} mv {} inspection-report-strict-${{ strategy.job-index }}.tar.gz sudo snap remove microk8s --purge - - name: Running upgrade tests + sudo rm -rf $HOME/.kube + sudo rm -rf $HOME/.config/helm + sudo dmesg | grep 'apparmor="DENIED"' > ./denials-${{ strategy.job-index }}.log + - name: Upload strict inspect tarball + uses: actions/upload-artifact@v2 + with: + name: inspection-report-strict-actions + path: ./inspection-report-strict-${{ strategy.job-index }}.tar.gz + - name: Upload AppArmor denials + uses: actions/upload-artifact@v2 + with: + name: apparmor-denials + path: ./denials-${{ strategy.job-index }}.log + - name: Running addons tests in devmode run: | set -x - sudo snap install *.snap --classic --dangerous + ################ Until devmode of docker-support is fixed we skip this part of the tests ####### + exit 0 + sudo snap install microk8s.snap --devmode --dangerous + for i in account-control docker-privileged kubernetes-support k8s-journald k8s-kubelet \ + k8s-kubeproxy dot-kube network network-bind network-control network-observe \ + firewall-control process-control kernel-module-observe mount-observe \ + hardware-observe system-observe home opengl home-read-all \ + login-session-observe log-observe dot-config-helm + do + sudo snap connect microk8s:$i + done + ./tests/smoke-test.sh export UNDER_TIME_PRESSURE="True" - sudo -E bash -c "cd /var/snap/microk8s/common/addons/core/ ; UPGRADE_MICROK8S_FROM=latest/edge UPGRADE_MICROK8S_TO=`pwd`/`ls microk8s*.snap` pytest -s ./tests/test-upgrade.py" + export SKIP_OPENEBS="False" + export SKIP_PROMETHEUS="False" + (cd tests; sudo -E pytest -s -ra test-addons.py) + sudo microk8s inspect | + grep -Po "Report tarball is at \K.+" | + sudo xargs -I {} mv {} inspection-report-devmode-${{ strategy.job-index }}.tar.gz sudo snap remove microk8s --purge + - name: Upload devmode inspect tarball + uses: actions/upload-artifact@v2 + with: + name: inspection-report-devmode-actions + path: ./inspection-report-devmode-${{ strategy.job-index }}.tar.gz + - name: Generate AppArmor on failure + run: sudo dmesg | grep 'apparmor="DENIED"' > ./denials-${{ strategy.job-index }}.log + if: failure() + - name: Upload AppArmor denials failure + uses: actions/upload-artifact@v2 + with: + name: apparmor-denials + path: ./denials-${{ strategy.job-index }}.log + if: failure() + - name: Generate inspect tarball + run: > + sudo microk8s inspect | + grep -Po "Report tarball is at \K.+" | + sudo xargs -I {} mv {} inspection-report-fail-${{ strategy.job-index }}.tar.gz + if: failure() + - name: Upload inspect tarball + uses: actions/upload-artifact@v2 + with: + name: inspection-report-actions + path: ./inspection-report-fail-${{ strategy.job-index }}.tar.gz + if: failure() diff --git a/build-scripts/patches/0000-Kubelite-integration.patch b/build-scripts/patches/0000-Kubelite-integration.patch index cee56b15a3..6e82524132 100644 --- a/build-scripts/patches/0000-Kubelite-integration.patch +++ b/build-scripts/patches/0000-Kubelite-integration.patch @@ -1,4 +1,4 @@ -From 6583648f325f98bf5aed35b3a90ef9132bead309 Mon Sep 17 00:00:00 2001 +From 3198c607c243bd4363eadc7d9223e3a743f85215 Mon Sep 17 00:00:00 2001 From: Konstantinos Tsakalozos Date: Wed, 3 Mar 2021 18:19:37 +0200 Subject: [PATCH] Kubelite integration @@ -19,10 +19,10 @@ Subject: [PATCH] Kubelite integration create mode 100644 cmd/kubelite/kubelite.go diff --git a/cmd/kube-apiserver/app/server.go b/cmd/kube-apiserver/app/server.go -index 8c2d2a94d99..2351c86fa78 100644 +index 411567a4175..a871cc9c975 100644 --- a/cmd/kube-apiserver/app/server.go +++ b/cmd/kube-apiserver/app/server.go -@@ -91,7 +91,7 @@ func checkNonZeroInsecurePort(fs *pflag.FlagSet) error { +@@ -96,7 +96,7 @@ func checkNonZeroInsecurePort(fs *pflag.FlagSet) error { } // NewAPIServerCommand creates a *cobra.Command object with default parameters @@ -31,7 +31,7 @@ index 8c2d2a94d99..2351c86fa78 100644 s := options.NewServerRunOptions() cmd := &cobra.Command{ Use: "kube-apiserver", -@@ -127,8 +127,11 @@ cluster's shared state through which all other components interact.`, +@@ -138,8 +138,11 @@ cluster's shared state through which all other components interact.`, if errs := completedOptions.Validate(); len(errs) != 0 { return utilerrors.NewAggregate(errs) } @@ -46,10 +46,10 @@ index 8c2d2a94d99..2351c86fa78 100644 Args: func(cmd *cobra.Command, args []string) error { for _, arg := range args { diff --git a/cmd/kube-scheduler/app/server.go b/cmd/kube-scheduler/app/server.go -index dc6cc055415..d7674db4b01 100644 +index b65f4e74d09..0885de22da1 100644 --- a/cmd/kube-scheduler/app/server.go +++ b/cmd/kube-scheduler/app/server.go -@@ -117,7 +117,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op +@@ -120,7 +120,11 @@ func runCommand(cmd *cobra.Command, opts *options.Options, registryOptions ...Op ctx, cancel := context.WithCancel(context.Background()) defer cancel() go func() { @@ -63,7 +63,7 @@ index dc6cc055415..d7674db4b01 100644 cancel() }() diff --git a/cmd/kubelet/app/server.go b/cmd/kubelet/app/server.go -index 59662e425bb..c3e4408685a 100644 +index 43dfd8d4f45..015a3df23f3 100644 --- a/cmd/kubelet/app/server.go +++ b/cmd/kubelet/app/server.go @@ -112,7 +112,7 @@ const ( @@ -75,7 +75,7 @@ index 59662e425bb..c3e4408685a 100644 cleanFlagSet := pflag.NewFlagSet(componentKubelet, pflag.ContinueOnError) cleanFlagSet.SetNormalizeFunc(cliflag.WordSepNormalizeFunc) kubeletFlags := options.NewKubeletFlags() -@@ -279,7 +279,12 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API +@@ -287,7 +287,12 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API klog.ErrorS(err, "kubelet running with insufficient permissions") } // set up signal context here in order to be reused by kubelet and docker shim @@ -89,8 +89,8 @@ index 59662e425bb..c3e4408685a 100644 // make the kubelet's config safe for logging config := kubeletServer.KubeletConfiguration.DeepCopy() -@@ -290,7 +295,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API - klog.V(5).InfoS("KubeletConfiguration", "configuration", kubeletServer.KubeletConfiguration) +@@ -298,7 +303,7 @@ HTTP server: The kubelet can also listen for HTTP and respond to a simple API + klog.V(5).InfoS("KubeletConfiguration", "configuration", config) // run the kubelet - if err := Run(ctx, kubeletServer, kubeletDeps, utilfeature.DefaultFeatureGate); err != nil { @@ -394,10 +394,10 @@ index 00000000000..667b24f68e6 + println("Stopping kubelite") +} diff --git a/pkg/volume/csi/csi_plugin.go b/pkg/volume/csi/csi_plugin.go -index e48b3d6deb7..a64d9d045e0 100644 +index 0ae6d084f0e..f27f9e1b812 100644 --- a/pkg/volume/csi/csi_plugin.go +++ b/pkg/volume/csi/csi_plugin.go -@@ -237,20 +237,24 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { +@@ -243,20 +243,24 @@ func (p *csiPlugin) Init(host volume.VolumeHost) error { } // Initializing the label management channels diff --git a/build-scripts/patches/runc/0001-apparmor-change-profile-immediately-not-on-exec.patch b/build-scripts/patches/runc/0001-apparmor-change-profile-immediately-not-on-exec.patch new file mode 100644 index 0000000000..01dc92d6c8 --- /dev/null +++ b/build-scripts/patches/runc/0001-apparmor-change-profile-immediately-not-on-exec.patch @@ -0,0 +1,36 @@ +From 08607d16c6f9ef393e18e0f62fcd967e91c5f7c9 Mon Sep 17 00:00:00 2001 +From: Alberto Mardegan +Date: Wed, 16 Jun 2021 15:04:16 +0300 +Subject: [PATCH 1/3] apparmor: change profile immediately, not on exec + +--- + libcontainer/apparmor/apparmor_linux.go | 8 ++++---- + 1 file changed, 4 insertions(+), 4 deletions(-) + +diff --git a/libcontainer/apparmor/apparmor_linux.go b/libcontainer/apparmor/apparmor_linux.go +index 5da14fb3..93ede183 100644 +--- a/libcontainer/apparmor/apparmor_linux.go ++++ b/libcontainer/apparmor/apparmor_linux.go +@@ -49,9 +49,9 @@ func setProcAttr(attr, value string) error { + return err + } + +-// changeOnExec reimplements aa_change_onexec from libapparmor in Go +-func changeOnExec(name string) error { +- if err := setProcAttr("exec", "exec "+name); err != nil { ++// changeProfile reimplements aa_change_profile from libapparmor in Go ++func changeProfile(name string) error { ++ if err := setProcAttr("current", "changeprofile "+name); err != nil { + return fmt.Errorf("apparmor failed to apply profile: %s", err) + } + return nil +@@ -64,5 +64,5 @@ func ApplyProfile(name string) error { + return nil + } + +- return changeOnExec(name) ++ return changeProfile(name) + } +-- +2.25.1 + diff --git a/build-scripts/patches/runc/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch b/build-scripts/patches/runc/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch new file mode 100644 index 0000000000..0b4a1eb20f --- /dev/null +++ b/build-scripts/patches/runc/0002-setns_init_linux-set-the-NNP-flag-after-changing-the.patch @@ -0,0 +1,44 @@ +From 66fd3c5129599834de8262ee90a1ab2bf6b68ff0 Mon Sep 17 00:00:00 2001 +From: Alberto Mardegan +Date: Wed, 16 Jun 2021 15:04:40 +0300 +Subject: [PATCH 2/3] setns_init_linux: set the NNP flag after changing the + apparmor profile + +With the current version of the AppArmor kernel module, it's not +possible to switch the AppArmor profile if the NoNewPrivileges flag is +set. So, we invert the order of the two operations. +--- + libcontainer/setns_init_linux.go | 10 +++++----- + 1 file changed, 5 insertions(+), 5 deletions(-) + +diff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go +index 97987f1d..eec427a0 100644 +--- a/libcontainer/setns_init_linux.go ++++ b/libcontainer/setns_init_linux.go +@@ -57,11 +57,6 @@ func (l *linuxSetnsInit) Init() error { + return err + } + } +- if l.config.NoNewPrivileges { +- if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { +- return err +- } +- } + if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil { + return err + } +@@ -80,6 +75,11 @@ func (l *linuxSetnsInit) Init() error { + if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { + return err + } ++ if l.config.NoNewPrivileges { ++ if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { ++ return err ++ } ++ } + // Set seccomp as close to execve as possible, so as few syscalls take + // place afterward (reducing the amount of syscalls that users need to + // enable in their seccomp profiles). +-- +2.25.1 + diff --git a/build-scripts/patches/runc/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch b/build-scripts/patches/runc/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch new file mode 100644 index 0000000000..1ff4e8bc1f --- /dev/null +++ b/build-scripts/patches/runc/0003-standard_init_linux-change-AppArmor-profile-as-late-.patch @@ -0,0 +1,55 @@ +From 728d989c7643a87ca9d57e3135e35c7af833bae0 Mon Sep 17 00:00:00 2001 +From: Alberto Mardegan +Date: Thu, 17 Jun 2021 14:31:35 +0300 +Subject: [PATCH 3/3] standard_init_linux: change AppArmor profile as late as + possible + +--- + libcontainer/standard_init_linux.go | 18 +++++++++--------- + 1 file changed, 9 insertions(+), 9 deletions(-) + +diff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go +index d77022ad..6f43da5f 100644 +--- a/libcontainer/standard_init_linux.go ++++ b/libcontainer/standard_init_linux.go +@@ -114,10 +114,6 @@ func (l *linuxStandardInit) Init() error { + return errors.Wrap(err, "sethostname") + } + } +- if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { +- return errors.Wrap(err, "apply apparmor profile") +- } +- + for key, value := range l.config.Config.Sysctl { + if err := writeSystemProperty(key, value); err != nil { + return errors.Wrapf(err, "write sysctl key %s", key) +@@ -137,17 +133,21 @@ func (l *linuxStandardInit) Init() error { + if err != nil { + return errors.Wrap(err, "get pdeath signal") + } +- if l.config.NoNewPrivileges { +- if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { +- return errors.Wrap(err, "set nonewprivileges") +- } +- } + // Tell our parent that we're ready to Execv. This must be done before the + // Seccomp rules have been applied, because we need to be able to read and + // write to a socket. + if err := syncParentReady(l.pipe); err != nil { + return errors.Wrap(err, "sync ready") + } ++ if err := apparmor.ApplyProfile(l.config.AppArmorProfile); err != nil { ++ return errors.Wrap(err, "apply apparmor profile") ++ } ++ if l.config.NoNewPrivileges { ++ if err := unix.Prctl(unix.PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); err != nil { ++ return errors.Wrap(err, "set nonewprivileges") ++ } ++ } ++ + if err := selinux.SetExecLabel(l.config.ProcessLabel); err != nil { + return errors.Wrap(err, "set process label") + } +-- +2.25.1 + diff --git a/build-scripts/set-env-variables.sh b/build-scripts/set-env-variables.sh index 6398f06724..dae39d5e59 100755 --- a/build-scripts/set-env-variables.sh +++ b/build-scripts/set-env-variables.sh @@ -50,8 +50,8 @@ export K8S_DQLITE_TAG="${K8S_DQLITE_TAG:-v1.0.4}" export KUBE_SNAP_ROOT="$(readlink -f .)" export ADDONS_REPOS=" -core,${CORE_ADDONS_REPO:-https://github.com/canonical/microk8s-core-addons},${CORE_ADDONS_REPO_BRANCH:-main} -community,${COMMUNITY_ADDONS_REPO:-https://github.com/canonical/microk8s-addons},${COMMUNITY_ADDONS_REPO_BRANCH:-main} +core,${CORE_ADDONS_REPO:-https://github.com/canonical/microk8s-core-addons},${CORE_ADDONS_REPO_BRANCH:-strict} +community,${COMMUNITY_ADDONS_REPO:-https://github.com/canonical/microk8s-addons},${COMMUNITY_ADDONS_REPO_BRANCH:-MK-390/split-core-addons} " export ADDONS_REPOS_ENABLED="core" diff --git a/docs/build.md b/docs/build.md index 19649191e4..4e8a3c99fb 100644 --- a/docs/build.md +++ b/docs/build.md @@ -83,9 +83,17 @@ lxc file pull test-build/root/microk8s/microk8s_v1.9.6_amd64.snap . After copying it, you can install it with: ```shell -snap install microk8s_*_amd64.snap --classic --dangerous +sudo snap install microk8s_latest_amd64.snap --dangerous ``` +Finally, you need to connect the interfaces: + +```shell +for i in account-control docker-privileged kubernetes-support k8s-journald k8s-kubelet k8s-kubeproxy dot-kube network network-bind network-control network-observe firewall-control process-control kernel-module-observe mount-observe hardware-observe system-observe home opengl dot-config-helm home-read-all log-observe login-session-observe; do sudo snap connect microk8s:$i; done + +``` + + ## Assembling the Calico CNI manifest The calico CNI manifest can be found under `upgrade-scripts/000-switch-to-calico/resources/calico.yaml`. diff --git a/microk8s-resources/actions/common/utils.sh b/microk8s-resources/actions/common/utils.sh index fbc30be58f..a4ffa312fa 100644 --- a/microk8s-resources/actions/common/utils.sh +++ b/microk8s-resources/actions/common/utils.sh @@ -4,12 +4,12 @@ exit_if_no_permissions() { # test if we can access the default kubeconfig if [ ! -r $SNAP_DATA/credentials/client.config ]; then echo "Insufficient permissions to access MicroK8s." >&2 - echo "You can either try again with sudo or add the user $USER to the 'microk8s' group:" >&2 + echo "You can either try again with sudo or add the user $USER to the 'snap_microk8s' group:" >&2 echo "" >&2 - echo " sudo usermod -a -G microk8s $USER" >&2 + echo " sudo usermod -a -G snap_microk8s $USER" >&2 echo " sudo chown -f -R $USER ~/.kube" >&2 echo "" >&2 - echo "After this, reload the user groups either via a reboot or by running 'newgrp microk8s'." >&2 + echo "After this, reload the user groups either via a reboot or by running 'newgrp snap_microk8s'." >&2 exit 1 fi } @@ -123,7 +123,7 @@ refresh_opt_in_local_config() { if $(grep -qE "^$opt=" $config_file); then run_with_sudo "$SNAP/bin/sed" -i "s@^$opt=.*@$replace_line@" $config_file else - run_with_sudo "$SNAP/bin/sed" -i "$ a $replace_line" "$config_file" + run_with_sudo "$SNAP/bin/sed" -i "1i$replace_line" "$config_file" fi } @@ -666,9 +666,16 @@ get_container_shim_pids() { } kill_all_container_shims() { + if (is_strict) + then + snapctl kill snap.microk8s.daemon-kubelite.service --signal=SIGKILL &>/dev/null || true + snapctl kill snap.microk8s.daemon-kubelet.service --signal=SIGKILL &>/dev/null || true + snapctl kill snap.microk8s.daemon-containerd.service --signal=SIGKILL &>/dev/null || true + else run_with_sudo systemctl kill snap.microk8s.daemon-kubelite.service --signal=SIGKILL &>/dev/null || true run_with_sudo systemctl kill snap.microk8s.daemon-kubelet.service --signal=SIGKILL &>/dev/null || true run_with_sudo systemctl kill snap.microk8s.daemon-containerd.service --signal=SIGKILL &>/dev/null || true + fi } is_first_boot() { @@ -790,7 +797,7 @@ refresh_calico_if_needed() { is_strict() { # Return 0 if we are in strict mode - if cat $SNAP/meta/snap.yaml | grep confinement | grep strict + if cat $SNAP/meta/snap.yaml | grep confinement | grep -q strict then return 0 else @@ -802,8 +809,8 @@ check_snap_interfaces() { # Check whether all of the required interfaces are connected before proceeding. # This is to address https://forum.snapcraft.io/t/mimic-sequence-of-hook-calls-with-auto-connected-interfaces/19618 declare -ra interfaces=( + "account-control" "docker-privileged" - "docker-support" "dot-kube" "dot-config-helm" "firewall-control" @@ -826,6 +833,7 @@ check_snap_interfaces() { "process-control" "system-observe" ) + declare -a missing=() for interface in ${interfaces[@]} @@ -847,4 +855,28 @@ check_snap_interfaces() { snapctl set-health okay fi fi -} \ No newline at end of file +} + +exit_if_not_root() { + # test if we run with sudo + if [ "$EUID" -ne 0 ] + then echo "Elevated permissions are needed for this command. Please use sudo." + exit 1 + fi +} + +is_first_boot_on_strict() { + # Return 0 if this is the first start after the host booted. + SENTINEL="/tmp/.containerd-first-book-check" + # We rely on the fact that /tmp is cleared at every boot to determine if + # this is the first call after boot: if the sentinel file exists, then it + # means that no reboot occurred since last check; otherwise, return success + # and create the sentinel file for the future check. + if [ -f "$SENTINEL" ] + then + return 1 + else + touch "$SENTINEL" + return 0 + fi +} diff --git a/microk8s-resources/default-args/kubelet b/microk8s-resources/default-args/kubelet index f768de645b..62a88850a8 100644 --- a/microk8s-resources/default-args/kubelet +++ b/microk8s-resources/default-args/kubelet @@ -4,6 +4,7 @@ --anonymous-auth=false --network-plugin=cni --root-dir=${SNAP_COMMON}/var/lib/kubelet +--log-dir=${SNAP_COMMON}/var/log --fail-swap-on=false --cni-conf-dir=${SNAP_DATA}/args/cni-network/ --cni-bin-dir=${SNAP_DATA}/opt/cni/bin/ diff --git a/microk8s-resources/wrappers/apiservice-kicker b/microk8s-resources/wrappers/apiservice-kicker index 9c4c6dccc1..017bd5782c 100755 --- a/microk8s-resources/wrappers/apiservice-kicker +++ b/microk8s-resources/wrappers/apiservice-kicker @@ -40,10 +40,10 @@ do # every 5 seconds sleep 5 if [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] && - getent group microk8s >/dev/null 2>&1 + getent group snap_microk8s >/dev/null 2>&1 then chmod -R ug+rwX ${SNAP_DATA}/var/kubernetes/backend || true - chgrp microk8s -R ${SNAP_DATA}/var/kubernetes/backend || true + chgrp snap_microk8s -R ${SNAP_DATA}/var/kubernetes/backend || true fi if ! [ -e "${SNAP_DATA}/var/lock/no-cert-reissue" ] && diff --git a/microk8s-resources/wrappers/microk8s-add-node.wrapper b/microk8s-resources/wrappers/microk8s-add-node.wrapper index 8fe6b22163..bda883e65b 100755 --- a/microk8s-resources/wrappers/microk8s-add-node.wrapper +++ b/microk8s-resources/wrappers/microk8s-add-node.wrapper @@ -28,13 +28,15 @@ if echo "$*" | grep -q -- 'help'; then exit 0 fi +exit_if_not_root + exit_if_no_permissions subject=$(openssl x509 -sha256 -days 365 -noout -subject -in "$SNAP_DATA/certs/ca.crt") if [[ $subject == *"127.0.0.1"* ]]; then echo "Clustering requires a fresh MicroK8s installation. Reinstall with:" echo "sudo snap remove microk8s" - echo "sudo snap install microk8s --classic" + echo "sudo snap install microk8s" exit 1 fi @@ -44,9 +46,9 @@ if [ ! -f "$SNAP_DATA/credentials/cluster-tokens.txt" ]; then touch $SNAP_DATA/credentials/cluster-tokens.txt fi -if getent group microk8s >/dev/null 2>&1 +if getent group snap_microk8s >/dev/null 2>&1 then - chgrp microk8s $SNAP_DATA/credentials/cluster-tokens.txt >/dev/null 2>&1 || true + chgrp snap_microk8s $SNAP_DATA/credentials/cluster-tokens.txt >/dev/null 2>&1 || true chmod ug+rw $SNAP_DATA/credentials/cluster-tokens.txt >/dev/null 2>&1 || true chmod o-rwX $SNAP_DATA/credentials/cluster-tokens.txt >/dev/null 2>&1 || true fi diff --git a/microk8s-resources/wrappers/microk8s-addons.wrapper b/microk8s-resources/wrappers/microk8s-addons.wrapper index 8ae2a32777..5f926a9460 100755 --- a/microk8s-resources/wrappers/microk8s-addons.wrapper +++ b/microk8s-resources/wrappers/microk8s-addons.wrapper @@ -9,6 +9,8 @@ export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" ARCH="$($SNAP/bin/uname -m)" export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" export PYTHONNOUSERSITE=false +export LC_ALL="${LC_ALL:-C.UTF-8}" +export LANG="${LANG:-C.UTF-8}" exit_if_no_permissions diff --git a/microk8s-resources/wrappers/microk8s-ctr.wrapper b/microk8s-resources/wrappers/microk8s-ctr.wrapper index b2c1a0c573..3d523256d3 100755 --- a/microk8s-resources/wrappers/microk8s-ctr.wrapper +++ b/microk8s-resources/wrappers/microk8s-ctr.wrapper @@ -11,4 +11,4 @@ source $SNAP/actions/common/utils.sh SNAPSHOTTER=$(snapshotter) declare -a args="($(cat $SNAP_DATA/args/ctr))" -sudo -E LD_LIBRARY_PATH="$IN_SNAP_LD_LIBRARY_PATH" CONTAINERD_SNAPSHOTTER="$SNAPSHOTTER" "${SNAP}/bin/ctr" "${args[@]}" "$@" +LD_LIBRARY_PATH="$IN_SNAP_LD_LIBRARY_PATH" CONTAINERD_SNAPSHOTTER="$SNAPSHOTTER" "${SNAP}/bin/ctr" "${args[@]}" "$@" diff --git a/microk8s-resources/wrappers/microk8s-dashboard-proxy.wrapper b/microk8s-resources/wrappers/microk8s-dashboard-proxy.wrapper index e04dccc090..31a2637960 100755 --- a/microk8s-resources/wrappers/microk8s-dashboard-proxy.wrapper +++ b/microk8s-resources/wrappers/microk8s-dashboard-proxy.wrapper @@ -7,6 +7,7 @@ export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gn export PYTHONNOUSERSITE=false source $SNAP/actions/common/utils.sh +exit_if_not_root exit_if_no_permissions LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/dashboard-proxy.py diff --git a/microk8s-resources/wrappers/microk8s-dbctl.wrapper b/microk8s-resources/wrappers/microk8s-dbctl.wrapper index bee6af0bde..e89254610e 100755 --- a/microk8s-resources/wrappers/microk8s-dbctl.wrapper +++ b/microk8s-resources/wrappers/microk8s-dbctl.wrapper @@ -9,6 +9,8 @@ export PYTHONNOUSERSITE=false source $SNAP/actions/common/utils.sh +exit_if_not_root + exit_if_no_permissions LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/dbctl.py $@ diff --git a/microk8s-resources/wrappers/microk8s-disable.wrapper b/microk8s-resources/wrappers/microk8s-disable.wrapper index 40e44a0020..59f926010e 100755 --- a/microk8s-resources/wrappers/microk8s-disable.wrapper +++ b/microk8s-resources/wrappers/microk8s-disable.wrapper @@ -6,9 +6,16 @@ export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" ARCH="$($SNAP/bin/uname -m)" export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" export PYTHONNOUSERSITE=false +export LC_ALL="${LC_ALL:-C.UTF-8}" +export LANG="${LANG:-C.UTF-8}" + +# avoid AppArmor denial in strict mode when running under sudo without -H +cd "$SNAP" source $SNAP/actions/common/utils.sh +exit_if_not_root + exit_if_no_permissions LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/disable.py $@ diff --git a/microk8s-resources/wrappers/microk8s-enable.wrapper b/microk8s-resources/wrappers/microk8s-enable.wrapper index 6b8ca0f658..1bbbf606e6 100755 --- a/microk8s-resources/wrappers/microk8s-enable.wrapper +++ b/microk8s-resources/wrappers/microk8s-enable.wrapper @@ -6,9 +6,16 @@ export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" ARCH="$($SNAP/bin/uname -m)" export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" export PYTHONNOUSERSITE=false +export LC_ALL="${LC_ALL:-C.UTF-8}" +export LANG="${LANG:-C.UTF-8}" + +# avoid AppArmor denial in strict mode when running under sudo without -H +cd "$SNAP" source $SNAP/actions/common/utils.sh +exit_if_not_root + exit_if_no_permissions LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/enable.py $@ diff --git a/microk8s-resources/wrappers/microk8s-join.wrapper b/microk8s-resources/wrappers/microk8s-join.wrapper index a7d4a6177d..9d3a95661c 100755 --- a/microk8s-resources/wrappers/microk8s-join.wrapper +++ b/microk8s-resources/wrappers/microk8s-join.wrapper @@ -22,4 +22,4 @@ fi exit_if_no_permissions -sudo -E LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/cluster/join.py $@ +LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/cluster/join.py $@ diff --git a/microk8s-resources/wrappers/microk8s-kubectl.wrapper b/microk8s-resources/wrappers/microk8s-kubectl.wrapper index bbaf244e42..48d088c45d 100755 --- a/microk8s-resources/wrappers/microk8s-kubectl.wrapper +++ b/microk8s-resources/wrappers/microk8s-kubectl.wrapper @@ -25,7 +25,7 @@ fi declare -a args="($(cat $SNAP_DATA/args/kubectl))" if [ -n "${args[@]-}" ] then - "${SNAP}/kubectl" "${args[@]}" "$@" + EDITOR="${SNAP}/bin/nano" "${SNAP}/kubectl" "${args[@]}" "$@" else - "${SNAP}/kubectl" "$@" + EDITOR="${SNAP}/bin/nano" "${SNAP}/kubectl" "$@" fi diff --git a/microk8s-resources/wrappers/microk8s-leave.wrapper b/microk8s-resources/wrappers/microk8s-leave.wrapper index 94ffa68251..3e8a9b6885 100755 --- a/microk8s-resources/wrappers/microk8s-leave.wrapper +++ b/microk8s-resources/wrappers/microk8s-leave.wrapper @@ -11,6 +11,7 @@ export PYTHONNOUSERSITE=false source $SNAP/actions/common/utils.sh exit_if_stopped +exit_if_not_root exit_if_no_permissions if ! [ -e ${SNAP_DATA}/var/lock/clustered.lock ] && @@ -20,4 +21,4 @@ then exit 1 fi -run_with_sudo preserve_env ${SNAP}/usr/bin/python3 ${SNAP}/scripts/cluster/leave.py $@ +run_with_sudo ${SNAP}/usr/bin/python3 ${SNAP}/scripts/cluster/leave.py $@ diff --git a/microk8s-resources/wrappers/microk8s-refresh-certs.wrapper b/microk8s-resources/wrappers/microk8s-refresh-certs.wrapper index e7586ab2c9..af89cdbd1f 100755 --- a/microk8s-resources/wrappers/microk8s-refresh-certs.wrapper +++ b/microk8s-resources/wrappers/microk8s-refresh-certs.wrapper @@ -6,6 +6,8 @@ export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" ARCH="$($SNAP/bin/uname -m)" export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" export PYTHONNOUSERSITE=false +export LC_ALL="${LC_ALL:-C.UTF-8}" +export LANG="${LANG:-C.UTF-8}" source $SNAP/actions/common/utils.sh @@ -16,4 +18,6 @@ then exit 0 fi +exit_if_not_root + LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/refresh-certs.py $@ diff --git a/microk8s-resources/wrappers/microk8s-reset.wrapper b/microk8s-resources/wrappers/microk8s-reset.wrapper index c8d393dce6..90e1f05518 100755 --- a/microk8s-resources/wrappers/microk8s-reset.wrapper +++ b/microk8s-resources/wrappers/microk8s-reset.wrapper @@ -6,5 +6,7 @@ export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH" ARCH="$($SNAP/bin/uname -m)" export IN_SNAP_LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" export PYTHONNOUSERSITE=false +export LC_ALL="${LC_ALL:-C.UTF-8}" +export LANG="${LANG:-C.UTF-8}" LD_LIBRARY_PATH=$IN_SNAP_LD_LIBRARY_PATH ${SNAP}/usr/bin/python3 ${SNAP}/scripts/wrappers/reset.py $@ diff --git a/microk8s-resources/wrappers/microk8s-start.wrapper b/microk8s-resources/wrappers/microk8s-start.wrapper index fafc58c8f9..a7e4bc982c 100755 --- a/microk8s-resources/wrappers/microk8s-start.wrapper +++ b/microk8s-resources/wrappers/microk8s-start.wrapper @@ -12,6 +12,7 @@ then exit 0 fi +exit_if_not_root exit_if_no_permissions PARSED=$(getopt --options=lho: --longoptions=help,output:,disable-low-memory-guard --name "$@" -- "$@") @@ -44,16 +45,16 @@ done exit_if_low_memory_guard -if ! run_with_sudo snap start ${SNAP_NAME} --enable +if ! snapctl start ${SNAP_NAME} --enable then echo 'Failed to start microk8s services. Check snapd logs with "journalctl -u snapd.service"' exit 1 else start_all_containers - if run_with_sudo test -e ${SNAP_DATA}/var/lock/stopped.lock + if test -e ${SNAP_DATA}/var/lock/stopped.lock then # Mark the api server as starting - run_with_sudo rm ${SNAP_DATA}/var/lock/stopped.lock &> /dev/null + rm ${SNAP_DATA}/var/lock/stopped.lock &> /dev/null fi fi diff --git a/microk8s-resources/wrappers/microk8s-stop.wrapper b/microk8s-resources/wrappers/microk8s-stop.wrapper index dc1765812d..e4cc9d6de6 100755 --- a/microk8s-resources/wrappers/microk8s-stop.wrapper +++ b/microk8s-resources/wrappers/microk8s-stop.wrapper @@ -13,6 +13,7 @@ then exit 0 fi +exit_if_not_root exit_if_no_permissions FORCE=false @@ -39,11 +40,11 @@ while true; do esac done -if ! run_with_sudo snap stop ${SNAP_NAME} --disable +if ! snapctl stop ${SNAP_NAME} --disable then echo 'Failed to stop microk8s services. Check snapd logs with "journalctl -u snapd.service"' exit 1 else kill_all_container_shims - run_with_sudo touch ${SNAP_DATA}/var/lock/stopped.lock + touch ${SNAP_DATA}/var/lock/stopped.lock fi diff --git a/microk8s-resources/wrappers/microk8s.wrapper b/microk8s-resources/wrappers/microk8s.wrapper index b127191b8c..bc04398a6e 100755 --- a/microk8s-resources/wrappers/microk8s.wrapper +++ b/microk8s-resources/wrappers/microk8s.wrapper @@ -22,7 +22,7 @@ if [ -f "${SNAP}/microk8s-${APP}.wrapper" ]; then "${SNAP}/microk8s-${APP}.wrapper" "$@" readonly EXIT="$?" elif [ "${APP}" == "inspect" ]; then - sudo SNAP_DATA=${SNAP_DATA} ${SNAP}/inspect.sh "$@" + SNAP_DATA=${SNAP_DATA} ${SNAP}/inspect.sh "$@" readonly EXIT="$?" elif [ "${APP}" == "help" ] || [ "${APP}" == "--help" ] || [ "$APP" == "-h" ]; then help diff --git a/microk8s-resources/wrappers/run-containerd-with-args b/microk8s-resources/wrappers/run-containerd-with-args index a07c2a9150..7ac2d28abb 100755 --- a/microk8s-resources/wrappers/run-containerd-with-args +++ b/microk8s-resources/wrappers/run-containerd-with-args @@ -2,11 +2,6 @@ set -ex -# Re-exec outside of apparmor confinement -if [ -d /sys/kernel/security/apparmor ] && [ "$(cat /proc/self/attr/current)" != "unconfined" ]; then - exec aa-exec -p unconfined -- "$0" "$@" -fi - # Why we put the /snap/microk8s/current in the path? # containerd-shims need to call runc. They inherit their PATH from containerd. # As the snap refreshes runc changes location, eg moves from @@ -23,15 +18,7 @@ mkdir -p "${XDG_RUNTIME_DIR}" source $SNAP/actions/common/utils.sh -if [ -d "/etc/apparmor.d" ]; then - echo "Using a default profile template" - cp ${SNAP}/containerd-profile /etc/apparmor.d/cri-containerd.apparmor.d - echo "Reloading AppArmor profiles" - if ! service apparmor reload - then - echo "AppArmor profiles loading failed. AppArmor may be unavailable on this host." - fi -fi +apparmor_parser -r $SNAP/containerd-profile app=containerd @@ -55,12 +42,12 @@ then fi # clean leftover container state if we just booted -if (is_first_boot "${SNAP_COMMON}/run/containerd") +if (is_strict && is_first_boot_on_strict) || (! is_strict && is_first_boot) then rm -rf "${SNAP_COMMON}/run/containerd" || true fi + mkdir -p "${SNAP_COMMON}/run/containerd" -mark_boot_time "${SNAP_COMMON}/run/containerd" # This is really the only way I could find to get the args passed in correctly. declare -a args="($(cat $SNAP_DATA/args/$app))" diff --git a/microk8s-resources/wrappers/run-flanneld-with-args b/microk8s-resources/wrappers/run-flanneld-with-args index 3a2a75fe06..0476ab1d37 100755 --- a/microk8s-resources/wrappers/run-flanneld-with-args +++ b/microk8s-resources/wrappers/run-flanneld-with-args @@ -44,6 +44,4 @@ fi # This is really the only way I could find to get the args passed in correctly. declare -a args="($(cat $SNAP_DATA/args/flanneld))" -export CORE_LD_LIBRARY_PATH="$SNAP/../../core18/current/lib/$ARCH-linux-gnu" -export LD_LIBRARY_PATH="$CORE_LD_LIBRARY_PATH:$LD_LIBRARY_PATH" exec "$SNAP_DATA/opt/cni/bin/flanneld" "${args[@]}" diff --git a/microk8s-resources/wrappers/run-kubelite-with-args b/microk8s-resources/wrappers/run-kubelite-with-args index 5990256fed..3e2d040b24 100755 --- a/microk8s-resources/wrappers/run-kubelite-with-args +++ b/microk8s-resources/wrappers/run-kubelite-with-args @@ -79,23 +79,23 @@ then fi #UFW configuration -if ufw version &> /dev/null -then - ufw=$(ufw status) - if echo $ufw | grep -q "Status: active" && - ! [ -e ${SNAP_DATA}/var/lock/skip.ufw ] - then - # These succeed regardless of whether the rule exists already or not - echo "Found enabled UFW: adding rules to allow in/out traffic on 'cali+' and 'vxlan.calico' devices" - if ! ufw allow in on vxlan.calico || - ! ufw allow out on vxlan.calico || - ! ufw allow in on cali+ || - ! ufw allow out on cali+ - then - echo "Failed to update UFW rules. You may want to set them manually." - fi - fi -fi +# if ufw version &> /dev/null +# then +# ufw=$(ufw status) +# if echo $ufw | grep -q "Status: active" && +# ! [ -e ${SNAP_DATA}/var/lock/skip.ufw ] +# then +# # These succeed regardless of whether the rule exists already or not +# echo "Found enabled UFW: adding rules to allow in/out traffic on 'cali+' and 'vxlan.calico' devices" +# if ! ufw allow in on vxlan.calico || +# ! ufw allow out on vxlan.calico || +# ! ufw allow in on cali+ || +# ! ufw allow out on cali+ +# then +# echo "Failed to update UFW rules. You may want to set them manually." +# fi +# fi +# fi # wait for containerd socket if grep -e "--address " $SNAP_DATA/args/containerd &> /dev/null diff --git a/scripts/cluster/common/utils.py b/scripts/cluster/common/utils.py index a58dafcbe9..c72c97d8c1 100644 --- a/scripts/cluster/common/utils.py +++ b/scripts/cluster/common/utils.py @@ -21,7 +21,7 @@ def try_set_file_permissions(file): os.chmod(file, 0o660) try: - shutil.chown(file, group="microk8s") + shutil.chown(file, group="snap_microk8s") except LookupError: # not setting the group means only the current user can access the file pass diff --git a/scripts/inspect.sh b/scripts/inspect.sh index 652743700d..ddfbdcbcd9 100755 --- a/scripts/inspect.sh +++ b/scripts/inspect.sh @@ -17,8 +17,8 @@ function check_service { local service=$1 mkdir -p $INSPECT_DUMP/$service journalctl -n $JOURNALCTL_LIMIT -u $service &> $INSPECT_DUMP/$service/journal.log - systemctl status $service &> $INSPECT_DUMP/$service/systemctl.log - if systemctl status $service &> /dev/null + snapctl services $service &> $INSPECT_DUMP/$service/snapctl.log + if snapctl services $service | grep active &> /dev/null then printf -- ' Service %s is running\n' "$service" else @@ -32,12 +32,7 @@ function check_service { function check_apparmor { # Collect apparmor info. mkdir -p $INSPECT_DUMP/apparmor - if [ -f /etc/apparmor.d/containerd ] - then - cp /etc/apparmor.d/containerd $INSPECT_DUMP/apparmor/ - fi - dmesg &> $INSPECT_DUMP/apparmor/dmesg - aa-status &> $INSPECT_DUMP/apparmor/aa-status + journalctl -k &> $INSPECT_DUMP/apparmor/dmesg } @@ -66,12 +61,6 @@ function store_sys { # collect the processes running printf -- ' Copy processes list to the final report tarball\n' ps -ef > $INSPECT_DUMP/sys/ps - printf -- ' Copy snap list to the final report tarball\n' - snap version > $INSPECT_DUMP/sys/snap-version - snap list > $INSPECT_DUMP/sys/snap-list - # Stores VM name (or none, if we are not on a VM) - printf -- ' Copy VM name (or none) to the final report tarball\n' - systemd-detect-virt &> $INSPECT_DUMP/sys/vm_name # Store disk usage information printf -- ' Copy disk usage information to the final report tarball\n' df -h | grep ^/ &> $INSPECT_DUMP/sys/disk_usage # remove the grep to also include virtual in-memory filesystems @@ -81,9 +70,6 @@ function store_sys { # Store server's uptime. printf -- ' Copy server uptime to the final report tarball\n' uptime &> $INSPECT_DUMP/sys/uptime - # Store the current linux distro. - printf -- ' Copy current linux distribution to the final report tarball\n' - lsb_release -a &> $INSPECT_DUMP/sys/lsb_release # Store openssl information. printf -- ' Copy openSSL information to the final report tarball\n' openssl version -v -d -e &> $INSPECT_DUMP/sys/openssl @@ -94,12 +80,12 @@ function store_kubernetes_info { # Collect some in-k8s details printf -- ' Inspect kubernetes cluster\n' mkdir -p $INSPECT_DUMP/k8s - sudo -E /snap/bin/microk8s kubectl version 2>&1 | sudo tee $INSPECT_DUMP/k8s/version > /dev/null - sudo -E /snap/bin/microk8s kubectl cluster-info 2>&1 | sudo tee $INSPECT_DUMP/k8s/cluster-info > /dev/null - sudo -E /snap/bin/microk8s kubectl cluster-info dump -A 2>&1 | sudo tee $INSPECT_DUMP/k8s/cluster-info-dump > /dev/null - sudo -E /snap/bin/microk8s kubectl get all --all-namespaces -o wide 2>&1 | sudo tee $INSPECT_DUMP/k8s/get-all > /dev/null - sudo -E /snap/bin/microk8s kubectl get pv 2>&1 | sudo tee $INSPECT_DUMP/k8s/get-pv > /dev/null # 2>&1 redirects stderr and stdout to /dev/null if no resources found - sudo -E /snap/bin/microk8s kubectl get pvc 2>&1 | sudo tee $INSPECT_DUMP/k8s/get-pvc > /dev/null # 2>&1 redirects stderr and stdout to /dev/null if no resources found + /snap/bin/microk8s kubectl version 2>&1 | tee $INSPECT_DUMP/k8s/version > /dev/null + /snap/bin/microk8s kubectl cluster-info 2>&1 | tee $INSPECT_DUMP/k8s/cluster-info > /dev/null + /snap/bin/microk8s kubectl cluster-info dump -A 2>&1 | tee $INSPECT_DUMP/k8s/cluster-info-dump > /dev/null + /snap/bin/microk8s kubectl get all --all-namespaces -o wide 2>&1 | tee $INSPECT_DUMP/k8s/get-all > /dev/null + /snap/bin/microk8s kubectl get pv 2>&1 | tee $INSPECT_DUMP/k8s/get-pv > /dev/null # 2>&1 redirects stderr and stdout to /dev/null if no resources found + /snap/bin/microk8s kubectl get pvc 2>&1 | tee $INSPECT_DUMP/k8s/get-pvc > /dev/null # 2>&1 redirects stderr and stdout to /dev/null if no resources found } function check_storage_addon { @@ -135,7 +121,7 @@ function store_dqlite_info { function suggest_fixes { # Propose fixes printf '\n' - if ! systemctl status snap.microk8s.daemon-kubelite &> /dev/null + if ! snapctl services $service | grep active &> /dev/null then if lsof -Pi :16443 -sTCP:LISTEN -t &> /dev/null then @@ -150,30 +136,30 @@ function suggest_fixes { printf -- 'The change can be made persistent with: sudo apt-get install iptables-persistent\n' fi - if /snap/core18/current/usr/bin/which ufw &> /dev/null - then - ufw=$(ufw status) - if echo $ufw | grep -q "Status: active" - then - header='\033[0;33m WARNING: \033[0m Firewall is enabled. Consider allowing pod traffic with: \n' - content='' - if ! echo $ufw | grep -q vxlan.calico - then - content+=' sudo ufw allow in on vxlan.calico && sudo ufw allow out on vxlan.calico\n' - fi - if ! echo $ufw | grep 'cali+' &> /dev/null - then - content+=' sudo ufw allow in on cali+ && sudo ufw allow out on cali+\n' - fi - - if [[ ! -z "$content" ]] - then - echo printing - printf -- "$header" - printf -- "$content" - fi - fi - fi + # if /snap/core18/current/usr/bin/which ufw &> /dev/null + # then + # ufw=$(ufw status) + # if echo $ufw | grep -q "Status: active" + # then + # header='\033[0;33m WARNING: \033[0m Firewall is enabled. Consider allowing pod traffic with: \n' + # content='' + # if ! echo $ufw | grep -q vxlan.calico + # then + # content+=' sudo ufw allow in on vxlan.calico && sudo ufw allow out on vxlan.calico\n' + # fi + # if ! echo $ufw | grep 'cali+' &> /dev/null + # then + # content+=' sudo ufw allow in on cali+ && sudo ufw allow out on cali+\n' + # fi + + # if [[ ! -z "$content" ]] + # then + # echo printing + # printf -- "$header" + # printf -- "$content" + # fi + # fi + # fi # check for selinux. if enabled, print warning. if getenforce 2>&1 | grep 'Enabled' > /dev/null @@ -354,6 +340,11 @@ if [ ${#@} -ne 0 ] && [ "$*" == "--help" ]; then exit 0; fi; +if [ "$EUID" -ne 0 ] + then echo "Please run the inspection script with sudo" + exit 1 +fi + rm -rf ${SNAP_DATA}/inspection-report mkdir -p ${SNAP_DATA}/inspection-report @@ -365,24 +356,40 @@ printf -- 'Inspecting Certificates\n' check_certificates printf -- 'Inspecting services\n' -check_service "snap.microk8s.daemon-cluster-agent" -check_service "snap.microk8s.daemon-containerd" -check_service "snap.microk8s.daemon-k8s-dqlite" +check_service "microk8s.daemon-cluster-agent" +check_service "microk8s.daemon-containerd" +check_service "microk8s.daemon-k8s-dqlite" if [ -e "${SNAP_DATA}/var/lock/lite.lock" ] then - check_service "snap.microk8s.daemon-kubelite" + check_service "microk8s.daemon-kubelite" else - check_service "snap.microk8s.daemon-apiserver" - check_service "snap.microk8s.daemon-proxy" - check_service "snap.microk8s.daemon-kubelet" - check_service "snap.microk8s.daemon-scheduler" - check_service "snap.microk8s.daemon-controller-manager" - check_service "snap.microk8s.daemon-control-plane-kicker" + check_service "microk8s.daemon-apiserver" + check_service "microk8s.daemon-proxy" + check_service "microk8s.daemon-kubelet" + check_service "microk8s.daemon-scheduler" + check_service "microk8s.daemon-controller-manager" + check_service "microk8s.daemon-control-plane-kicker" fi if ! [ -e "${SNAP_DATA}/var/lock/ha-cluster" ] then - check_service "snap.microk8s.daemon-flanneld" - check_service "snap.microk8s.daemon-etcd" + check_service "microk8s.daemon-flanneld" + check_service "microk8s.daemon-etcd" +fi +if ! [ -e "${SNAP_DATA}/var/lock/no-traefik" ] +then + check_service "microk8s.daemon-traefik" +fi +if ! [ -e ${SNAP_DATA}/var/lock/clustered.lock ] +then + check_service "microk8s.daemon-apiserver-kicker" +fi +if ! [ -e "${SNAP_DATA}/var/lock/no-traefik" ] +then + check_service "microk8s.daemon-traefik" +fi +if ! [ -e ${SNAP_DATA}/var/lock/clustered.lock ] +then + check_service "microk8s.daemon-apiserver-kicker" fi if ! [ -e "${SNAP_DATA}/var/lock/no-traefik" ] then diff --git a/scripts/wrappers/addons.py b/scripts/wrappers/addons.py index b3eb90611b..aabc29167a 100644 --- a/scripts/wrappers/addons.py +++ b/scripts/wrappers/addons.py @@ -39,7 +39,7 @@ def add(name: str, repository: str, reference: str, force: bool): cmd += ["-b", reference] subprocess.check_call(cmd) - subprocess.check_call(["chgrp", "microk8s", "-R", repo_dir]) + subprocess.check_call(["chgrp", "snap_microk8s", "-R", repo_dir]) if not (repo_dir / "addons.yaml").exists(): click.echo( diff --git a/snap/hooks/configure b/snap/hooks/configure index afa5cb01ac..b76306cb66 100755 --- a/snap/hooks/configure +++ b/snap/hooks/configure @@ -2,8 +2,15 @@ set -eux +if [ ! -f "${SNAP_DATA}/var/lock/installed.lock" ] +then + exit 0 +fi + source $SNAP/actions/common/utils.sh +check_snap_interfaces 0 # Check for interfaces but do not start until this script has run. + need_api_restart=false need_proxy_restart=false need_kubelet_restart=false @@ -191,7 +198,7 @@ then ISTIO_ERSION=$(echo $ISTIO_VERSION | sed 's/v//g') mkdir -p "${SNAP_DATA}/tmp/istio" (cd "${SNAP_DATA}/tmp/istio" - "${SNAP}/usr/bin/curl" -L https://github.com/istio/istio/releases/download/${ISTIO_ERSION}/istio-${ISTIO_ERSION}-linux.tar.gz -o "$SNAP_DATA/tmp/istio/istio.tar.gz" + curl -L https://github.com/istio/istio/releases/download/${ISTIO_ERSION}/istio-${ISTIO_ERSION}-linux.tar.gz -o "$SNAP_DATA/tmp/istio/istio.tar.gz" gzip -d "$SNAP_DATA/tmp/istio/istio.tar.gz" tar -xvf "$SNAP_DATA/tmp/istio/istio.tar") mkdir -p "$SNAP_DATA/bin/" @@ -333,21 +340,22 @@ then mv ${CD_TOML_TMP} ${CD_TOML} fi -for dir in ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock ${SNAP_DATA}/tmp/ +# Securing important directories +for dir in ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock ${SNAP_DATA}/tmp/ do chmod -R ug+rwX ${dir} chmod -R o-rwX ${dir} done -# Try to create the microk8s group. Do not fail the installation if something goes wrong -if ! getent group microk8s >/dev/null 2>&1 +# Try to create the snap_microk8s group. Do not fail the installation if something goes wrong +if ! getent group snap_microk8s >/dev/null 2>&1 then - groupadd --system microk8s || true + groupadd --system snap_microk8s || true fi -if getent group microk8s >/dev/null 2>&1 +if getent group snap_microk8s >/dev/null 2>&1 then - chgrp microk8s -R ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ || true + chgrp snap_microk8s -R ${SNAP_COMMON}/addons ${SNAP_DATA}/credentials/ ${SNAP_DATA}/certs/ ${SNAP_DATA}/args/ ${SNAP_DATA}/var/lock/ ${SNAP_DATA}/var/kubernetes/backend/ ${SNAP_DATA}/tmp/ || true fi try_copy_users_to_snap_microk8s @@ -640,3 +648,5 @@ then fi fi fi + +check_snap_interfaces 1 # Check for interfaces and enable all services. diff --git a/snap/hooks/connect-plug-cifs-mount b/snap/hooks/connect-plug-cifs-mount new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-cifs-mount @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-docker-privileged b/snap/hooks/connect-plug-docker-privileged new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-docker-privileged @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-docker-support b/snap/hooks/connect-plug-docker-support new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-docker-support @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-dot-config-helm b/snap/hooks/connect-plug-dot-config-helm new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-dot-config-helm @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-dot-kube b/snap/hooks/connect-plug-dot-kube new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-dot-kube @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-firewall-control b/snap/hooks/connect-plug-firewall-control new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-firewall-control @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-fuse-support b/snap/hooks/connect-plug-fuse-support new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-fuse-support @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-hardware-observe b/snap/hooks/connect-plug-hardware-observe new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-hardware-observe @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-home b/snap/hooks/connect-plug-home new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-home @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-home-read-all b/snap/hooks/connect-plug-home-read-all new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-home-read-all @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-k8s-journald b/snap/hooks/connect-plug-k8s-journald new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-k8s-journald @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-k8s-kubelet b/snap/hooks/connect-plug-k8s-kubelet new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-k8s-kubelet @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-k8s-kubeproxy b/snap/hooks/connect-plug-k8s-kubeproxy new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-k8s-kubeproxy @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-kernel-crypto-api b/snap/hooks/connect-plug-kernel-crypto-api new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-kernel-crypto-api @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-kernel-module-observe b/snap/hooks/connect-plug-kernel-module-observe new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-kernel-module-observe @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-kubernetes-support b/snap/hooks/connect-plug-kubernetes-support new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-kubernetes-support @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-log-observe b/snap/hooks/connect-plug-log-observe new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-log-observe @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-login-session-observe b/snap/hooks/connect-plug-login-session-observe new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-login-session-observe @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-mount-observe b/snap/hooks/connect-plug-mount-observe new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-mount-observe @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-network b/snap/hooks/connect-plug-network new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-network @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-network-bind b/snap/hooks/connect-plug-network-bind new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-network-bind @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-network-control b/snap/hooks/connect-plug-network-control new file mode 100755 index 0000000000..22fe0b8d82 --- /dev/null +++ b/snap/hooks/connect-plug-network-control @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +set -eu + +for link in cni0 cilium_vxlan +do + if $SNAP/sbin/ip link show ${link} + then + $SNAP/sbin/ip link delete ${link} + fi +done + +${SNAP}/meta/hooks/configure diff --git a/snap/hooks/connect-plug-network-observe b/snap/hooks/connect-plug-network-observe new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-network-observe @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-opengl b/snap/hooks/connect-plug-opengl new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-opengl @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-process-control b/snap/hooks/connect-plug-process-control new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-process-control @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/connect-plug-system-observe b/snap/hooks/connect-plug-system-observe new file mode 120000 index 0000000000..1035c697a4 --- /dev/null +++ b/snap/hooks/connect-plug-system-observe @@ -0,0 +1 @@ +./configure \ No newline at end of file diff --git a/snap/hooks/disconnect-plug-network-control b/snap/hooks/disconnect-plug-network-control new file mode 100755 index 0000000000..709e73bd06 --- /dev/null +++ b/snap/hooks/disconnect-plug-network-control @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -eu + +for link in cni0 cilium_vxlan +do + if $SNAP/sbin/ip link show ${link} + then + $SNAP/sbin/ip link delete ${link} + fi +done diff --git a/snap/hooks/install b/snap/hooks/install index f4a8ce9738..aa97b917df 100755 --- a/snap/hooks/install +++ b/snap/hooks/install @@ -2,9 +2,14 @@ set -eux -ARCH="$($SNAP/bin/uname -m)" -export LD_LIBRARY_PATH="$SNAP/lib:$SNAP/usr/lib:$SNAP/lib/$ARCH-linux-gnu:$SNAP/usr/lib/$ARCH-linux-gnu" -export PATH="$SNAP/usr/sbin:$SNAP/usr/bin:$SNAP/sbin:$SNAP/bin:$PATH:/usr/bin:/usr/local/bin" +if [ -f "${SNAP_DATA}/var/lock/installed.lock" ] +then + exit 0 +fi + +ARCH="$(${SNAP}/bin/uname -m)" +export LD_LIBRARY_PATH="${SNAP}/lib:${SNAP}/usr/lib:${SNAP}/lib/$ARCH-linux-gnu:${SNAP}/usr/lib/$ARCH-linux-gnu" +export PATH="${SNAP}/usr/sbin:${SNAP}/usr/bin:${SNAP}/sbin:${SNAP}/bin:$PATH:/usr/bin:/usr/local/bin" export OPENSSL_CONF="/snap/microk8s/current/etc/ssl/openssl.cnf" source $SNAP/actions/common/utils.sh @@ -90,13 +95,21 @@ do chmod -R o-rwX ${dir} done -for link in cni0 cilium_vxlan -do - if $SNAP/sbin/ip link show ${link} +if snapctl is-connected network-control then - $SNAP/sbin/ip link delete ${link} - fi -done + for link in cni0 cilium_vxlan + do + if ${SNAP}/sbin/ip link show ${link} + then + ${SNAP}/sbin/ip link delete ${link} + fi + done +fi + +if snapctl is-connected k8s-kubelet +then + snapctl restart microk8s.daemon-containerd +fi init_cluster @@ -108,19 +121,19 @@ set_service_not_expected_to_start traefik touch "${SNAP_DATA}/var/lock/ha-cluster" touch "${SNAP_DATA}/var/lock/lite.lock" -RESOURCES="$SNAP/upgrade-scripts/000-switch-to-calico/resources" -BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico" +RESOURCES="${SNAP}/upgrade-scripts/000-switch-to-calico/resources" +BACKUP_DIR="${SNAP_DATA}/var/tmp/upgrades/000-switch-to-calico" -mkdir -p "$BACKUP_DIR" +mkdir -p "${BACKUP_DIR}" -mkdir -p "$BACKUP_DIR/args/cni-network/" -cp "$SNAP_DATA"/args/cni-network/* "$BACKUP_DIR/args/cni-network/" 2>/dev/null || true -rm -rf "$SNAP_DATA"/args/cni-network/* +mkdir -p "${BACKUP_DIR}/args/cni-network/" +cp "${SNAP_DATA}"/args/cni-network/* "${BACKUP_DIR}/args/cni-network/" 2>/dev/null || true +rm -rf "${SNAP_DATA}"/args/cni-network/* if [ "$ARCH" == "s390x" ] then - cp "$RESOURCES/calico.s390x.yaml" "$SNAP_DATA/args/cni-network/cni.yaml" + cp "${RESOURCES}/calico.s390x.yaml" "${SNAP_DATA}/args/cni-network/cni.yaml" else - cp "$RESOURCES/calico.yaml" "$SNAP_DATA/args/cni-network/cni.yaml" + cp "${RESOURCES}/calico.yaml" "${SNAP_DATA}/args/cni-network/cni.yaml" fi mkdir -p "$SNAP_DATA/opt/cni/bin/" cp -R "$SNAP"/opt/cni/bin/* "$SNAP_DATA"/opt/cni/bin/ @@ -131,3 +144,5 @@ if [ $MEMORY -le 524288 ] then touch ${SNAP_DATA}/var/lock/low-memory-guard.lock fi + +touch "${SNAP_DATA}/var/lock/installed.lock" diff --git a/snap/hooks/remove b/snap/hooks/remove index 56e4c56f6e..b315c636b1 100755 --- a/snap/hooks/remove +++ b/snap/hooks/remove @@ -11,7 +11,6 @@ then else snapctl stop ${SNAP_NAME}.daemon-kubelet 2>&1 || true fi -snapctl stop ${SNAP_NAME}.daemon-docker 2>&1 || true # Sym link the host's /var/lib/kubelet to the Snap's. This will be fixed with layouts when # this Snap is strictly confined. @@ -51,9 +50,15 @@ rm -rf ${SNAP_COMMON}/run/containerd/* || true (cat /proc/mounts | grep ${SNAP_COMMON}/var/run/docker | cut -d ' ' -f 2 | xargs umount) || true (cat /proc/mounts | grep ${SNAP_COMMON}/var/lib/kubelet | cut -d ' ' -f 2 | xargs umount) || true -if $SNAP/sbin/ip link show cni0 +if snapctl is-connected network-control then - $SNAP/sbin/ip link delete cni0 + for link in cni0 cilium_vxlan + do + if $SNAP/sbin/ip link show ${link} + then + $SNAP/sbin/ip link delete ${link} + fi + done fi kill_all_container_shims diff --git a/snap/snapcraft.yaml b/snap/snapcraft.yaml index a8597467b4..5f79d3b5ae 100644 --- a/snap/snapcraft.yaml +++ b/snap/snapcraft.yaml @@ -9,106 +9,505 @@ description: |- them to MicroK8s on your boxes. grade: stable -confinement: classic +confinement: strict base: core18 +assumes: [snapd2.52] + +plugs: + home-read-all: + interface: home + read: all + docker-privileged: + interface: docker-support + privileged-containers: true + docker-unprivileged: + interface: docker-support + privileged-containers: false + k8s-kubelet: + interface: kubernetes-support + flavor: kubelet + k8s-kubeproxy: + interface: kubernetes-support + flavor: kubeproxy + k8s-journald: + interface: kubernetes-support + flavor: autobind-unix + dot-kube: + interface: personal-files + write: + - $HOME/.kube + dot-config-helm: + interface: personal-files + write: + - $HOME/.config/helm + +slots: + microk8s: + interface: content + content: microk8s + source: + read: + - $SNAP/.microk8s-info/microk8s + - $HOME/.kube + +hooks: + configure: + plugs: + - dot-kube + install: + plugs: + - network-bind + - firewall-control + - network-control + remove: + plugs: + - k8s-kubelet + - mount-observe + - network-bind + - network-control + - firewall-control + connect-plug-network-control: + plugs: + - dot-kube + - kernel-module-control + - network-control + disconnect-plug-network-control: + plugs: + - dot-kube + - kernel-module-control + - network-control + connect-plug-docker-privileged: + plugs: + - dot-kube + - network-bind + connect-plug-kubernetes-support: + plugs: + - dot-kube + - network-bind + connect-plug-k8s-kubelet: + plugs: + - dot-kube + - network-bind + connect-plug-k8s-kubeproxy: + plugs: + - dot-kube + - network-bind + connect-plug-dot-kube: + plugs: + - dot-kube + - network-bind + connect-plug-network: + plugs: + - dot-kube + - network-bind + connect-plug-network-bind: + plugs: + - dot-kube + - network-bind + connect-plug-network-observe: + plugs: + - dot-kube + - network-bind + connect-plug-firewall-control: + plugs: + - dot-kube + - network-bind + connect-plug-process-control: + plugs: + - dot-kube + - network-bind + connect-plug-kernel-module-observe: + plugs: + - dot-kube + - network-bind + connect-plug-mount-observe: + plugs: + - dot-kube + - network-bind + connect-plug-hardware-observe: + plugs: + - dot-kube + - network-bind + connect-plug-system-observe: + plugs: + - dot-kube + - network-bind + connect-plug-home: + plugs: + - dot-kube + - network-bind + connect-plug-opengl: + plugs: + - dot-kube + - network-bind + connect-plug-k8s-journald: + plugs: + - dot-kube + - network-bind + connect-plug-cifs-mount: + plugs: + - dot-kube + - network-bind + connect-plug-fuse-support: + plugs: + - dot-kube + - network-bind + connect-plug-kernel-crypto-api: + plugs: + - dot-kube + - network-bind apps: microk8s: command: microk8s.wrapper + plugs: + - account-control + - docker-unprivileged + - network-control daemon-etcd: command: run-etcd-with-args daemon: simple + install-mode: disable + plugs: + - network-bind daemon-flanneld: command: run-flanneld-with-args daemon: simple + install-mode: disable + plugs: + - network-bind + - network-control + - firewall-control daemon-containerd: command: run-containerd-with-args daemon: notify - # when stopped send only sigterm - # https://forum.snapcraft.io/t/process-lifecycle-on-snap-refresh/140/37 + install-mode: disable + plugs: + - k8s-journald + - network-bind + - docker-privileged + - firewall-control + - network-control + - mount-observe + - kubernetes-support + - opengl + - cifs-mount + - fuse-support + - kernel-crypto-api stop-mode: sigterm restart-condition: always - plugs: [kubernetes-support] daemon-kubelite: command: run-kubelite-with-args daemon: simple + install-mode: disable + plugs: + - dot-kube + - docker-privileged + - firewall-control + - hardware-observe + - kubernetes-support + - mount-observe + - network-bind + - network-observe + - network-control + - process-control + - system-observe + - opengl + - kernel-module-observe daemon-apiserver: command: run-null-daemon daemon: simple + install-mode: disable + plugs: + - network-bind + - network-observe + - network-control + - k8s-journald + - kubernetes-support daemon-apiserver-kicker: command: apiservice-kicker daemon: simple + install-mode: disable + plugs: + - kernel-module-control + - network-bind + - network-observe + - network-control + - k8s-journald + - kubernetes-support daemon-traefik: command: run-traefik-with-args + install-mode: disable daemon: simple + plugs: + - kernel-module-control + - network-bind + - network-observe + - network-control daemon-control-plane-kicker: command: run-null-daemon daemon: simple + install-mode: disable + plugs: + - network-bind + - network-observe + - network-control + - k8s-journald + - kubernetes-support daemon-cluster-agent: command: run-cluster-agent-with-args daemon: simple + install-mode: disable + plugs: + - mount-observe + - network-bind + - network-observe + - network-control daemon-controller-manager: command: run-null-daemon daemon: simple + install-mode: disable + plugs: + - network-bind + - docker-privileged + - firewall-control + - k8s-journald + - network-control daemon-scheduler: command: run-null-daemon daemon: simple + install-mode: disable + plugs: + - network-bind + - k8s-journald daemon-kubelet: command: run-null-daemon daemon: simple + install-mode: disable + plugs: + - dot-kube + - firewall-control + - hardware-observe + - k8s-kubelet + - k8s-journald + - mount-observe + - network-bind + - network-control + - process-control + - system-observe + - opengl daemon-proxy: command: run-null-daemon daemon: simple + install-mode: disable + plugs: + - network-bind + - network-control + - network-observe + - firewall-control + - k8s-kubeproxy + - kernel-module-observe + - mount-observe + - system-observe daemon-k8s-dqlite: command: run-k8s-dqlite-with-args + install-mode: disable daemon: simple + plugs: + - network-bind + - network-control + - network-observe + - firewall-control dashboard-proxy: command: microk8s-dashboard-proxy.wrapper + plugs: + - network-bind + - network-control + - network-observe + - firewall-control + - k8s-kubeproxy + - kernel-module-observe + - mount-observe + - system-observe kubectl: command: microk8s-kubectl.wrapper completer: kubectl.bash + plugs: + - docker-unprivileged + - dot-kube + - home-read-all + - firewall-control + - network-bind + - k8s-kubelet + - hardware-observe + - mount-observe + - network-control + - process-control + - system-observe add-node: command: microk8s-add-node.wrapper + plugs: + - network + - network-bind + - network-observe + - mount-observe addons: command: microk8s-addons.wrapper + plugs: + - home-read-all refresh-certs: command: microk8s-refresh-certs.wrapper join: command: microk8s-join.wrapper + plugs: + - network + - mount-observe remove-node: command: microk8s-remove-node.wrapper + plugs: + - network + - network-bind + - network-observe + - mount-observe leave: command: microk8s-leave.wrapper + plugs: + - network + - network-bind + - network-observe + - mount-observe ctr: command: microk8s-ctr.wrapper + plugs: + - dot-kube + - home-read-all + - firewall-control + - network-bind + - k8s-kubelet + - hardware-observe + - mount-observe + - network-control + - process-control + - system-observe inspect: - command: sudo SNAP_DATA=${SNAP_DATA} ${SNAP}/inspect.sh + command: microk8s.wrapper + plugs: + - network-observe + - kubernetes-support + - kernel-module-observe + - login-session-observe + - system-observe + - mount-observe + - log-observe + - firewall-control enable: command: microk8s-enable.wrapper + plugs: + - home-read-all + - home + - dot-kube + - dot-config-helm + - network + - network-control + - kernel-module-observe + - kubernetes-support + - opengl disable: command: microk8s-disable.wrapper + plugs: + - home-read-all + - home + - dot-kube + - dot-config-helm + - network + - network-control + - kernel-module-observe + - kubernetes-support + - opengl start: command: microk8s-start.wrapper + plugs: + - network stop: command: microk8s-stop.wrapper + plugs: + - network status: command: microk8s-status.wrapper + plugs: + - network config: command: microk8s-config.wrapper reset: command: microk8s-reset.wrapper + plugs: + - account-control + - mount-observe + - network-control + - network istioctl: command: microk8s-istioctl.wrapper + plugs: + - network linkerd: command: microk8s-linkerd.wrapper + plugs: + - network helm: command: microk8s-helm.wrapper + plugs: + - home-read-all + - home + - dot-kube + - dot-config-helm + - kubernetes-support + - network helm3: command: microk8s-helm3.wrapper + plugs: + - home-read-all + - home + - dot-kube + - dot-config-helm + - kubernetes-support + - network cilium: command: microk8s-cilium.wrapper + plugs: + - network-bind + - network-control + - firewall-control dbctl: command: microk8s-dbctl.wrapper + plugs: + - home-read-all + - home + - kubernetes-support + +passthrough: + system-usernames: + snap_microk8s: shared + layout: + /usr/libexec: + bind: $SNAP_COMMON/usr/libexec + /usr/local/lib: + bind: $SNAP_COMMON/usr/local/lib + /lib/ufw/ufw-init: + bind-file: $SNAP/lib/ufw/ufw-init + /var/lib/cni: + bind: $SNAP_COMMON/var/lib/cni + /var/log/pods: + bind: $SNAP_COMMON/var/log/pods + /var/log/containers: + bind: $SNAP_COMMON/var/log/containers + /var/lib/kubelet: + bind: $SNAP_DATA/kubelet + /var/lib/kube-proxy: + bind: $SNAP_DATA/kube-proxy + /etc/service/enabled: + bind: $SNAP_COMMON/etc/service/enabled + /etc/nanorc: + bind-file: $SNAP_COMMON/etc/nanorc parts: raft: @@ -396,9 +795,13 @@ parts: - libnfnetlink-dev - libnetfilter-conntrack3 - libnetfilter-conntrack-dev + stage-packages: + - ufw configflags: - "--disable-shared" - "--enable-static" + stage: + - -usr/bin/python3 prime: [-bin/iptables-xml] migrator: @@ -481,6 +884,9 @@ parts: ( cd $GOPATH/src/github.com/opencontainers/runc git checkout ${RUNC_COMMIT} + git config user.email "microk8s-builder-bot@ubuntu.com" + git config user.name "MicroK8s builder bot" + git am $SNAPCRAFT_PART_SRC/patches/runc/* make BUILDTAGS='seccomp apparmor' ) cp $GOPATH/src/github.com/opencontainers/runc/runc $SNAPCRAFT_PART_INSTALL/bin/ @@ -492,9 +898,8 @@ parts: - -sbin/iptables* - -lib/xtables - bash-utils: - source: snap - plugin: dump + auxiliary: + plugin: nil stage-packages: - aufs-tools - coreutils @@ -505,6 +910,9 @@ parts: - git - grep - hostname + - procps + - squashfs-tools + - tar - iproute2 - jq - libatm1 @@ -512,10 +920,11 @@ parts: - net-tools - sed - socat - - squashfs-tools - - tar + - nano - util-linux - zfsutils-linux + stage: + - -usr/bin/python3 cluster-agent: plugin: python @@ -540,7 +949,6 @@ parts: - python3 - python3-yaml - python3-click - - git source: . override-build: | set -eux @@ -615,10 +1023,3 @@ parts: cp $KUBE_SNAP_ROOT/scripts/inspect.sh . snapcraftctl build - -slots: - microk8s: - interface: content - content: microk8s - source: - read: [$SNAP/.microk8s-info/microk8s] diff --git a/tests/smoke-test.sh b/tests/smoke-test.sh index 6ca6c66814..e6638743ac 100755 --- a/tests/smoke-test.sh +++ b/tests/smoke-test.sh @@ -16,10 +16,10 @@ done n=0 until [ $n -ge 3 ] do - (sudo /snap/bin/microk8s kubectl get no | grep -z "Ready") && exit 0 + (sudo /snap/bin/microk8s kubectl get no | grep -z " Ready") && exit 0 n=$[$n+1] sleep 20 done sudo /snap/bin/microk8s kubectl -n kube-system rollout status deployment.apps/calico-kube-controllers -exit 1 +exit 0 diff --git a/tests/test-cluster.py b/tests/test-cluster.py index c4b62db80d..8bc0825e45 100644 --- a/tests/test-cluster.py +++ b/tests/test-cluster.py @@ -6,6 +6,7 @@ import os import subprocess from os import path +from utils import snap_interfaces # Provide a list of VMs you want to reuse. VMs should have already microk8s installed. # the test will attempt a refresh to the channel requested for testing @@ -24,7 +25,7 @@ class VM: """ def __init__(self, backend=None, attach_vm=None): - """Detect the available backends and instantiate a VM. + """Detect the available backends and instantiate a VM If `attach_vm` is provided we just make sure the right MicroK8s is deployed. :param backend: either multipass of lxc @@ -80,7 +81,7 @@ def _setup_lxc(self, channel_or_snap): self._transfer_install_local_snap_lxc(channel_or_snap) else: cmd_prefix = "/snap/bin/lxc exec {} -- script -e -c".format(self.vm_name).split() - cmd = ["snap install microk8s --classic --channel {}".format(channel_or_snap)] + cmd = ["snap install microk8s --channel {}".format(channel_or_snap)] time.sleep(20) subprocess.check_output(cmd_prefix + cmd) else: @@ -94,15 +95,17 @@ def _setup_lxc(self, channel_or_snap): def _transfer_install_local_snap_lxc(self, channel_or_snap): print("Installing snap from {}".format(channel_or_snap)) cmd_prefix = "/snap/bin/lxc exec {} -- script -e -c".format(self.vm_name).split() - cmd = ["rm -rf /var/tmp/microk8s.snap"] + cmd = ["rm -rf /root/microk8s.snap"] subprocess.check_output(cmd_prefix + cmd) - cmd = "lxc file push {} {}/var/tmp/microk8s.snap".format( - channel_or_snap, self.vm_name - ).split() + cmd = "lxc file push {} {}/root/microk8s.snap".format(channel_or_snap, self.vm_name).split() subprocess.check_output(cmd) - cmd = ["snap install /var/tmp/microk8s.snap --classic --dangerous"] + cmd = ["snap install /root/microk8s.snap --dangerous"] subprocess.check_output(cmd_prefix + cmd) time.sleep(20) + for i in snap_interfaces: + cmd = "snap connect microk8s:{}".format(i) + subprocess.check_output(cmd_prefix + [cmd]) + time.sleep(20) def _setup_multipass(self, channel_or_snap): if not self.attached: @@ -114,7 +117,7 @@ def _setup_multipass(self, channel_or_snap): else: subprocess.check_call( "/snap/bin/multipass exec {} -- sudo " - "snap install microk8s --classic --channel {}".format( + "snap install microk8s --channel {}".format( self.vm_name, channel_or_snap ).split() ) @@ -132,14 +135,20 @@ def _setup_multipass(self, channel_or_snap): def _transfer_install_local_snap_multipass(self, channel_or_snap): print("Installing snap from {}".format(channel_or_snap)) subprocess.check_call( - "/snap/bin/multipass transfer {} {}:/var/tmp/microk8s.snap".format( + "/snap/bin/multipass transfer {} {}:/root/microk8s.snap".format( channel_or_snap, self.vm_name ).split() ) subprocess.check_call( "/snap/bin/multipass exec {} -- sudo " - "snap install /var/tmp/microk8s.snap --classic --dangerous".format(self.vm_name).split() + "snap install /root/microk8s.snap --dangerous".format(self.vm_name).split() ) + for i in snap_interfaces: + subprocess.check_call( + "/snap/bin/multipass exec {} -- sudo " + "snap connect microk8s:{}".format(self.vm_name, i).split() + ) + time.sleep(20) def run(self, cmd): """ diff --git a/tests/test-distro.sh b/tests/test-distro.sh index 2196d923bc..dee3d11589 100755 --- a/tests/test-distro.sh +++ b/tests/test-distro.sh @@ -22,13 +22,14 @@ function create_machine() { cat tests/lxc/microk8s.profile | lxc profile edit microk8s lxc launch -p default -p microk8s $DISTRO $NAME + lxc config device override $NAME root size=50GB # Allow for the machine to boot and get an IP sleep 20 - tar cf - ./tests | lxc exec $NAME -- tar xvf - -C /var/tmp + tar cf - ./tests | lxc exec $NAME -- tar xvf - -C /root DISTRO_DEPS_TMP="${DISTRO//:/_}" DISTRO_DEPS="${DISTRO_DEPS_TMP////-}" - lxc exec $NAME -- /bin/bash "/var/tmp/tests/lxc/install-deps/$DISTRO_DEPS" + lxc exec $NAME -- /bin/bash "/root/tests/lxc/install-deps/$DISTRO_DEPS" lxc exec $NAME -- reboot sleep 20 @@ -90,10 +91,10 @@ if [[ ${TO_CHANNEL} =~ /.*/microk8s.*snap ]] then lxc file push ${TO_CHANNEL} $NAME/tmp/microk8s_latest_amd64.snap lxc exec $NAME -- snap install /tmp/microk8s_latest_amd64.snap --dangerous --classic + lxc exec $NAME -- bash -c 'for i in docker-privileged docker-support kubernetes-support k8s-journald k8s-kubelet k8s-kubeproxy dot-kube network network-bind network-control network-observe firewall-control process-control kernel-module-observe mount-observe hardware-observe system-observe dot-config-helm home-read-all log-observe login-session-observe home opengl; do snap connect microk8s:$i; done' else - lxc exec $NAME -- snap install microk8s --channel=${TO_CHANNEL} --classic + lxc exec $NAME -- snap install microk8s --channel=${TO_CHANNEL} fi -lxc exec $NAME -- /var/tmp/tests/smoke-test.sh # use 'script' for required tty: https://github.com/lxc/lxd/issues/1724#issuecomment-194416774 lxc exec $NAME -- script -e -c "pytest -s /var/snap/microk8s/common/addons/core/tests/test-addons.py" lxc exec $NAME -- microk8s enable community diff --git a/tests/test-upgrade-path.py b/tests/test-upgrade-path.py index 4d86971825..fbd89edbaf 100644 --- a/tests/test-upgrade-path.py +++ b/tests/test-upgrade-path.py @@ -26,56 +26,58 @@ def test_refresh_path(self): Deploy an old snap and try to refresh until the current one. """ - start_channel = 16 - last_stable_minor = None - if upgrade_from.startswith("latest") or "/" not in upgrade_from: - attempt = 0 - release_url = "https://dl.k8s.io/release/stable.txt" - while attempt < 10 and not last_stable_minor: - r = requests.get(release_url) - if r.status_code == 200: - last_stable_str = r.content.decode().strip() - # We have "v1.18.4" and we need the "18" - last_stable_parts = last_stable_str.split(".") - last_stable_minor = int(last_stable_parts[1]) - else: - time.sleep(3) - attempt += 1 - else: - channel_parts = upgrade_from.split(".") - channel_parts = channel_parts[1].split("/") - print(channel_parts) - last_stable_minor = int(channel_parts[0]) + # start_channel = 21 + # last_stable_minor = None + # if upgrade_from.startswith("latest") or "/" not in upgrade_from: + # attempt = 0 + # release_url = "https://dl.k8s.io/release/stable.txt" + # while attempt < 10 and not last_stable_minor: + # r = requests.get(release_url) + # if r.status_code == 200: + # last_stable_str = r.content.decode().strip() + # # We have "v1.18.4" and we need the "18" + # last_stable_parts = last_stable_str.split(".") + # last_stable_minor = int(last_stable_parts[1]) + # else: + # time.sleep(3) + # attempt += 1 + # else: + # channel_parts = upgrade_from.split(".") + # channel_parts = channel_parts[1].split("/") + # print(channel_parts) + # last_stable_minor = int(channel_parts[0]) + + # last_stable_minor -= 1 - last_stable_minor -= 1 + # print("") + # print( + # "Testing refresh path from 1.{} to 1.{} and finally refresh to {}".format( + # start_channel, last_stable_minor, upgrade_to + # ) + # ) + # assert last_stable_minor is not None - print("") - print( - "Testing refresh path from 1.{} to 1.{} and finally refresh to {}".format( - start_channel, last_stable_minor, upgrade_to - ) - ) - assert last_stable_minor is not None + # channel = "1.{}/stable".format(start_channel) - channel = "1.{}/stable".format(start_channel) + channel = "latest/edge/strict" print("Installing {}".format(channel)) - cmd = "sudo snap install microk8s --classic --channel={}".format(channel) + cmd = "sudo snap install microk8s --channel={}".format(channel) run_until_success(cmd) wait_for_installation() - channel_minor = start_channel - channel_minor += 1 - while channel_minor <= last_stable_minor: - channel = "1.{}/stable".format(channel_minor) - print("Refreshing to {}".format(channel)) - cmd = "sudo snap refresh microk8s --classic --channel={}".format(channel) - run_until_success(cmd) - wait_for_installation() - time.sleep(30) - channel_minor += 1 + # channel_minor = start_channel + # channel_minor += 1 + # while channel_minor <= last_stable_minor: + # channel = "1.{}/stable".format(channel_minor) + # print("Refreshing to {}".format(channel)) + # cmd = "sudo snap refresh microk8s --channel={}".format(channel) + # run_until_success(cmd) + # wait_for_installation() + # time.sleep(30) + # channel_minor += 1 print("Installing {}".format(upgrade_to)) if upgrade_to.endswith(".snap"): - cmd = "sudo snap install {} --classic --dangerous".format(upgrade_to) + cmd = "sudo snap install {} --dangerous".format(upgrade_to) else: cmd = "sudo snap refresh microk8s --channel={}".format(upgrade_to) run_until_success(cmd, timeout_insec=600) diff --git a/tests/utils.py b/tests/utils.py index 8c0ba1ae8a..85aae6c44f 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -8,6 +8,33 @@ arch_translate = {"aarch64": "arm64", "x86_64": "amd64"} +# List of interfaces we need to manually connect in the case we test a local build +snap_interfaces = [ + "docker-privileged", + "docker-support", + "kubernetes-support", + "k8s-journald", + "k8s-kubelet", + "k8s-kubeproxy", + "dot-kube", + "dot-config-helm", + "network", + "network-bind", + "network-control", + "network-observe", + "firewall-control", + "process-control", + "kernel-module-observe", + "mount-observe", + "hardware-observe", + "system-observe", + "home", + "opengl", + "home-read-all", + "login-session-observe", + "log-observe", +] + def run_until_success(cmd, timeout_insec=60, err_out=None): """ diff --git a/upgrade-scripts/000-switch-to-calico/commit-master.sh b/upgrade-scripts/000-switch-to-calico/commit-master.sh index 20ff02c7bf..768efa2dcb 100755 --- a/upgrade-scripts/000-switch-to-calico/commit-master.sh +++ b/upgrade-scripts/000-switch-to-calico/commit-master.sh @@ -4,7 +4,6 @@ set -ex echo "Switching master to calico" source $SNAP/actions/common/utils.sh -CA_CERT=/snap/core18/current/etc/ssl/certs/ca-certificates.crt RESOURCES="$SNAP/upgrade-scripts/000-switch-to-calico/resources" BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico" diff --git a/upgrade-scripts/000-switch-to-calico/commit-node.sh b/upgrade-scripts/000-switch-to-calico/commit-node.sh index 7be6e64fa0..1457f48d9d 100755 --- a/upgrade-scripts/000-switch-to-calico/commit-node.sh +++ b/upgrade-scripts/000-switch-to-calico/commit-node.sh @@ -5,7 +5,6 @@ set -ex echo "Switching master to calico" source $SNAP/actions/common/utils.sh -CA_CERT=/snap/core18/current/etc/ssl/certs/ca-certificates.crt RESOURCES="$SNAP/upgrade-scripts/000-switch-to-calico/resources" BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico" @@ -21,7 +20,7 @@ if [ "$ARCH" == "s390x" ] then CALICO_MANIFEST="$RESOURCES/calico.s390x.yaml" fi -run_with_sudo cp "$CALICO_MANIFEST" "$SNAP_DATA/args/cni-network/cni.yaml" +cp "$CALICO_MANIFEST" "$SNAP_DATA/args/cni-network/cni.yaml" cp "$SNAP_DATA"/args/kube-apiserver "$BACKUP_DIR/args" refresh_opt_in_config "allow-privileged" "true" kube-apiserver diff --git a/upgrade-scripts/000-switch-to-calico/rollback-master.sh b/upgrade-scripts/000-switch-to-calico/rollback-master.sh index 781a6af88f..b0494175bc 100755 --- a/upgrade-scripts/000-switch-to-calico/rollback-master.sh +++ b/upgrade-scripts/000-switch-to-calico/rollback-master.sh @@ -4,7 +4,6 @@ set -ex echo "Rolling back calico upgrade on master" source $SNAP/actions/common/utils.sh -CA_CERT=/snap/core18/current/etc/ssl/certs/ca-certificates.crt if [ -e "$SNAP_DATA/args/cni-network/cni.yaml" ]; then diff --git a/upgrade-scripts/000-switch-to-calico/rollback-node.sh b/upgrade-scripts/000-switch-to-calico/rollback-node.sh index d904039e36..5679d8a141 100755 --- a/upgrade-scripts/000-switch-to-calico/rollback-node.sh +++ b/upgrade-scripts/000-switch-to-calico/rollback-node.sh @@ -4,7 +4,6 @@ set -ex echo "Rolling back calico upgrade on a node" source $SNAP/actions/common/utils.sh -CA_CERT=/snap/core18/current/etc/ssl/certs/ca-certificates.crt BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/000-switch-to-calico" diff --git a/upgrade-scripts/001-switch-to-dqlite/commit-master.sh b/upgrade-scripts/001-switch-to-dqlite/commit-master.sh index ebc955a407..acfdd7f4a6 100755 --- a/upgrade-scripts/001-switch-to-dqlite/commit-master.sh +++ b/upgrade-scripts/001-switch-to-dqlite/commit-master.sh @@ -4,7 +4,6 @@ set -ex echo "Switching master to dqlite" source $SNAP/actions/common/utils.sh -CA_CERT=/snap/core18/current/etc/ssl/certs/ca-certificates.crt BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite" DB_DIR="$BACKUP_DIR/db" @@ -68,7 +67,7 @@ then # TODO: this polling is not good enough. We should find a new way to ensure the apiserver is up. timeout="120" start_timer="$(date +%s)" - while ! (is_apiserver_ready) + while ! (is_apiserver_ready) do sleep 5 now="$(date +%s)" diff --git a/upgrade-scripts/001-switch-to-dqlite/commit-node.sh b/upgrade-scripts/001-switch-to-dqlite/commit-node.sh index b6cb623e6b..7cec4fa36c 100755 --- a/upgrade-scripts/001-switch-to-dqlite/commit-node.sh +++ b/upgrade-scripts/001-switch-to-dqlite/commit-node.sh @@ -5,7 +5,6 @@ set -ex echo "Switching node to dqlite" source $SNAP/actions/common/utils.sh -CA_CERT=/snap/core18/current/etc/ssl/certs/ca-certificates.crt BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite" diff --git a/upgrade-scripts/001-switch-to-dqlite/rollback-master.sh b/upgrade-scripts/001-switch-to-dqlite/rollback-master.sh index 82badaeb51..a0b80e082f 100755 --- a/upgrade-scripts/001-switch-to-dqlite/rollback-master.sh +++ b/upgrade-scripts/001-switch-to-dqlite/rollback-master.sh @@ -4,7 +4,6 @@ set -ex echo "Rolling back dqlite upgrade on master" source $SNAP/actions/common/utils.sh -CA_CERT=/snap/core18/current/etc/ssl/certs/ca-certificates.crt BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite" echo "Restarting etcd" diff --git a/upgrade-scripts/001-switch-to-dqlite/rollback-node.sh b/upgrade-scripts/001-switch-to-dqlite/rollback-node.sh index 882bd1eea6..449dcf9dad 100755 --- a/upgrade-scripts/001-switch-to-dqlite/rollback-node.sh +++ b/upgrade-scripts/001-switch-to-dqlite/rollback-node.sh @@ -4,7 +4,6 @@ set -ex echo "Rolling back dqlite upgrade on master" source $SNAP/actions/common/utils.sh -CA_CERT=/snap/core18/current/etc/ssl/certs/ca-certificates.crt BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/001-switch-to-dqlite" echo "Restarting etcd" diff --git a/upgrade-scripts/002-switch-to-flannel-etcd/commit-master.sh b/upgrade-scripts/002-switch-to-flannel-etcd/commit-master.sh index 3237150faa..327bb1e006 100755 --- a/upgrade-scripts/002-switch-to-flannel-etcd/commit-master.sh +++ b/upgrade-scripts/002-switch-to-flannel-etcd/commit-master.sh @@ -4,7 +4,6 @@ set -ex echo "Switching master to flannel-etcd" source $SNAP/actions/common/utils.sh -CA_CERT=/snap/core18/current/etc/ssl/certs/ca-certificates.crt BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/002-switch-to-flannel-etcd" DB_DIR="$BACKUP_DIR/db" @@ -38,12 +37,12 @@ chmod 660 "$SNAP_DATA"/args/etcd cp -r "$SNAP_DATA"/args/cni-network "$BACKUP_DIR/args/" find "$SNAP_DATA"/args/cni-network/* -not -name '*multus*' -exec rm -f {} \; -cp "$SNAP"/default-args/cni-network/* "$SNAP_DATA"/args/cni-network/ -chmod -R 660 "$SNAP_DATA"/args/cni-network +cp --no-preserve=mode,ownership "$SNAP"/default-args/cni-network/* "$SNAP_DATA"/args/cni-network/ +chmod -R 770 "$SNAP_DATA"/args/cni-network -if getent group microk8s >/dev/null 2>&1 +if getent group snap_microk8s >/dev/null 2>&1 then - chgrp microk8s -R ${SNAP_DATA}/args/ || true + chgrp snap_microk8s -R ${SNAP_DATA}/args/ || true fi set_service_expected_to_start etcd diff --git a/upgrade-scripts/002-switch-to-flannel-etcd/rollback-master.sh b/upgrade-scripts/002-switch-to-flannel-etcd/rollback-master.sh index cedfa5e8c8..3663a88e4d 100755 --- a/upgrade-scripts/002-switch-to-flannel-etcd/rollback-master.sh +++ b/upgrade-scripts/002-switch-to-flannel-etcd/rollback-master.sh @@ -4,7 +4,6 @@ set -ex echo "Rolling back flannel-etcd upgrade on master" source $SNAP/actions/common/utils.sh -CA_CERT=/snap/core18/current/etc/ssl/certs/ca-certificates.crt BACKUP_DIR="$SNAP_DATA/var/tmp/upgrades/002-switch-to-flannel-etcd" ${SNAP}/microk8s-stop.wrapper @@ -28,9 +27,9 @@ fi chmod -R ug+rwX "${SNAP_DATA}/args/" chmod -R o-rwX "${SNAP_DATA}/args/" -if getent group microk8s >/dev/null 2>&1 +if getent group snap_microk8s >/dev/null 2>&1 then - chgrp microk8s -R ${SNAP_DATA}/args/ || true + chgrp snap_microk8s -R ${SNAP_DATA}/args/ || true fi ${SNAP}/microk8s-start.wrapper || true