From aebdb7bc7144904d1f6bc57a51c0afb8d39f9893 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Wed, 11 Sep 2024 14:31:31 -0400 Subject: [PATCH 01/54] Adjust helm-project-operator/helm-locker imports --- go.mod | 2 +- go.sum | 8 +++----- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index bff5b48c..8e46adb1 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/rancher/prometheus-federator go 1.22.3 replace ( + github.com/rancher/helm-project-operator => github.com/rancher/helm-project-operator v0.2.2-rc.1.0.20240911141850-1140ae4aace0 k8s.io/api => k8s.io/api v0.25.4 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.25.4 k8s.io/apimachinery => k8s.io/apimachinery v0.25.4 @@ -62,7 +63,6 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.39.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/rancher/helm-locker v0.0.1 // indirect github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc // indirect github.com/rubenv/sql-migrate v1.3.0 // indirect github.com/sirupsen/logrus v1.9.0 // indirect diff --git a/go.sum b/go.sum index 3e514a8c..547e939f 100644 --- a/go.sum +++ b/go.sum @@ -338,7 +338,7 @@ github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjY github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU= github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= github.com/onsi/gomega v1.20.1 h1:PA/3qinGoukvymdIDV8pii6tiZgC8kbmJO6Z5+b002Q= @@ -368,10 +368,8 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rancher/client-go v1.25.4-rancher1 h1:9MlBC8QbgngUkhNzMR8rZmmCIj6WNRHFOnYiwC2Kty4= github.com/rancher/client-go v1.25.4-rancher1/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= -github.com/rancher/helm-locker v0.0.1 h1:v/m7Uu5wGivn+FQn5/xMuUG2L+CzocSzD7sjM7+/74E= -github.com/rancher/helm-locker v0.0.1/go.mod h1:PRThM9wL4o7MXJwUDeAk/+9s1vpmbRbacnGm+HoGqbY= -github.com/rancher/helm-project-operator v0.2.1 h1:EYqqyYgOOqCsNBAzTub9nJmjkmspfhLhO2u8iHPDLM4= -github.com/rancher/helm-project-operator v0.2.1/go.mod h1:8z5cRg/aOcxpIUWhLoW/bBUtxR8coJSnZT26k3bZa+g= +github.com/rancher/helm-project-operator v0.2.2-rc.1.0.20240911141850-1140ae4aace0 h1:LJXfxKzKCVSvDvRHO8lzyM1FUyqiT/2UgS31UAEFHvU= +github.com/rancher/helm-project-operator v0.2.2-rc.1.0.20240911141850-1140ae4aace0/go.mod h1:HkQq2yAWVGoZ0Q6jUlNTJaI2J8mar/PF8Ur2PN9nmYY= github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc h1:29VHrInLV4qSevvcvhBj5UhQWkPShxrxv4AahYg2Scw= github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc/go.mod h1:dEfC9eFQigj95lv/JQ8K5e7+qQCacWs1aIA6nLxKzT8= github.com/rancher/wrangler v1.0.2 h1:0JGv62gF2OkYUoR0fsr99Za63fquFeKTHE2z9kAFVsE= From 0b47461e34b395010a1c47f9fd9428d3c3b5ca24 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Wed, 11 Sep 2024 15:15:24 -0400 Subject: [PATCH 02/54] Add improved cross arch compile script from BRO --- scripts/build | 33 ++++++++++++++++++++++++++++----- scripts/version | 5 +++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/scripts/build b/scripts/build index 73d7d55a..694e5e98 100755 --- a/scripts/build +++ b/scripts/build @@ -7,14 +7,37 @@ cd $(dirname $0)/.. ./scripts/build-chart +echo "Starting binary build"; + +ARCHES=( "$ARCH" ) +# Set CROSS_ARCH to build for the other architecture +if [ "$CROSS_ARCH" == "true" ]; then + case "$ARCH" in + amd64) XARCH=arm64 ;; + arm64) XARCH=amd64 ;; + *) echo "Unsupported ARCH of $ARCH" 1>&2 ; exit 1 + esac + ARCHES+=( "$XARCH" ) +fi + +echo "Building for Arch: ${ARCHES[*]}" + mkdir -p bin if [ "$(uname)" = "Linux" ]; then OTHER_LINKFLAGS="-extldflags -static -s" fi LINKFLAGS="-X github.com/rancher/prometheus-federator/pkg/version.Version=$VERSION" LINKFLAGS="-X github.com/rancher/prometheus-federator/pkg/version.GitCommit=$COMMIT $LINKFLAGS" -CGO_ENABLED=0 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o bin/prometheus-federator -if [ "$CROSS" = "true" ] && [ "$ARCH" = "amd64" ]; then - GOOS=darwin go build -ldflags "$LINKFLAGS" -o bin/prometheus-federator-darwin - GOOS=windows go build -ldflags "$LINKFLAGS" -o bin/prometheus-federator-windows -fi +for A in "${ARCHES[@]}" ; do + GOARCH="$A" CGO_ENABLED=0 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o "bin/prometheus-federator-$A" + # Set CROSS to build for other OS'es + if [ "$CROSS" = "true" ]; then + for OS in darwin windows ; do + GOARCH="$A" GOOS=$OS go build -ldflags "$LINKFLAGS" -o "bin/prometheus-federator-$OS-$A" + done + fi +done + +cd bin +ln -sf "./prometheus-federator-$ARCH" "./prometheus-federator" +cd .. \ No newline at end of file diff --git a/scripts/version b/scripts/version index 5f357a62..c66a1f33 100755 --- a/scripts/version +++ b/scripts/version @@ -16,6 +16,11 @@ else VERSION="${COMMIT}${DIRTY}" fi +ARCH=$TARGET_ARCH +if [ -z "$ARCH" ]; then + ARCH=$(go env GOHOSTARCH) +fi + TAG=${TAG:-${VERSION}} REPO=${REPO:-rancher} From 8572b3c457ce1e0c9e633df20a281630b4a39da3 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Wed, 11 Sep 2024 15:15:40 -0400 Subject: [PATCH 03/54] Add build script verbosity --- Makefile | 1 - scripts/build-chart | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index f944f016..e7720925 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,6 @@ $(TARGETS): .DEFAULT_GOAL := default # Charts Build Scripts - pull-scripts: ./scripts/pull-scripts diff --git a/scripts/build-chart b/scripts/build-chart index 0ea7c4b0..df9e9796 100755 --- a/scripts/build-chart +++ b/scripts/build-chart @@ -10,4 +10,6 @@ VERSION=$(find ./charts/${CHART} -type d -maxdepth 1 -mindepth 1 | tr - \~ | sor helm package charts/${CHART}/${VERSION} --destination bin/${CHART} base64 -i bin/${CHART}/${CHART}-${VERSION}.tgz > bin/${CHART}/${CHART}.tgz.base64 -rm bin/${CHART}/${CHART}-${VERSION}.tgz \ No newline at end of file +rm bin/${CHART}/${CHART}-${VERSION}.tgz + +echo "Completed ${CHART} build process." \ No newline at end of file From 5ad4f62cb61f03678b52b6a1ae421a2c694933a5 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Wed, 11 Sep 2024 15:16:01 -0400 Subject: [PATCH 04/54] Improve docker file consistenty for helm builds --- package/Dockerfile | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/package/Dockerfile b/package/Dockerfile index d6997751..9230eb0a 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -1,9 +1,23 @@ -FROM registry.suse.com/bci/golang:1.22 AS helm -RUN zypper -n install git -RUN git -C / clone --branch release-v3.9.0 --depth=1 https://github.com/rancher/helm +# Image that provides cross compilation tooling. +FROM --platform=$BUILDPLATFORM rancher/mirrored-tonistiigi-xx:1.3.0 AS xx + +FROM --platform=$BUILDPLATFORM registry.suse.com/bci/golang:1.22 AS helm + +# Clone repository once, and reuse it for target archs. +ARG HELM_VERSION=release-v3.9.0 +ADD --keep-git-dir=true https://github.com/rancher/helm.git#${HELM_VERSION} /helm +RUN cd /helm && go mod download + +COPY --from=xx / / + +# Cross-compile instead of emulating the compilation on the target arch. +ARG TARGETPLATFORM +RUN xx-go --wrap && mkdir -p /run/lock RUN make -C /helm -FROM registry.suse.com/bci/golang:1.22 as builder +RUN xx-verify --static /helm/bin/helm + +FROM registry.suse.com/bci/golang:1.22 AS builder WORKDIR /usr/src/app COPY --from=helm ./helm/bin/helm /usr/local/bin/ RUN zypper -n install git vim less file curl wget patch @@ -12,7 +26,7 @@ RUN go mod download COPY . . RUN make build -FROM registry.suse.com/bci/bci-micro:15.5 +FROM registry.suse.com/bci/bci-micro:15.6 RUN echo 'prometheus:x:1000:1000::/home/prometheus:/bin/bash' >> /etc/passwd && \ echo 'prometheus:x:1000:' >> /etc/group && \ mkdir /home/prometheus && \ From a3733c54d5335cbc3c2a9bb3e9524c890f0a4d91 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Thu, 12 Sep 2024 14:08:50 -0400 Subject: [PATCH 05/54] bump chart version --- packages/prometheus-federator/charts/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/prometheus-federator/charts/Chart.yaml b/packages/prometheus-federator/charts/Chart.yaml index b7f2297a..51f5984e 100755 --- a/packages/prometheus-federator/charts/Chart.yaml +++ b/packages/prometheus-federator/charts/Chart.yaml @@ -16,4 +16,4 @@ dependencies: description: Prometheus Federator icon: https://raw.githubusercontent.com/rancher/prometheus-federator/main/assets/logos/prometheus-federator.svg name: prometheus-federator -version: 0.4.2 +version: 0.4.3-rc.1 From 6bf04f6187bf6ba6563c2e39233e1e5e55d4a036 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Thu, 12 Sep 2024 14:09:26 -0400 Subject: [PATCH 06/54] Update deps to use new upstream chart --- .../dependencies/helmProjectOperator/dependency.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/prometheus-federator/generated-changes/dependencies/helmProjectOperator/dependency.yaml b/packages/prometheus-federator/generated-changes/dependencies/helmProjectOperator/dependency.yaml index e8b94722..5a1d67de 100644 --- a/packages/prometheus-federator/generated-changes/dependencies/helmProjectOperator/dependency.yaml +++ b/packages/prometheus-federator/generated-changes/dependencies/helmProjectOperator/dependency.yaml @@ -1,3 +1,3 @@ url: https://github.com/rancher/helm-project-operator.git subdirectory: charts/helm-project-operator -commit: 54630179aeae78d5bb28688ee2262a0ec362ec65 +commit: 1f8d3f40a2708b8a616934aac6b3b30d81eaab32 From 34318c04c6998ceef6579939f51ec5d4d5f2c7bb Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Thu, 12 Sep 2024 14:10:00 -0400 Subject: [PATCH 07/54] make charts --- .../prometheus-federator-0.4.3-rc.1.tgz | Bin 0 -> 20617 bytes .../0.4.3-rc.1/Chart.yaml | 19 ++ .../prometheus-federator/0.4.3-rc.1/README.md | 120 +++++++++ .../0.4.3-rc.1/app-README.md | 27 +++ .../charts/helmProjectOperator/Chart.yaml | 15 ++ .../charts/helmProjectOperator/README.md | 77 ++++++ .../charts/helmProjectOperator/app-readme.md | 20 ++ .../charts/helmProjectOperator/questions.yaml | 43 ++++ .../helmProjectOperator/templates/NOTES.txt | 2 + .../templates/_helpers.tpl | 75 ++++++ .../templates/cleanup.yaml | 82 +++++++ .../templates/clusterrole.yaml | 57 +++++ .../templates/configmap.yaml | 14 ++ .../templates/deployment.yaml | 124 ++++++++++ .../helmProjectOperator/templates/psp.yaml | 68 ++++++ .../helmProjectOperator/templates/rbac.yaml | 32 +++ .../system-namespaces-configmap.yaml | 62 +++++ .../templates/validate-psp-install.yaml | 7 + .../charts/helmProjectOperator/values.yaml | 228 ++++++++++++++++++ .../0.4.3-rc.1/questions.yaml | 43 ++++ .../0.4.3-rc.1/templates/NOTES.txt | 3 + .../0.4.3-rc.1/templates/_helpers.tpl | 66 +++++ .../0.4.3-rc.1/values.yaml | 94 ++++++++ index.yaml | 23 ++ 24 files changed, 1301 insertions(+) create mode 100644 assets/prometheus-federator/prometheus-federator-0.4.3-rc.1.tgz create mode 100644 charts/prometheus-federator/0.4.3-rc.1/Chart.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/README.md create mode 100644 charts/prometheus-federator/0.4.3-rc.1/app-README.md create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/Chart.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/README.md create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/app-readme.md create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/questions.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/NOTES.txt create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/_helpers.tpl create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/cleanup.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/clusterrole.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/configmap.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/deployment.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/psp.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/rbac.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/validate-psp-install.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/values.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/questions.yaml create mode 100644 charts/prometheus-federator/0.4.3-rc.1/templates/NOTES.txt create mode 100644 charts/prometheus-federator/0.4.3-rc.1/templates/_helpers.tpl create mode 100644 charts/prometheus-federator/0.4.3-rc.1/values.yaml diff --git a/assets/prometheus-federator/prometheus-federator-0.4.3-rc.1.tgz b/assets/prometheus-federator/prometheus-federator-0.4.3-rc.1.tgz new file mode 100644 index 0000000000000000000000000000000000000000..7926f1ea89a78f3af892ed32b8877a8956f04a3d GIT binary patch literal 20617 zcmZtN1CK6D&@kxn%o*FZZQHhO+xDEXZQHi3Gq!De?tQ*@-_2%|?LW{-rPEzqbrnH0 z6e`gFSpX^^8Y4+%MiWUnc3CejHe(ibMpG3wD{U1nb_I2Hb~z1OYa@G8FI6Q6K1nlM zJD}@cKNp<#PRiZ>HpR~8+?ZGD?&1)C~QXY5bI4(O`I*-NyP~JO-FjSbbvl`V9Dw)g@?TLeCu2gRQ zMX}dzgasH8C4>741A z9hHWdxJF10{d%PCyNXz!TY1$JlS?6BGvr5C1*@T10mbBzp>WrrI9agsVrF%250=7 z-RU1$r7Pjsa1kkoL#(wYXJu+~lti1fD^8^7LO!`+dFYk`UIOLP4E15znb6A~I`wI- z7T~eLZGuq>(9DtX#d7fugN<^OPnoP#SwBXIhOl1&|L=39rD4#={dO5skfUhcJY4-j03m_^{V(pEQuK3;gke3ENavnHRG z1!7GMI=a7Kl+9nea~4MCA7^Ier;|FP%Yom>9QHQCu5S{ngUNimh{GhM28yb2=r>x3 zo7PT3Cv&hYvDImj*?Mb^u-Ui*{!==b)OU zJ@|^h2hU{;2_2`^E~x4ahXY+U25mqOw^Gf+)zdfI0u94KPo}Fj2eoLj#a!17%ww3P z=^eC)AHN99IG7reCFkJ{#DuBbi77?g8!1MtG(dzKbyAlS(YjRqnPn!t4HCrZj>LTE0bs@ zON_Tp!Vqds!cxgBGizU}I|v^bvb1!&+^#2r(q1=I-ymU=-#q#W&Q!0v#<09=i`&c(t(Cz)K59i>`1XsXie-q zWXVk7&LZ;9ztqav?hL^vxrrF18hrYIDGO!XMsh5TcDOy8u8pAy*<1Qlmf3l@c}5|8 z!PhLV5BEL$TBK#!Scpx*GOad7P)BE$GT3Y@*Mua#Z~l}uAM4&lutxY^7scnWijB!^ z9jmjD>ldWQZs(s7;Tt(~s`0_z>o{8Dx_@N7u43*CaNEp}GU!^0Zqc`_|MU1tTv=rf zR!?&Pe9kUjkj2S19jGA@eX^OTRD43u4 zYzafyK(b7;4yjc^W!ceEydo#nyi^Y)G)Ss^1gJR%s?a}2g-Zk^)Z&6G`!Odmn@i#J zI82f*kU_NS9*DvvEe!%s4F67?@nB|FI`9$5`2oEU61IMb^UvLp?`|KN!HDG@+dm4gx44-~Vw5rGA(v9Cv_@9)`BU0X za$TAt)|FCb;xyW)oHLe?zu}Z*+$14E9OsGc4!ubd-npwd0S@sar=u)ou|vfZgd}`6 z!e7|ro>2iERIf$z<~az@8^Hj9c`bFuN@Kn4J=7L(00ACe`#oSfdHyv&{&B;_+4TAK zWX3My-lzndKdXO-hQ`H^2jwl+GP9Lx++Uh2z7O zkQ&44O>*o;ZfTuDQ$4EYOcGfx+~_!RZfIGC1%m017dU_!6L;@sta*NOE|xk2KA#1X z`>38iqaL!FWkpF-xcaKuIGQwRVXTCC(ebw8fEqP_5EzT;GORD7pp)8(6@ z?nVQ%eE9evmx;4hmZ`(#NN%SaKC48ft!EnKHIMeg80_;r#X`0o`t@bV(3{zU1Y`@? zBe$2zGA|h3Fe6c^dd8yxcVu_7a_SR}NR`Ggl%q^gBqb}co? zC{x#=CeLxd2no_`^(`{3#LSJC-)0GKfwyfeM@x4c#g)N;3jLcHS39l`l<7q+`kYKfXU$^W{GPkJry1@fHunKa6+lv;cw6$A{tw zxDUW$dj3y8!P(M{Pb_hQUxJ^LnO6^Si_R(W7fRKDCA98cD+`RrXCk2SPBs%0Lm39* z!nbHopoR-_U@x#|OfL3}B?ns??>xA{pXLeY#iefM#4?2g88;a=Uva;%VP0r?_6nMJ zWd!t!3uBmnqdg|Rf#W=AQe7MbD(|J>Qr9mq)IL#f?bDB?a7pl6mxGKECM0L-X#-K! zix2i>HJjrn?x-(knUsB^iFt(drMGI1T1dDHr!mCrYmULQ97U&>4N?I8$ zocnIi3^=PQOeZZ%nV@|06Cst_Q&a)7T8X;{%%>{<9h58Zq_FT;kcG6^-i)YUeLmLDHgw%1wsak0ba)QILALiI($6T$QGCm(h9h?N7uM^*5M14Y$J!x4JVIj~E-}mz+=pmZs&N z>z$vFQ#Pw?dzmf^-G3A`O>FE#d&Cz8Wq7+favkh%RhTh2Y^_d#w;^(3jRz&^SydqX z|4Rz&W0375=gfB8^%C6dYp#mb#? zJ;k{;&9LG`@zDArv0(A?R87s**hoEi(9= zH!j-GQ}zLd)Pln3yc|E~ee1g&7DT?>b`Tw?q2U%Xavkd8s}bAI*joelbQBvHDg{q; zjHhh<5FtxKr)A9fCkkCUbV1oPpGM1(bpuld+)F4Q$nrc>tOTOf@6+7J?2r22((IEo3`Hh$i zrAf;)cmm`)f{MT5#D+A)R{FeX5Xa{Ff2~5;1@&g7T+r>;Nl(vn`2OM1;;exi>Tqh= zvMlF;E8>VPdh*5*#zDq1kU)hsj1}nnBzxZrK36qD<*8gmbI+Tk%x6Nj;%07@phY0DPH^Q*s8A2{Ds z2$2U>j%@WCb`qwlM2Eky_{VVsJT4K{W)f|r7?Nh5(^PRh4Iit@H_D*|cW#ye-|3XM z3vt6nYJ%|@qWMweX4$veWO{Ttke$0Q)M>~~2Yf2oRXZuRi#&B>9P13S>Gkx+`PbT* zMB-T|67}D=HQ_5wvv|fjp}Vk|FCvY&8S@(rcDJKWH2B;Ci0>k#3C8094V*=YM+?Sojl`yA}i2M5*FrH5d4SqCB{lo03H%*ESgYTY5b!`66 zY>2h>qWe;XSEkufS8=8dNaOg9{T7_}OxXph7T3{7y)kuXV2;Yi134)1vzPrjy0hxz z^&tgw%Rwo-?#d`+%O6=(z6e)2mmE<;6wp%D^srQ2i+wLP-x8`o%PF z*@|K0^1ws$-T%g@$QEi+*aXvW(nRae)O$p#5UIQktFk}Zm8a32Ng7QTu!*l(U5hFBJysK+i;V4dj%RbesY zW7Om;-&m%j)BVHTM;8sXUZACL0x41mXNNUUpExr>#?jEJsUtj${x1~doP%9wc77FR zTg1Zt`3ZbKZ{+wfzI?um(A=`67_SFD`*m}x>o}=7J6z;4A!jxCQ&{6b;Vkd2p3R>t zY?_&}JlZE=NIK__3Tyec*Zq0Sqb^a9-;Jks!aAgvgbMVG_}>`tzVSFzq6=q<2+MNZ z1?72{_q+ZGXW`@nea3wti0fp((Tv#8+%8j z!5Jo9NC~Y@Sw}>=F&mV@ah!3>xn)MIbgpdmOH6IYcCTq#M(i>@3p$X}smXTW$3^tZ zLN25znkp_6-WlE2#TI7b0bu#i6j?N4gSY_n_o ztD%3;3RqXVNLE`ajHryOxCr6i1=vqa(Y&NFI4ir9E&;&27^zASy?(rqqv@u^POuLz zo+`vgm*FzQsm^CH>OwTeogUXc~euzL-Kfoi@@JI4))!C zg5s=%0#HsjXhytx0{=kb95-j}j{a_0N37-Y268+GbWdihY+Vb9JgteIdyR8}E@4ED zh;ue$*eQlSdTPFb5PH_c&TgNYh$+)+6*a4wA>n#Be1@K>*5NWev1ll{VlT8zIUYy- zKq3TOzUqa<)Se|VB|TClc*rHqIF3~+VJjussVrcxnhEQML{H{~Cnlb@Ii4nJ3|^ui zzAo4SsG}_&$jP?Fw>3gmiVI?nlAV1I+=Ah^BepMnw>TfHK4e7XE8;l{s57Tn=YT4S45^}g4Z6@P1uPt;x%2Jehs%DrEg=xlj|Y8 zq>X$_2x}D*&bn>(x4pPrF>QbSZ-9qCs;EdElbbhBU9*}XtW2^8-wktOM(Y~5tSR$^ zaSITB`DIVp*x+SbWZTs3Xls;7qzWTsdxtv4z@CJ&r!+sM!&A@1!@l^rN_M+~`U~eF zp(}XKIbKs)qWk7dqtna+VAX8fdG!~a!L{TZVhDOJACy6)`r00B65jAkSq}VZd4I6+ zD0x@z9F_E`nk&i1u|sL!&UOJW7;78s%kUM_dWC=Gr8MxeA3Vwfi1Cw_rXHvXUNK6n z&y)-LN7XLXQ1E#w%sf209|VSs@(pNeZUDuRPZVI$m>E5 zI%Yb9a9fTB(LjI`%^r@bgp@MlvVXKiIAvt=~-#VszQe9 z>A#YBxf@Hh!nh?lM^7KW5-0^PrWq7$Wv;d9lYC3(E1ujFbZO~JtedOw!2P{IbQBH< zdwu1L6n{3>U(w0+p3ShIc!ZlhGp1x#vm=pV`J2-19n&9@RVQn=H{UTqwm&miL=81S zU)N1`;gr#+n88MejByfg=)pylP)-Om7FsZ{2!#4n0nFoSe!9M^NRPL7r?RC!o2mvxVvf{rfIw) ztbr2Ibg5&54lqT*M@?#|5KN{?eF_Y*u|rpIsAd9k-*pjO;f>F7Bb7UA2q!?b@bXPz zYYW)wY(3M5+Xg}!@%OC9Fe5eg1}#J0c2HJkhAC+yXg9Dtz3h$t9Biz0@j((s|CdJGG5W zIfQqW$QFkV6bKt;w`db`F|@~OhfwVsAOgYlF?Hrqc&(@=^=|&n8cC%VghGwNg2RfYS%%v*P=yCKG{FxF z*UgP;15(ZR-qva!E&K&b1sPBIS-jwKHK{xsDHQQN7m0qVY(EYl{3MlhL|meNDCAE6 zRsK7b*;MXclYKlJ@3w)&LN1fk7nuS^%0-My8~xN1f1? zd>W?7KprQ>q}%dVU2-Y)!+gy$cfGHlUDMYtAa z?+~Os{ab6yu1bs^c5ErP|Bq+hIHoQGdUC*;*!1s-)FNPW!F$GS`P>&10{doo^i#IL z(Wc`XcI_pm*evr7@vcfVj2d=Yr$+NS&C*Qa~k~{N$Ot2ChIGbxY zgI1e!8H4qYLjHsafhe~SpApV22s?@(QTw+7n$mu+1CTjo(P57x+ovI+e)%%|>nIoh5$lN-hlJ;WPh%Lh!c4QM!zO_BI+0#an}hhQIx{E;u{x&OP22 zj}LS~hjxfoU(5@*zJ_z~FB>#)c6LneDcv&e3}cy#^8l4vUEVLuwU8!FiVcn|8lj3{TnwGfK2{5n0Pi&0F1Btc zUN^(GZ(L0S`XwWuyT*#p9sg`XqRq;JaS0AWePvcwMN%Dw{UR_8A9c{d46YNOvFQg!>$KJ3FVY2OJwH6&eq1kq zWEsV3Nh@)Qz1fv*(U5hISGJIHp%nAZGl3M)9z0@KF;6*z57SjzqDyv~f=n?39#z6N z#)jt1^W+?CP)Z$&uHhjtLrShO0|N1(X@Psw1(&Ik*_2a?SCgHsk0MFd z0Q`2J@X`HyMdLK9`{NVS`_!3Z&#B`9{KbiglXWsvSy)+YF6FgZHdU1x9v|m{(TdrH zO$Z9?D9h|^7n=P<iuG^|-%yk4+sN9g4^hU`r@*g`!;tpn)TDQ^+!}UQ|>Iqpc;7e*xJt{>)?%|>Qa#f*aSffe;h&K@JPZG z%79?BzkLsXJl+;x_G?3q&W;d!a+j+^NN8;WsJpFe14{YYy_toRr{>!u=;Wrpr^(zx zk(uuAAY=`2XDeXuh+_6PAij_HR~PVL;_F@kIFA;nqZ#br2ZB;WKC~@^vk7I3;qX0w z0lO_SJNeN~d~gvPGjI|cTNpE03p4if6xQV`WrVcQ6W8*n9JEN^Z`;%@S$v|bU_?f3 zf(d@KiKE9U*l;hm5>oxczH4>y@6Psb%dDQR43M$&Zrn83uuQ7UQm%l@JV+pB+8kOI zgm!S7+;uIpXpASs7{%obHyf}bUJ^*D7bt&=z2_L`*zI)d+e%HFEMxp))JxV@KH3zT z^&kY4W+OFw)!h+jcSfKKuK(T|>Gy{i3uL42ulgW9FVi+>;k6321B zIpex52x1AUQ%wvl;}rBzDfU2R$Zci4BEfc2tWuYcKxj-cf-LU$4)800m;lO;3gyi^ zKa8RC^Ni@*V2@cdt5!;?R4O_Y#62iF+l!%r0bFVA05?CGLLWAQsC9)p27Y*7EO!-K zSS>Vu+}IWkYFuk>v1a9l+#9dk)zRe$O#A|rP^C+N8t-JZ%>Qq*zSSWPWzCXg60;Re zFAIr2MYd#-x3FJTuEG5&Y$y`w$cjqH&|z7mW`wOIWvhkv z{Le96Lg7CHj#|`dW5>qR`kpq?GK|pd<7Y7H-Me*I#h0V{gQSD@l5fWI;Pw zLL|}!u+ZfoS*oPIVdKSaJ610EDk@Pf-kuot7Y)-*bQv3H=sh8!=xe3nRyLk=0sSp!FNXOuXUZ7-Op8qB+N)di0_ex zt-(kaYM2f(@vqi`J3A8^vm^4yHD#J=2&_hc0(E7knzi0K2-exgo96_u(@s5FA9n2a zohfer27J-ki#w3Ku0eYk%llJ0lm-qvpAY!D4r}(S9dHc(y}C}F+5HId9W%-68A0AQ z*-dgFZVPhE3afx->=$iHp@z$dKm?giczD;Tm%0Teg|CE>xG(Kqd1T|2|g~!Jz@q_Dm1) zt8B$^yqjrT0?u3}>Y0u*#iSI%1?_%ft?~+Y3QI@?F(Kjs0x?vJ;{8!wDk$<3+x!3Z zGYCgV72@389@W_WLZC%vel-eyp#lNBM=lzl1f-I;-*L0gfcMFxWComP!14L?{11SC z%3_|U(^hm&b;7;~S;A*mS#okuKH%FZFb-`3=Ln01gV#dJyhWbkr?aUaTx|**Gq_g{ z(&*EMn~TH5YLVQiGi7caDrQ&RnpZ)hWG#dyjQXC!m<}-rQ(RIr(?oO?eJ03<$0zaY zjdcj<=YJUrSb0HJz#x z_5}6uO}90+24ZjzaTwH*nwr}%S$5n6J-Ou-@0D}K&0_y;q?51Kj z-?PZ388(9k>xl`W#=n5jP@XI3hB^$Hi~?UmF#rGTNl>=CkcA`T+iT=@6Ixc4%{rnh z8SIRDM^?VGr8w?@V~Fr0*@`V89TJwfyT<-mH1l+wqc+LoC&4KPGa+T*%QcWWZH_m? z-%Gb72~s+YXf+8dQ%A+MW{6Xe=#MSu zK^IU=9QO{DoW5L6%wA9q!&I2C`pOSXyyI+O0-e8gV7g1lU%h=}K+TZ$?7JslaqAvkj?M5C1(2m|PCU)1f?yfF zT)a3a*t)0CJwxWqMtbdWAVUe0XHRt@m)Pi*YS4i)ee8D;eJ-u$ihZZP3ux4`1&j)8 zu?yKor(3dG<(^}yaqRTCl-gh;EoM*EJq=Y#EzT-_5B7d5Kn1x@i0XEshn=}*QCD`! zMRA$hymi>pL`fA@s<`Bu`PE~u zWlqJLbR&yf0sFpHROm_eN1#4{AD>qj1pYAnKk)fQ)A)lxnK}Ez9nzSu5-pyMqDH?8 z@&Qw70>~gMuWJ6FPAwky<+)5CR?U2GW7gS=m(O1yem|0SlRZL|Rb?=w%4yM*Vtdz>cN zWB!$gYU$Jvq@qt2#B1!!T4O@C`(;LvNu`-#h>Gylu0CoyFN~y~%O7>V-UcG^dA$kd zj|*P*;k+wqxk$+_`hhUk#sh=uq=m6hPBzP5kvM6 zic7l3@N&h{PjJ=m6*o92P5P8*)YOQZ!d~`;O0$%wc3coky2d3(4{BwmdSJQx=9^VT zMs(mjkQ~Mn$qpGi)>{l;=Jd7~eVs^3C?5DA6>j*DK7>PLYI;Py{+k73?nF=H>~CCVbP1$Pql25B`Dp291UxDrT;Zr5)*P@}C0)kqtf#{$Hn zOB<22i{-)txIthZqI2a8p?wLmjk?8;24M??5@QYd?zC|V+ttW-DYN{B)v;Wj-KU#w zOj`ke{|Ijl5Bhc)I^{kT86x+1zOqAxGaMBUmKnpBGk-^?ue(km^AY!A98Lwp zE{toB)Z#W;=+GZNkhj5YMe_17V`gdX-r<8w0Km|`SyN^rJnl zWQCGXDZz6?tFGu1NOe`A{g}hxV!(p z;Pk%W=^XaQ;rn|n-vb1=y*wW`001U_hrJ{QW7x}PwB0IJE}Y8we;K?XNy)k<=&FB^ zKG~bL2}5Yocq*uOlm3%m$@$M+B+%cpe8|}x^BLq}g+C!N&oxM|HW^06(0X@k@tXaK z^FI%OGp=c9aqF^bgL-kIRO}8`|_5b zyVmvp$S-!bSt{o@}F@!(j8(dNj z9}8mHop*R5$nJ6xm|0L+wMnDlOnsjI|K(nR*#E=53^0eSMY$=C2AfZX%YE0#NLCA4 z&9UpAUG3u0bZ9eu$e9!B>0d8U`YGlznq6`DTV*ndXYp9N30*VFpuuf z#d5gadAAcF{@>0EzJO{zLm%z>Ds1rCbWR+s4fK}VN2No!Qc-B7i;8h=WlK_xEBBFu zKHEpoW`#$|L+gKh7q)O^oC4!4dX_VlTwV@hR|zkuOPZfwE6i18+FMBICi4=n;dAN<+;zo&sH@dt>W<%+6yaE-QdCh1!vBa7WJq9zK?Bv{(xh=I*r8q5V=j zN!x&J*9>Rp1`66!yveH7hdz+9RlHlvefPm+dVhXwsz`M&Y zK;k1bQy8Tx`NPo*w=nv8-WZuKql{{0neyYDyc?wxcU{Cdzb#Y8qaC)xlLs8(iFb0! zmB#1qB;BM}UXyOJ#K@zcHmV2NM<{F3F%&;)^@k(^I z;P2DRTzuqrluKJCOC;z31*&;XY$_b*1M=8psJAj_lU+ppl;3KOovkuBD=jfwY_pnS{=ngHAzqB&%qWCnNxe9#u_g*NCK95alta6 z%sKbI6{oE>AtkS89D@d-4W{&x2%(sf?&tPz_AM>B+zeYU{&0Hf`K!CGuKYQ8AQ~?lvrJ#ztiW@2BkYIXhR7grKE_R zEk}0|=)x+`U&Fx4=HG0IhS-0V^TS$$>24p{yM@5t=fiCfe^e2JAnuv&G{H88{VRso z14_RK_p92w2|6#46WN!1d%t$;cQY%~b>gOZosFI@a!1K*cNWCJ<|sJ;z}8n?UO zGLyp=uFKN4j~!(JOUfCU4S@aR#^!d$MDOROyw9f7gIvy(@jezmTSZ!IwMu7YAd)^%+D>;on#X{8OKe zvSv&(H2g!_{aIvX9mkWX=>*i+Cx@{3@gcf+=d{Vlu5ji0(*MAP}P$=&XIqeZMvbp9n3Y&Vl zn`_QTjTTg0>AK6oJx+-Feh+WF6M950m$ERu-e&erqCmk)wPC zA9U=|``v}xYf;-^LM^0sKq5uVV}SuVnBIhss;%ej+rsi)=lVY38b&w99KdTwt~;*X zL6803`G2mwhp03Esai`U>7VW?=cWEDjpi7t13_zEf@u|R)&&iBy#JiFNz?K%yveq{ zst&WP%?y~$^k^pw)gRMSN8IhV1HQTuxHAEO)>oloZ5e}Zs{@|S&y9wE-;Up1e#+yY z8Y31F2f_vPr7Ol#`$eLwzqpm17KBjMQ>mFB@r0%CK^>jL8Ea4d{qr#pM=OB>a# z@BMB69ePrk^ z`nYo*bIiZ@G%ZW6AYxwrLTNE`HHHXqdLqI>geY{b7_E%#Ey$Y`os_Si&-vvrQT8(t zA~4r&Ygk%8OmcA6e81yt8p`JW$l2b@Hw~W62LR@MCIaY3U8^BIZGPq%vLE-3#`BN* z1?DW~PvEdt;lcR`jl54(`25CBh_n`L#0#8A>{tg)KayxGNB*sBt{z*KSZosV zAi2`QZwV^LT)FVcG!S$rWHf75>w@U%`lAlZ=(zpcTTc(0x;ne1)t zz>)lL_w3l?)kT69Ttsfe4^3p*=`l^#st7np<<>)l5oU+5L(A7<9jfGi&*$&;V*J!I zdgFlpceA=UJ8#bbyuME+Eq(ySzQVzh(ZXsVKQ8_KcQWV!c+Uyj4Ai4yC8{4k2e?^C zaqeAnF;<{%F*C7=>2oM9^Y@s-?l2@f=+w_w(9avkiQNb>k|)JiJJV9_qg)u=C*A~< z{NYoXRoxf2-H$>UVQ(+J4Rp9w)zjnv|HZy5LiTDHhjPL`#1fuzBL{CZd8R zHJv{BD4gqZz&3zn({b+x8 zw0hs2#avs^?zu%4`+p_kVS|4_-rdjOslQx>IYE`r9_{&H@gVf$C%SWuck?6L?rW2} z%HamHg;91MtrO*VZ07nXSYB1&U65`~q}Ffm3gOz_1?1J`i{3oE++AO?s;WI~-~5{6 z&G_(z%{FY>giZU8Dx0zm>{;!13l&ylsrh)HUi!?=!F~BBhZvjN1qC-}zh&j%xc@xd zSYit8+;)SD_K8retpc-4 zhwsY9sWS+&A&8grqn>mbK0KKF@87Yi1^CoI^f$=@peEa44{TD{mh8GWEN5fiLMOrE zVASpHdEYR5h>!%AIH@46yDVd^ixgPyH`V18c)uQ=Jh4C}RDtUQf4!^!9j(C9o6}0u zZ?byo`Z)={{D1=WcS!`+An|_&u8M@9*_+`F#>yonPO_^Yd|VL?IC1;r*iLd#USV5&gE!XMd;v;7vd)f4BSK1;n113iC?xoOv;1 zaW_!GuNaKU)Z89s@#0@OQ~Nf#-eps^>N7@_bh@2g7=;JsrNVd^fgoZ#+koV?&NK+u z7+(8vfb>*uGdn3g^f`9>giqhDU?%~n^}oFXP%YDU?)7gK{w_U8u>ijQ%hl(9|Id7e ztzyk*A4f5dU4G@sL#WlIuK29`MfIC|1;%B2E>9oPXU*{%AFNMSJ$@kM4dn3LHcSC3N*!sTzZWhNb8D-wz@^OEitZ>MJ}d;DQ!Y4z}3>`h^@Q2z)vLZ=xF?5 zaV~73piBqtLrPy@Ao+WrfNfa(2>(^Qd7HO>+fRN9It7JA^^#N8|FW{#@ z6_&n>qOsJ#)q;b>R`zVeaTEYJ!$o2x%DU*b+iEzBOkn5U2{n(^_{_n9 z?{?s`PO?2#wzXXbn%9@E=M%TpWnLb&cQJJXCYb_vv~`=^vOAl1=Imw=;v1iRTCH)f<7G?!A9D8CS%-3pbAM^WNI_#wZV-tW1cmaw2=yQHN;L{c^9lGQjWSj(bH! zUY2Zadm_=mL!D~Bnp#)I8BG@ca|S#=TPD)YrF@!Yu^JHkc^q7(m}3Qp8**xP`X&2M zSHcGhIyEx5loXAlQ|mF9=EPbeo9stR*7j_zs#Np4Qnd8zDF7!|CAAv6W*Ta92tYu< zXMYb!K02a65O@N_KUi^Vud;3GBp?64951xYt<;COb&$1|VE|OHB)8!MyL*1t6U}-* zchp97>QC`@%x0(qzwiBor`>ZFPZzn*nwDt1dxzCw=f8M;?;_Xov{C*>I=Aei1E?R1$vcE_%A<>%@@d=ki|(_I49(R*nAG3Da`!kf%}$8+$2R_vb? zbzk5Skkkq?pt8`?z%5v_!2r7_PP=S7eceSN?ybWHd8>eU0EPqW-?Zz_zuD)b?Y9&3 z%odj9au*$D6t>Pi{MbnO+WT<{_(903QkjJaqlRcx({1PTjQ;g_KCZ|Fe4Qj870-X} zjlW-O^$9FdUfx&;>!yQW*bzVS{ezjI<@?71xO$sJdR|@cmXV#*Cp5lW>#Z!uI7>UJ zc;~5-tjBj@wiU~M{`4QAI>EZ=&0;C4%wf9m3HHrrH{v$;sHk4uHwJ;2L;03075 zFG_&|2>jasz9sXU^wvH;$2?9U@5(f5&Lqb#1V!hZ2214j^gz<%*_rfp)spY&6?oLj%kxDG#78)F~au{&^PDolIX!RwV=bFM!7CG z;Z+UZAM&FIQC`$@W>&61a4Fjx|Svz_s(81tPVM9A? zgyQU+Nyxvl>pA8s8vjkaK_unT(O?4U&TvnJQlF>Vd4QhE<>>_j2Zp2B9~j$Rc5wAM zXt}(QHu!x-eU5#I1ZP>ZsR3O|$R~wH*)@B88qR>rDkZW?NcGUo>5nfaXgLtt6RAMR&iOli^Xw+bqu;+RrkJ5O&THx zQ^-$Cu*igeOaeq$oF>a;n|O^xpxt61!zs@0_X0u?{5vpUp`;wE6kgF>xR@83=I<%; z6vA}(Z7)PqvN~wtSWuc5{B7_3|BE%RLB#$qy)T&Y!hnDpTN98+gq-_< z_|m>goJDL><08U;oHc_=7xnwS@?#MM;Q?OeR>XmiZkv)u0~P_BEsiLfvH&rRWj}@^ zZZU_dW5^bhg!~XFXwc+olU=sfefD)Q9;FfU6_0T>=%2~*fBUt>nxCp~d2uNe5Rs96 z!EY@#E+a_&bs@}bwz`1gHJJnJ%!D~V-F6hNEgTjybJVmyW2&DDmOx@hnrT7<@fb>h z;2>0oovy#3x+0~3No<;_ac?62Bf1*IfC$ywhXh8GFsNs3nlT%{uFfc0BK(&u!k_rd zV$T3CNx(0_T~dCQqb#{Y;L#Wx*5$mwn9%q%G>KeRV-f8lHyJ0(MtE35v$0Wv+?~iQ zk2w=~T!vYx%pcjDP7_MbiG*TPoLa_y;5L}mi$7J>N{c+9-fW^NLmsQ5Dt=I<=snm%HS6YLM7N}Ja8WdTITTW`;b`OqzB}*` z@(xIXAygv%+UKAUp&0DW6JY=>yB$s3bZ_Y+0reGjYXOv?b{hr~JUlS&7J@>W@+_mu-J0kUST# zZQ=wV5LGoET@1rGsLC^T4T4?2V2G#Z%-`G%MbEjc#OdW~D`gp4?S>=@ro##Tdawe4YVs5wiU zcer!t;Up82;ezsfxMlD@*}s`H$%6jEuKBPNUv&U|_nX^nH(Wm&kROD&_Qb48Z#GOm zZRuNarEj_Lm2MS+=QRy#jhyUSXWJ77(iWKrJ?}VRhUSm|RWb$1F7_f!;>Pt`Hgb(c zju0~Y_9awR8d=IEOSHlJ+#TmNyFKAKl`DsQl}SB{PHh8`76nkwvLqQJDW8J$u95KY zDNk&^PKSJ4LxN0*YekIsG!+sApe^dw2zQq&-_6|WPOYJXup@*3(k!Q10w~ajFz(wX z=y{+)1-rGai^>#*jIJJiR3~$X`@x}vW#-w+g>jM7bis2))Ervv189-?`?|E1M0#*w z*y#Pu0Cf;r^>E`Z;WBWc-Hix&9z(r?unGwPsvW21SX)tKtM=-w5)A=zc)%^EH+U^e zJ@8n(|Dc64?*IYK5JB2+8Xo`q7;1v(Hkl`-iZ~OGeWH zY^eX$2yy+LHuT^ij=r%qA2#o4rMSc5XSTa}o41%b@kJq|c85QN2L>TJ;>o9EyuC)+ z*gRsDfYgy2EPo6}tMw{rpyBqd`N1571PV6TQpj75vW4P=HO)5_vQG_L1A&%^JR}ST z>n2dG!q%FASK3bXuUdw#T$o|t{3{k>MbOr%%rmo*wi1TH{5;&D+B-nWo|!2j`3X@k zGqbI=$b{!C=nl;VgmYp}Z=K$`gRr3Tx>}i2{l~qXy6*{D%xTWjo-5}+cHGKf70vAa zaBtMG5a<&Fa*fRhNhb0cvqS;~wrUY%f5H-4+UdKC=g)yt`Qv1zjj(V`!yvyoPNbT) z*=L2puPMt{8CN=_S_&wAV`nRhld~n09=+UlYeDEz@RX!XX^B!ZXxJ&$^r?k|WQkZT zSZGpj6wC3_IX;++)CwBGccD{L4!`o`3$?xXH28f*+S0Ki8~-|4jHKkX;e8j zFxWQv-He*iN&G&R+`F?PWa*)%?if(hG}L5NPrJ<8W}$WI4mk?)6Dk zjxbie%L!r23ALG>qct~kp_rjKhFO6E&3B_XJ4E!29V4{-&BwljRBw*q8YFGR-Hn}V z*iW!eZsrWhQTrs5f*kF`t-cPc3%ZG0v*#9wfTjZn^3odt_C16TLjk`ZMGY5Vb3+a*8ql(o zbU1~qVcMMA1(4ehh7qyId>0c zIU%zH<31Q1Ykb)-FNbY`%XvM%)of!}=5V_W4ez@rs+osT91fYXv!(Lf!V=sKwFiAK zZsp7CY0>SQu?kUuU~>pgSD_8r!<_8j2bFD7U8VblNZYS8==xSD!WO*dmOfj80UW4H z$~1}yEQcb8Ez)X%WVAnM^=|N)Tag0+%Qcx$Jb?g6u;79N2PJ+6uRFxEYTlt38M`jY zDN~8$6Gnpm$mA>7b62R?;K4U!kdv>x1+^I;`_1^NA07l+_j;0vnL^6fNLDn+0;>*F z0TCK@btu;a26AQuFs<35usV@ZJ}o?==#N6rt(Qu%;4yG9TjY0UvQSpuj{mCCznDsB zhDS(D!zw%OTvoMzQ2nw7O(V-k(W&zA_OiHQ2@aQsF9f+|oi~xaDZmd82dFM@x8!UT zu+hhh`k;NU3)2ItWFNvBv^6I-68-QB2zv>WV3%y`OUn=m1fGg(LM=fH(bH1Qi zYxuktD8Sw%tkAo}gH$%5^}={y3Ifn*5YDZuOEQ|Gmt1l*7;x$19zbuG5P=AW5h2qs zJ+-0unCqy=SvfhecJfAuKznPrw-p3XxTAarmJsAtAXFRD;dDVGopsnmqe0zty3K+K zA}QLnP$aENx{4f)r3l(p!N3ad-oJGz$WU+Q%odhnWNcFa#+5n_aS%4`xDY%?&1YfF z#57!C#d!@pTQ;9A&1>gk;Er|NX8@xC2dw5kLl`>Sj^Hu$&s9(lJc=D2#<5?7oecPY zU{nYyFvMxevX!%LQS0*gMA24pJi-WD=;TUw!@Im@qHT0k5{=hFsu*yFzy!=^mb4z*mC%GiZ?IIMEa@+Y88W^$M_ z1z$@?G#y)_1#+MDMR*KwR?NHbl-Bs5 zq*~z(?V$c#fQSTU3*33DUoSRsWNS6Bb3?t-@F*Z404fmVx|wlzbe@y&PSYjL6Sl_0 z=)+0i=_tAdBh+o}>TPs87!1t+*L`lI+i|`4_V|{Z+SszZdYNItE>Of4ZTHcZAhG4$ zf!Fnt?Pf@SN{q5hYvkyNercO1`W+@!JMWa`=BU{1Ugr+M-Tw-*sx-{EMi#IW?$JYc?q;)qnpJ}S{v*~cbaP2 zW#A!upP_%C4}w9+9E_0kHaw)(qG@F0@fNeVcgcdM6JjZ1RG(ewhS4|Jn0H+0Xs zL%U~ei?JRO!#HFLHVYRSPv9Us=p!6A_S6~QlD|43xy6N%6(C^??X%5e66Cmczpg2A zBA5F+u(`imK6e|6+X~wSTk*;Kk#d!oLqmKy_T2FwSkAs-}PT#iEEbII2fv@ z$eH#+t-GewKW!5~RqOl=_kx?(g{y^TZ!b?wB%{|xUO2xZ$MNwK-{F`j!|~U^t=5sk zlCFA}itgaWH6S#*T@N*>14ZNr5L7~|Rlw{z+v3!B6ff3nq-!cs`CI|(?OiKm$K=XW zT${~^wx`SQ&mOu>QbchxSpAx_LCDf(fjp~K!m?f%o@*wJY!(L4i$x)Fuq?Wz*IVCg zU3Q1X_bzL`{p2Z@zv|Cj^FL`(49rgilE<$42 z0Nq-yC6HPD(xn0Cv@-kS0&)1! zruAYe{f@~`!Lj6k7#`lv)9OkDc;_@!5kMbc-N#4v*%Gx(^8z0W=fmavTKncWagCZl z##}XnrW2Eri8KNoQQF9G0l#$fRb^u1GN+{kK3w-os!G`~@{My+RvyT%Fa&y8FhmU`++K6Dx*nWY}pI1u(&AkU~_yDMz!mHd+N?hI8K!pgx!$t zRiXs|!s2T;Txpu8RHiEa)Litp`?;(9S5Ly(?Hya?zvE{8pOeq^e?HR3@8>8QkR_Ge zki`*MOE4qiNZE+=fN=KUl1d~8yX%PbzRleYbSt@l@NwJ$xZRSu~hXu|n{4_h;PNCMUZkBLSpAYfC?U` z%>wG17LV1|aj~l5vFcoU$J)bC2VB3@YXRVCiktOXh(OHJU17meu7g9Fa=j}gpnG`} z5H^bGmxY9#^M?h*lCzr%5RezWBOoAl&H **Important Note: Prometheus Federator is designed to be deployed alongside an existing Prometheus Operator deployment in a cluster that has already installed the Prometheus Operator CRDs.** + +By default, the chart is configured and intended to be deployed alongside [rancher-monitoring](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/), which deploys Prometheus Operator alongside a Cluster Prometheus that each Project Monitoring Stack is configured to federate namespace-scoped metrics from by default. + +## Pre-Installation: Using Prometheus Federator with Rancher and rancher-monitoring + +If you are running your cluster on [Rancher](https://rancher.com/) and already have [rancher-monitoring](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/) deployed onto your cluster, Prometheus Federator's default configuration should already be configured to work with your existing Cluster Monitoring Stack; however, here are some notes on how we recommend you configure rancher-monitoring to optimize the security and usability of Prometheus Federator in your cluster: + +### Ensure the cattle-monitoring-system namespace is placed into the System Project (or a similarly locked down Project that has access to other Projects in the cluster) + +Prometheus Operator's security model expects that the namespace it is deployed into (`cattle-monitoring-system`) has limited access for anyone except Cluster Admins to avoid privilege escalation via execing into Pods (such as the Jobs executing Helm operations). In addition, deploying Prometheus Federator and all Project Prometheus stacks into the System Project ensures that the each Project Prometheus is able to reach out to scrape workloads across all Projects (even if Network Policies are defined via Project Network Isolation) but has limited access for Project Owners, Project Members, and other users to be able to access data they shouldn't have access to (i.e. being allowed to exec into pods, set up the ability to scrape namespaces outside of a given Project, etc.). + +### Configure rancher-monitoring to only watch for resources created by the Helm chart itself + +Since each Project Monitoring Stack will watch the other namespaces and collect additional custom workload metrics or dashboards already, it's recommended to configure the following settings on all selectors to ensure that the Cluster Prometheus Stack only monitors resources created by the Helm Chart itself: + +``` +matchLabels: + release: "rancher-monitoring" +``` + +The following selector fields are recommended to have this value: +- `.Values.alertmanager.alertmanagerSpec.alertmanagerConfigSelector` +- `.Values.prometheus.prometheusSpec.serviceMonitorSelector` +- `.Values.prometheus.prometheusSpec.podMonitorSelector` +- `.Values.prometheus.prometheusSpec.ruleSelector` +- `.Values.prometheus.prometheusSpec.probeSelector` + +Once this setting is turned on, you can always create ServiceMonitors or PodMonitors that are picked up by the Cluster Prometheus by adding the label `release: "rancher-monitoring"` to them (in which case they will be ignored by Project Monitoring Stacks automatically by default, even if the namespace in which those ServiceMonitors or PodMonitors reside in are not system namespaces). + +> Note: If you don't want to allow users to be able to create ServiceMonitors and PodMonitors that aggregate into the Cluster Prometheus in Project namespaces, you can additionally set the namespaceSelectors on the chart to only target system namespaces (which must contain `cattle-monitoring-system` and `cattle-dashboards`, where resources are deployed into by default by rancher-monitoring; you will also need to monitor the `default` namespace to get apiserver metrics or create a custom ServiceMonitor to scrape apiserver metrics from the Service residing in the default namespace) to limit your Cluster Prometheus from picking up other Prometheus Operator CRs; in that case, it would be recommended to turn `.Values.prometheus.prometheusSpec.ignoreNamespaceSelectors=true` to allow you to define ServiceMonitors that can monitor non-system namespaces from within a system namespace. + +In addition, if you modified the default `.Values.grafana.sidecar.*.searchNamespace` values on the Grafana Helm subchart for Monitoring V2, it is also recommended to remove the overrides or ensure that your defaults are scoped to only system namespaces for the following values: +- `.Values.grafana.sidecar.dashboards.searchNamespace` (default `cattle-dashboards`) +- `.Values.grafana.sidecar.datasources.searchNamespace` (default `null`, which means it uses the release namespace `cattle-monitoring-system`) +- `.Values.grafana.sidecar.plugins.searchNamespace` (default `null`, which means it uses the release namespace `cattle-monitoring-system`) +- `.Values.grafana.sidecar.notifiers.searchNamespace` (default `null`, which means it uses the release namespace `cattle-monitoring-system`) + +### Increase the CPU / memory limits of the Cluster Prometheus + +Depending on a cluster's setup, it's generally recommended to give a large amount of dedicated memory to the Cluster Prometheus to avoid restarts due to out-of-memory errors (OOMKilled), usually caused by churn created in the cluster that causes a large number of high cardinality metrics to be generated and ingested by Prometheus within one block of time; this is one of the reasons why the default Rancher Monitoring stack expects around 4GB of RAM to be able to operate in a normal-sized cluster. However, when introducing Project Monitoring Stacks that are all sending `/federate` requests to the same Cluster Prometheus and are reliant on the Cluster Prometheus being "up" to federate that system data on their namespaces, it's even more important that the Cluster Prometheus has an ample amount of CPU / memory assigned to it to prevent an outage that can cause data gaps across all Project Prometheis in the cluster. + +> Note: There are no specific recommendations on how much memory the Cluster Prometheus should be configured with since it depends entirely on the user's setup (namely the likelihood of encountering a high churn rate and the scale of metrics that could be generated at that time); it generally varies per setup. + +## How does the operator work? + +1. On deploying this chart, users can create ProjectHelmCharts CRs with `spec.helmApiVersion` set to `monitoring.cattle.io/v1alpha1` (also known as "Project Monitors" in the Rancher UI) in a **Project Registration Namespace (`cattle-project-`)**. +2. On seeing each ProjectHelmChartCR, the operator will automatically deploy a Project Prometheus stack on the Project Owner's behalf in the **Project Release Namespace (`cattle-project--monitoring`)** based on a HelmChart CR and a HelmRelease CR automatically created by the ProjectHelmChart controller in the **Operator / System Namespace**. +3. RBAC will automatically be assigned in the Project Release Namespace to allow users to view the Prometheus, Alertmanager, and Grafana UIs of the Project Monitoring Stack deployed; this will be based on RBAC defined on the Project Registration Namespace against the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) (see below for more information about configuring RBAC). + +### What is a Project? + +In Prometheus Federator, a Project is a group of namespaces that can be identified by a `metav1.LabelSelector`; by default, the label used to identify projects is `field.cattle.io/projectId`, the label used to identify namespaces that are contained within a given [Rancher](https://rancher.com/) Project. + +### Configuring the Helm release created by a ProjectHelmChart + +The `spec.values` of this ProjectHelmChart resources will correspond to the `values.yaml` override to be supplied to the underlying Helm chart deployed by the operator on the user's behalf; to see the underlying chart's `values.yaml` spec, either: +- View to the chart's definition located at [`rancher/prometheus-federator` under `charts/rancher-project-monitoring`](https://github.com/rancher/prometheus-federator/blob/main/charts/rancher-project-monitoring) (where the chart version will be tied to the version of this operator) +- Look for the ConfigMap named `monitoring.cattle.io.v1alpha1` that is automatically created in each Project Registration Namespace, which will contain both the `values.yaml` and `questions.yaml` that was used to configure the chart (which was embedded directly into the `prometheus-federator` binary). + +### Namespaces + +As a Project Operator based on [rancher/helm-project-operator](https://github.com/rancher/helm-project-operator), Prometheus Federator has three different classifications of namespaces that the operator looks out for: +1. **Operator / System Namespace**: this is the namespace that the operator is deployed into (e.g. `cattle-monitoring-system`). This namespace will contain all HelmCharts and HelmReleases for all ProjectHelmCharts watched by this operator. **Only Cluster Admins should have access to this namespace.** +2. **Project Registration Namespace (`cattle-project-`)**: this is the set of namespaces that the operator watches for ProjectHelmCharts within. The RoleBindings and ClusterRoleBindings that apply to this namespace will also be the source of truth for the auto-assigned RBAC created in the Project Release Namespace (see more details below). **Project Owners (admin), Project Members (edit), and Read-Only Members (view) should have access to this namespace**. +> Note: Project Registration Namespaces will be auto-generated by the operator and imported into the Project it is tied to if `.Values.global.cattle.projectLabel` is provided (which is set to `field.cattle.io/projectId` by default); this indicates that a Project Registration Namespace should be created by the operator if at least one namespace is observed with that label. The operator will not let these namespaces be deleted unless either all namespaces with that label are gone (e.g. this is the last namespace in that project, in which case the namespace will be marked with the label `"helm.cattle.io/helm-project-operator-orphaned": "true"`, which signals that it can be deleted) or it is no longer watching that project (because the project ID was provided under `.Values.helmProjectOperator.otherSystemProjectLabelValues`, which serves as a denylist for Projects). These namespaces will also never be auto-deleted to avoid destroying user data; it is recommended that users clean up these namespaces manually if desired on creating or deleting a project +> Note: if `.Values.global.cattle.projectLabel` is not provided, the Operator / System Namespace will also be the Project Registration Namespace +3. **Project Release Namespace (`cattle-project--monitoring`)**: this is the set of namespaces that the operator deploys Project Monitoring Stacks within on behalf of a ProjectHelmChart; the operator will also automatically assign RBAC to Roles created in this namespace by the Project Monitoring Stack based on bindings found in the Project Registration Namespace. **Only Cluster Admins should have access to this namespace; Project Owners (admin), Project Members (edit), and Read-Only Members (view) will be assigned limited access to this namespace by the deployed Helm Chart and Prometheus Federator.** +> Note: Project Release Namespaces are automatically deployed and imported into the project whose ID is specified under `.Values.helmProjectOperator.projectReleaseNamespaces.labelValue` (which defaults to the value of `.Values.global.cattle.systemProjectId` if not specified) whenever a ProjectHelmChart is specified in a Project Registration Namespace +> Note: Project Release Namespaces follow the same orphaning conventions as Project Registration Namespaces (see note above) +> Note: if `.Values.projectReleaseNamespaces.enabled` is false, the Project Release Namespace will be the same as the Project Registration Namespace + +### Helm Resources (HelmChart, HelmRelease) + +On deploying a ProjectHelmChart, the Prometheus Federator will automatically create and manage two child custom resources that manage the underlying Helm resources in turn: +- A HelmChart CR (managed via an embedded [k3s-io/helm-contoller](https://github.com/k3s-io/helm-controller) in the operator): this custom resource automatically creates a Job in the same namespace that triggers a `helm install`, `helm upgrade`, or `helm uninstall` depending on the change applied to the HelmChart CR; this CR is automatically updated on changes to the ProjectHelmChart (e.g. modifying the values.yaml) or changes to the underlying Project definition (e.g. adding or removing namespaces from a project). +> **Important Note: If a ProjectHelmChart is not deploying or updating the underlying Project Monitoring Stack for some reason, the Job created by this resource in the Operator / System namespace should be the first place you check to see if there's something wrong with the Helm operation; however, this is generally only accessible by a Cluster Admin.** +- A HelmRelease CR (managed via an embedded [rancher/helm-locker](https://github.com/rancher/helm-locker) in the operator): this custom resource automatically locks a deployed Helm release in place and automatically overwrites updates to underlying resources unless the change happens via a Helm operation (`helm install`, `helm upgrade`, or `helm uninstall` performed by the HelmChart CR). +> Note: HelmRelease CRs emit Kubernetes Events that detect when an underlying Helm release is being modified and locks it back to place; to view these events, you can use `kubectl describe helmrelease -n `; you can also view the logs on this operator to see when changes are detected and which resources were attempted to be modified + +Both of these resources are created for all Helm charts in the Operator / System namespaces to avoid escalation of privileges to underprivileged users. + +### RBAC + +As described in the section on namespaces above, Prometheus Federator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: +- ClusterRoleBindings +- RoleBindings in the Project Release Namespace + +On observing a change to one of those types of bindings, the Helm Project Operator will check whether the `roleRef` that the the binding points to matches a ClusterRole with the name provided under `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin`, `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit`, or `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view`; by default, these roleRefs correspond will correspond to `admin`, `edit`, and `view` respectively, which are the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles). + +> Note: for Rancher RBAC users, these [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) directly correlate to the `Project Owner`, `Project Member`, and `Read-Only` default Project Role Templates. + +If the `roleRef` matches, the Helm Project Operator will filter the `subjects` of the binding for all Users and Groups and use that to automatically construct a RoleBinding for each Role in the Project Release Namespace with the same name as the role and the following labels: +- `helm.cattle.io/project-helm-chart-role: {{ .Release.Name }}` +- `helm.cattle.io/project-helm-chart-role-aggregate-from: ` + +By default, the `rancher-project-monitoring` (the underlying chart deployed by Prometheus Federator) creates three default Roles per Project Release Namespace that provide `admin`, `edit`, and `view` users to permissions to view the Prometheus, Alertmanager, and Grafana UIs of the Project Monitoring Stack to provide least privilege; however, if a Cluster Admin would like to assign additional permissions to certain users, they can either directly assign RoleBindings in the Project Release Namespace to certain users or created Roles with the above two labels on them to allow Project Owners to control assigning those RBAC roles to users in their Project Registration namespaces. + +### Advanced Helm Project Operator Configuration + +|Value|Configuration| +|---|---------------------------| +|`helmProjectOperator.valuesOverride`| Allows an Operator to override values that are set on each ProjectHelmChart deployment on an operator-level; user-provided options (specified on the `spec.values` of the ProjectHelmChart) are automatically overridden if operator-level values are provided. For an exmaple, see how the default value overrides `federate.targets` (note: when overriding list values like `federate.targets`, user-provided list values will **not** be concatenated) | +|`helmProjectOperator.projectReleaseNamespaces.labelValues`| The value of the Project that all Project Release Namespaces should be auto-imported into (via label and annotation). Not recommended to be overridden on a Rancher setup. | +|`helmProjectOperator.otherSystemProjectLabelValues`| Other namespaces that the operator should treat as a system namespace that should not be monitored. By default, all namespaces that match `global.cattle.systemProjectId` will not be matched. `cattle-monitoring-system`, `cattle-dashboards`, and `kube-system` are explicitly marked as system namespaces as well, regardless of label or annotation. | +|`helmProjectOperator.releaseRoleBindings.aggregate`| Whether to automatically create RBAC resources in Project Release namespaces +|`helmProjectOperator.releaseRoleBindings.clusterRoleRefs.`| ClusterRoles to reference to discover subjects to create RoleBindings for in the Project Release Namespace for all corresponding Project Release Roles. See RBAC above for more information | +|`helmProjectOperator.hardenedNamespaces.enabled`| Whether to automatically patch the default ServiceAccount with `automountServiceAccountToken: false` and create a default NetworkPolicy in all managed namespaces in the cluster; the default values ensure that the creation of the namespace does not break a CIS 1.16 hardened scan | +|`helmProjectOperator.hardenedNamespaces.configuration`| The configuration to be supplied to the default ServiceAccount or auto-generated NetworkPolicy on managing a namespace | +|`helmProjectOperator.helmController.enabled`| Whether to enable an embedded k3s-io/helm-controller instance within the Helm Project Operator. Should be disabled for RKE2/K3s clusters before v1.23.14 / v1.24.8 / v1.25.4 since RKE2/K3s clusters already run Helm Controller at a cluster-wide level to manage internal Kubernetes components | +|`helmProjectOperator.helmLocker.enabled`| Whether to enable an embedded rancher/helm-locker instance within the Helm Project Operator. | diff --git a/charts/prometheus-federator/0.4.3-rc.1/app-README.md b/charts/prometheus-federator/0.4.3-rc.1/app-README.md new file mode 100644 index 00000000..99fa7ca1 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/app-README.md @@ -0,0 +1,27 @@ +# Prometheus Federator + +This chart deploys an operator that manages Project Monitoring Stacks composed of the following set of resources that are scoped to project namespaces: +- [Prometheus](https://prometheus.io/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Alertmanager](https://prometheus.io/docs/alerting/latest/alertmanager/) (managed externally by [Prometheus Operator](https://github.com/prometheus-operator/prometheus-operator)) +- [Grafana](https://github.com/helm/charts/tree/master/stable/grafana) (deployed via an embedded Helm chart) +- Default PrometheusRules and Grafana dashboards based on the collection of community-curated resources from [kube-prometheus](https://github.com/prometheus-operator/kube-prometheus/) +- Default ServiceMonitors that watch the deployed Prometheus, Grafana, and Alertmanager + +Since this Project Monitoring Stack deploys Prometheus Operator CRs, an existing Prometheus Operator instance must already be deployed in the cluster for Prometheus Federator to successfully be able to deploy Project Monitoring Stacks. It is recommended to use [`rancher-monitoring`](https://rancher.com/docs/rancher/v2.6/en/monitoring-alerting/) for this. For more information on how the chart works or advanced configurations, please read the `README.md`. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + ​ +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. \ No newline at end of file diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/Chart.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/Chart.yaml new file mode 100644 index 00000000..c4d14e1d --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/Chart.yaml @@ -0,0 +1,15 @@ +annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Helm Project Operator + catalog.cattle.io/kube-version: '>=1.16.0-0' + catalog.cattle.io/namespace: cattle-helm-system + catalog.cattle.io/os: linux,windows + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1 + catalog.cattle.io/rancher-version: '>= 2.6.0-0' + catalog.cattle.io/release-name: helm-project-operator +apiVersion: v2 +appVersion: 0.2.1 +description: Helm Project Operator +name: helmProjectOperator +version: 0.2.1 diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/README.md b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/README.md new file mode 100644 index 00000000..9623ff22 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/README.md @@ -0,0 +1,77 @@ +# Helm Project Operator + +## How does the operator work? + +1. On deploying a Helm Project Operator, users can create ProjectHelmCharts CRs with `spec.helmApiVersion` set to `dummy.cattle.io/v1alpha1` in a **Project Registration Namespace (`cattle-project-`)**. +2. On seeing each ProjectHelmChartCR, the operator will automatically deploy the embedded Helm chart on the Project Owner's behalf in the **Project Release Namespace (`cattle-project--dummy`)** based on a HelmChart CR and a HelmRelease CR automatically created by the ProjectHelmChart controller in the **Operator / System Namespace**. +3. RBAC will automatically be assigned in the Project Release Namespace to allow users to based on Role created in the Project Release Namespace with a given set of labels; this will be based on RBAC defined on the Project Registration Namespace against the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) (see below for more information about configuring RBAC). + +### What is a Project? + +In Helm Project Operator, a Project is a group of namespaces that can be identified by a `metav1.LabelSelector`; by default, the label used to identify projects is `field.cattle.io/projectId`, the label used to identify namespaces that are contained within a given [Rancher](https://rancher.com/) Project. + +### What is a ProjectHelmChart? + +A ProjectHelmChart is an instance of a (project-scoped) Helm chart deployed on behalf of a user who has permissions to create ProjectHelmChart resources in a Project Registration namespace. + +Generally, the best way to think about the ProjectHelmChart model is by comparing it to two other models: +1. Managed Kubernetes providers (EKS, GKE, AKS, etc.): in this model, a user has the ability to say "I want a Kubernetes cluster" but the underlying cloud provider is responsible for provisioning the infrastructure and offering **limited view and access** of the underlying resources created on their behalf; similarly, Helm Project Operator allows a Project Owner to say "I want this Helm chart deployed", but the underlying Operator is responsible for "provisioning" (deploying) the Helm chart and offering **limited view and access** of the underlying Kubernetes resources created on their behalf (based on configuring "least-privilege" Kubernetes RBAC for the Project Owners / Members in the newly created Project Release Namespace). +2. Dynamically-provisioned Persistent Volumes: in this model, a single resource (PersistentVolume) exists that allows you to specify a Storage Class that actually implements provisioning the underlying storage via a Storage Class Provisioner (e.g. Longhorn). Similarly, the ProjectHelmChart exists that allows you to specify a `spec.helmApiVersion` ("storage class") that actually implements deploying the underlying Helm chart via a Helm Project Operator (e.g. [`rancher/prometheus-federator`](https://github.com/rancher/prometheus-federator)). + +### Configuring the Helm release created by a ProjectHelmChart + +The `spec.values` of this ProjectHelmChart resources will correspond to the `values.yaml` override to be supplied to the underlying Helm chart deployed by the operator on the user's behalf; to see the underlying chart's `values.yaml` spec, either: +- View to the chart's definition located at [`rancher/helm-project-operator` under `charts/project-operator-example`](https://github.com/rancher/helm-project-operator/blob/main/charts/project-operator-example) (where the chart version will be tied to the version of this operator) +- Look for the ConfigMap named `dummy.cattle.io.v1alpha1` that is automatically created in each Project Registration Namespace, which will contain both the `values.yaml` and `questions.yaml` that was used to configure the chart (which was embedded directly into the `helm-project-operator` binary). + +### Namespaces + +All Helm Project Operators have three different classifications of namespaces that the operator looks out for: +1. **Operator / System Namespace**: this is the namespace that the operator is deployed into (e.g. `cattle-helm-system`). This namespace will contain all HelmCharts and HelmReleases for all ProjectHelmCharts watched by this operator. **Only Cluster Admins should have access to this namespace.** +2. **Project Registration Namespace (`cattle-project-`)**: this is the set of namespaces that the operator watches for ProjectHelmCharts within. The RoleBindings and ClusterRoleBindings that apply to this namespace will also be the source of truth for the auto-assigned RBAC created in the Project Release Namespace (see more details below). **Project Owners (admin), Project Members (edit), and Read-Only Members (view) should have access to this namespace**. +> Note: Project Registration Namespaces will be auto-generated by the operator and imported into the Project it is tied to if `.Values.global.cattle.projectLabel` is provided (which is set to `field.cattle.io/projectId` by default); this indicates that a Project Registration Namespace should be created by the operator if at least one namespace is observed with that label. The operator will not let these namespaces be deleted unless either all namespaces with that label are gone (e.g. this is the last namespace in that project, in which case the namespace will be marked with the label `"helm.cattle.io/helm-project-operator-orphaned": "true"`, which signals that it can be deleted) or it is no longer watching that project (because the project ID was provided under `.Values.helmProjectOperator.otherSystemProjectLabelValues`, which serves as a denylist for Projects). These namespaces will also never be auto-deleted to avoid destroying user data; it is recommended that users clean up these namespaces manually if desired on creating or deleting a project +> Note: if `.Values.global.cattle.projectLabel` is not provided, the Operator / System Namespace will also be the Project Registration Namespace +3. **Project Release Namespace (`cattle-project--dummy`)**: this is the set of namespaces that the operator deploys Helm charts within on behalf of a ProjectHelmChart; the operator will also automatically assign RBAC to Roles created in this namespace by the Helm charts based on bindings found in the Project Registration Namespace. **Only Cluster Admins should have access to this namespace; Project Owners (admin), Project Members (edit), and Read-Only Members (view) will be assigned limited access to this namespace by the deployed Helm Chart and Helm Project Operator.** +> Note: Project Release Namespaces are automatically deployed and imported into the project whose ID is specified under `.Values.helmProjectOperator.projectReleaseNamespaces.labelValue` (which defaults to the value of `.Values.global.cattle.systemProjectId` if not specified) whenever a ProjectHelmChart is specified in a Project Registration Namespace +> Note: Project Release Namespaces follow the same orphaning conventions as Project Registration Namespaces (see note above) +> Note: if `.Values.projectReleaseNamespaces.enabled` is false, the Project Release Namespace will be the same as the Project Registration Namespace + +### Helm Resources (HelmChart, HelmRelease) + +On deploying a ProjectHelmChart, the Helm Project Operator will automatically create and manage two child custom resources that manage the underlying Helm resources in turn: +- A HelmChart CR (managed via an embedded [k3s-io/helm-contoller](https://github.com/k3s-io/helm-controller) in the operator): this custom resource automatically creates a Job in the same namespace that triggers a `helm install`, `helm upgrade`, or `helm uninstall` depending on the change applied to the HelmChart CR; this CR is automatically updated on changes to the ProjectHelmChart (e.g. modifying the values.yaml) or changes to the underlying Project definition (e.g. adding or removing namespaces from a project). +> **Important Note: If a ProjectHelmChart is not deploying or updating the underlying Project Monitoring Stack for some reason, the Job created by this resource in the Operator / System namespace should be the first place you check to see if there's something wrong with the Helm operation; however, this is generally only accessible by a Cluster Admin.** +- A HelmRelease CR (managed via an embedded [rancher/helm-locker](https://github.com/rancher/helm-locker) in the operator): this custom resource automatically locks a deployed Helm release in place and automatically overwrites updates to underlying resources unless the change happens via a Helm operation (`helm install`, `helm upgrade`, or `helm uninstall` performed by the HelmChart CR). +> Note: HelmRelease CRs emit Kubernetes Events that detect when an underlying Helm release is being modified and locks it back to place; to view these events, you can use `kubectl describe helmrelease -n `; you can also view the logs on this operator to see when changes are detected and which resources were attempted to be modified + +Both of these resources are created for all Helm charts in the Operator / System namespaces to avoid escalation of privileges to underprivileged users. + +### RBAC + +As described in the section on namespaces above, Helm Project Operator expects that Project Owners, Project Members, and other users in the cluster with Project-level permissions (e.g. permissions in a certain set of namespaces identified by a single label selector) have minimal permissions in any namespaces except the Project Registration Namespace (which is imported into the project by default) and those that already comprise their projects. Therefore, in order to allow Project Owners to assign specific chart permissions to other users in their Project namespaces, the Helm Project Operator will automatically watch the following bindings: +- ClusterRoleBindings +- RoleBindings in the Project Release Namespace + +On observing a change to one of those types of bindings, the Helm Project Operator will check whether the `roleRef` that the the binding points to matches a ClusterRole with the name provided under `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin`, `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit`, or `helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view`; by default, these roleRefs correspond will correspond to `admin`, `edit`, and `view` respectively, which are the [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles). + +> Note: for Rancher RBAC users, these [default Kubernetes user-facing roles](https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles) directly correlate to the `Project Owner`, `Project Member`, and `Read-Only` default Project Role Templates. + +If the `roleRef` matches, the Helm Project Operator will filter the `subjects` of the binding for all Users and Groups and use that to automatically construct a RoleBinding for each Role in the Project Release Namespace with the same name as the role and the following labels: +- `helm.cattle.io/project-helm-chart-role: {{ .Release.Name }}` +- `helm.cattle.io/project-helm-chart-role-aggregate-from: ` + +By default, the `project-operator-example` (the underlying chart deployed by Helm Project Operator) does not create any default roles; however, if a Cluster Admin would like to assign additional permissions to certain users, they can either directly assign RoleBindings in the Project Release Namespace to certain users or created Roles with the above two labels on them to allow Project Owners to control assigning those RBAC roles to users in their Project Registration namespaces. + +### Advanced Helm Project Operator Configuration + +|Value|Configuration| +|---|---------------------------| +|`valuesOverride`| Allows an Operator to override values that are set on each ProjectHelmChart deployment on an operator-level; user-provided options (specified on the `spec.values` of the ProjectHelmChart) are automatically overridden if operator-level values are provided. For an exmaple, see how the default value overrides `federate.targets` (note: when overriding list values like `federate.targets`, user-provided list values will **not** be concatenated) | +|`projectReleaseNamespaces.labelValues`| The value of the Project that all Project Release Namespaces should be auto-imported into (via label and annotation). Not recommended to be overridden on a Rancher setup. | +|`otherSystemProjectLabelValues`| Other namespaces that the operator should treat as a system namespace that should not be monitored. By default, all namespaces that match `global.cattle.systemProjectId` will not be matched. `kube-system` is explicitly marked as a system namespace as well, regardless of label or annotation. | +|`releaseRoleBindings.aggregate`| Whether to automatically create RBAC resources in Project Release namespaces +|`releaseRoleBindings.clusterRoleRefs.`| ClusterRoles to reference to discover subjects to create RoleBindings for in the Project Release Namespace for all corresponding Project Release Roles. See RBAC above for more information | +|`hardenedNamespaces.enabled`| Whether to automatically patch the default ServiceAccount with `automountServiceAccountToken: false` and create a default NetworkPolicy in all managed namespaces in the cluster; the default values ensure that the creation of the namespace does not break a CIS 1.16 hardened scan | +|`hardenedNamespaces.configuration`| The configuration to be supplied to the default ServiceAccount or auto-generated NetworkPolicy on managing a namespace | +|`helmController.enabled`| Whether to enable an embedded k3s-io/helm-controller instance within the Helm Project Operator. Should be disabled for RKE2 clusters since RKE2 clusters already run Helm Controller to manage internal Kubernetes components | +|`helmLocker.enabled`| Whether to enable an embedded rancher/helm-locker instance within the Helm Project Operator. | diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/app-readme.md b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/app-readme.md new file mode 100644 index 00000000..fd551467 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/app-readme.md @@ -0,0 +1,20 @@ +# Helm Project Operator + +This chart installs the example [Helm Project Operator](https://github.com/rancher/helm-project-operator) onto your cluster. + +## Upgrading to Kubernetes v1.25+ + +Starting in Kubernetes v1.25, [Pod Security Policies](https://kubernetes.io/docs/concepts/security/pod-security-policy/) have been removed from the Kubernetes API. + +As a result, **before upgrading to Kubernetes v1.25** (or on a fresh install in a Kubernetes v1.25+ cluster), users are expected to perform an in-place upgrade of this chart with `global.cattle.psp.enabled` set to `false` if it has been previously set to `true`. +​ +> **Note:** +> In this chart release, any previous field that was associated with any PSP resources have been removed in favor of a single global field: `global.cattle.psp.enabled`. + ​ +> **Note:** +> If you upgrade your cluster to Kubernetes v1.25+ before removing PSPs via a `helm upgrade` (even if you manually clean up resources), **it will leave the Helm release in a broken state within the cluster such that further Helm operations will not work (`helm uninstall`, `helm upgrade`, etc.).** +> +> If your charts get stuck in this state, please consult the Rancher docs on how to clean up your Helm release secrets. +Upon setting `global.cattle.psp.enabled` to false, the chart will remove any PSP resources deployed on its behalf from the cluster. This is the default setting for this chart. +​ +As a replacement for PSPs, [Pod Security Admission](https://kubernetes.io/docs/concepts/security/pod-security-admission/) should be used. Please consult the Rancher docs for more details on how to configure your chart release namespaces to work with the new Pod Security Admission and apply Pod Security Standards. \ No newline at end of file diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/questions.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/questions.yaml new file mode 100644 index 00000000..054361a7 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/questions.yaml @@ -0,0 +1,43 @@ +questions: +- variable: global.cattle.psp.enabled + default: "false" + description: "Flag to enable or disable the installation of PodSecurityPolicies by this chart in the target cluster. If the cluster is running Kubernetes 1.25+, you must update this value to false." + label: "Enable PodSecurityPolicies" + type: boolean + group: "Security Settings" +- variable: helmController.enabled + label: Enable Embedded Helm Controller + description: 'Note: If you are running this chart in an RKE2 cluster, this should be disabled.' + type: boolean + group: Helm Controller +- variable: helmLocker.enabled + label: Enable Embedded Helm Locker + type: boolean + group: Helm Locker +- variable: projectReleaseNamespaces.labelValue + label: Project Release Namespace Project ID + description: By default, the System Project is selected. This can be overriden to a different Project (e.g. p-xxxxx) + type: string + required: false + group: Namespaces +- variable: releaseRoleBindings.clusterRoleRefs.admin + label: Admin ClusterRole + description: By default, admin selects Project Owners. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: admin + required: false + group: RBAC +- variable: releaseRoleBindings.clusterRoleRefs.edit + label: Edit ClusterRole + description: By default, edit selects Project Members. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: edit + required: false + group: RBAC +- variable: releaseRoleBindings.clusterRoleRefs.view + label: View ClusterRole + description: By default, view selects Read-Only users. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: view + required: false + group: RBAC diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/NOTES.txt b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/NOTES.txt new file mode 100644 index 00000000..32baeebc --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/NOTES.txt @@ -0,0 +1,2 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "helm-project-operator.namespace" . }} get pods -l "release={{ $.Release.Name }}" diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/_helpers.tpl b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/_helpers.tpl new file mode 100644 index 00000000..194214cb --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/_helpers.tpl @@ -0,0 +1,75 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Helm Project Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "helm-project-operator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "helm-project-operator.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "helm-project-operator.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "helm-project-operator.labels" -}} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "helm-project-operator.name" . }} +chart: {{ template "helm-project-operator.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end -}} + +{{/* Replica Default - Allow setting to 0, or default to 1 */}} +{{- define "replicaDefault" -}} +{{- if (eq 0 (int .Values.replicas)) -}} +{{ .Values.replicas }} +{{- else -}} +{{ default .Values.replicas 1 }} +{{- end -}} +{{- end -}} diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/cleanup.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/cleanup.yaml new file mode 100644 index 00000000..98675642 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/cleanup.yaml @@ -0,0 +1,82 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "helm-project-operator.name" . }}-cleanup + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} + annotations: + "helm.sh/hook": pre-delete + "helm.sh/hook-delete-policy": before-hook-creation, hook-succeeded, hook-failed +spec: + template: + metadata: + name: {{ template "helm-project-operator.name" . }}-cleanup + labels: {{ include "helm-project-operator.labels" . | nindent 8 }} + app: {{ template "helm-project-operator.name" . }} + spec: + serviceAccountName: {{ template "helm-project-operator.name" . }} +{{- if .Values.cleanup.securityContext }} + securityContext: {{ toYaml .Values.cleanup.securityContext | nindent 8 }} +{{- end }} + initContainers: + - name: add-cleanup-annotations + image: {{ template "system_default_registry" . }}{{ .Values.cleanup.image.repository }}:{{ .Values.cleanup.image.tag }} + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + command: + - /bin/sh + - -c + - > + echo "Labeling all ProjectHelmCharts with helm.cattle.io/helm-project-operator-cleanup=true"; + EXPECTED_HELM_API_VERSION={{ .Values.helmApiVersion }}; + IFS=$'\n'; + for namespace in $(kubectl get namespaces -l helm.cattle.io/helm-project-operated=true --no-headers -o=custom-columns=NAME:.metadata.name); do + for projectHelmChartAndHelmApiVersion in $(kubectl get projecthelmcharts -n ${namespace} --no-headers -o=custom-columns=NAME:.metadata.name,HELMAPIVERSION:.spec.helmApiVersion); do + projectHelmChartAndHelmApiVersion=$(echo ${projectHelmChartAndHelmApiVersion} | xargs); + projectHelmChart=$(echo ${projectHelmChartAndHelmApiVersion} | cut -d' ' -f1); + helmApiVersion=$(echo ${projectHelmChartAndHelmApiVersion} | cut -d' ' -f2); + if [[ ${helmApiVersion} != ${EXPECTED_HELM_API_VERSION} ]]; then + echo "Skipping marking ${namespace}/${projectHelmChart} with cleanup annotation since spec.helmApiVersion: ${helmApiVersion} is not ${EXPECTED_HELM_API_VERSION}"; + continue; + fi; + kubectl label projecthelmcharts -n ${namespace} ${projectHelmChart} helm.cattle.io/helm-project-operator-cleanup=true --overwrite; + done; + done; +{{- if .Values.cleanup.resources }} + resources: {{ toYaml .Values.cleanup.resources | nindent 12 }} +{{- end }} +{{- if .Values.cleanup.containerSecurityContext }} + securityContext: {{ toYaml .Values.cleanup.containerSecurityContext | nindent 12 }} +{{- end }} + containers: + - name: ensure-subresources-deleted + image: {{ template "system_default_registry" . }}{{ .Values.cleanup.image.repository }}:{{ .Values.cleanup.image.tag }} + imagePullPolicy: IfNotPresent + command: + - /bin/sh + - -c + - > + SYSTEM_NAMESPACE={{ .Release.Namespace }} + EXPECTED_HELM_API_VERSION={{ .Values.helmApiVersion }}; + HELM_API_VERSION_TRUNCATED=$(echo ${EXPECTED_HELM_API_VERSION} | cut -d'/' -f0); + echo "Ensuring HelmCharts and HelmReleases are deleted from ${SYSTEM_NAMESPACE}..."; + while [[ "$(kubectl get helmcharts,helmreleases -l helm.cattle.io/helm-api-version=${HELM_API_VERSION_TRUNCATED} -n ${SYSTEM_NAMESPACE} 2>&1)" != "No resources found in ${SYSTEM_NAMESPACE} namespace." ]]; do + echo "waiting for HelmCharts and HelmReleases to be deleted from ${SYSTEM_NAMESPACE}... sleeping 3 seconds"; + sleep 3; + done; + echo "Successfully deleted all HelmCharts and HelmReleases in ${SYSTEM_NAMESPACE}!"; +{{- if .Values.cleanup.resources }} + resources: {{ toYaml .Values.cleanup.resources | nindent 12 }} +{{- end }} +{{- if .Values.cleanup.containerSecurityContext }} + securityContext: {{ toYaml .Values.cleanup.containerSecurityContext | nindent 12 }} +{{- end }} + restartPolicy: OnFailure + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} + {{- if .Values.cleanup.nodeSelector }} + {{- toYaml .Values.cleanup.nodeSelector | nindent 8 }} + {{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} + {{- if .Values.cleanup.tolerations }} + {{- toYaml .Values.cleanup.tolerations | nindent 8 }} + {{- end }} diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/clusterrole.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/clusterrole.yaml new file mode 100644 index 00000000..60ed263b --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/clusterrole.yaml @@ -0,0 +1,57 @@ +{{- if and .Values.global.rbac.create .Values.global.rbac.userRoles.create }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "helm-project-operator.name" . }}-admin + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-admin: "true" + {{- end }} +rules: +- apiGroups: + - helm.cattle.io + resources: + - projecthelmcharts + - projecthelmcharts/finalizers + - projecthelmcharts/status + verbs: + - '*' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "helm-project-operator.name" . }}-edit + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-edit: "true" + {{- end }} +rules: +- apiGroups: + - helm.cattle.io + resources: + - projecthelmcharts + - projecthelmcharts/status + verbs: + - 'get' + - 'list' + - 'watch' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ template "helm-project-operator.name" . }}-view + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + {{- if .Values.global.rbac.userRoles.aggregateToDefaultRoles }} + rbac.authorization.k8s.io/aggregate-to-view: "true" + {{- end }} +rules: +- apiGroups: + - helm.cattle.io + resources: + - projecthelmcharts + - projecthelmcharts/status + verbs: + - 'get' + - 'list' + - 'watch' +{{- end }} diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/configmap.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/configmap.yaml new file mode 100644 index 00000000..d4def157 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/configmap.yaml @@ -0,0 +1,14 @@ +## Note: If you add another entry to this ConfigMap, make sure a corresponding env var is set +## in the deployment of the operator to ensure that a Helm upgrade will force the operator +## to reload the values in the ConfigMap and redeploy +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "helm-project-operator.name" . }}-config + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +data: + hardened.yaml: |- +{{ .Values.hardenedNamespaces.configuration | toYaml | indent 4 }} + values.yaml: |- +{{ .Values.valuesOverride | toYaml | indent 4 }} diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/deployment.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/deployment.yaml new file mode 100644 index 00000000..a5386aae --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/deployment.yaml @@ -0,0 +1,124 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +spec: + replicas: {{ include "replicaDefault" . }} + selector: + matchLabels: + app: {{ template "helm-project-operator.name" . }} + release: {{ $.Release.Name | quote }} + template: + metadata: + labels: {{ include "helm-project-operator.labels" . | nindent 8 }} + app: {{ template "helm-project-operator.name" . }} + spec: + containers: + - name: {{ template "helm-project-operator.name" . }} + image: "{{ template "system_default_registry" . }}{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: "{{ .Values.image.pullPolicy }}" + args: + - {{ template "helm-project-operator.name" . }} + - --namespace={{ template "helm-project-operator.namespace" . }} + - --controller-name={{ template "helm-project-operator.name" . }} + - --values-override-file=/etc/helmprojectoperator/config/values.yaml +{{- if .Values.global.cattle.systemDefaultRegistry }} + - --system-default-registry={{ .Values.global.cattle.systemDefaultRegistry }} +{{- end }} +{{- if .Values.global.cattle.url }} + - --cattle-url={{ .Values.global.cattle.url }} +{{- end }} +{{- if .Values.global.cattle.projectLabel }} + - --project-label={{ .Values.global.cattle.projectLabel }} +{{- end }} +{{- if not .Values.projectReleaseNamespaces.enabled }} + - --system-project-label-values={{ join "," (append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId) }} +{{- else if and (ne (len .Values.global.cattle.systemProjectId) 0) (ne (len .Values.projectReleaseNamespaces.labelValue) 0) (ne .Values.projectReleaseNamespaces.labelValue .Values.global.cattle.systemProjectId) }} + - --system-project-label-values={{ join "," (append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId) }} +{{- else if len .Values.otherSystemProjectLabelValues }} + - --system-project-label-values={{ join "," .Values.otherSystemProjectLabelValues }} +{{- end }} +{{- if .Values.projectReleaseNamespaces.enabled }} +{{- if .Values.projectReleaseNamespaces.labelValue }} + - --project-release-label-value={{ .Values.projectReleaseNamespaces.labelValue }} +{{- else if .Values.global.cattle.systemProjectId }} + - --project-release-label-value={{ .Values.global.cattle.systemProjectId }} +{{- end }} +{{- end }} +{{- if .Values.global.cattle.clusterId }} + - --cluster-id={{ .Values.global.cattle.clusterId }} +{{- end }} +{{- if .Values.releaseRoleBindings.aggregate }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs.admin }} + - --admin-cluster-role={{ .Values.releaseRoleBindings.clusterRoleRefs.admin }} +{{- end }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs.edit }} + - --edit-cluster-role={{ .Values.releaseRoleBindings.clusterRoleRefs.edit }} +{{- end }} +{{- if .Values.releaseRoleBindings.clusterRoleRefs.view }} + - --view-cluster-role={{ .Values.releaseRoleBindings.clusterRoleRefs.view }} +{{- end }} +{{- end }} +{{- end }} +{{- if .Values.hardenedNamespaces.enabled }} + - --hardening-options-file=/etc/helmprojectoperator/config/hardening.yaml +{{- else }} + - --disable-hardening +{{- end }} +{{- if .Values.debug }} + - --debug + - --debug-level={{ .Values.debugLevel }} +{{- end }} +{{- if not .Values.helmController.enabled }} + - --disable-embedded-helm-controller +{{- else }} + - --helm-job-image={{ template "system_default_registry" . }}{{ .Values.helmController.job.image.repository }}:{{ .Values.helmController.job.image.tag }} +{{- end }} +{{- if not .Values.helmLocker.enabled }} + - --disable-embedded-helm-locker +{{- end }} +{{- if .Values.additionalArgs }} +{{- toYaml .Values.additionalArgs | nindent 10 }} +{{- end }} + env: + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + ## Note: The below two values only exist to force Helm to upgrade the deployment on + ## a change to the contents of the ConfigMap during an upgrade. Neither serve + ## any practical purpose and can be removed and replaced with a configmap reloader + ## in a future change if dynamic updates are required. + - name: HARDENING_OPTIONS_SHA_256_HASH + value: {{ .Values.hardenedNamespaces.configuration | toYaml | sha256sum }} + - name: VALUES_OVERRIDE_SHA_256_HASH + value: {{ .Values.valuesOverride | toYaml | sha256sum }} +{{- if .Values.resources }} + resources: {{ toYaml .Values.resources | nindent 12 }} +{{- end }} +{{- if .Values.containerSecurityContext }} + securityContext: {{ toYaml .Values.containerSecurityContext | nindent 12 }} +{{- end }} + volumeMounts: + - name: config + mountPath: "/etc/helmprojectoperator/config" + serviceAccountName: {{ template "helm-project-operator.name" . }} +{{- if .Values.securityContext }} + securityContext: {{ toYaml .Values.securityContext | nindent 8 }} +{{- end }} + nodeSelector: {{ include "linux-node-selector" . | nindent 8 }} +{{- if .Values.nodeSelector }} +{{- toYaml .Values.nodeSelector | nindent 8 }} +{{- end }} + tolerations: {{ include "linux-node-tolerations" . | nindent 8 }} +{{- if .Values.tolerations }} +{{- toYaml .Values.tolerations | nindent 8 }} +{{- end }} + volumes: + - name: config + configMap: + name: {{ template "helm-project-operator.name" . }}-config diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/psp.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/psp.yaml new file mode 100644 index 00000000..73dcc456 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/psp.yaml @@ -0,0 +1,68 @@ +{{- if .Values.global.cattle.psp.enabled }} +apiVersion: policy/v1beta1 +kind: PodSecurityPolicy +metadata: + name: {{ template "helm-project-operator.name" . }}-psp + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +{{- if .Values.global.rbac.pspAnnotations }} + annotations: {{ toYaml .Values.global.rbac.pspAnnotations | nindent 4 }} +{{- end }} +spec: + privileged: false + hostNetwork: false + hostIPC: false + hostPID: false + runAsUser: + # Permits the container to run with root privileges as well. + rule: 'RunAsAny' + seLinux: + # This policy assumes the nodes are using AppArmor rather than SELinux. + rule: 'RunAsAny' + supplementalGroups: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + fsGroup: + rule: 'MustRunAs' + ranges: + # Forbid adding the root group. + - min: 0 + max: 65535 + readOnlyRootFilesystem: false +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "helm-project-operator.name" . }}-psp + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +rules: +{{- if semverCompare "> 1.15.0-0" .Capabilities.KubeVersion.GitVersion }} +- apiGroups: ['policy'] +{{- else }} +- apiGroups: ['extensions'] +{{- end }} + resources: ['podsecuritypolicies'] + verbs: ['use'] + resourceNames: + - {{ template "helm-project-operator.name" . }}-psp +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: {{ template "helm-project-operator.name" . }}-psp + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ template "helm-project-operator.name" . }}-psp +subjects: + - kind: ServiceAccount + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} +{{- end }} diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/rbac.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/rbac.yaml new file mode 100644 index 00000000..b1c40920 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/rbac.yaml @@ -0,0 +1,32 @@ +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ template "helm-project-operator.name" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: "cluster-admin" # see note below +subjects: +- kind: ServiceAccount + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ template "helm-project-operator.name" . }} + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} + app: {{ template "helm-project-operator.name" . }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: {{ toYaml .Values.global.imagePullSecrets | nindent 2 }} +{{- end }} +# --- +# NOTE: +# As of now, due to the fact that the k3s-io/helm-controller can only deploy jobs that are cluster-bound to the cluster-admin +# ClusterRole, the only way for this operator to be able to perform that binding is if it is also bound to the cluster-admin ClusterRole. +# +# As a result, this ClusterRoleBinding will be left as a work-in-progress until changes are made in k3s-io/helm-controller to allow us to grant +# only scoped down permissions to the Job that is deployed. diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml new file mode 100644 index 00000000..f4c85254 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/system-namespaces-configmap.yaml @@ -0,0 +1,62 @@ +{{- if .Values.systemNamespacesConfigMap.create }} +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "helm-project-operator.name" . }}-system-namespaces + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +data: + system-namespaces.json: |- + { +{{- if .Values.projectReleaseNamespaces.enabled }} +{{- if .Values.projectReleaseNamespaces.labelValue }} + "projectReleaseLabelValue": {{ .Values.projectReleaseNamespaces.labelValue | quote }}, +{{- else if .Values.global.cattle.systemProjectId }} + "projectReleaseLabelValue": {{ .Values.global.cattle.systemProjectId | quote }}, +{{- else }} + "projectReleaseLabelValue": "", +{{- end }} +{{- else }} + "projectReleaseLabelValue": "", +{{- end }} +{{- if not .Values.projectReleaseNamespaces.enabled }} + "systemProjectLabelValues": {{ append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId | toJson }} +{{- else if and (ne (len .Values.global.cattle.systemProjectId) 0) (ne (len .Values.projectReleaseNamespaces.labelValue) 0) (ne .Values.projectReleaseNamespaces.labelValue .Values.global.cattle.systemProjectId) }} + "systemProjectLabelValues": {{ append .Values.otherSystemProjectLabelValues .Values.global.cattle.systemProjectId | toJson }} +{{- else if len .Values.otherSystemProjectLabelValues }} + "systemProjectLabelValues": {{ .Values.otherSystemProjectLabelValues | toJson }} +{{- else }} + "systemProjectLabelValues": [] +{{- end }} + } +--- +{{- if (and .Values.systemNamespacesConfigMap.rbac.enabled .Values.systemNamespacesConfigMap.rbac.subjects) }} +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: {{ template "helm-project-operator.name" . }}-system-namespaces + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - "" + resources: + - configmaps + resourceNames: + - "{{ template "helm-project-operator.name" . }}-system-namespaces" + verbs: + - 'get' +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: {{ template "helm-project-operator.name" . }}-system-namespaces + namespace: {{ template "helm-project-operator.namespace" . }} + labels: {{ include "helm-project-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: {{ template "helm-project-operator.name" . }}-system-namespaces +subjects: {{ .Values.systemNamespacesConfigMap.rbac.subjects | toYaml | nindent 2 }} +{{- end }} +{{- end }} diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/validate-psp-install.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/validate-psp-install.yaml new file mode 100644 index 00000000..a30c59d3 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/validate-psp-install.yaml @@ -0,0 +1,7 @@ +#{{- if gt (len (lookup "rbac.authorization.k8s.io/v1" "ClusterRole" "" "")) 0 -}} +#{{- if .Values.global.cattle.psp.enabled }} +#{{- if not (.Capabilities.APIVersions.Has "policy/v1beta1/PodSecurityPolicy") }} +#{{- fail "The target cluster does not have the PodSecurityPolicy API resource. Please disable PSPs in this chart before proceeding." -}} +#{{- end }} +#{{- end }} +#{{- end }} diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/values.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/values.yaml new file mode 100644 index 00000000..63fae45a --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/values.yaml @@ -0,0 +1,228 @@ +# Default values for helm-project-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Helm Project Operator Configuration + +global: + cattle: + clusterId: "" + psp: + enabled: false + projectLabel: field.cattle.io/projectId + systemDefaultRegistry: "" + systemProjectId: "" + url: "" + rbac: + ## Create RBAC resources for ServiceAccounts and users + ## + create: true + + userRoles: + ## Create default user ClusterRoles to allow users to interact with ProjectHelmCharts + create: true + ## Aggregate default user ClusterRoles into default k8s ClusterRoles + aggregateToDefaultRoles: true + + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + +helmApiVersion: dummy.cattle.io/v1alpha1 + +## valuesOverride overrides values that are set on each ProjectHelmChart deployment on an operator-level +## User-provided values will be overwritten based on the values provided here +valuesOverride: {} + +## projectReleaseNamespaces are auto-generated namespaces that are created to host Helm Releases +## managed by this operator on behalf of a ProjectHelmChart +projectReleaseNamespaces: + ## Enabled determines whether Project Release Namespaces should be created. If false, the underlying + ## Helm release will be deployed in the Project Registration Namespace + enabled: true + ## labelValue is the value of the Project that the projectReleaseNamespace should be created within + ## If empty, this will be set to the value of global.cattle.systemProjectId + ## If global.cattle.systemProjectId is also empty, project release namespaces will be disabled + labelValue: "" + +## otherSystemProjectLabelValues are project labels that identify namespaces as those that should be treated as system projects +## i.e. they will be entirely ignored by the operator +## By default, the global.cattle.systemProjectId will be in this list +otherSystemProjectLabelValues: [] + +## releaseRoleBindings configures RoleBindings automatically created by the Helm Project Operator +## in Project Release Namespaces where underlying Helm charts are deployed +releaseRoleBindings: + ## aggregate enables creating these RoleBindings off aggregating RoleBindings in the + ## Project Registration Namespace or ClusterRoleBindings that bind users to the ClusterRoles + ## specified under clusterRoleRefs + aggregate: true + + ## clusterRoleRefs are the ClusterRoles whose RoleBinding or ClusterRoleBindings should determine + ## the RoleBindings created in the Project Release Namespace + ## + ## By default, these are set to create RoleBindings based on the RoleBindings / ClusterRoleBindings + ## attached to the default K8s user-facing ClusterRoles of admin, edit, and view. + ## ref: https://kubernetes.io/docs/reference/access-authn-authz/rbac/#user-facing-roles + ## + clusterRoleRefs: + admin: admin + edit: edit + view: view + +hardenedNamespaces: + # Whether to automatically manage the configuration of the default ServiceAccount and + # auto-create a NetworkPolicy for each namespace created by this operator + enabled: true + + configuration: + # Values to be applied to each default ServiceAccount created in a managed namespace + serviceAccountSpec: + secrets: [] + imagePullSecrets: [] + automountServiceAccountToken: false + # Values to be applied to each default generated NetworkPolicy created in a managed namespace + networkPolicySpec: + podSelector: {} + egress: [] + ingress: [] + policyTypes: ["Ingress", "Egress"] + +## systemNamespacesConfigMap is a ConfigMap created to allow users to see valid entries +## for registering a ProjectHelmChart for a given Project on the Rancher Dashboard UI. +## It does not need to be enabled for a non-Rancher use case. +systemNamespacesConfigMap: + ## Create indicates whether the system namespaces configmap should be created + ## This is a required value for integration with Rancher Dashboard + create: true + + ## RBAC provides options around the RBAC created to allow users to be able to view + ## the systemNamespacesConfigMap; if not specified, only users with the ability to + ## view ConfigMaps in the namespace where this chart is deployed will be able to + ## properly view the system namespaces on the Rancher Dashboard UI + rbac: + ## enabled indicates that we should deploy a RoleBinding and Role to view this ConfigMap + enabled: true + ## subjects are the subjects that should be bound to this default RoleBinding + ## By default, we allow anyone who is authenticated to the system to be able to view + ## this ConfigMap in the deployment namespace + subjects: + - kind: Group + name: system:authenticated + +nameOverride: "" + +namespaceOverride: "" + +replicas: 1 + +image: + repository: rancher/helm-project-operator + tag: v0.2.1 + pullPolicy: IfNotPresent + +helmController: + # Note: should be disabled for RKE2 clusters since they already run Helm Controller to manage internal Kubernetes components + enabled: true + + job: + image: + repository: rancher/klipper-helm + tag: v0.7.0-build20220315 + +helmLocker: + enabled: true + +# Additional arguments to be passed into the Helm Project Operator image +additionalArgs: [] + +## Define which Nodes the Pods are scheduled on. +## ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} + +## Tolerations for use with node taints +## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ +## +tolerations: [] +# - key: "key" +# operator: "Equal" +# value: "value" +# effect: "NoSchedule" + +resources: {} + # limits: + # memory: 500Mi + # cpu: 1000m + # requests: + # memory: 100Mi + # cpu: 100m + +containerSecurityContext: {} + # allowPrivilegeEscalation: false + # capabilities: + # drop: + # - ALL + # privileged: false + # readOnlyRootFilesystem: true + +securityContext: {} + # runAsGroup: 1000 + # runAsUser: 1000 + # supplementalGroups: + # - 1000 + +debug: false +debugLevel: 0 + +cleanup: + image: + repository: rancher/shell + tag: v0.1.19 + pullPolicy: IfNotPresent + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + containerSecurityContext: {} + # allowPrivilegeEscalation: false + # capabilities: + # drop: + # - ALL + # privileged: false + # readOnlyRootFilesystem: true + + securityContext: + runAsNonRoot: false + runAsUser: 0 + + resources: {} + # limits: + # memory: 500Mi + # cpu: 1000m + # requests: + # memory: 100Mi + # cpu: 100m diff --git a/charts/prometheus-federator/0.4.3-rc.1/questions.yaml b/charts/prometheus-federator/0.4.3-rc.1/questions.yaml new file mode 100644 index 00000000..87cf1339 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/questions.yaml @@ -0,0 +1,43 @@ +questions: +- variable: global.cattle.psp.enabled + default: "false" + description: "Flag to enable or disable the installation of PodSecurityPolicies by this chart in the target cluster. If the cluster is running Kubernetes 1.25+, you must update this value to false." + label: "Enable PodSecurityPolicies" + type: boolean + group: "Security Settings" +- variable: helmProjectOperator.helmController.enabled + label: Enable Embedded Helm Controller + description: 'Note: If you are running Prometheus Federator in an RKE2 / K3s cluster before v1.23.14 / v1.24.8 / v1.25.4, this should be disabled.' + type: boolean + group: Helm Controller +- variable: helmProjectOperator.helmLocker.enabled + label: Enable Embedded Helm Locker + type: boolean + group: Helm Locker +- variable: helmProjectOperator.projectReleaseNamespaces.labelValue + label: Project Release Namespace Project ID + description: By default, the System Project is selected. This can be overriden to a different Project (e.g. p-xxxxx) + type: string + required: false + group: Namespaces +- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.admin + label: Admin ClusterRole + description: By default, admin selects Project Owners. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: admin + required: false + group: RBAC +- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.edit + label: Edit ClusterRole + description: By default, edit selects Project Members. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: edit + required: false + group: RBAC +- variable: helmProjectOperator.releaseRoleBindings.clusterRoleRefs.view + label: View ClusterRole + description: By default, view selects Read-Only users. This can be overridden to a different ClusterRole (e.g. rt-xxxxx) + type: string + default: view + required: false + group: RBAC \ No newline at end of file diff --git a/charts/prometheus-federator/0.4.3-rc.1/templates/NOTES.txt b/charts/prometheus-federator/0.4.3-rc.1/templates/NOTES.txt new file mode 100644 index 00000000..f551f366 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/templates/NOTES.txt @@ -0,0 +1,3 @@ +{{ $.Chart.Name }} has been installed. Check its status by running: + kubectl --namespace {{ template "prometheus-federator.namespace" . }} get pods -l "release={{ $.Release.Name }}" + diff --git a/charts/prometheus-federator/0.4.3-rc.1/templates/_helpers.tpl b/charts/prometheus-federator/0.4.3-rc.1/templates/_helpers.tpl new file mode 100644 index 00000000..15ea4e5c --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/templates/_helpers.tpl @@ -0,0 +1,66 @@ +# Rancher +{{- define "system_default_registry" -}} +{{- if .Values.global.cattle.systemDefaultRegistry -}} +{{- printf "%s/" .Values.global.cattle.systemDefaultRegistry -}} +{{- end -}} +{{- end -}} + +# Windows Support + +{{/* +Windows cluster will add default taint for linux nodes, +add below linux tolerations to workloads could be scheduled to those linux nodes +*/}} + +{{- define "linux-node-tolerations" -}} +- key: "cattle.io/os" + value: "linux" + effect: "NoSchedule" + operator: "Equal" +{{- end -}} + +{{- define "linux-node-selector" -}} +{{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} +beta.kubernetes.io/os: linux +{{- else -}} +kubernetes.io/os: linux +{{- end -}} +{{- end -}} + +# Helm Project Operator + +{{/* vim: set filetype=mustache: */}} +{{/* Expand the name of the chart. This is suffixed with -alertmanager, which means subtract 13 from longest 63 available */}} +{{- define "prometheus-federator.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 50 | trimSuffix "-" -}} +{{- end }} + +{{/* +Allow the release namespace to be overridden for multi-namespace deployments in combined charts +*/}} +{{- define "prometheus-federator.namespace" -}} + {{- if .Values.namespaceOverride -}} + {{- .Values.namespaceOverride -}} + {{- else -}} + {{- .Release.Namespace -}} + {{- end -}} +{{- end -}} + +{{/* Create chart name and version as used by the chart label. */}} +{{- define "prometheus-federator.chartref" -}} +{{- replace "+" "_" .Chart.Version | printf "%s-%s" .Chart.Name -}} +{{- end }} + +{{/* Generate basic labels */}} +{{- define "prometheus-federator.labels" }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +app.kubernetes.io/instance: {{ .Release.Name }} +app.kubernetes.io/version: "{{ replace "+" "_" .Chart.Version }}" +app.kubernetes.io/part-of: {{ template "prometheus-federator.name" . }} +chart: {{ template "prometheus-federator.chartref" . }} +release: {{ $.Release.Name | quote }} +heritage: {{ $.Release.Service | quote }} +{{- if .Values.commonLabels}} +{{ toYaml .Values.commonLabels }} +{{- end }} +{{- end }} diff --git a/charts/prometheus-federator/0.4.3-rc.1/values.yaml b/charts/prometheus-federator/0.4.3-rc.1/values.yaml new file mode 100644 index 00000000..103c1604 --- /dev/null +++ b/charts/prometheus-federator/0.4.3-rc.1/values.yaml @@ -0,0 +1,94 @@ +# Default values for helm-project-operator. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +# Prometheus Federator Configuration + +global: + cattle: + psp: + enabled: false + systemDefaultRegistry: "" + projectLabel: field.cattle.io/projectId + clusterId: "" + systemProjectId: "" + url: "" + rbac: + pspEnabled: true + pspAnnotations: {} + ## Specify pod annotations + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp + ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl + ## + # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*' + # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default' + # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default' + + ## Reference to one or more secrets to be used when pulling images + ## ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/ + ## + imagePullSecrets: [] + # - name: "image-pull-secret" + +helmProjectOperator: + enabled: true + + # ensures that all resources created by subchart show up as prometheus-federator + helmApiVersion: monitoring.cattle.io/v1alpha1 + + nameOverride: prometheus-federator + + helmController: + # Note: should be disabled for RKE2 clusters since they already run Helm Controller to manage internal Kubernetes components + enabled: true + + helmLocker: + enabled: true + + ## valuesOverride overrides values that are set on each Project Prometheus Stack Helm Chart deployment on an operator level + ## all values provided here will override any user-provided values automatically + valuesOverride: + + federate: + # Change this to point at all Prometheuses you want all your Project Prometheus Stacks to federate from + # By default, this matches the default deployment of Rancher Monitoring + targets: + - rancher-monitoring-prometheus.cattle-monitoring-system.svc:9090 + + image: + repository: rancher/prometheus-federator + tag: v0.3.5 + pullPolicy: IfNotPresent + + # Additional arguments to be passed into the Prometheus Federator image + additionalArgs: [] + + ## Define which Nodes the Pods are scheduled on. + ## ref: https://kubernetes.io/docs/user-guide/node-selection/ + ## + nodeSelector: {} + + ## Tolerations for use with node taints + ## ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/ + ## + tolerations: [] + # - key: "key" + # operator: "Equal" + # value: "value" + # effect: "NoSchedule" + + resources: {} + # limits: + # memory: 500Mi + # cpu: 1000m + # requests: + # memory: 100Mi + # cpu: 100m + + securityContext: {} + # allowPrivilegeEscalation: false + # readOnlyRootFilesystem: true + + debug: false + debugLevel: 0 \ No newline at end of file diff --git a/index.yaml b/index.yaml index 36b0ef06..31c7f8d0 100755 --- a/index.yaml +++ b/index.yaml @@ -1,6 +1,29 @@ apiVersion: v1 entries: prometheus-federator: + - annotations: + catalog.cattle.io/certified: rancher + catalog.cattle.io/display-name: Prometheus Federator + catalog.cattle.io/namespace: cattle-monitoring-system + catalog.cattle.io/os: linux,windows + catalog.cattle.io/permits-os: linux,windows + catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1 + catalog.cattle.io/release-name: prometheus-federator + apiVersion: v2 + appVersion: 0.3.5 + created: "2024-09-12T14:09:16.571722-04:00" + dependencies: + - condition: helmProjectOperator.enabled + name: helmProjectOperator + repository: file://./charts/helmProjectOperator + version: 0.2.1 + description: Prometheus Federator + digest: d17b79b337568b320c06e34582cdbdc13bddcceb74d700572f044d1e7938f569 + icon: https://raw.githubusercontent.com/rancher/prometheus-federator/main/assets/logos/prometheus-federator.svg + name: prometheus-federator + urls: + - assets/prometheus-federator/prometheus-federator-0.4.3-rc.1.tgz + version: 0.4.3-rc.1 - annotations: catalog.cattle.io/certified: rancher catalog.cattle.io/display-name: Prometheus Federator From 406c8e7df3cf6ee944a35ae8df93a459ab6e4f5e Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 20 Sep 2024 15:18:33 -0400 Subject: [PATCH 08/54] Add ability to manually select rancher-project-monitoring chart version --- scripts/build-chart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build-chart b/scripts/build-chart index df9e9796..9d379118 100755 --- a/scripts/build-chart +++ b/scripts/build-chart @@ -6,7 +6,7 @@ source $(dirname $0)/version cd $(dirname $0)/.. CHART=rancher-project-monitoring -VERSION=$(find ./charts/${CHART} -type d -maxdepth 1 -mindepth 1 | tr - \~ | sort -rV | tr \~ - | head -n1 | cut -d'/' -f4) +VERSION=${CHART_VERSION:-$(find ./charts/${CHART} -type d -maxdepth 1 -mindepth 1 | tr - \~ | sort -rV | tr \~ - | head -n1 | cut -d'/' -f4)} helm package charts/${CHART}/${VERSION} --destination bin/${CHART} base64 -i bin/${CHART}/${CHART}-${VERSION}.tgz > bin/${CHART}/${CHART}.tgz.base64 From 04810b2c908ae6c06b6c9c75139a9099cab1db45 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 23 Sep 2024 10:28:28 -0400 Subject: [PATCH 09/54] Add version output note --- scripts/build-chart | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/build-chart b/scripts/build-chart index 9d379118..29be3b03 100755 --- a/scripts/build-chart +++ b/scripts/build-chart @@ -12,4 +12,4 @@ helm package charts/${CHART}/${VERSION} --destination bin/${CHART} base64 -i bin/${CHART}/${CHART}-${VERSION}.tgz > bin/${CHART}/${CHART}.tgz.base64 rm bin/${CHART}/${CHART}-${VERSION}.tgz -echo "Completed ${CHART} build process." \ No newline at end of file +echo "Completed ${CHART} (${VERSION}) build process." \ No newline at end of file From c618eda68a07790c8280f0fc93f855c63d299b1e Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 23 Sep 2024 10:29:00 -0400 Subject: [PATCH 10/54] Ensure docker file can utilize version config method --- package/Dockerfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/package/Dockerfile b/package/Dockerfile index 9230eb0a..aa8d3da0 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -18,6 +18,11 @@ RUN make -C /helm RUN xx-verify --static /helm/bin/helm FROM registry.suse.com/bci/golang:1.22 AS builder + +# Allow chart version config +ARG CHART_VERSION +ENV CHART_VERSION=$CHART_VERSION + WORKDIR /usr/src/app COPY --from=helm ./helm/bin/helm /usr/local/bin/ RUN zypper -n install git vim less file curl wget patch From 5b76e33e29a67f710b45b60ea51ea642da0801ab Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 27 Sep 2024 14:03:56 -0400 Subject: [PATCH 11/54] Update to use new stable helm-project-operator --- go.mod | 3 +-- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 8e46adb1..c95c1a52 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/rancher/prometheus-federator go 1.22.3 replace ( - github.com/rancher/helm-project-operator => github.com/rancher/helm-project-operator v0.2.2-rc.1.0.20240911141850-1140ae4aace0 k8s.io/api => k8s.io/api v0.25.4 k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.25.4 k8s.io/apimachinery => k8s.io/apimachinery v0.25.4 @@ -12,7 +11,7 @@ replace ( ) require ( - github.com/rancher/helm-project-operator v0.2.1 + github.com/rancher/helm-project-operator v0.3.0 github.com/rancher/wrangler v1.0.2 github.com/rancher/wrangler-cli v0.0.0-20220624114648-479c5692ba22 github.com/spf13/cobra v1.6.1 diff --git a/go.sum b/go.sum index 547e939f..6c3162e9 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJf github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/rancher/client-go v1.25.4-rancher1 h1:9MlBC8QbgngUkhNzMR8rZmmCIj6WNRHFOnYiwC2Kty4= github.com/rancher/client-go v1.25.4-rancher1/go.mod h1:8trHCAC83XKY0wsBIpbirZU4NTUpbuhc2JnI7OruGZw= -github.com/rancher/helm-project-operator v0.2.2-rc.1.0.20240911141850-1140ae4aace0 h1:LJXfxKzKCVSvDvRHO8lzyM1FUyqiT/2UgS31UAEFHvU= -github.com/rancher/helm-project-operator v0.2.2-rc.1.0.20240911141850-1140ae4aace0/go.mod h1:HkQq2yAWVGoZ0Q6jUlNTJaI2J8mar/PF8Ur2PN9nmYY= +github.com/rancher/helm-project-operator v0.3.0 h1:9JtzgQVUnwgr9btUNFybPUuChAKOL3ye6855ATAtQyM= +github.com/rancher/helm-project-operator v0.3.0/go.mod h1:HkQq2yAWVGoZ0Q6jUlNTJaI2J8mar/PF8Ur2PN9nmYY= github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc h1:29VHrInLV4qSevvcvhBj5UhQWkPShxrxv4AahYg2Scw= github.com/rancher/lasso v0.0.0-20221227210133-6ea88ca2fbcc/go.mod h1:dEfC9eFQigj95lv/JQ8K5e7+qQCacWs1aIA6nLxKzT8= github.com/rancher/wrangler v1.0.2 h1:0JGv62gF2OkYUoR0fsr99Za63fquFeKTHE2z9kAFVsE= From dba355013d60d826736d53d46d82fac9cc09c8d1 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 27 Sep 2024 16:24:44 -0400 Subject: [PATCH 12/54] update e2e ci --- .github/workflows/e2e-ci.yaml | 2 +- .github/workflows/e2e/scripts/install-monitoring.sh | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 0de3bf88..28583dfe 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -48,7 +48,7 @@ jobs: matrix: k3s_version: # k3d version list k3s | sed 's/+/-/' | sort -h - - ${{ github.event.inputs.k3s_version || 'v1.28.4-k3s2' }} + - ${{ github.event.inputs.k3s_version || 'v1.28.14-k3s1' }} steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/e2e/scripts/install-monitoring.sh b/.github/workflows/e2e/scripts/install-monitoring.sh index d94f503b..6c4a7bee 100755 --- a/.github/workflows/e2e/scripts/install-monitoring.sh +++ b/.github/workflows/e2e/scripts/install-monitoring.sh @@ -13,6 +13,9 @@ helm version helm repo add ${HELM_REPO} https://charts.rancher.io helm repo update +echo "Create required \`cattle-fleet-system\` namespace" +kubectl create namespace cattle-fleet-system + echo "Installing rancher monitoring crd with :\n" helm search repo ${HELM_REPO}/rancher-monitoring-crd --versions --max-col-width=0 | head -n 2 From 7c0c92dbd42405a8c95dc154a30357fe84660bc6 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Wed, 2 Oct 2024 15:45:12 -0400 Subject: [PATCH 13/54] remove use of `AbsaOSS/k3d-action` action --- .github/workflows/e2e-ci.yaml | 14 ++-- .github/workflows/e2e/scripts/install-k3d.sh | 17 +++++ .../workflows/e2e/scripts/setup-cluster.sh | 66 +++++++++++++++++++ 3 files changed, 88 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/e2e/scripts/install-k3d.sh create mode 100644 .github/workflows/e2e/scripts/setup-cluster.sh diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 28583dfe..08daa184 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -71,15 +71,11 @@ jobs: REPO=${REPO} TAG=${TAG} ./scripts/build; REPO=${REPO} TAG=${TAG} ./scripts/package; - - name: Provision k3d Cluster - uses: AbsaOSS/k3d-action@v2 - # k3d will automatically create a network named k3d-test-cluster-1 with the range 172.18.0.0/16 - with: - cluster-name: "e2e-ci-prometheus-federator" - args: >- - --agents 1 - --network "nw01" - --image docker.io/rancher/k3s:${{matrix.k3s_version}} + name : Install k3d + run : ./.github/workflows/e2e/scripts/install-k3d.sh + - + name : Setup k3d cluster + run : CLUSTER_NAME=e2e-ci-prometheus-federator K3S_VERSION=${{ matrix.k3s_version }} ./.github/workflows/e2e/scripts/setup-cluster.sh - name: Import Images Into k3d run: | diff --git a/.github/workflows/e2e/scripts/install-k3d.sh b/.github/workflows/e2e/scripts/install-k3d.sh new file mode 100644 index 00000000..1aa640e6 --- /dev/null +++ b/.github/workflows/e2e/scripts/install-k3d.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -e +set -x + +K3D_URL=https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh +DEFAULT_K3D_VERSION=v5.7.4 + +install_k3d(){ + local k3dVersion=${K3D_VERSION:-${DEFAULT_K3D_VERSION}} + echo -e "Downloading k3d@${k3dVersion} see: ${K3D_URL}" + curl --silent --fail ${K3D_URL} | TAG=${k3dVersion} bash +} + +install_k3d + +k3d version \ No newline at end of file diff --git a/.github/workflows/e2e/scripts/setup-cluster.sh b/.github/workflows/e2e/scripts/setup-cluster.sh new file mode 100644 index 00000000..33049db7 --- /dev/null +++ b/.github/workflows/e2e/scripts/setup-cluster.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +set -e + +source ./scripts/version + +if [ -z "$CLUSTER_NAME" ]; then + echo "CLUSTER_NAME must be specified when setting up a cluster" + exit 1 +fi + +if [ -z "$K3S_VERSION" ]; then + echo "K3S_VERSION must be specified when setting up a cluster, use $(k3d version list k3s) to find valid versions" + exit 1 +fi + +# waits until all nodes are ready +wait_for_nodes(){ + timeout=120 + start_time=$(date +%s) + echo "wait until all agents are ready" + while : + do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ $elapsed_time -ge $timeout ]; then + echo "Timeout reached, exiting..." + exit 1 + fi + + readyNodes=1 + statusList=$(kubectl get nodes --no-headers | awk '{ print $2}') + # shellcheck disable=SC2162 + while read status + do + current_time=$(date +%s) + elapsed_time=$((current_time - start_time)) + if [ $elapsed_time -ge $timeout ]; then + echo "Timeout reached, exiting..." + exit 1 + fi + if [ "$status" == "NotReady" ] || [ "$status" == "" ] + then + readyNodes=0 + break + fi + done <<< "$(echo -e "$statusList")" + # all nodes are ready; exit + if [[ $readyNodes == 1 ]] + then + break + fi + sleep 1 + done +} + +k3d cluster delete "$CLUSTER_NAME" || true +k3d cluster create "$CLUSTER_NAME" --image "docker.io/rancher/k3s:${K3S_VERSION}" + +wait_for_nodes + +echo "$CLUSTER_NAME ready" + +kubectl cluster-info --context "k3d-${CLUSTER_NAME}" +kubectl config use-context "k3d-${CLUSTER_NAME}" +kubectl get nodes -o wide \ No newline at end of file From a5748d04e4a4af8f7ce43e1d30fc77c86377e0fd Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Wed, 2 Oct 2024 15:51:21 -0400 Subject: [PATCH 14/54] fix perms --- .github/workflows/e2e/scripts/install-k3d.sh | 0 .github/workflows/e2e/scripts/setup-cluster.sh | 0 2 files changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 .github/workflows/e2e/scripts/install-k3d.sh mode change 100644 => 100755 .github/workflows/e2e/scripts/setup-cluster.sh diff --git a/.github/workflows/e2e/scripts/install-k3d.sh b/.github/workflows/e2e/scripts/install-k3d.sh old mode 100644 new mode 100755 diff --git a/.github/workflows/e2e/scripts/setup-cluster.sh b/.github/workflows/e2e/scripts/setup-cluster.sh old mode 100644 new mode 100755 From fc3162b6d2f82f7dfda1b516875260836694aa52 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Wed, 2 Oct 2024 15:57:33 -0400 Subject: [PATCH 15/54] Set cluster name as env var --- .github/workflows/e2e-ci.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 08daa184..f2631bc4 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -37,6 +37,7 @@ env: DEFAULT_SLEEP_TIMEOUT_SECONDS: 10 KUBECTL_WAIT_TIMEOUT: 300s DEBUG: ${{ github.event.inputs.debug || false }} + CLUSTER_NAME: 'e2e-ci-prometheus-federator' permissions: contents: write @@ -75,15 +76,15 @@ jobs: run : ./.github/workflows/e2e/scripts/install-k3d.sh - name : Setup k3d cluster - run : CLUSTER_NAME=e2e-ci-prometheus-federator K3S_VERSION=${{ matrix.k3s_version }} ./.github/workflows/e2e/scripts/setup-cluster.sh + run : K3S_VERSION=${{ matrix.k3s_version }} ./.github/workflows/e2e/scripts/setup-cluster.sh - name: Import Images Into k3d run: | - k3d image import ${REPO}/prometheus-federator:${TAG} -c e2e-ci-prometheus-federator; + k3d image import ${REPO}/prometheus-federator:${TAG} -c $CLUSTER_NAME; - name: Setup kubectl context run: | - kubectl config use-context k3d-e2e-ci-prometheus-federator; + kubectl config use-context "k3d-$CLUSTER_NAME"; - name: Set Up Tmate Debug Session if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.enable_tmate == 'true' }} From c1d47bcdf1607129afa22e2df7533377cbc276c1 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 15:35:55 -0400 Subject: [PATCH 16/54] Adjust ci to use new webhook project label method --- .github/workflows/e2e/scripts/create-project-namespace.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e/scripts/create-project-namespace.sh b/.github/workflows/e2e/scripts/create-project-namespace.sh index f012e7b1..ac78e519 100755 --- a/.github/workflows/e2e/scripts/create-project-namespace.sh +++ b/.github/workflows/e2e/scripts/create-project-namespace.sh @@ -8,6 +8,7 @@ cd $(dirname $0)/../../../.. kubectl create namespace e2e-prometheus-federator || true kubectl label namespace e2e-prometheus-federator field.cattle.io/projectId=p-example --overwrite +kubectl annotate namespace e2e-prometheus-federator field.cattle.io/projectId=local:p-example --overwrite sleep "${DEFAULT_SLEEP_TIMEOUT_SECONDS}" if ! kubectl get namespace cattle-project-p-example; then echo "ERROR: Expected cattle-project-p-example namespace to exist after ${DEFAULT_SLEEP_TIMEOUT_SECONDS} seconds, not found" From 1356de87f2c0c1682c8f335ad1285cb341a17c1b Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 18:47:11 -0400 Subject: [PATCH 17/54] adjust e2e build step --- .github/workflows/e2e-ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index f2631bc4..57edf557 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -67,10 +67,10 @@ jobs: run: | sudo wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_amd64 -O /usr/bin/yq && sudo chmod +x /usr/bin/yq; - - name: Perform CI + name: Perform pre-e2e image build run: | - REPO=${REPO} TAG=${TAG} ./scripts/build; - REPO=${REPO} TAG=${TAG} ./scripts/package; + REPO=${REPO} TAG=${TAG} make build; + REPO=${REPO} TAG=${TAG} make package; - name : Install k3d run : ./.github/workflows/e2e/scripts/install-k3d.sh From 872c950761d667b5886a32cbe6900fd441e746be Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 18:48:03 -0400 Subject: [PATCH 18/54] Remove some drone stuff and adjust dev tags --- scripts/version | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/version b/scripts/version index c66a1f33..f36a6c68 100755 --- a/scripts/version +++ b/scripts/version @@ -8,7 +8,7 @@ if [ -n "$(git status --porcelain --untracked-files=no)" ]; then fi COMMIT=$(git rev-parse --short HEAD) -GIT_TAG=${DRONE_TAG:-$(git tag -l --contains HEAD | head -n 1)} +GIT_TAG=$(git tag -l --contains HEAD | head -n 1) if [[ -z "$DIRTY" && -n "$GIT_TAG" ]]; then VERSION=$GIT_TAG @@ -25,6 +25,6 @@ TAG=${TAG:-${VERSION}} REPO=${REPO:-rancher} if echo $TAG | grep -q dirty; then - TAG=dev + TAG="dev-$COMMIT" fi IMAGE=${IMAGE:-$REPO/prometheus-federator:${TAG}} \ No newline at end of file From c1dfbdda899a836681c9346fab37ee9343eeec62 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 18:48:58 -0400 Subject: [PATCH 19/54] make scripts more verbose --- scripts/build | 9 ++++++--- scripts/build-chart | 4 ++-- scripts/package | 5 ++++- 3 files changed, 12 insertions(+), 6 deletions(-) diff --git a/scripts/build b/scripts/build index 694e5e98..be738bfd 100755 --- a/scripts/build +++ b/scripts/build @@ -7,7 +7,7 @@ cd $(dirname $0)/.. ./scripts/build-chart -echo "Starting binary build"; +echo "Starting \`prometheus-federator\` binary build:"; ARCHES=( "$ARCH" ) # Set CROSS_ARCH to build for the other architecture @@ -20,7 +20,7 @@ if [ "$CROSS_ARCH" == "true" ]; then ARCHES+=( "$XARCH" ) fi -echo "Building for Arch: ${ARCHES[*]}" +echo "Building for Arch(s): ${ARCHES[*]}" mkdir -p bin if [ "$(uname)" = "Linux" ]; then @@ -34,10 +34,13 @@ for A in "${ARCHES[@]}" ; do if [ "$CROSS" = "true" ]; then for OS in darwin windows ; do GOARCH="$A" GOOS=$OS go build -ldflags "$LINKFLAGS" -o "bin/prometheus-federator-$OS-$A" + echo "Built \`prometheus-federator-$OS-$A\`" done fi done cd bin ln -sf "./prometheus-federator-$ARCH" "./prometheus-federator" -cd .. \ No newline at end of file +cd .. + +echo "Completed \`prometheus-federator\` binary build." \ No newline at end of file diff --git a/scripts/build-chart b/scripts/build-chart index 29be3b03..27bc7e84 100755 --- a/scripts/build-chart +++ b/scripts/build-chart @@ -5,8 +5,8 @@ source $(dirname $0)/version cd $(dirname $0)/.. -CHART=rancher-project-monitoring -VERSION=${CHART_VERSION:-$(find ./charts/${CHART} -type d -maxdepth 1 -mindepth 1 | tr - \~ | sort -rV | tr \~ - | head -n1 | cut -d'/' -f4)} +CHART=${CHART:-rancher-project-monitoring} +VERSION=${CHART_VERSION:-$(find ./charts/${CHART} -maxdepth 1 -mindepth 1 -type d | tr - \~ | sort -rV | tr \~ - | head -n1 | cut -d'/' -f4)} helm package charts/${CHART}/${VERSION} --destination bin/${CHART} base64 -i bin/${CHART}/${CHART}-${VERSION}.tgz > bin/${CHART}/${CHART}.tgz.base64 diff --git a/scripts/package b/scripts/package index e86450e5..b28ce5cd 100755 --- a/scripts/package +++ b/scripts/package @@ -5,6 +5,8 @@ source $(dirname $0)/version cd $(dirname $0)/.. +echo "Starting \`prometheus-federator\` packaging:"; + mkdir -p dist/artifacts cp bin/prometheus-federator dist/artifacts/prometheus-federator${SUFFIX} @@ -14,5 +16,6 @@ if [ -e ${DOCKERFILE}.${ARCH} ]; then DOCKERFILE=${DOCKERFILE}.${ARCH} fi +echo "Building \`${DOCKERFILE}\` with name \`${IMAGE}\`:"; docker build -f ${DOCKERFILE} -t ${IMAGE} . -echo Built ${IMAGE} +echo "Completed building ${IMAGE} container image" From b20571b96cf20ba8f9d33031dc991ff3baeee4ce Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 18:49:35 -0400 Subject: [PATCH 20/54] expand docker build step args --- package/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/package/Dockerfile b/package/Dockerfile index aa8d3da0..c33e79d7 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -21,7 +21,9 @@ FROM registry.suse.com/bci/golang:1.22 AS builder # Allow chart version config ARG CHART_VERSION -ENV CHART_VERSION=$CHART_VERSION +ARG TAG='' +ARG REPO='' +ENV CHART_VERSION=$CHART_VERSION TAG=$TAG REPO=$REPO WORKDIR /usr/src/app COPY --from=helm ./helm/bin/helm /usr/local/bin/ From 4ac14587c8aaad72a31bddb1bb0cb5d8ea91649f Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 18:50:09 -0400 Subject: [PATCH 21/54] Add branch tag helper --- .github/scripts/branch-tags.sh | 55 ++++++++++++++++++++++++++++++++++ .github/workflows/e2e-ci.yaml | 19 +++++++++++- 2 files changed, 73 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/branch-tags.sh diff --git a/.github/scripts/branch-tags.sh b/.github/scripts/branch-tags.sh new file mode 100644 index 00000000..162478a0 --- /dev/null +++ b/.github/scripts/branch-tags.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Exit immediately if a command exits with a non-zero status +set -e + +# Function to get the previous tag +getPreviousTag() { + local tagPrefix="$1" + # List all tags and filter ones that start with tagPrefix, sort by creation date + git tag --sort=-creatordate | grep "^${tagPrefix}" | head -n 1 +} + +# Determine if we're in a GitHub Actions environment +if [ -n "$GITHUB_REF" ] && [ -n "$GITHUB_SHA" ]; then + # Use GHA environment variables + ref="$GITHUB_REF" + commitSha="${GITHUB_SHA:0:7}" +else + # Fallback to local Git repo + if [ ! -d ".git" ]; then + echo "This script must be run from the root of a Git repository or GitHub Actions." + exit 1 + fi + ref=$(git symbolic-ref HEAD) + commitSha=$(git rev-parse --short HEAD) +fi + +branchTag="" +branchStaticTag="" +prevTag="" + +if [ "$ref" == "refs/heads/main" ]; then + branchTag="head" + branchStaticTag="main-${commitSha}" + prevTag=$(getPreviousTag "main-") +elif [[ "$ref" == refs/heads/release/* ]]; then + version="${ref#refs/heads/release/}" # Extract "vX.0" + branchTag="${version}-head" + branchStaticTag="${version}-head-${commitSha}" + prevTag=$(getPreviousTag "${version}-head-") +else + gitTag=$(git tag -l --contains HEAD | head -n 1) + if [[ -n "$gitTag" ]]; then + branchTag="${gitTag}" + branchStaticTag="${gitTag}-${commitSha}" + else + branchTag="dev-${commitSha}" + branchStaticTag="dev-${commitSha}" + fi +fi + +# Output the results +echo "branch_tag=${branchTag}" +echo "branch_static_tag=${branchStaticTag}" +echo "prev_static_tag=${prevTag}" \ No newline at end of file diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 57edf557..417af6f6 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -32,7 +32,6 @@ env: YQ_VERSION: v4.25.1 E2E_CI: true REPO: rancher - TAG: dev APISERVER_PORT: 8001 DEFAULT_SLEEP_TIMEOUT_SECONDS: 10 KUBECTL_WAIT_TIMEOUT: 300s @@ -43,8 +42,26 @@ permissions: contents: write jobs: + prebuild-env: + name: Prebuild needed Env vars + runs-on: ubuntu-latest + steps: + - name: Check out the repository to the runner + uses: actions/checkout@v4 + - name: Set Branch Tag and Other Variables + id: set-vars + run: bash ./.github/scripts/branch-tags.sh >> $GITHUB_OUTPUT + outputs: + branch_tag: ${{ steps.set-vars.outputs.branch_tag }} + branch_static_tag: ${{ steps.set-vars.outputs.branch_static_tag }} + prev_tag: ${{ steps.set-vars.outputs.prev_tag }} e2e-prometheus-federator: + needs: [ + prebuild-env, + ] runs-on: ubuntu-latest + env: + TAG: ${{ needs.prebuild-env.outputs.branch_static_tag }} strategy: matrix: k3s_version: From b2c0e533fc1776959621dc065cca8bdcde40bfdc Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 19:00:46 -0400 Subject: [PATCH 22/54] Enable more e2e ci tests --- .github/workflows/e2e-ci.yaml | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 417af6f6..18825e67 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -124,6 +124,9 @@ jobs: - name: Check if Project Registration Namespace is auto-created on namespace detection run: ./.github/workflows/e2e/scripts/create-project-namespace.sh; + - + name: Create Project Monitoring Stack via ProjectHelmChart CR + run: ./.github/workflows/e2e/scripts/create-projecthelmchart.sh; # Commenting out for failure in CI but not locally # - @@ -154,9 +157,9 @@ jobs: # - # name: Validate Project Alertmanager # run: ./.github/workflows/e2e/scripts/validate-project-alertmanager.sh; - # - - # name: Delete Project Prometheus Stack - # run: ./.github/workflows/e2e/scripts/delete-projecthelmchart.sh; + - + name: Delete Project Prometheus Stack + run: ./.github/workflows/e2e/scripts/delete-projecthelmchart.sh; - name: Uninstall Prometheus Federator run: ./.github/workflows/e2e/scripts/uninstall-federator.sh; From 1a767413de7c82ffab02e42e25b01808194a1d5c Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 19:32:16 -0400 Subject: [PATCH 23/54] Rename ci example to more clear names --- .github/workflows/e2e/scripts/create-projecthelmchart.sh | 4 ++-- examples/{ci-example.yaml => ci/project-helm-chart.yaml} | 0 examples/{example.yaml => project-helm-chart.yaml} | 0 3 files changed, 2 insertions(+), 2 deletions(-) rename examples/{ci-example.yaml => ci/project-helm-chart.yaml} (100%) rename examples/{example.yaml => project-helm-chart.yaml} (100%) diff --git a/.github/workflows/e2e/scripts/create-projecthelmchart.sh b/.github/workflows/e2e/scripts/create-projecthelmchart.sh index 04775c90..918e106c 100755 --- a/.github/workflows/e2e/scripts/create-projecthelmchart.sh +++ b/.github/workflows/e2e/scripts/create-projecthelmchart.sh @@ -7,9 +7,9 @@ source $(dirname $0)/entry cd $(dirname $0)/../../../.. if [[ "${E2E_CI}" == "true" ]]; then - kubectl apply -f ./examples/ci-example.yaml + kubectl apply -f ./examples/ci/project-helm-chart.yaml else - kubectl apply -f ./examples/example.yaml + kubectl apply -f ./examples/project-helm-chart.yaml fi sleep ${DEFAULT_SLEEP_TIMEOUT_SECONDS}; diff --git a/examples/ci-example.yaml b/examples/ci/project-helm-chart.yaml similarity index 100% rename from examples/ci-example.yaml rename to examples/ci/project-helm-chart.yaml diff --git a/examples/example.yaml b/examples/project-helm-chart.yaml similarity index 100% rename from examples/example.yaml rename to examples/project-helm-chart.yaml From 3dadce36c73c56658a729bdd85c13d3580ad5c49 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 20:43:49 -0400 Subject: [PATCH 24/54] update project-namespace creation --- .../workflows/e2e/scripts/create-project-namespace.sh | 10 +++++++--- examples/ci/namespace.yaml | 8 ++++++++ examples/ci/project.yaml | 8 ++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 examples/ci/namespace.yaml create mode 100644 examples/ci/project.yaml diff --git a/.github/workflows/e2e/scripts/create-project-namespace.sh b/.github/workflows/e2e/scripts/create-project-namespace.sh index ac78e519..212441cb 100755 --- a/.github/workflows/e2e/scripts/create-project-namespace.sh +++ b/.github/workflows/e2e/scripts/create-project-namespace.sh @@ -6,9 +6,13 @@ source $(dirname $0)/entry cd $(dirname $0)/../../../.. -kubectl create namespace e2e-prometheus-federator || true -kubectl label namespace e2e-prometheus-federator field.cattle.io/projectId=p-example --overwrite -kubectl annotate namespace e2e-prometheus-federator field.cattle.io/projectId=local:p-example --overwrite +USE_RANCHER=${USE_RANCHER:-"false"} +if [ "$USE_RANCHER" = "true" ]; then + kubectl apply -f ./examples/ci/project.yaml +fi + +kubectl apply -f ./examples/ci/namespace.yaml + sleep "${DEFAULT_SLEEP_TIMEOUT_SECONDS}" if ! kubectl get namespace cattle-project-p-example; then echo "ERROR: Expected cattle-project-p-example namespace to exist after ${DEFAULT_SLEEP_TIMEOUT_SECONDS} seconds, not found" diff --git a/examples/ci/namespace.yaml b/examples/ci/namespace.yaml new file mode 100644 index 00000000..ef8eb7ec --- /dev/null +++ b/examples/ci/namespace.yaml @@ -0,0 +1,8 @@ +apiVersion: v1 +kind: Namespace +metadata: + annotations: + field.cattle.io/projectId: local:p-example + labels: + field.cattle.io/projectId: p-example + name: e2e-prometheus-federator diff --git a/examples/ci/project.yaml b/examples/ci/project.yaml new file mode 100644 index 00000000..4f4f3a37 --- /dev/null +++ b/examples/ci/project.yaml @@ -0,0 +1,8 @@ +apiVersion: management.cattle.io/v3 +kind: Project +metadata: + name: p-example + namespace: local +spec: + clusterName: local + displayName: PromFed Example From e4673b54ddad98056fd9d427b476810b396e8d4c Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 20:44:25 -0400 Subject: [PATCH 25/54] Enable the rest of the e2e ci steps --- .github/workflows/e2e-ci.yaml | 47 +++++++++---------- .../scripts/validate-project-alertmanager.sh | 4 +- ...validate-project-grafana-dashboard-data.sh | 12 ++--- .../validate-project-grafana-dashboards.sh | 4 +- .../validate-project-grafana-datasource.sh | 8 ++-- .../scripts/validate-project-monitoring.sh | 6 +-- .../validate-project-prometheus-alerts.sh | 4 +- .../validate-project-prometheus-targets.sh | 4 +- 8 files changed, 42 insertions(+), 47 deletions(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 18825e67..a36271e0 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -127,36 +127,31 @@ jobs: - name: Create Project Monitoring Stack via ProjectHelmChart CR run: ./.github/workflows/e2e/scripts/create-projecthelmchart.sh; - - # Commenting out for failure in CI but not locally + - + name: Check if the Project Prometheus Stack is up + run: ./.github/workflows/e2e/scripts/validate-project-monitoring.sh; # - - # name: Create Project Monitoring Stack via ProjectHelmChart CR - # run: ./.github/workflows/e2e/scripts/create-projecthelmchart.sh; - # - - # name: Check if the Project Prometheus Stack is up - # run: ./.github/workflows/e2e/scripts/validate-project-monitoring.sh; - # - # name: Wait for 8 minutes for enough scraping to be done to continue # run: | # for i in {1..48}; do sleep 10; echo "Waited $((i*10)) seconds for metrics to be populated"...; done; - # - - # name: Validate Project Prometheus Targets - # run: ./.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh; - # - - # name: Validate Project Grafana Datasources - # run: ./.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh; - # - - # name: Validate Project Grafana Dashboards - # run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh; - # #- - # #name: Validate Project Grafana Dashboard Data - # #run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh; - # - - # name: Validate Project Prometheus Alerts - # run: ./.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh; - # - - # name: Validate Project Alertmanager - # run: ./.github/workflows/e2e/scripts/validate-project-alertmanager.sh; + - + name: Validate Project Prometheus Targets + run: ./.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh; + - + name: Validate Project Grafana Datasources + run: ./.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh; + - + name: Validate Project Grafana Dashboards + run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh; + - + name: Validate Project Grafana Dashboard Data + run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh; + - + name: Validate Project Prometheus Alerts + run: ./.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh; + - + name: Validate Project Alertmanager + run: ./.github/workflows/e2e/scripts/validate-project-alertmanager.sh; - name: Delete Project Prometheus Stack run: ./.github/workflows/e2e/scripts/delete-projecthelmchart.sh; diff --git a/.github/workflows/e2e/scripts/validate-project-alertmanager.sh b/.github/workflows/e2e/scripts/validate-project-alertmanager.sh index 8ec0454b..fdd7205f 100755 --- a/.github/workflows/e2e/scripts/validate-project-alertmanager.sh +++ b/.github/workflows/e2e/scripts/validate-project-alertmanager.sh @@ -14,9 +14,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts | yq -P - > ${tmp_alerts_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts | yq -P - > ${tmp_alerts_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_alerts_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_alerts_yaml} fi if [[ $(yq '. | length' "${tmp_alerts_yaml}") != "1" ]]; then diff --git a/.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh b/.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh index bdb7e697..86b3d897 100755 --- a/.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh +++ b/.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh @@ -16,9 +16,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml} fi dashboards=$(yq '.[].uri' ${tmp_dashboards_yaml}) @@ -27,9 +27,9 @@ dashboards=$(yq '.[].uri' ${tmp_dashboards_yaml}) for dashboard in ${dashboards[@]}; do dashboard_uid=$(yq ".[] | select(.uri==\"${dashboard}\") | .uid" ${tmp_dashboards_yaml}); if [[ -z "${RANCHER_TOKEN}" ]]; then - dashboard_json=$(curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/dashboards/uid/${dashboard_uid} | yq '.dashboard' -) + dashboard_json=$(curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/dashboards/uid/${dashboard_uid} | yq '.dashboard' -) else - dashboard_json=$(curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/dashboards/uid/${dashboard_uid} -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq '.dashboard' -) + dashboard_json=$(curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/dashboards/uid/${dashboard_uid} -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq '.dashboard' -) fi # TODO: Fix this to actually recursively utilize Grafana dashboard's yaml structure # Today, it just looks for .expr entries in .panels[], .panels[].panels[], and .rows[].panels[], which should cover all dashboards in Monitoring today @@ -147,9 +147,9 @@ for query_key in $(yq "keys" ${tmp_queries_yaml} | cut -d' ' -f2-); do EOF )" if [[ -z "${RANCHER_TOKEN}" ]]; then - query_response=$(curl -s "${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/ds/query" -H 'content-type: application/json' --data-raw "${query_body}") + query_response=$(curl -s "${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/ds/query" -H 'content-type: application/json' --data-raw "${query_body}") else - query_response=$(curl -s "${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/ds/query" -H 'content-type: application/json' --data-raw "${query_body}" -k -H "Authorization: Bearer ${RANCHER_TOKEN}") + query_response=$(curl -s "${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/ds/query" -H 'content-type: application/json' --data-raw "${query_body}" -k -H "Authorization: Bearer ${RANCHER_TOKEN}") fi if [[ "$(echo ${query_response} | yq '.message == "bad request data"')" == "true" ]]; then # echo "QUERY: ${query}" diff --git a/.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh b/.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh index fe1ff5ac..187678fe 100755 --- a/.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh +++ b/.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh @@ -14,9 +14,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml} fi expected_dashboards=( diff --git a/.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh b/.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh index 1cff7781..078a8e5c 100755 --- a/.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh +++ b/.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh @@ -14,9 +14,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/datasources | yq -P - > ${tmp_datasources_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/datasources | yq -P - > ${tmp_datasources_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/datasources -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_datasources_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/datasources -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_datasources_yaml} fi if [[ $(yq '. | length' ${tmp_datasources_yaml}) != "1" ]]; then @@ -25,8 +25,8 @@ if [[ $(yq '. | length' ${tmp_datasources_yaml}) != "1" ]]; then exit 1 fi -if [[ $(yq '.[0].url' ${tmp_datasources_yaml}) != "http://cattle-project-p-example-m-prometheus.cattle-project-p-example:9090/" ]]; then - echo "ERROR: Expected the only datasource to be configured to point to Project Prometheus at Kubernetes DNS http://cattle-project-p-example-m-prometheus.cattle-project-p-example:9090/" +if [[ $(yq '.[0].url' ${tmp_datasources_yaml}) != "http://cattle-project-p-example-m-prometheus.cattle-project-p-example-monitoring:9090/" ]]; then + echo "ERROR: Expected the only datasource to be configured to point to Project Prometheus at Kubernetes DNS http://cattle-project-p-example-m-prometheus.cattle-project-p-example-monitoring:9090/" cat ${tmp_datasources_yaml} exit 1 fi diff --git a/.github/workflows/e2e/scripts/validate-project-monitoring.sh b/.github/workflows/e2e/scripts/validate-project-monitoring.sh index 63095e3b..44d6a518 100755 --- a/.github/workflows/e2e/scripts/validate-project-monitoring.sh +++ b/.github/workflows/e2e/scripts/validate-project-monitoring.sh @@ -6,17 +6,17 @@ source $(dirname $0)/entry cd $(dirname $0)/../../../.. -if ! kubectl -n cattle-project-p-example rollout status statefulset alertmanager-cattle-project-p-example-m-alertmanager --timeout="${KUBECTL_WAIT_TIMEOUT}"; then +if ! kubectl -n cattle-project-p-example-monitoring rollout status statefulset alertmanager-cattle-project-p-example-m-alertmanager --timeout="${KUBECTL_WAIT_TIMEOUT}"; then echo "ERROR: Project Alertmanager did not roll out" exit 1; fi -if ! kubectl -n cattle-project-p-example rollout status statefulset prometheus-cattle-project-p-example-m-prometheus --timeout="${KUBECTL_WAIT_TIMEOUT}"; then +if ! kubectl -n cattle-project-p-example-monitoring rollout status statefulset prometheus-cattle-project-p-example-m-prometheus --timeout="${KUBECTL_WAIT_TIMEOUT}"; then echo "ERROR: Project Prometheus did not roll out" exit 1; fi -if ! kubectl -n cattle-project-p-example rollout status deployment cattle-project-p-example-monitoring-grafana --timeout="${KUBECTL_WAIT_TIMEOUT}"; then +if ! kubectl -n cattle-project-p-example-monitoring rollout status deployment cattle-project-p-example-monitoring-grafana --timeout="${KUBECTL_WAIT_TIMEOUT}"; then echo "ERROR: Project Grafana did not roll out" exit 1 fi diff --git a/.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh b/.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh index 0d9b3ceb..1e8d99f9 100755 --- a/.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh +++ b/.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh @@ -16,9 +16,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts | yq -P - > ${tmp_rules_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts | yq -P - > ${tmp_rules_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_rules_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_rules_yaml} fi yq '.data.alerts' ${tmp_rules_yaml} > ${tmp_alert_rules_yaml} diff --git a/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh b/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh index 12223d29..5335bdfe 100755 --- a/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh +++ b/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh @@ -16,9 +16,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets | yq -P - > ${tmp_targets_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets | yq -P - > ${tmp_targets_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_targets_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_targets_yaml} fi yq '.data.activeTargets[] | {.labels.job: .health}' ${tmp_targets_yaml} > ${tmp_targets_up_yaml}; From b1a80331f12c5462393008bc79359e21b7c37e77 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 21:07:46 -0400 Subject: [PATCH 26/54] pause for deployments --- .github/workflows/e2e-ci.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index a36271e0..483d2634 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -127,6 +127,10 @@ jobs: - name: Create Project Monitoring Stack via ProjectHelmChart CR run: ./.github/workflows/e2e/scripts/create-projecthelmchart.sh; + - + name: Wait for a few minutes for chart installs + run: | + for i in {1..12}; do sleep 10; echo "Waited $((i*10)) seconds for metrics to be populated"...; done; - name: Check if the Project Prometheus Stack is up run: ./.github/workflows/e2e/scripts/validate-project-monitoring.sh; From ee0b238a4aebf69deaf6dd69c50e7c1214308fd3 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Fri, 4 Oct 2024 21:24:50 -0400 Subject: [PATCH 27/54] fix log and expand timeout for create projecthelmchart step --- .github/workflows/e2e-ci.yaml | 2 +- .github/workflows/e2e/scripts/create-projecthelmchart.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 483d2634..7de21763 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -126,7 +126,7 @@ jobs: run: ./.github/workflows/e2e/scripts/create-project-namespace.sh; - name: Create Project Monitoring Stack via ProjectHelmChart CR - run: ./.github/workflows/e2e/scripts/create-projecthelmchart.sh; + run: DEFAULT_SLEEP_TIMEOUT_SECONDS=20 ./.github/workflows/e2e/scripts/create-projecthelmchart.sh; - name: Wait for a few minutes for chart installs run: | diff --git a/.github/workflows/e2e/scripts/create-projecthelmchart.sh b/.github/workflows/e2e/scripts/create-projecthelmchart.sh index 918e106c..5ce0611b 100755 --- a/.github/workflows/e2e/scripts/create-projecthelmchart.sh +++ b/.github/workflows/e2e/scripts/create-projecthelmchart.sh @@ -14,7 +14,7 @@ fi sleep ${DEFAULT_SLEEP_TIMEOUT_SECONDS}; if ! kubectl get -n cattle-monitoring-system job/helm-install-cattle-project-p-example-monitoring; then - echo "ERROR: Helm Install Job for Project Monitoring Stack was never created after ${KUBECTL_WAIT_TIMEOUT} seconds" + echo "ERROR: Helm Install Job for Project Monitoring Stack was never created after ${DEFAULT_SLEEP_TIMEOUT_SECONDS} seconds" exit 1 fi From f040af98bf9fd6e0be9c25e4ebf3a06bccb16690 Mon Sep 17 00:00:00 2001 From: "Dan P." Date: Fri, 4 Oct 2024 22:20:55 -0400 Subject: [PATCH 28/54] update artifacts script sort artifacts by type --- .../e2e/scripts/generate-artifacts.sh | 27 ++++++++++++++----- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/.github/workflows/e2e/scripts/generate-artifacts.sh b/.github/workflows/e2e/scripts/generate-artifacts.sh index 8a9f2363..be1bf508 100755 --- a/.github/workflows/e2e/scripts/generate-artifacts.sh +++ b/.github/workflows/e2e/scripts/generate-artifacts.sh @@ -38,17 +38,30 @@ MANIFEST_DIRECTORY=${ARTIFACT_DIRECTORY}/manifests LOG_DIRECTORY=${ARTIFACT_DIRECTORY}/logs # Manifests - mkdir -p ${MANIFEST_DIRECTORY} -kubectl get pods -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/monitoring_pods.yaml || true -kubectl get pods -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/project_pods.yaml || true + kubectl get namespaces -o yaml > ${MANIFEST_DIRECTORY}/namespaces.yaml || true -kubectl get projecthelmchart -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/projecthelmcharts.yaml || true -kubectl get helmcharts -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/helmcharts.yaml || true -kubectl get helmreleases -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/helmreleases.yaml || true -# Logs +## cattle-monitoring-system ns manifests +kubectl get helmcharts -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/helmcharts/cattle-monitoring-system.yaml || true +kubectl get helmreleases -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/helmreleases/cattle-monitoring-system.yaml || true +kubectl get daemonset -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/daemonsets/cattle-monitoring-system.yaml || true +kubectl get deployment -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/deployments/cattle-monitoring-system.yaml || true +kubectl get job -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/jobs/cattle-monitoring-system.yaml || true +kubectl get statefulset -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/statefulsets/cattle-monitoring-system.yaml || true +kubectl get pods -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-monitoring-system.yaml || true +## cattle-project-p-example ns manifests +kubectl get projecthelmchart -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/projecthelmcharts/cattle-project-p-example.yaml || true +kubectl get statefulset -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/statefulsets/cattle-project-p-example.yaml || true +kubectl get pods -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-project-p-example.yaml || true + +## cattle-project-p-example-monitoring ns manifests +kubectl get deployment -n cattle-project-p-example-monitoring -o yaml > ${MANIFEST_DIRECTORY}/deployments/cattle-project-p-example-monitoring.yaml || true +kubectl get statefulset -n cattle-project-p-example-monitoring -o yaml > ${MANIFEST_DIRECTORY}/statefulsets/cattle-project-p-example-monitoring.yaml || true +kubectl get pods -n cattle-project-p-example-monitoring -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-project-p-example-monitoring.yaml || true + +# Logs mkdir -p ${LOG_DIRECTORY}/rancher-monitoring ## Rancher Monitoring From 36d4b5037f192c7708e7e662c2cc758bb5c414a4 Mon Sep 17 00:00:00 2001 From: "Dan P." Date: Fri, 4 Oct 2024 22:30:17 -0400 Subject: [PATCH 29/54] fix dirs --- .github/workflows/e2e/scripts/generate-artifacts.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/e2e/scripts/generate-artifacts.sh b/.github/workflows/e2e/scripts/generate-artifacts.sh index be1bf508..dd2c30e4 100755 --- a/.github/workflows/e2e/scripts/generate-artifacts.sh +++ b/.github/workflows/e2e/scripts/generate-artifacts.sh @@ -39,6 +39,13 @@ LOG_DIRECTORY=${ARTIFACT_DIRECTORY}/logs # Manifests mkdir -p ${MANIFEST_DIRECTORY} +mkdir -p ${MANIFEST_DIRECTORY}/helmcharts +mkdir -p ${MANIFEST_DIRECTORY}/helmreleases +mkdir -p ${MANIFEST_DIRECTORY}/daemonsets +mkdir -p ${MANIFEST_DIRECTORY}/deployments +mkdir -p ${MANIFEST_DIRECTORY}/jobs +mkdir -p ${MANIFEST_DIRECTORY}/statefulsets +mkdir -p ${MANIFEST_DIRECTORY}/pods kubectl get namespaces -o yaml > ${MANIFEST_DIRECTORY}/namespaces.yaml || true From a97ae110e8eb624705d777de6e30e932f81bae8b Mon Sep 17 00:00:00 2001 From: "Dan P." Date: Fri, 4 Oct 2024 22:41:22 -0400 Subject: [PATCH 30/54] capture a list of all helmcharts --- .github/workflows/e2e/scripts/generate-artifacts.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e/scripts/generate-artifacts.sh b/.github/workflows/e2e/scripts/generate-artifacts.sh index dd2c30e4..c1a13cb5 100755 --- a/.github/workflows/e2e/scripts/generate-artifacts.sh +++ b/.github/workflows/e2e/scripts/generate-artifacts.sh @@ -48,6 +48,7 @@ mkdir -p ${MANIFEST_DIRECTORY}/statefulsets mkdir -p ${MANIFEST_DIRECTORY}/pods kubectl get namespaces -o yaml > ${MANIFEST_DIRECTORY}/namespaces.yaml || true +kubectl get helmcharts -A > ${MANIFEST_DIRECTORY}/helmcharts-list.txt || true ## cattle-monitoring-system ns manifests kubectl get helmcharts -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/helmcharts/cattle-monitoring-system.yaml || true From 77fc8e7615827c979496b86734ade80f81067a93 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 11:55:51 -0400 Subject: [PATCH 31/54] Update helm-project-operator chart sources --- packages/prometheus-federator/charts/Chart.yaml | 2 +- .../dependencies/helmProjectOperator/dependency.yaml | 5 ++--- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/prometheus-federator/charts/Chart.yaml b/packages/prometheus-federator/charts/Chart.yaml index 51f5984e..a17f8d31 100755 --- a/packages/prometheus-federator/charts/Chart.yaml +++ b/packages/prometheus-federator/charts/Chart.yaml @@ -12,7 +12,7 @@ dependencies: - condition: helmProjectOperator.enabled name: helmProjectOperator repository: file://./charts/helmProjectOperator - version: 0.2.1 + version: 0.3.1 description: Prometheus Federator icon: https://raw.githubusercontent.com/rancher/prometheus-federator/main/assets/logos/prometheus-federator.svg name: prometheus-federator diff --git a/packages/prometheus-federator/generated-changes/dependencies/helmProjectOperator/dependency.yaml b/packages/prometheus-federator/generated-changes/dependencies/helmProjectOperator/dependency.yaml index 5a1d67de..41a0fa0c 100644 --- a/packages/prometheus-federator/generated-changes/dependencies/helmProjectOperator/dependency.yaml +++ b/packages/prometheus-federator/generated-changes/dependencies/helmProjectOperator/dependency.yaml @@ -1,3 +1,2 @@ -url: https://github.com/rancher/helm-project-operator.git -subdirectory: charts/helm-project-operator -commit: 1f8d3f40a2708b8a616934aac6b3b30d81eaab32 +url: https://github.com/rancher/helm-project-operator/releases/download/v0.3.1/helm-project-operator-0.3.1.tgz +doNotRelease: true \ No newline at end of file From b4c08fc447697eb99c06b0450bf9a911e04a4f9c Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 12:35:12 -0400 Subject: [PATCH 32/54] update charts app version --- packages/prometheus-federator/charts/Chart.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/prometheus-federator/charts/Chart.yaml b/packages/prometheus-federator/charts/Chart.yaml index a17f8d31..5809c282 100755 --- a/packages/prometheus-federator/charts/Chart.yaml +++ b/packages/prometheus-federator/charts/Chart.yaml @@ -7,7 +7,7 @@ annotations: catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1 catalog.cattle.io/release-name: prometheus-federator apiVersion: v2 -appVersion: 0.3.5 +appVersion: 0.4.3-rc.1 dependencies: - condition: helmProjectOperator.enabled name: helmProjectOperator From 117a97f7d81425c5710b1a4f932c3cfb21732276 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 12:49:22 -0400 Subject: [PATCH 33/54] Remove unnecessary condition... ...if this were ever false then the chart does nothing. --- packages/prometheus-federator/charts/Chart.yaml | 3 +-- packages/prometheus-federator/charts/values.yaml | 2 -- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/packages/prometheus-federator/charts/Chart.yaml b/packages/prometheus-federator/charts/Chart.yaml index 5809c282..eea5005c 100755 --- a/packages/prometheus-federator/charts/Chart.yaml +++ b/packages/prometheus-federator/charts/Chart.yaml @@ -9,8 +9,7 @@ annotations: apiVersion: v2 appVersion: 0.4.3-rc.1 dependencies: -- condition: helmProjectOperator.enabled - name: helmProjectOperator +- name: helmProjectOperator repository: file://./charts/helmProjectOperator version: 0.3.1 description: Prometheus Federator diff --git a/packages/prometheus-federator/charts/values.yaml b/packages/prometheus-federator/charts/values.yaml index 103c1604..d725c52e 100755 --- a/packages/prometheus-federator/charts/values.yaml +++ b/packages/prometheus-federator/charts/values.yaml @@ -32,8 +32,6 @@ global: # - name: "image-pull-secret" helmProjectOperator: - enabled: true - # ensures that all resources created by subchart show up as prometheus-federator helmApiVersion: monitoring.cattle.io/v1alpha1 From 8120c5be6d430e4a88272348e128f810728bf897 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 12:50:01 -0400 Subject: [PATCH 34/54] Update image version --- packages/prometheus-federator/charts/values.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/prometheus-federator/charts/values.yaml b/packages/prometheus-federator/charts/values.yaml index d725c52e..3d04e3ee 100755 --- a/packages/prometheus-federator/charts/values.yaml +++ b/packages/prometheus-federator/charts/values.yaml @@ -56,7 +56,7 @@ helmProjectOperator: image: repository: rancher/prometheus-federator - tag: v0.3.5 + tag: v0.4.3-rc.1 pullPolicy: IfNotPresent # Additional arguments to be passed into the Prometheus Federator image From a4cd2f6904eacefa5b4ce280e25f696953d809bb Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 13:22:03 -0400 Subject: [PATCH 35/54] improve ci scripts verbosity --- scripts/validate | 3 ++- scripts/validate-chart | 2 +- scripts/validate-charts | 2 +- scripts/validate-ci | 1 + scripts/validate-packages | 1 + 5 files changed, 6 insertions(+), 3 deletions(-) diff --git a/scripts/validate b/scripts/validate index 35a87f00..feb8b87d 100755 --- a/scripts/validate +++ b/scripts/validate @@ -3,7 +3,8 @@ set -e cd $(dirname $0)/.. -echo Running validation +echo "Running validation" PACKAGES="$(go list ./...)" echo Running: go fmt test -z "$(go fmt ${PACKAGES} | tee /dev/stderr)" +echo "Validate passed" \ No newline at end of file diff --git a/scripts/validate-chart b/scripts/validate-chart index 37473add..11677faf 100755 --- a/scripts/validate-chart +++ b/scripts/validate-chart @@ -3,7 +3,7 @@ set -e cd $(dirname $0)/.. - +echo "Validating newest prometheus-federator chart" CHART=prometheus-federator VERSION=$(find ./charts/${CHART} -type d -maxdepth 1 -mindepth 1 | tr - \~ | sort -rV | tr \~ - | head -n1 | cut -d'/' -f4) diff --git a/scripts/validate-charts b/scripts/validate-charts index f481bdeb..44b2cfe3 100755 --- a/scripts/validate-charts +++ b/scripts/validate-charts @@ -3,7 +3,7 @@ set -e cd $(dirname $0)/.. -echo Running chart validation +echo Running general chart validation ./scripts/pull-scripts ./bin/charts-build-scripts validate --local \ No newline at end of file diff --git a/scripts/validate-ci b/scripts/validate-ci index e7981cf8..38121d5e 100755 --- a/scripts/validate-ci +++ b/scripts/validate-ci @@ -3,6 +3,7 @@ set -e cd $(dirname $0)/.. +echo "Verifying code is generated and repo is clean" go generate source ./scripts/version diff --git a/scripts/validate-packages b/scripts/validate-packages index 7e580657..509720e2 100755 --- a/scripts/validate-packages +++ b/scripts/validate-packages @@ -1,6 +1,7 @@ #!/bin/bash set -e +echo "Validating packages..." cd $(dirname $0)/.. monitoring_pkg_path=packages/rancher-project-monitoring/package.yaml From 9e081a9f661e95978041b9c039dd364fe0d2f3fa Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 13:22:10 -0400 Subject: [PATCH 36/54] make charts --- .../prometheus-federator-0.4.3-rc.1.tgz | Bin 20617 -> 20583 bytes .../0.4.3-rc.1/Chart.yaml | 7 +++---- .../charts/helmProjectOperator/Chart.yaml | 7 +++++-- .../templates/_helpers.tpl | 9 --------- .../templates/deployment.yaml | 4 +++- .../charts/helmProjectOperator/values.yaml | 4 ++-- .../0.4.3-rc.1/values.yaml | 4 +--- index.yaml | 11 +++++------ 8 files changed, 19 insertions(+), 27 deletions(-) diff --git a/assets/prometheus-federator/prometheus-federator-0.4.3-rc.1.tgz b/assets/prometheus-federator/prometheus-federator-0.4.3-rc.1.tgz index 7926f1ea89a78f3af892ed32b8877a8956f04a3d..0f2e6e0c26e2fd217dc302dcf59c0494c20f8360 100644 GIT binary patch delta 20471 zcmYh?19P5T*e>8WX>8lJZ8vFb+g5`+w%ORW)!4Sv*hb?f&wl5d+27v3VXawnuIoIm zs6z150`NpZz`={4y_0rx?vcT{;>u1%k5DfC`bvUlL#s7S`n#!}j*{<-Yqz*F45bPZ ztQwd^X6x_Ow}7wXH!81vA27s|xDxXPqZ~ zvyEVRqWW|8k3VCtgd`*+B0hiNzJ!E@9lv~Ce?Gi=0W=A{!oqyQeEttlhr2}ig2Mjp z4+5_LRCaog)#$$;-^Po)8qAD*N(*z&g_UK+=*LlrqgUBbpSXy(&T3DOs0@~zaLg{U zC2=ZaD|+svyK0!b3r(IU$4!)qXdDT6Rq>N>v{wgH+biB3Skt-Ig&n>5m8{A)Tb#Oy z$H|R80uR^wNONi~#r(JQzw}*er3H4FMxx6V<4zd(s1eecs!0yTqx+gAv#X;9;)_?l z%|t^BofFD8#j1=ja&|d07D0nJQlex06g5#ObTPl?4qsLL_><-Q&`WEKeI_45Ik@7b zpCqFCM_I@GO2ZH1d8x%p7fmICB1Q6Swejq1I8~2r+O;POFt4@z z1y7wx=RB|H+l5#&@@2yKkn7?g5)ok`$3+m$7)swfNcnz)Vs0ZgJ(IfQ+?<+nJ|n3V z@TM?1mm<7QT0^LV(+W)n@qe6;Ehvsttz?yoI!zs1r*0<_?DoV!Xx9DSX?ei$2NxwK&aAm&^7Tmne?)8ityl%~NK^ zUYu#_jK&vfu+PAKDo=31i@Vp{F-UR_q*{cVH!;>WLq1+g!M{-JRC1=^R08;8z3yIb zm!;EJUVNod`Ny5vx#?7HICBs;3OhaR2pd}zTF?rw_TS-?k^`l*cz&BUN23~Ls8PFF z7T9Pv$}X#rKZIiG5@C@aPP|v3p363tr0UnpJZ};+!P95-!wa7@L{mj+W0Uv-%dPV6 zC$Jtyk^XvD#F0TPV^3@Ij2JwOJ*k5^%Q(^(xGAu(C^y{l(Xe#57K+3rcCG-Z&NKHugHg3^B|(lo30xMZ`jL%C2P;vEDkMblhSV6fN;%!l#kdM}q$$;kQopHDCdJ|(a2Q}{D2XkS z*^)WR>Y!G2jle(*Uo)ci(g$&f2C)2*g0lQ{9Vt1o$!9f<@TYcVl1gU-qOAL54B=)* zEmbYjvu5NvA=tsppI1;$QHV@bBwf}wq4hAb7Jjt6)t+lLfDq|TG``|)-`=BL}y za?6jDDrAT4a?26hv{=yTJxDN~5cx2&P_t34Cc&Bckxd60o$s!beTq3KGGSumDr`O@ znJ(Rq*YSw)d*{9bCx!Z~Li09r;Q&~fntWiTbm$L}*~}b@g#pN?oEh)g@tI&6ga`2w z?8yvr4e#g_XaxlzB!xV|iz3X$w&8+&+30ARSIM(y_VACU6z;N`lQ4RW9H6Z z3CWf>{KkvF5ciC{l4dL!7tGkb$?6zMVbN%d)8&0TEfdUvZR^8$PV^>o1zydBgaqK= z%T_hj%+=M;?{`ZE<%i6K`dSE0E(-I^2hkE&YD&+_0HY{oN2=cl2e;KJeIcGyT6;3V zjDpQO``l`M==|<|ELk$tqMoGeWZCiWpNL=o7Jf9{m#Y{EkCv-}9tYnRVlwLI>P2tgQj7UV@i$MO#bKPD?)wtI^((jbFyD88b=9Ah=-gTTa?CGON8^Kg- zp>|TKWnJphC&?ChkA9Bxi>MWnR_2@!FhUPExmpk71xtg6w?v8SzrAN$RJ<|qEX<9+ zTkrNm8SRreMu(imRGw1!x9x#g5Yn*}tF&=mc%T7?0g%iX4bwl+`~I5I9Z1dZImZ~U z$Crn_s5~hRX8by!G5L%|X^bFo;p4@bB0uSM&YDWHJ9a6TpNaXaQT#!FZhUWdd122G zvO&QxPP%E&75M(5$Py>UejDAs@c5c_PnMEjSLP4io9k@Rl} zd0aUB7WK?;wJ^1LHna@2tIyhv$I%m3raHKCK6f7}uSW}K-l7LqEd^?ME;E_YabGqo z=dTI+TlK6;5feiKX3o0*J;kS^*Iy))zfU{Nm5|Q;F)rpZ-rO%koU828fxqSh zqfl*-H~fI!ydboHKPyGCRpza+d}L3nT3Rz6Fr|vQBExB>T8zw$Qft^~&dI~0)LIfn zO;cWQo1}Q;R>tY$Dt{LJFipF4^R&YJmwn8ZWjhKo_PYEU%#v_<*kI?nDdZR3Mc9N z?4?_a1wGC9%?1?@)NMq&6VBtPaGr_-)K*RvN4vxL~EAE1-Vn8iq6YqS}^+6`R z_`9h*ma_9t+p+Wg8AuLXrf3*j3e}S0^Pl#^A}IR|)LMcje^7!_GihqRBGdlz-@L^4 z4Rm{L8JkItJi|e!{iL)!B$853Riaq`Wsq0tiJI4J4Xw;gxuN%vplYps8Y~CYo&Wa9 z4$*hEg5*GN)Q=|g!-Wr^(*1{HV42I4`0fJN8{o)yGt z-#`m%2_!=HYdB#7g9V@-@OilM=3H!PJqnZu-)pCxmghQ~@{7}UWSte+{ltT& z28Ch4d7S9))G<(|&-DqBCVDOWLVKh#6x+DbMek~0HBK)eDcxhPy0{6mzU4w6Sna0v zOCq|`&z;H?OkNiMaQs|O`6@2q$Xc2Wn$GLyrDDPp&{Zw2BcgK{J_d^Ug!w5yI#w8| z7?fs$jHl=n)Y4%DoD5GO@T8SNCKqFofpJy9!6YmK?!Z7%8{F$*6xbD9D zTA0`Kv+f`ygtp3eSX=7nZwCG_q@zwtpDvqcr_HZ=TfHl%29`j|_`4e7QRR5;0L<4B z!7s$z^wV+UHF=1;Br9yv0e`UUtnwDQNcM~`;c*0m?#2TAW@&0t`iwIVqczpIy?KBxWqL$e(xw5oc70=8Gm`~gYepE z!N9G+Lls86i7>!&vfqi!aXe0C5NUx;yc4_wuo5H+B0nX5q} zx}=8oK@vTr-Q3!uI7M{~qFq0vLq>$sN1N}1pt;7{IaGS<6+^h@d{0`ItZ$M^~>HBRS?F9}TYkJA7O2up)~c=#c#_vZ%e zV6%YR7~pj;IxtKDfqb7>#r82mo{~x1n71?vQ$B1-#XP@O+mUk%M-lR;*ff~cS*c7h zR8zo#no??!P$cmR&1aDkua;51Ey)SSqPgWe z6l3|cwV8&3%UI#2WhzoBqSawdS*o~07s_RK;^3|^&_RRaF%F5Y$!m$#z{ts$jw~i4 zd?+tmn{!x>&q&dj-r zGKNwVIWK`4H$IF9h-daak7*1fZ$b+SGh14qrvS)!z|vmk8P({f%dt{ap+Q$F-#Bu_ zxL+k750CpbcQ;igLDc~c8q_Ej;V*rij->^ct};7O$dGJ=aBgk(4R1l}GksHqra3o; zBx?1IUt;}zbQFGh2vMg^C|U$!u0dcvvV>@k4GJ&9e;pixLd-=QVjW4v2Fbn;h~X7I z)oQ5M0M4245(;E3vx$1n3hrnJc|ELuOMCDhJI&qPx=$U5^adDIO0rCxVFX(zS4CUx z8af};M=BFVp$G>?&VF-)jrV#GrpXJ0K%V1>8vERwMzbpAJQWowj5A7fmTekFZdw!7 zl;bYJ%`PZ@OXetHct+M<9*$?N^o^Kav>rgh1B}^A{&=QT6yHAmfyXuYu$i|Jg>P(g z;#H!hS~;NjR1CKi+6@U&+`w4Gm!Rl`p6+bB$`Du1P}0`EawBIB7|+A8P?H@^}G zP#KI`kY{*w)9qD>yqhs}d2tlWsrm`!+PON%^~5_3YE+a&1)RQ!WmLY~A4LWzB#CDk z0Z79E7OQX=1j9-du_&e^6^hsaQW!XZU1L9{GHgA#-bA`- zm|{<~7LXwIxCtwDn<8x)lo(sx>}uLnIQL91n^$40-k;COCs=^pIY)^arS6 zMS+9>>XWqZ6R9u+-nyn`vXwHX+)a?IbVyViz}rJPiFbOVySJ>o#$XQmh2Po)FgVXe zbFkX#BohWU?>OmWlS<2R44qHVC|M?0Ua-vD6|e8-=s!1`#bYJ!Yco%a=f6>0=RYd246fNa8R|WO~aPY1xm7N-{+7oKZ$Us-28Q|dHa~$|7V;S0oU>Fk6})FOm;GEMy6Sfg)H-#N!Y5yO ztCAHXJv5zkSbPhGMyvPD)v#1HCV}cEEqip9Jv8>If}7&XVASjOXK$*dKr!DH6&zYK z|CeRqMJB57r%*ctQ7hPi=)`*;vK~;@gAe00D1CU;jQ#3pYI+ye9+b>MpF?snx*dkzI9&B}1?)wbi zn5#H#IsCL;^G$v$Sdsh0`un(WB+6Q6M-W_&806b>KM|Y^MnEqvxh@G|hp$5g+2%=A zBB>LmyVAt&$?ig zCQ(duz2~vsvbM8IrIquClTbC{s0Mo(rwbnFz4Pl^dp*Wvo+`*;z7d0GaG_RQ$Z-I`ohPkNBD4hxp*{_jAJX!2^g-_*q!LHb)h=KbEniExA zCI@Wzn>-vvy!}eD@`8Frq#MO_#ibQF#B^?_Qy*Op2X#+uR5!%60hLu|3_{5mIVbX0 z^F!R3OaPzyklkuYo=5`WhTACIU%cO`d0^l`SkusjFH`9L7K4-6bVtAC67a^LW z44mi>?$#onE!WI2CPR@uk-zNo!n&GHrv7>*1uuVK-^YwYJIZ*Z^pOl%T@QI|`6Z2R zKblf^r-QzW-t}Eq3hbS zR-iF+%dbXul1jn7t&TA`<>q_-L(XI?g=N4Aee^6umL^`jU?b>kU-?h3U1H;tIJ>l} zFyup>Tr%D)FNZvZe3Jvb?ogLvq@6Qc<`OIS;x_iI;4{7UAMtJCu4^NUd%>NO(5=W1 z+xg9N)}jDXzAQB~II2}5y-LV|+PySaN`R&xO4~lKHakF%VVP(9XBT(->t6pj@RC7t z85pYm&gAQ1<~9szW!WKX%nB14`x_na0}^%y9ozy(yE|U7inhta*CU9_+cpFtkOPb@ zsnpQ>z$CJNT)~X=lye$}U|3aRQ>;QZvrJHp)(%mkp{PCmO0Ah|D19ycsyX$N43M|4 z$NetTQ23=qaf#%$IUf?TZWopP(vo~@e4l2!Vs2x+59ZOCG!pNMjH)8j_{P`}zJgvf zNY8~4+2M?Z!ssJGgMB9VS)@Sv4Jj9oO6D+G0Dg;dQFN_#>H}+X4v~jzt7br44FMBP zjrFhI6Rx2R28CfeXNmqVlf8e{SpWy2crUx9q2Wp9hV0Qx;G0i}M5|XID>sjtI5}pK zp{1pso5z4#*kfb6pTl>~O!lUdhc4YruT!K8siUrm57}{3F}HN&QS$=$KO7 zaEC^y;8&E@wqa$#=_0S1Ue8hIF58Ov<9&2C{Hq)*g}7(i)Az%2GBw@Wo^_CXJv2|3 zce$nt25c27i7ji(IObh1gHvXUCST0BL=qbrme9Tvy)bSR2Z2wS<|eS^L{dPu7{4)S zrfuTZa+!1TA>sn$bt)sq3798TE^WEw)0ve|PZ7f|hQ8y~IOXC( z)@+w*(ePxc`%Wa06U5gs(%J>~MV36I_MqaEeJT|X%Ojq@`P;v%ZVU~!kYP{qHjyi? zWyvx=&Mf?^o%t}m`MKS@kV1+FRnz&N&>K(J!iz%I6HFk(mi|{c41l6UnE%&Nyqq-C zXeQ}uU|%|@Jxa_A&e9o!FnEKcLgPbyEd!zogiW~v-hb9y^BZEEUz7^_E&7=DuUe}{ zF!%xkP8J$J76bPcrA<$Nt448PgVUf3bP?&WP))7{6RTjL_X;k>y0(`)WDGj8b#WKzuq&PBi7yBDkOKP9 zFZUBiefN7g8fo`uc(tQx z-vD_F0!3P+cO{k$wZ1oG0p_-wrYbW+MH^YCiR12Jdk|fatrd18dV*kLfCLPdM9J0} zk$%!_r4(!cyMX9{RI0HYQJ$u& z{ge1vu;$^D3GevmMYs>aBBDxuusx+*OtBI^=SSwRt%Aya_Txlo&1-8^>1RCa^_4K#o+@UXm1!mFID+!7UHMxSC->Rf`WHTuB z#`1(I#x3@jD&ObQ4&DUaupx1-g(R^|C)|2VbO5{8-zN@D6Ep&^m<%B*eYC{uQTEP3 zKc+Q|M(isj84)HHvieKC@o zH}L8%@nojhcgc1%qTn@f3)GKt( z^8vIoL%*HR1cO`mhmlE*8a8f}s4mTbt|PT))J@K{TgaB2!>|s+eo}uA`H`_>A!h`a z$XhAxH)2T-)4J*s|AA<{yc@W*Z3QP`x#egq@pVO1n9)3R$#);qkOFRn8EZ>hQOMyp z($vb~uW|{zD-wT@E`EbY+;YZg$OMD$JO+TeNQhg1NLC3ntkbaS1z5;?{!-6|m+-`> zgujop*0T?n1r7g3YJ!NKU!6u8@a9@1m^74vet*};j2xmLV*p?_L&+$L7*l8Z+&5L0 z;?pAaZFH!!<`WD0o%i^{EbKk0`I?-JJzMX)%<=?oChO7v~bf18fsGrnzszgAA zMle-5&h(T*(P8K!izJ8&N?^8as0YLv(4gcGV&_*5q^d;K4w}n|&E0U>17;vj2ucpP zEN>)#MLX7r`L!bBBuvaQ`{L}Z;{$z@cyiahua8K{#M(yoh4h>lj68DKtFI=f;TMhj zt)g&2m_dS@OzO*_tl0P1D4PnmVvfjM1j~5 zu|hs!yWcw*UKweD=qb1TQ}XLuR7~uiaQsVy__9k4^>T?~>@N3njkP(#WFLGDit}w( zaiVOU4-P6|4x0(Y+y&^u5IIMCq&V+;{nBtW>diU9w{=blUJcgO6HKp)J>UYvT)Q+T zA$XnaYk@Jfb%ydH*2g20t9du*Kb!nA$`4qWJ+3XP{RqRXv3j{t_##FQnx%NDc+ zP7%_vC}37C2@cbF=-)YCfetCf1xsr#f`rq=b%8Hr(lo-h=fAoKA8wrTJ%~BtPo`i0 z_X5^f<@c0n+GafEj>4AJDMbw#O$@FJAe{(($i)J_jhL%k$hspcn@>GgiG5`lM*~`ehaD>BXs3x0x=Ks*sZLVSX{Mp0 zO1MV3FjDr3aRG6{~3UuT01Q3u!To= z-t?7ta%#H+g@_`NW?WB}=a*HQO9UO%ELP-)WX8H)y9k2tS>xs?xdCSf3MDjQ82fxM)Yoj_r7Fgl{zbDBb-V_>yvj9$ zc8mXOzJX=nA7(z$g~i@GJ+fENsa% z7po+8KhoLXRPi(QjpAUx>^udgctMGNNGbFSK2Y>b82lC2x;pK5lfXuE!Uv zoyk@NOt-yH84O-)8!Ss*sD()=nv1g4i}VA6v&<}}rf#~FY6ag&2(>5bwp-dlAS-TQ z=>d=;d#m#T#2JJ`&|t=`Tjqdfk5mMAmeP!7X7S7;+Nj`qAY}fCc{>v4FX{1G`l^0T z64V4+#C){6>QU)1a|UMloU|HHE8&*YAx$7RT2Xtp80%bUP9Jyj?X%Q2 zxHX#vQyL0)PW~H&Z!xi{MR-tE;K$5eswW~s`W#2%Lqjt21$Y(rIeX`G;|MEigMr4BKZG^X2vaI$aF3_oVD6AyzL7KNu$W_;tY0+l8yH7mkS+#8i4Zk^Xce0DKt_E9zzQ z@7eXQvbLZ=nINVhPFnmN#Lo^hUq=WC{Il78VtKQ885}pE4kxik5xfFFYW~CIHupb`%U3ax$gHJZ zXhDyVKiKnoGkVmdxAKB*L{Pg3eA zXn-Ekd-s`{E*yE(ns>InA=SDN$XwSR2KVU{roHuFs{oVxRSxK>U&qB$4IH-0=L(Jx zuFIvrlKqr*=hX8}HB=5!>*X3G05~ah(zADk$40m)Tg}#;_FrA-y$iiQXaCpq z$@vNL2h96)YfX>V2|>Revl16Qe%qD%dk3>M;JE|c=%gFx`I}(Kf7!Y>S#~#AoKMr1 zRSP`9FSl#9Fb9nj3fu?R2~6ylM=*<18fIigeIXBZbp#qy^u>RUg;O}nwx!v(8%R$z zd!qj~39hG_7bhqG*nv9n1ge%FIgrGyI8@JD%F&NL;;*t?Eg()ZqEb3=Bl%*<7gz$3 zin*S8*_VN6l!-;&yTm}s34Ov};vkU`t2u#Ad&ybFamONb8Q&dM*$E)`^Y0-X5kn%+ zD2Jt^_iXaCb*|F8n+1SSm5RU$K>21P5B~M#=i@Q6S)exQOr2fFjos0-{Vh9Y&Kk%N zMt?_R%7hw(D=n*?X{PuS;iSJOw`cs*7vCiC>n9z@hk=q!u-e9!TU^M0GiR9XCtmV= z5z;h&%(ZU`2~p`(F?T|f;oY4pnk+7w`m~9Qv9WfkL!m$Lv(9wO^fa_5fp`Po9^g;r zZT&n*dfn~i@d>EdtoNiPv3S+xwH_J_Ucs*7MKH&Dxbf$TV}KJgOc4*gud3?W$I6bI zV5YXd6ufk;xLznKaQ)^wJw~Y6NQpLeV3yFO88!Q_;0B@5DdrC(+cV-6@IVT^6!TGwzX0Bj-&E}3cN*C=&1KYNJ3cPf7!3*y6}W_N zsKZhCRp3Vo6@X)r9+d4Vck0ag^8CNn#GNX3TC*c7|6x3qH+WwgwlxL_86%f?Mi45o2$+H2bw zV`#BQ2%MlLMy!t7aFu1AW;SBV7LBsRKBf#op1?+v2!WCFt*DR%0cS|f&L6K?SqTeb zy?A$xrDuSAKNnNu?iNMb$Pyp?M+!=?{Q)&EzwLNoInJ&t8-=~j*^^06Y}3=i+D;c6 z`EEknJ@c;DaPLxBwg)e`5N&PLR%R&}dH>bS{>HEQJ$9`VB9&g!xq2gzf-{9J`j2{3HL(7Ft~PMHk3P%b2o_ zj>9zKiqf(#V!W!WVZg3qLmF%;Fz2or74)65F+HZ24_sD$z$Zy+lpe4)wza{tWxvrB z@Oui#AV>U3Zf7wEpy%kk7EO!!B2((0pLMBP@zYmnQTyqW?40U-^5-1uCcLz}8K%vz zGwT4M(LAUF0rzIZ^SWb+dzXK=B)V^Z;`5)5e3p5CTxS%|ZUftXwRD)t4oBes?ns`F z&hfnwyJa&3Wm1H~A?tX%qr6gBZ&EC;&0|KN3$wsdD*Xtd%8qN_p-wEX*Hr|pAm$51 zo+vAVG?Ysk{EBGYf=@)7l0rV*Q+_^&Dkr2=&=LaWR&@jSCD$c5>WFsAxW~gb)>YI; znaTzn&lD6kOvHR%MLY7uOX}7G3X&Q-&8ynRfBQhMq^?~;ybkR_IN=f| z5AX#@qz7%6^9N{Dy0VI^12Dk`5pB^SWzVrBt`#$yzxA6P8eb#JNqa=bVfRr1AdarT=mbx#Uu(#}c6o)HJ%R_t=r?v-p|91m+YMK6up+{G(~yuci2j`)__4 z5P3~HaP~cHI8JLh-OOblUTjhozTUtuWjGAjn9Eok=#;&g=5{_-KO&NZGOD@7h%por z;`}_p(KIu}xsK0GAy>_nmI7BpW}Uk3tNBkZW8*Fr4g5pJSq82XzUPzSpCT-hwghHk zFP_szx(zMUqQ7H{i6jLRcmFSc8tjK=ekitfVqwp)6ozaE<>*KgAfcujIoe2!$xy3sHyIwGm7DhI*6PE5e+DXl=r^v@})1 zfqV4q8SSDm^E+{UokVbaovDsAnrHhMM~)|%SVp$Xdgia1Ux;;LFdW0SONcR;akWWkj!~-I zOs9^*cVh|mBTa04@#K-FEGAUo_+Dg~6kAi=8`VHF@4QXz@HGPOaGY+oxI`uzpuKt5g)1#e{Bp21S(!ip)vXg%0GC#A$p@9EfCh*ua!?}PI_mI zo8X}3o8Yn(OG7PmqL@NL?=bBwt~y}yd6^Y>nfaS4ZS;A}u~Eg%5}6$D$2aK< z!sD)*M+)eHdm{MQCIJCD4K>E9BVFt00DKEHh=fSQ{s43sF=lij;+H&^$003wyAx+0 zqslHi`Vq6T3%D3wg5&i7C_E2%8_S%28Ecyyv9_W*gzoV^Z7Kh_v7uytBD}OxanQ(N{$7|5)$?q zj0^{^CX6{N22c|Q7sT98^d?&fcdy?A5`XAfW&yCVy-mM^h`7+E=vsbDF{A%6>D0t# zU7{=zv(&ZZ@g>;lxUSn&c|-|Qg|4IO=)RmDC~PeA=0q^t|0DQxt!bd3dRaZ%ZC~6? zBnCeBAHCw9rq)Xt=tCLb%Lo#HMqYA+9OQrJ;i||@uKP{{w9JySmqybt6Pt{^?f>Bw z&;>A&{+VIxm!;)Ht zd$uf(s4@tJWpW)TM)G-1|5uEOui0gsY{&1kb7G9My&;+)$(TVN(SQKmhtBjB!`wVe zAp0pBD)Y~6io!@Z|JeriN9jj`$~tuu6sB*& zOwrxRe&YCYu23Y=D+ktO!%xPF@DujMi_deN&oSMJvU0S>B(~$> zPvXn%SWmkRKZ+M;q=su=j@TWeFrfD4BYIVp$8`WZv*hAzBN<;ND54s(*{Y)Y@wm0e zBCg$pY7QojC@J9H?#rXQsg#>>{j)EqZc5qCi5~=Fp3CnM2EL|VPDQ@{PHh~IumHk< zhsXWH2+u&)rOE5j=f^uP&aQ#~<4Q>13CcxA9J0nG&*w^SbN;N3_H}YB&_uL4p!!P$ zFV2-smo zDgQrIKos#^{=4)!)q~`}`rUOR1)lp_@tYeIKDJjO^zjnefQXcu1EWb*SQ&P{ zf5^~eLSG@$=<>$yFw89_f#2J|7m#1zzf)Dk_Md_6YashcAb|vk0)27j3uSPe$%mKI z74l*J4GcYRRet)5ml0tuSC1Do8rQM_m2%9`RD`aeWv7=?oRnu1awR$6$ZHQN$5uD@ z#5e#1T#1|3nbx{~bAOu-b(l80gxh%0cq^1{NkiNi2GQYiKRx3n2aYl>)y8LJO>oyH&>eP!w62uU@GYZx=49dw9 zb{qi#{_j*@pcFsxVA&XP4Tw*-zCIFpYycqoo4CzFKO$Aa`4=wZCV@h00tAj%v#ZKM z!_QHUbi~HaCvC~EJR>w@1-HR8?XE|=P|tvBoicVK%u1ey;^fRivx9wVaMnWLSMrKS ze_C}j+I>4IW0JnZ^D;Qi2=asty$ z^Zqj@=`;4Y8`(wBUZhEaf(*1rlm~PURR#30BJa2O{PSfoJKg_4v&nG#eMvC@^Gsf7 z_kdbMCpkehhP?lK>1NwkTu$+2w|A3q!VBG+EgECti5`1j6V*McprbF3Y%{VZa7L!(C2gQk+(i zzRlo5KgF5TLZYeKj)_?PT~d!X=CWqF)ZFpiPkPZqDMPUF`UhM80v1G^!1Mh_ zv)0_4k^b-3LlzbmBctDbE>BY{FIIhme*ca#Zs+QHotzwJP+f5-nsw(R67nltHv}Je z6GZYqUmxFASXe-QU%L{N2~+&-&b==fcT$?dGsfS- z^FgcIy$azUMYwEk4PjPDUmBSPB*_aNYl?L!v1XJrojeja0&ovy#v=(NW$Rfc6zq*= z!33sgdXGaCC(2rL6Y_)aV>gfB%$;)%R-o2^n_CdwGIQr{AFx&UwKQSF0s2@tPhq*w z&p&G3FxGzZcarwn6jrNFp<7z+jLo`UTqC*q!LsZm;N>UrSEr{C&xR_?tDy=hfv1e% zhry}bIek|rs{EEC*i#Hn#@q!CGfSQpPSs)-ScZ^m)U{fVIwAzzuvKbX@NZ0UEww;o z9v;T2AFHVrqrm9V00YOBPHzAhaT5trw<}Aas-pJ^oW>+8YRx88t7+U{U3vue`BEThAAR;t?um$92ZFk7kU0vX> zpmdNjc7Hz`HeXPthxI16BiNtvZ_kNqNa~RIS+jXtuzuTLDH=W%g+u*um{ zD$E~vEKo;e?xJZdHFCG&A-7jN-EbZOfo}Q8t;so;ytdj*M$pI{+`B-7LwPWD#bP}Q zitfrjz=nty5b#0O_R90Ov+o~@l|K!%am+u^-Nb~P+86NmNa3?w`Jod?k0^d6Atrp@ z4UR;~$^Q1+0n$3z?nuqSVF7&RrgS}@qOC6T;;_Aots9hW2G;#ww|2?xX4RdmSC75+ z^LLirl5n@oH1l;54vSHKLa~r540DA3HeqrL@b@MX4@scGFuy9Pn)HXyWPdSO9qN|w z#jPp+^H{4Dul0kv!7$=f&)r|Mu^+g%;id__Kex7j;*cn-w1Zp;EbrmVd7BWHf&Loe~xL=|~I zBtnPA5^ACHpYv14LEU4GUq4BG6V6tcQg$6$X5X&X zGy!8`6p_XA@L4_p~N;-lHKAVXlM042MkO~(C0ef2A z4lm6_eu2mBNN`SSaCBqw*0|E5Pjq=kik+Z8A}?#8f=d1sd-V!abG;Fdt&sNrcB_=* zv)}rRMEHb1|J$vKp36J_aEmM237{VK&`BJiCKn+M>7QmNg?#MWPb8E;Nb2bxPQ8LA ze-8MQ{nOLOL?=GFuRI#@f{jL|7O+kkm`~s>7NiJSLy+c7b1U;lVcpFK#I&ZlGHX#J z{ww$do#G~_xTTuF*Nz@87O^w$YWEupMCeLnaXA=Sh}8p~sek9)=KF$fqe1nvb9o?l zVwNR=-)0II5qfy`un5Nj~CO-M*1qk2~6An}Ugqu={6~iLPV$@yod%kD@hqI>9wrwmy?KHw$ znQ>rdgB%DB6*qRkM=8z*awcG-6*qz{IkY;d^6)u^{4sAQ|Zr=9K0Y;zUTEo zK!PDLY!8lXF1xt-?KYiU$Qge;V>}@}CcxQNt*C=nl(YFfMKoI(BxbRTr~S8l(sjSq zzi8-bY|mpQ7h_u(4*u5ZL^OV)IQ^f~a;_Pku7XfbDiF{TMA{@2vOm_`rq6Ql zUEm!358t0Z_p%cP$fY^5ex+EZ!acQrB7GmG%4VB*2qjXfFcV}Kdy+Dc!i4h=Q!9GXYW_1n@sO1WFae{DkT-5yzgy zk5!D`m?jtljCvH#<%%}WkftLI|2FfQlw6xI>87S_%k7dy&52$ig61y?#2O&*wo6CD zH~xh>Kubu{muS13(KG>HLn~x&hbX5JN7H_0{815REM)_%5y$TvjXyW@hD)54L*7%lFvf9D02J#4M;){ z3zH$vKaAnd&K#XxrQ*kx!><*L!kDdi!0oP&6^7QwN#@vS=17;vB_;(ek4Q`Zn?B?_ z+$esSGvv*2oMb3ISC`)vl2MEB!$;`#VN=-c>D?Znhc-_-|dV$*{9dcBb_#q~s zN&29+#+p6%gUSCBj1P10x^W6u^=4vu!cmugHK|y~oa;q71amMxS z4Mey$yNM#B4I_Lo6>@^u!__8E00L1}!h; z>$JvB@ZDr$HRM*?9`=@+v$T1KJBL1+W@37@q&z>`GWd|}-z=D9L4RS_eAtPvI)J|W z&26@S8*Uhl$d5u?dtz3lFB>MGw)CyI(zjgrO1BEZ^O^?LBPYAo+4h8iv_+;uFFFpG zq4|@4luSXgi@gYwxN-fKja*}qBZSPpeF;^SMwW8P5^eB4cgMMBx2HU(a_x|>GO0(= znQb7_q5#TSmLy{&5z|qYe5GF(01U(Nls9?9Yby1n2kkQqnkLqOMa6dSdu*^JLyD%&?_8K4eAs~&FLC0qtBw7U@@&ts@Z5LOuhK(*t4 z)EsLoifr9pomHYCKn@SM<@5%xWvLe)j}20eymAfl%FYTh=#8K)rc6_wDO4hdAy}=j zvkxJ8!$UuslYRDC%JuLNR{4t2bOamfe>FnfaHkDDIEbTfZB2*Gds->(u=u&{Zrm#j8JTRafi^LJk60xjb>s%iAA`}VUnLDR+@Uo;n1hf&K?Yk2 zdCO6@P@J-+`KChlnPF=n&=Qe{gu!6l6sl3!S`zR|+o}F_%g~hzGYp)6#X_tI+B%ha zW;W7Rx-giZhdWez2PoNdGbJQHA?jsjwzU?S@SFwRp}Bx?PR!}8(>r$%mQ-GUS1WU> z|75UJ_dOwt1ZhdHD&oa<4T8AO97>C>}*AGa<*d9qnF!m zEeKr+o|2R)Em3L)4LhZpKDBUvkSr0)CCkln007n=oH3}5(mqd1np>%SW(MZcDrnFt zIiDK{!vxUiY$50A#2t_}hm2H&G^!jM7;KyTZbr@MH2x4v?%i1tvh+|>cMPa$8fr4C zr(I@kv(UP9ha3g@i53JKWYic-ja^wn6i&x0+f&EzdnKt)poFj}??{Y%eazSlo=V;B%LMUb^j$u}yK=a)w z&JGcM6UPWGfAg{LAk~{=*n^~vxVwpS4f_cW$<2ZxIclHe`cnFhm=%`p0N|Mb*8|`o zT{42a0hCXXuUL-qJXKqNMU;J?5Jab9#fF_UYg>3JP%g;PKHTc-u)3g|xHWrjfe2_i za3EiKBf!3g@L?$652L8z0&H%`VMPO4mXZ#qkTp!3bGrm{FPMUp1? zTZfgQX3f^6%*WA)oYnne`%7z829r)+sfcfyX&>PE>eTAH6GVo8j;eWe+miIgIyxa> zxb&aI)C~c<*VO1FpU(|VqNuzQ8-X)9w13;eW0OO}Fzuz>-DEb6VRDSMw%F2Ov~IZx zfn`P1XRq3dKUNV#Hf>XA>XLOe4hnlqk@A_hvIVL>2vChNf{k^DC@|jOfVL2VkXS3t zd=I!Kv>WxZ!Z0R((YNH_-~vb^&2`lQrJJM}0#u!HA#uGOHbT!$M9#GUBi)sz37S}x zSTZ6RRU$w1c|bL>omVDnxAkp_B<~RZA^VFRt=xxn9`@W7A~tyNEg9wH z_uhiqjF0_x^3)Fx08f1Z0hpB)F4ZAv&YXSo~GXj{_Y*|>H$S9u{ z9#QlMp%>OmrC9J7xR@>SJ2P1*D{setRq0<$B{ai-BP3>Fl^u63tJ*)Pep!R2k>#W4 zOnG>FSzNIMhs(nkg51*QO=NEi@WaCas>|CgIU5CR^zouTXy5C?^nfbahp@(M&54af zKfD6MUcw~UCENN^Gw8s&%g`v`iuDaqi&*lUFKN~qKCcA|u(t^-^e*url})H$7!OQA z02&Q{!nt*INk&ujl1q*T11^2s1L*A%A`rnaB4iq-r#2KHa~<_KD<>z`PTmL+Xm1Vo zwt@f(ca-nI5`x?cgla=NoGxgjvkseRG^(3Uw^=YjBt^RxilkM^R*|Ez6hXT#7+B%m z`?oFy8S2e~*}_tcOl%6kq*BKr4#K7#7lP-1sQE0dnV5zvtT?ZMXUpc(rFrdK?Ax)9 z`wU<-;DFWKX9z=Q+Yvm5{<#Y2fk&~!!#MGau#*A*4~z;y1%^1SShjZ7EoxmppDEfZ zjy#h!q};{_p``>+$SwbKMT2>V7l0YEO`PMaMjs5s>y+BT-rGVZZc8f^G-vs>OACm9 zqv&GRzz%!-w_@0gXWF5b%Tk%R5D$k{j#>T;w8=~kbEe>H>4>IoTCWn3W84j)AwhxR zz*z~)*e!strjt3f+N{=+?vT9MM5Abg*wl6e*jna>2*Y+^Do?F()BE?6-Jc&-EoqI~ zGbX2+j(ll?A_d(C;@xue4>(%x(J{-t=(0~LeMnhFHD z0p%v+*DPRKRb}#4Oq*r>6~3)O?z6rKj{(k#c^9728XuHYE4-l{)Sn9wk-%($J8$*t zu1q1{@1%g~RGwzPga}wTJx}tf)dQ6NyoCKbZqFXRR-PW#u-bS~h z(a8M2?{gd7PU^+AC%5Fx#+K#P%M1&4fg-jjyN|X6i7oFAysnpIH$(bU3X@ca?d8aHlN45TXw-zcZCLWXTz}ZOH+a)bBb_j~a!kQ5s z7Fc7bg^3~)8}GwnOj=PBvY!KgTLG2C-Uw8MEA8G3DAIoPumP3u5ummQ2f5G(2jDyu zc|tYIDY&rAe(qr7Rg+tA6MBhR+MdSz&R6?s&!0bBP-qd##FV<|NJ(Z|@u7LIS_6pfCx$+d(W;3Gg>D3SC58Wmy zqPQ8Xe$Ck+WNEWNp4BR0SuYIFHIqg*3j^rIvJg2~7Twb8t!=h0yTjspmo?vh_7uxs z_2;hnpR_1O<}56K+3q=@Tk^k8Pmi1R|4$!2{Q67&=coAe=6}Af@|&%z0rgy~7?L-g z&)q%lc}Rqj_tv76j^2RoLVgCX2WUegq$sPC_~GM!kD@D0B{<=D-un5Fytx!9ae3Y3 z(pJXaXj?H-Fe|h=a`m*23XzW7A0wN6ZjHoK#&X}63H*zH1~|by&Mq%v0><^KK;glG zO%*RUt?l4|>|+=_B{Rv?!fgN=xm%!cU>mwzTce4fH6GcRE4v7ZWg~QJxt3QhDjatj z1%uv%{VEmLN(WfDaX}TrqmXRHMX9p2zfsGQP2%YP{ICBFK3A}kL14>8Ers5Sc!#zZ z77l|=zp6NYZ>p3?I51mGB->lVb26hV1I}4x_QwU{@Z(ME#Zvkmlb?ZO z$pJAuyq%}jl?d?8X{aKAKES$LV<>E^4-#KvXLN(p?p?v+%PvSH*K=cKHEJdkZSC8Xa${E~;2bdIe#(?sbq zx%M$*FtbBa*d!K_8)ShgIu*{Kr@;qMphX1?7`L`!g|$@U=;uO!9|$PKrW3727*6BR zAj|9t>NsR!$|cu6ecKZON5wF7-iGiwG15)Rmr8{QYac9vSx z*N4!5GwKF88V0I(sf;Fhxn(cB!s4RDgU#_t7}c)(?WsF2;W$-Z5OzblSBVw?2#c@X zaHVOUQkkmwb92$(?&q%ZUp)zDw|8un|4y3qe@?&D|M^58zn|l1L{?OCLl(!RmtaQ3 zk+Lxv0O1_KC6!1HcGoc(e3#LApv(#Bi|1f}4m7Y0ujM!E?{KDkIFyq=(YIQwK$`4cjn10S|#RtFtC?fUDmqj^o6w*g(MWebP` zAM}&4PZ)o5zoqWdCtEZj+H{NAkei^q1%`?k7K!>1W`8&nKG=|GkoD)DwDBsVx8^hS zD_2_bd=8ZyvGr=zV#|UeNI484v2!3m1&_040rgFb$7<`iSk>@YbuPVQ?O~_`u3ze2 z0C<|>X59-Bh*`QTELh5Qa41u*cZCFWCvO76MlnJCYawCh{BZ%X;_RjZ1ms2U2ndLs vbAYH)BQTuoUkV62=goll+&_^0D1ucct^l8 delta 20466 zcmZtN19PBV&@kvYnb@{Hv2EM7ZQn5_#>CFVwmq>YwrxAv=X>|v+S;o91zpv3>YP5^ zS0CqtMiqc2@B#j=c&+W!TfNV!J1X3Q(L8%94R)mw3jdVtiyNPPU0w= z_7yO#@v6g&_wUct=j4t4dl0P)I74z?50^M@dpQQr`aW>JYsWBjxRT>4wGbNFj6XV~ zyN%qxc?{;npE{9lxp{baeC|$AK78HW7CxR|4#sY;;{YQ5*SnLk!JqN*MBi5@FSlbw zKLN=&fzkC=Kbhk4kIgyX-pcm2?a_#%_E|O8#7bo`DG~Omzw`=aOE}T(*pzZM0*CYj zDAM<4(`Lu^G@9ZPnjzVYE0KCHs^UHFr4{$guAQ`20}>EF5iv=Rb)<4TG^e_p6$o7yw6FR&To ztu-c3osX50%wuRR3g*I0Q5aqf)7A72$Qt~x=L&BC!Xc{LeQI;a3Ds1p>~du`;;2#8 znq4T?Xt?az39yl9Z3QYESu&BEVuEL}P$>-de>= zzgUhU35zR!+WzQ=obtIyY`Cbj;~viPy^9JhC0e3w>NyumbOFEopaN`DJ|B@vak|E! z++^rk7lX!xb`$7u{|3omJ8EUs@8^W`nz$H zO@Z?}hQ9nDoRFpvGJ{D4TTkJ~OfxZ^vP|J>;b@r!ZCqiuf`~|Nl;5~zF~k+Rc%z)0 zl|@F%bo#4=Q~~UVNMvy!8B1}7i4zKUB%r@bk>y;BaCrWBqpi1#NpK z$QS0DT&0#dcE2DPYiiim`SGY?@zj|;`*-GUa!O$$sXe+B^qInOdo}FhGO;q4!moog zOiH@1uo91PwTZN0`7m@W8^;P)gC3P#PAs=hq|%s#CATU(ObYrJMe%_Td)@Mr6`-ut z4#M(H5}{5n{kQOCwx~v}kX5kkUF2fjQ#i81#nC=#P2cSn3JAd&tW07<}{$Gw%U-g{W*4k}(k}izCJi zk1`EhvCfRfY;^O~TR(}H;OX=MVc;OOdR9$;FdXEpK4=wou$g8Cp_Z}H4rCAkb}UV; zF{nw4J?5gOZwAXeRe!fh;^0wm($UO_A~~0!FD6X&T3k8e##kw0u?{NSxSh6`gxP1guQC@f69i>fXVLLsr9}oZ4V?T#l4rh3sdj1sQ8Oi|mv`v0guH(3L>|@HDH8D~=w~o!f{}kYUEA zkD`5az?;e-#a_4i2sw2cgL5`+roejFy{&12*gUFTljza3V{P#Yp1q^|jNt2=%G!p3 zX533X!s0}=Lu^CpGGN6*1$eNE{tGC!cCkN3@=b0aMXQ9I*k#T{8?ludPNg4g&0=Wz z*?{UJb0o*>1n&7(VM~9@m5al64{4v|u94u4t879V4WxJ4GFAzMf-B7T+^-#FmG1 z<0@1oa-)alyH~-^Y`%iiUcmhY)@8r>&zSg`lI6F_?)K9NM*T`aBv7lTlrst1GWn?j zzMP_0_#qeYFtQj|Ql5>|)d(PFZ(&wwEEdErpX%4^P!@gjyww(slCZ~NS0HDZy}%r^ z;3ZPfvi04M(|3Xd3y>bqV~H3_m1xzVG%KnuI9W-QW&gG))&~m>k}mxV(ij6%5Rk3P zEeaNDdCHx2mz|i!4JaNRgh??3{uHaY0ikkDO@$&5C%l$m+MS$|$!#!ZRN+vfH;wXc zr;0qf!_dP(ZR11ykTHWd$3yRsF^_A?9sK-@tk+F8^>o4dgQA24o;lx}XZA}yKxMHw zLCBDmj6lCp9f=V+K7s5X@4Kcnt0zH%m6*>>ZRco-YU&BfDbS6qiJ8z1lcFG--yPE^ z$>PA~Dv9w!yACqDY(!F0&;?+p8Ast3;Nj6(kxlxnG^;r8-=BuYI{J(Q#VZ`!Ix)f| z#?+JQIl#%PXZ<}JQ)r_*mT*+e=AU6%DNFLu%4Zr$6hw}NlGGs0T}c=s2r!A+Jvc9< zzX$L<3jWMS0t~zS+&wVPIp6%Z>czi#eO?ZgMZe$gP8vV2_x-kdC=CBvU9Q9wD`ZgrnF3h9>%E)`|jqozDw@^Dz@3%O~pSZ$S#|Y@eP8)VwHEgR)pCDXv za^#*qfEOewjVET30WCEO&zC#lcMO{k*?~Kyl}!p=<&e4yS!Ah5z0=GnsHXcN%Gv8X-%W*3@(bOHW$$LrG(nCd$~SZO@C&ejip& z3uL#fI000A{XgIp#LkefiB!#LPdcEnO2}Pr3B3 zCJ=8YDVB1zuuqQzMm{W-WMJ!{o;lq#R=L55M(K&lm6M)z_OWsFQx|X86!%@xT<0f&F_O@`deSWjST4EWo`MD8=pxr!} z2h8FNJH)DGQXSl)_#H;mrSqL4_^>1INn;a=l}bQ_tLw(e(qhI{q^Z}drh-?{kIUT$ z&S>hClk!nTX~}s_t3VX;zw=fv{Q|Q{4P}dr`QH4A?l_3ox~a}<^;Mu$)>jPS>6ydD z#WK@399lUjYO0X%$c9oe`^BNH4jpCY1`ITKjrd1Ml4ohGQ)nkzE5od_#j}_>t%5;z(&_N>S`48E4dZVP>?OfM6=T zqzi^uO&Yc;*J<))x23KrVaB$`Ae<;IdND)+c*tHMuitZNd7r&R$McuIv7`z93I5LJ-d&_kx<{n% zXcc`{uzJ_ctZ<%hiQo0tav9iIfC?PcsbAr?U==rX-*#Zvuzc(>Yc{S7!AWpk0Np*_ zqifCNp;ZbeDtfJ94WyO0mb`yIl-3t-X=S)pb1z{;%p$zx`aj99-) zgOSeNU&4VXXnp)ds~IIX4lc=AUB_X?iu$z0D3F|`S5$=1^_}R#dzv=xa}OYF_MtHIeTy~)!%tku#pE*K&!vr?~>_BZHp+a>nx4Aw5$jt*C=f3= zHZsM|rXI+dN`b$H`H2=M&8RI1uK3*679n@Y39MK@d)#iHP#*^9C!N!z!7l$*$~E2s z@vom=66P(W8B#UcGAho=@4mDhiR5xYvGOO}_i=7@bbT>SK*cBZSzjM?MR1?UA9}J2 zc~Hf{m+4_sD#A#1ho|>_cRIpnW1pQywkD`jZg$xTO;XFyZfUK8&#Sx=xVRRP4sih z8;)XqRdhVUGRAHL9sE_|8|mB2kX}yWeFMdiiB9p<&99;qNtpCZ+5f~~iw91r8)j1J zIWsTeNXa4Kh(Gs~rn_nT`e$6o-CLXjw*6*bhotrg&(82w`S5CS7^VDix6kwT7tdmjtO1 z9jUbeA4W1FA_2gqZO6Kh3#o)BKIg?3M;r$o`;!bNtZq2pz&F|FM(ClU9wwVDyp)uY zZ3$@vDL8yq6d7T31qTqx=zSVd?@wBTij5) zP{%J#VqL)2yzig6e%Y9iNj~UCqW}A_A%3E3lt^D8_7E}mL#C53XL+W>?R3(OhMb-S z^8@CfP4W84@+AOoFDnXXE-!=?nHFSOvorP4`ULhRgm-((a}h3KQID5pPy*jSbcE5Y zwYO78K6ELzKmE2vD`WFMr$TIO<~-)hyfe)AJBl)F!Rkjg9o7+iCQDAywYm3SYfWg| z1G80MZz;h^AG{q-FkRH{F7~Kcns$ph^nk@7=%ycX==>3G@~+upMrh#0YH4A=^(=S1 z+5P@FFU9I9Qjfv661$Pe#M~-qkE0>8Uh*xmGS}M4l(4Pz&7-o4|DT64h#*kfntJSO zB26%0rZJ}YQ&gn_^(aN@xKrUMyoMX4wZ@;979!9?by~JaEQ4m|buyBI<&Yb)7#N;& zKiFeE_w`un!ckq5rE_3N^}w0(N%}z+su*J*5I2mF{;O!;|1R=*l3jL#Y92 zNapKJgW`_L$GTs@`V{LQ)3|OY4q)Z*BEs_De8s5B6=+e}1~abFMH@_2!LZgtpg!o} z2Tf#RLqKEw6_MiD+nVf>_5`z4P&g?Z9k|yUq^Y14WrL0?PdMVo&E_vIVbyTv<8tJh zz+vd1hxvN!vi=EcHl?}smI_x$JQhcV!LmmD(>6{wMMw+zwqseF;tt|T7^s6Yc|we+ zLP#zBRj21V*6@3sifo zAQXHZpa&ZlU`?)gBUn}63vzX&GP1PXs(*}7OCeJsT}P#9>D3<?jrf_g_)Aw87#Q?^u=FrQ!>r_MD*}hmq6G-{I5Uii zlYJDNb?sWZB7>O!LcvZrIdrFHmf$u-Egc^2A$M~B9^A#3&U6r4STz+9bRlLvt*v() zBsFG*i=HK9F9pAeXznT==U&&c2XKc?uuzvq`z8#?WdBfQE8XzEIf=Q`BMI`q^wLRK zf%cYEg`Jf66$9Bb5(l7@oVrLxSe4?>s?4~)T=zz}h$QbCFztXqT_pPtrN@TmbeO5; znl5pQb;^rrdn-)JO^W0Q-_?ds3N^IppAiY&pRgzpL5&dQrb`<)HCa=)`5G`8hn9wC z2lucrx3L;9$)}AlYpr$V99vH{DX(e^O)~32OKP{vIUzF)+X86)BX}cL(+f;EY1~;F zXV^MUt=<##Ot>ZbmJDFU<72I$cXOC$1>7i6bQRpDe3N?3b4@IyeV_`VDRLO3iaF#g zh_IqDl?$@yS!&woIF?^WEQ%SExQRHwtq16aR+z;oR}@g8>DC{yET(O-F}Le<2ArVnG<^OAKQ^eZhZVaZYPf_WQrqZ6cO) z_yRfafX=Zj)%6Qu(fehw6Ymjj@OiArzY<)HSoTVxcYv4HGZ?W~RqWKpk*T-}qjq7V zx;YBIr{i1bv04p2^F6DkvK#Jf)40tJg#GG_Tli(?zH0d;4u8gak zWUo4lyJRk+7ZN>|9iEtY)Z%oMsM&vpx%V_{|4kchc}q#LA+ezuvZN4!@(4aQm;FnK zy8dcK1|WcOfUHd+;|8VdGr6~0d|1~h=-FMGZ*CughL-s1?ETP!CO4aQT&={*LM7y4 z9TW!(VJ0>^P%*oR$7hp;NYOXCT?wpK>`N;Gd3uD zo(azmSr)+>x`5O?^G>dyY7VLY*OpF$;?&ZEE?}VBFr!lQPF=6hl8I;}`dKTnbbj4b z01aKgjLLa!O|jj-N4Hop_OEXM^vs3KmEaJcxG>J--r5MEl0qy@-DVisc^4L6>n>b@ zel~e2nlXrc!P6oIKUVgN+AdiNT)&m^T9aCkePh9{5eiP8K}k%mZ`N_q)T>xN#%u9k z0^njC9&}%8GwlTPI-c(BondJ~mbeXmbi9kWxCJ_}GAZ{XN-5GOBg1}b5po7&dSvUa znSIV|U^ICA+{Sz>XSot9@tpD+oFtliN_!12F*lXs+`FixpEh&#bJyUd8TkYa+=WZL zy-$rymRw1p3RC!r3Lb&l!sTl3)paN4>%aPkW+Z2zhSYV3qa7BGsK5R%RfTnyzlH=_ zDY^M8?Y>Nenk;CsR2LY9W@L^MjtGHQXVEcRbc~SB#(?!)Q-?n z*6S|p+T?#n10f+}w(Hc_pGt?aX609(aPPKl z(5-D6Sbf`)E?zPM)Q+b=c$RbA}TWx)0v>K9xs%Cme_<_}Gpw%p~aO21p{#&!OB` zu=PPib34v--Xs#trW+TA*wl`ah)Ig)ACxz(%VcTp|LcvHy0ca)j#yE0cJ&A-gH!Qg zo5R4D=h&FtD>St~5y;=e78gIpy1R+=-P{O9N8y2S)RsR=3uNK^5*u6TS_}J(N4nfL zXHI4_-xnQJxGdh-H2Wl50-SB!pZ$h~*ndpokknOyyk9ifhf_zRV+R}WF~vzfV+I#e z!#E?+S!%<A`09N28auDF!Ir*|dctj+VeClF z?S89}=S{T5$w6xR2>MkVFK-9qAGzskAtqx7u?IUyfgwm#>|9V7hxKPFfkv)6==7Q} zfl!lUU;aw%MlQ4mKp`QGET>QYeGhRI78#-*LzAjz4lmp?&!xMp;cd#N0ny z<4sv+MfzOiDbW>)LGG+iAXVVA*Lh%GSbzOn`OP*Eyo?6*8+CKNCzv0p(hm@Y;(i@J z_AIziQkQt~7JJx4IYWC|;dhUB|)@Rav;lvLc3ionw=uIx)$+qnRe)%z`$UNg3`MN?Z zj23>DCRl>&oGD!*q(k35%Nq?sh0Uix$Y!jDZj0AFAJE5uhkv~K)J+Tic(9lTVcQu< zAT>?14|fR-*lpQ{6|7-eKeS})}hy-qEHr-nOI6#!)uxXugzLLY9bxOgcuqwDmpeI zUuj)T^)y<1qXC7hOtW@%JY40du0?VJRHVH9PoN*HcvF-7S;G~mBPE>1H~U;s^_bcLqFhIesh(@C`H;|Dh!@2=)F385~QjglZXDgZWbhG7~kARPFyYn~lWCh6qn6o~YCz4+6mqZ9z0FI` z1Mpbwycu44QJrnSrOW5^?v^2F&mP(Oon;o^ z&uAL)ag`3i-k#Yb#V3!WqB0sRHun8@A~3(^$Ws`*wv-kSQO`xqHc}e9{BCpAfU0D2 z$Ka*?!?!VCd>B#KGE*1a8uy9)2aD7&;}>^J*uN1oLFm*UYpLBR)RdY%46ym~?{kAt zY6Fsi$@usOB1HHrFuS2Psj4@d)$98N^lLVLa0051eTdFet>P<25;c?TJ0_KM;J~}A zLe4T9QfK^wDVYu%E7ln#7>VCP6altqmq&3{PE_lGIcr$kWpD%BT)0bqZ@qdUNXW)* zkDirUR8V0iIC?`~INU0(du%S#)*kkafUOgxXx&(eP}<;?oWKsH6813Fr?f@NY7Ete zlFqq#Nd;MD6z-$o1Y*=~8w;dvJX-piKLow=dRu^K2ZYVc;7IF1ZE`Ll4kYg=3ecL| zMTi4nxbWV)1REC9hGJ@Z3Qm$!YEFVdy=qzFUv?m5sAe=|7ZcQ^A#^H36bIX2gejm& zQPhDx-z2(BpPL>yx7gO9Z zpz45E{I1S%t`3rWp57nI{Onl&A*Pohf1F}i2|9FOygUlh+0wBJqjKTV$jZfA_2C(Gcv%Z{ zHCR|Gu`vAVhpqx`Z~orir<(c#;(PeM^uBLR{XELPPof2D==$3PKwy+m_v}g#Y(v>& zIQ>o@A+8F|4?p!1Z(YTQ4V}e@XNOIf!%Y0WMD)0enV>E8CA2-u`z_OUTGsUb%-vI$ zF`=S2zy;sg#xdgMuX>bP3#=dHSVA*yLs9p=fD95L#E=;t#iz1;aTvaVFLp3F+jm#@;gHhBy5 zr9u9PL%jft1*@e>ITD@9PmeO-Tl2_h$e z3!uaJ@XZWj>3%*S`_(yMS4{!x#pKGxVgo@u{bEzySejU%h&tnTJg+~tRTcW{Dsx2&+t=&<(oosAa+{;kzEJI0|c zTak@nH)H5$qA;e&{h8w{=v9-i^SBQihy>ZUrV%!BToA3AWKzO#>cs(a|HT>i-+4o+ z5NiAr=NDR})Tdk)Xu-BP`XNH<0?BU{+~g9Hn>!o{i_PRxv@;V?pD|g zBg#O4EA*~PojfmKCd{mMvSDso$B%NOPHzAiw5`zUuId7@_#_4jQQriHjD9iH$5X?o zmYkhavQ%&*@DS4>9R3OPIcd|T4j-6I79@36UtLpu%VU6u+emjOtIfT5)n9m(WRa2!7l|K>r5H3%Kf~2ufal)iOdY zrjm$D7|EV`H<)+7#|dIH&o7lAzTnoe!(W06V-J#txMbwk4O@`lJOy7Hw?5UhXLLSP z7?H7{3?RQm8Z`%_oT_6x%Emw02yJeTYEJ!CIH)SoQb%Gl{?6A>VXj*4u7P5ks=s_l zfH-Q`r}yQ+ZQUdt=Lx7I6q`D|2FvZ}cYw3HIbuMoYI z0PT|%?(e%kZ5Iz!Q0NNp3mANu4(GmdmSd)^7c9uQoO_8KKbus;cein1>0!CvS@fdT zgR|Z#`oq)DZz18EP{7aSdG#oWn5V zi&=6zy6{1qtm!G<4sq-eh>OqrD;ZJ*ozv<&WOuNp8fb{PPr{RTlOu-YSVv%YtdIO& zvS>8Y$-E(nU?Cgzz(Ad1S`6iiaWlGHevUtmBP@!X5OE8I9I8$Aa;G636#0hh^Z&9I zr2YLe37$^RN&vT47`)Kjze>?RR1nzOch!6&BA2@Qh?{!&ei_?O{)zYSeQ+`{^Z6|> zZaKr-ZYMUaHtJA_D(SnWA~m+H0DKq+#$k-&?c=a=@>xn-G$~Mhwm0-bs*gir2Y1Uu z8^77|aC4ek&ruq;r%bQF#B6EU@F`0ESq`BKqrIUr0T_^juqC9lGEBvmFeih2d3_V# zpVO2s)p9zB^R64UIqXjAu4ymU}gu4KI}h#&3Q*&4!0TRx+qL z1GuBK2bY~TxSGhpU8G?!`|9fMgA`eDql}bR=X~cbWtVe>`7WR?lM(nD_2ekgd!q3j zDpAuP`B!lDj?tUo%%haCP-88&b7AOEwf&S#q)3Q_8}{g($N$4gAm^jst8wQ*TX7qT z-2ILt8z$Ke>ug3xh3kKPhlcW=!`3BRQ6d4tmQGC1Pmvo97?~BeE68pX@RJ&CnRzZ& z68L>iAtHAai+02eC^!-xnmfnQEE6?OI%IcmL`R$~#MFV07a$h&**-siow+ATk~3gM zt4ms&IVmkSLIq2`AY~AO~a`{Y(?6I?1#z*UN z+NEB@zvDO<@u_tn{x+H4S9aA^C^xyN`rkVEFM{OfI3sJ=hwioKm`9!4Cl>;789ID5 zxH2S3Wff}pluJt*Bqgff5Nab1$D_M-x_3`udWQY31367AR)~8CmEGMM($(R`?644Z zG6!qxwp+)IS+;isOINV0+RJjwgrUNP-W>u^goBL#AZO;x;&%gOryce-$z$ICX!EWX*87)H_L(W2Vu$OX~#-rwE z)#OK6^Vsi*E*733ayqL_^>xM(;l>A{2OiN$-G`pQV`HYsRhz|+rs&E_x^yu5hiL=v)?Q*iH<2x4nmcMFLm_LGI(Ppb)9`` z@ORgdNB}eXqb#qdd>n(h=d|);?sHjO8phG*LNgqkBIVdAzhjG<>6&9kXrI)lw(344 zc?po?--9vjqu@p=>A9yh&CTg11Ly55m|K^KHfp(S@W|~@h^R*C@<{C7=|K+$-e&PS z`E(>R&5>;j?wiz8Zjoh!chiMY+t2HwLxLk_Uj40wo2^Lht}@yvZ` ztO5DM9wacUpzTNAJOe5Osn`XDvDXBPN9aLVe^f+c^RFdq4O6Q|y+jrx@g-4|+%KMW zVTPI$Dp6Lo4g^WX=2s)>=SoHT@Pi;c#imPthV~@L)$0}A8HUXgOAgoNdCeOY~nNR_r(k3&wlr+s?+Ut*;@sQv!S^}A8V-#d<jh9u1{Pb=IM#lwR~82Dxp^Fs3>P@I&~Z~m>KXy zllP9x+-cGDMKh77#~lCx&nyMMG=xYl{}S`Mz~M8gL^=J>F8y31`wM6Izj&#uqY0Mf zBRXxxZ5)-Kv>R)0JQ#jwJ3|zw65?-X%Z7EuZO|7@lv%`);T1dYX_v;p^9sv$Ec1eP=si7X4PY%8$c{^0$>FiY1~A8Xf2taYdz_iJa)XU)8! z;I})xRs{HZJ;*}I8s9B>=KYt*!BK%z8wgSysZqcCxJoI*=0b5u4;zgfXRNhKhT`Sc z=39H^_x8Je#7~1B9-#BrBVP9-f$m;!9ASX>!p*lJkGI#|>i0LZfa7-3PZRjFMvScr zHg3G~nSbegAxX)4rkH9!P~JEiHi$#$QhCd0x03!tR>^q}9b~W{Q~an|oHOZ^VFjNd zF%MNJPqyjCMX>tU%kf&hi8G(K-^bh&u$1am(F_Gbx;Ni*V?eUS$IN~NQtwC1H_?9M zaSY1bdMo&PVah`S=(pYDb6(DJ$Nz(@IM}DGLMQ+Kd6h{WWjk%Oh!lai<2E7^sb@3o zVYYc_NqMVIpcvEu;a|VtKc$GVU{;;Edxt_CuBU++`Q;UBbeez*?Md4IS5^h${*SCO z#2&N}?z^!?3vyaEnrO)uA#0*Ou_BA9J8HtCIJ?YY+3@M1^kM~ zSG7%~TuFGbgNA8&aa~HCJLis*G0RuTc9B=vQ~Q4?6|P8moFdZ{W~K{`d~P;!#~(g$ z*HnN1X1MbV(`KBj(eSP{5+$o5(pCwZX|%f+n4Y^KjLF;wsBT>NTWWZuLe*`RlEW8= z<>!#lHI{ikqle#z2*5?ze|{?ekijMK1aYFdAuxlI1c7CqrE_)tX4u|yF)kCJPN8*T z=-<@0L_~~bA1czuhPys0cWgb=NzyT7-!jMByo7=El4!7QHs)B(eE45BMdOt2frq^G zKWr++&7S8q*&QnJhdQ70Ee!88()IA_T%pxYt2+sRKw0SnYJ5IKnuFACLy^NC!F%;g za?;)E*eW~{8GVTYOZzA!C)qh(PF*Nb$H~jDc1g--l?X>@Kib%OU|2j`_m4YSZ(jSJ z1G;l(8p?;G*ROBM*U$_Rw2I_cCvW_M=!+Q>REG2tn#Bd`*AvQ4 zw08UzQIouu3|-Gw_%<(INThqdv2iy#-(MnQ3}fE84SLCvfA9Qt&^;+$L)nrJU?2mTitjA{WvtCH?d9)c}zGsd^UO*9{? zPz0@V<}AQb?iLksc=w&kjvZIXpkt9|I8d)AE+K z+@8J-v-y49pFWPSrvqMYkLRQB^S)g?yxqUs_+Ki?^T3OU%b_2xiMt#bo34XtheP5sdn|F3(!S8X`a=__|V8~F9Gw*eM_E@~LW zGufFc)bexZ{O8jywg0WhdG+-ugSY6R++&`D{|rHdyo@~5DfnF=e8~vG;LNt=^zIH4 zFVwN>-5dMs-euSA!8h=h>38dKc?w8=-6ByRVICA_Sm+2Qp%e#-NX`R zsmh3NwS{Xg)%DQc)=8|25}oR#YVzwcckV|JbeN6!F1Kf&&jW4X-}<~e;qB^pJ#lcR z+;2g&Q~~uhpr5r_Xv-#a1B0I=ZC8uiH3m7KFO4K~npmYX(>)GrRtd?A7B9wJR`+vx zhWh^{Wc9SN@fd@BL#UDu10R>xYau^as%CmJwbKrnPI#6N!8fJj0+|ciYUOj!9*z2X zkIMl$J&SuL9Vlq%=4q@t88V(#b7SZ%h4efm>G|5b^hxLvJ)6(O_I{cA_T0F}8Q=0H zv1uP^+4Wx*&5a!5&wpj$h~DWe*jSF*fDmpXzXlO4WEl9H;@qXq<;@mT1%i z4|lqGo3c&S_BFc9vN^8|v#QPjrZPNRDMAf~^)-;UdhJ2aF9oknzrkxO&~esHz&A9$ z?@vyQ2fv;VUR*y*YV!PEe|VyC+0dd8I20D=!i{~$=9Ja| z@zZ#gfHC|;dv|Sdy@t(=|7GYV31X~yEsf4DveV~VWly>eyq$mnk{z-#f~|Rm@dTXC z4XuyoEHbJMN!*REEW>FJPxf85#mBam)t_^yn8#l*+AQ4lA%a|9$Oup&itUTWi+{Ig z6-)~cOIJ>&{j-^=dzp!mSZX#jt!!?`IJv4mfET~pBf?>~Gs6taF3acO-!3cbDg=t7V&^1+r*@OIEQ^$JxJ_C zijg`jI^Ud-ZXM#r;yLsoq811r"z3RLZP7iy2?5<W&I=3^g?ui0Zk@M7Co4?QS%mKS{d$8&}3 zy@^`O==YYMp7Z?l>t5KW&PQ9b&-HQ4g(dyAdt_0-dm;fY`%_M=w<9V1fOzG7A zwl5AZQZHem2lq&)0IJ=N4!N5=fFH~rM%})@LXz#dmgB2vbzVkrO1?h&yLMws7~lRH z$gRl}yS#n8zBpr3Q@`D~{IVdJ^yLqms#~)Sn+O^8l&`IL12K`m0~B zGC7vr&NGL$Z@42w`$i~LS3uaOA$DZp)ffiZ62;5=(~h|g?(I(h_XAnd{(dvq3oy<6 zMo+fK?OUU=`?KXyw~&Q<1)BtqhgGw^?Q_ZEDM}Xn$5|D5#dQH^MKs@Pr=cb{-{)!X z@SYVWp#stX^!-KS*H9S_pg*mhYS3VP-|=}EeD(?h9^je?whia-P;#c}YH@%680+L= zSoQ`(a9dDBg&otMP!-ln!HF>E?e8X@v>sZPoUjogq~J=D=aQ;N+3vzcRjlHO&6 zA9M6;%-nQog)W;?i6RZXm_BsikMVlgG&`uIDm!^&$I`Aag<_LCz^YAzKi;Ast1oj@ zGE+*ff9Teawtj^Uj;&uX(`HK4%iHZ2@U-_alot@-eS7wC7+slH+r#_$y1P#$2zdHD z>ifDfD?SrSn6Z~45eX_E7J45z69U|yyQPhld4^wy7k76budDhN^;>6pzLwF?8~hH} z2Dd&$^a|Hox85KCN8)d|C$fi((*eutzA^!&U~J~b)-cP*fbz-e=dqOz+ma>UVVb0) zjjVzw0tjzaro9LxQM;)+6z>)0euVn)>epSA`%>GfF`0q4;j4Q>##Tjp$?xibtLtx? z1;+O6-t~g7`CDn$@AtW}D zJ(U~SK!HN;vC8NQ8^9J+(g;Ol;iZ~WNB<{I$?kUk!TV;`5(YwFLJv-Fz~!T(<#LPs zW`T#Q)mSGX{OlKfSMxRITJ0Sg8NfikN1-$W5Sel#t1kYrb7xjkt3E`m# zoO5_1LIbdyzTK>B_t`ru^ZiBR_V-6_=cdDE^Gg6-j8|y`!M@~=9U}HYiG9K+iN+1S z+6{k&ao7|zR<$#7J?!7CuD#I^VgA5pz8XAZ2UUHsp_?TqnVsD6s?*T-_c1;S8%gGA zr~P`}USt9X&nB>ItO&k5m%l|p*-^@Onb#@U`-!jnf%mpy_!ZIElMGfT{sp`oytL7L z3j#iO?O*efuN#6BgUTPzNQm!_!he0I5g@nP2-(Ki@5)YX! z;H@*rD+{V5R`UMWw+~W-x-M{*XM*}P(hOK~S>37W|An9EzIiYmQNq3sH;M1@S>N!% zDh(bhPl&E?rg+fRfNIfu_ozBIB<$gddqPHCkZNwZC(|TAA8$P$Us1ywN*4KX3_3Ga zBHGEVa+GPg1O&ew1eYjfTO;6y9GM@z%l*@n^o4=_9T{9qj=|ZkeHTo3Xd{_L@d;?l z*&Q!e6l;Byixt1$19&;AzbkR8CScYEzKMwV9d5pp_xGug1n@QBme|*HlMg=O z4rZIC7i&Y@+bEh#u)bAsq&5%(JG(wt63x5cHr4;?)*caTnorUOe%$zrOn79^9nJ9^ zH_X%dbPsC4&%E>bT}LkG>Y)9KbOK&(>eF`azZy|rfoHB|LnO0nl+6zFyUy4p?z|j> z+czTlG=?*f8b(j8A7=cVAOvGMF9eRBuu8pSVji=+f>PQ+hBTJin)vz4wpb81q^V~u zM^9U5q}?^RV9#Yxx1b0Ry=(TJd6zr<^u6{%URfei+#X_sOd>Wpd#|f0@4)t_EAWYw zS)n=w6-EoyqORA-YKd7|O%aN~f&cm;tfL^>TN4Db{i7Vc83Bg;Xt}D~p|Y*27!CUz$Ul)9y^x z!t!k9OW(lPuMgJ_vzk>qe*oLX$N9?JR*tcbCJAAb^1o?X%V?-0LOkTBG>P~?y0N!& zEm$b)ZuRR|`SW+p3*hbX_A%1vmAJJ_kahcgaBwgu4gO6O(DLo~XJ(Dj#@F|N*E!@x zg>Kn}?BJ28@Pv!kvmsMgC>5TbN>x|)$^3*{e=p)bp{7Mt(3B4Fhid#%k{c-XpK zt19@cZNfAY!_|jxnD`|G{MjYzPjvsWdeGiyy?lqe$dV@Cr^SXKSETd<-mQc+g+81yf>m23-D&HvWTpptSK z=&(WbCV56fX-|IJdjjA`^11rKpn>5S4!b7yXKma)j@qt|sPE9A zx*gM&4RhX)1ZY^helOv(@)Swn2kRPkKB;ZLADPxg_NP!D{Q+<&L_SRcQC8=%61fIG zV^LW580c`S`Khsbc0of~2M}CG4YC27LBwY45l4$CDS&)B80J zxoVH8r~Y`fdh90xrlp{NrVIZCY>Krw(pdNAR?H`%pm;}IpQ~R$lK$mNoY81~3d3hQ z4bh$fcXG7hBvM^4C~WSe<#5bgJ02{F!htf`fC1(S45UDD5^KOu)Lzn@lT*PZHcVD| zG?4xfTMGJ#4Ab3%0zsG1uWw_PJ{7;B!6f!aJ2z($nd~8;0lUL(@ZLW&Cc+R){VRKht|1H&q93c z$1T3=6TAol-=9fS2}G65!_1GNqopRrmc3Vc(||FvULRPZGgFs8MNVyxbjkxBSoF+p z2}yJeE&uGWf@xSh2y7xu+5CMXs{XTfojahEb@?GLhdLU zkiQtZ;Q~tR6Cl$;8Z^m70&t@`*tkw>>=@ro##Tdawe4YVs5wiUcer!t;Up82;ezsf zxMlD@*}s`H$%6jEuKBPNUv&U|_nX^nH(Wm&kROD&_Qb48Z#GOmZRuNarEj_Lm2MS+ z=QRy~YK@%iT4&o62GSOp2tDsOV20+8|5Y*t$u9OHOyb7%TQ+iyMUD_M`}QSNRT^2! zB}=ry``jJpHM>3GIh8Aie3eN(icW0Q1epgRmoigaFbkr&_}cr{-8&QDm$3>Z}qC0djc2EvGklElWN3cx;ez;FW8C zkXLqAkU?(*Z82q<@=T!;*$=^Lg`Ir}$?G5b(VXnF&r+`Yhp@^^M$-XosQ=Xnas8b( z^xzn2dG!q%FA zSK3bXuUdw#T$o|t{3{k>MbOr%%rmo*wi1TH{5;&D+B-nWo|!2j`3X@kGqbI=$b{!C z=nl;VgmYp}Z=K$`gRr3Tx>}i2{l~qXy6*{D%xTWjo-5}+cHGKf70vAaaBtLqun_1I z19FYc2uUXL8M8zJ1-5DtWPidETH5Kmi|5aQQ~Bd$rH!y~Ov50*IZmXSw%KQe!LKRH zR~c72q*@9nePd@Uij%V?lODa?c56ZCQt*_dOlgTyGicZ;)%2-_gJg+VELd)q0|2o0 z;EX}_l=gXA(A-MpQ!_A^RzZV*PRaQUyUu4!qtV$y&eMrIAZ-pAsR(IQIW{oZHu>F* zn$b!8K9=0Ovm#{ap{DK_P}4NjWK>VP%-Uw5b?FW{3i1;z2sX&5F_s#;vVBgh*-`4st*fFXiqgvuO;IW308smJXwJ%S{L@E22KTY%Bg)WenN0O`)kv*3~#D>@7sfr{2mI zsQMs4HO2@w)*Yh2c!LAlLI^@)Ej9B!;Fi#C)XNI}m_*-@gM)J*jWpL)2b6A-VhB)m z%7w)BcGw6#HxW61*8+@mSDGehVo_qjh-6fW{LtqC)x>sQnXKK`wd6FD{iP zP&*FHqRE^k*UoCj!AvgM0|nF2AdcQh0m)HTKnA42)5=Bul+4AAA+v{G5}7%74`w+b zvjgKk7#wSS*)T7MZGp>qJ-*d!V_4>JyA2KRyCSya|ljXp$*x?ob2BRm2Fa8rTc|Q+pjd}`c^2y7QE(`K3jqT9H>jm zG>Ql;ha!h9(rSTZv_EL|Zt$5~kpltCHJMO6fdELb;DQ4OC4L64JH)eU-k}&7yDrHo zQ;FmgMuPr-$mA>7b62R?;K4U!kdv>x1+^I;`_1^NA07l+_j;0vnL^6fNLDn+0;>*F z0TCK@btu;a26AQuFs<35usV@ZJ}o?==#N6rt(Qu%;4yG9TjY0UvQSpuj{mCCznDsB zhDS(D!zw%OTvoMzQ2nw7O(V-k(W&zA_OiHQ2@aQkhc5)VWt}&Xy(z#C4+p3&Z@1)Z z6tL09i~68_uM5)ys$?I+8niVhHWK~t3J7}%lVF!@>r2g`1M4nBqkt>s*GDa4!E?T# zS!?*b7AU~pB&^W8#Di2eq4mOeU+@q4=2VsK;43Ik9%~Mur6d!eTw%p|4Ln;mpDxX7=VIWFb=+qFqX7qhtmZyL7&_aI;4$>iRZtH+iX9%tv0sFp z4ETRwR0t|C#A(U0m9uV9>+<50H+0XsL%U~ei?JRO!#HFLHVYRSPv9Us=p!6A_S6~QlD|43xy6N%6(C^??X%5e z66Cmczpg2ABA5F+u(`imK6e|6+X~wSTk*;Kk#d!oLqmK^<1kMlGmNj-97HPPlS>8R-%-S-hl2xegdxtXhR~TD9hvc z;p6{|qAN`$IN^BS`uUK&z7Q#KdEMl~R>t0FTQO2FE3_KAdfLN6qyzWIz-FIYBk_c> z-1lt)|Dpj-FptxV^O%5fy(&<6a9~r#%S~%LI3W8N22aUgilE<$420Nq-yC6HPD z(xn0Cv@-jD;{tK`(WdocDgBPgPrpMQz5_)1QcS^iPj89jM zrNSE#K`gBR1O;mc1ZZAes3ti(O|9wcedrl=gADtDDqbq1NnUK(3$L)aDDhx(d=f^r z>wbHG>ds3zPL&sg-H`58q6GlL;%hftX_}{0rYio_T=cj5xvTtFPr}*l9b4tU<7WMz zlh5^kKGMhU=O`MGC6(Ne#SvLcFeBnf*@*OjaQ5JmN+bun>xlHe&FCyp=7jXcb1(-Q z*oN2goAq}%Q$8HZ$)D(3tyS`giS;u%KE{<~QBU(m4luA^j51l2O52DX933r}jse6y zR~K$XdS^KGj=7n;^r~PZG7-WsqKIJ0gAwUflnJT4>eYbSt@l@NwJ$xZRSu~hXu|n{ z4_h;3HqDqaxaI#+%5O&U+0r9E%=;zPp&*#tQ&u{JX R{{jF2|Np`EWKsZn0RXm)6#)PM diff --git a/charts/prometheus-federator/0.4.3-rc.1/Chart.yaml b/charts/prometheus-federator/0.4.3-rc.1/Chart.yaml index 51f5984e..eea5005c 100644 --- a/charts/prometheus-federator/0.4.3-rc.1/Chart.yaml +++ b/charts/prometheus-federator/0.4.3-rc.1/Chart.yaml @@ -7,12 +7,11 @@ annotations: catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1 catalog.cattle.io/release-name: prometheus-federator apiVersion: v2 -appVersion: 0.3.5 +appVersion: 0.4.3-rc.1 dependencies: -- condition: helmProjectOperator.enabled - name: helmProjectOperator +- name: helmProjectOperator repository: file://./charts/helmProjectOperator - version: 0.2.1 + version: 0.3.1 description: Prometheus Federator icon: https://raw.githubusercontent.com/rancher/prometheus-federator/main/assets/logos/prometheus-federator.svg name: prometheus-federator diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/Chart.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/Chart.yaml index c4d14e1d..4d81896c 100644 --- a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/Chart.yaml +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/Chart.yaml @@ -9,7 +9,10 @@ annotations: catalog.cattle.io/rancher-version: '>= 2.6.0-0' catalog.cattle.io/release-name: helm-project-operator apiVersion: v2 -appVersion: 0.2.1 +appVersion: v0.3.1 description: Helm Project Operator +maintainers: +- email: dan.pock@suse.com + name: Dan Pock name: helmProjectOperator -version: 0.2.1 +version: 0.3.1 diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/_helpers.tpl b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/_helpers.tpl index 194214cb..97dd6b36 100644 --- a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/_helpers.tpl +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/_helpers.tpl @@ -64,12 +64,3 @@ heritage: {{ $.Release.Service | quote }} {{ toYaml .Values.commonLabels }} {{- end }} {{- end -}} - -{{/* Replica Default - Allow setting to 0, or default to 1 */}} -{{- define "replicaDefault" -}} -{{- if (eq 0 (int .Values.replicas)) -}} -{{ .Values.replicas }} -{{- else -}} -{{ default .Values.replicas 1 }} -{{- end -}} -{{- end -}} diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/deployment.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/deployment.yaml index a5386aae..33b81e72 100644 --- a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/deployment.yaml +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/templates/deployment.yaml @@ -6,7 +6,9 @@ metadata: labels: {{ include "helm-project-operator.labels" . | nindent 4 }} app: {{ template "helm-project-operator.name" . }} spec: - replicas: {{ include "replicaDefault" . }} + {{- if .Values.replicas }} + replicas: {{ .Values.replicas }} + {{- end }} selector: matchLabels: app: {{ template "helm-project-operator.name" . }} diff --git a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/values.yaml b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/values.yaml index 63fae45a..796bf5f1 100644 --- a/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/values.yaml +++ b/charts/prometheus-federator/0.4.3-rc.1/charts/helmProjectOperator/values.yaml @@ -129,8 +129,8 @@ namespaceOverride: "" replicas: 1 image: - repository: rancher/helm-project-operator - tag: v0.2.1 + repository: ghcr.io/rancher/helm-project-operator + tag: v0.3.0 pullPolicy: IfNotPresent helmController: diff --git a/charts/prometheus-federator/0.4.3-rc.1/values.yaml b/charts/prometheus-federator/0.4.3-rc.1/values.yaml index 103c1604..3d04e3ee 100644 --- a/charts/prometheus-federator/0.4.3-rc.1/values.yaml +++ b/charts/prometheus-federator/0.4.3-rc.1/values.yaml @@ -32,8 +32,6 @@ global: # - name: "image-pull-secret" helmProjectOperator: - enabled: true - # ensures that all resources created by subchart show up as prometheus-federator helmApiVersion: monitoring.cattle.io/v1alpha1 @@ -58,7 +56,7 @@ helmProjectOperator: image: repository: rancher/prometheus-federator - tag: v0.3.5 + tag: v0.4.3-rc.1 pullPolicy: IfNotPresent # Additional arguments to be passed into the Prometheus Federator image diff --git a/index.yaml b/index.yaml index 31c7f8d0..97db9f39 100755 --- a/index.yaml +++ b/index.yaml @@ -10,15 +10,14 @@ entries: catalog.cattle.io/provides-gvr: helm.cattle.io.projecthelmchart/v1alpha1 catalog.cattle.io/release-name: prometheus-federator apiVersion: v2 - appVersion: 0.3.5 - created: "2024-09-12T14:09:16.571722-04:00" + appVersion: 0.4.3-rc.1 + created: "2024-10-07T13:17:48.590665-04:00" dependencies: - - condition: helmProjectOperator.enabled - name: helmProjectOperator + - name: helmProjectOperator repository: file://./charts/helmProjectOperator - version: 0.2.1 + version: 0.3.1 description: Prometheus Federator - digest: d17b79b337568b320c06e34582cdbdc13bddcceb74d700572f044d1e7938f569 + digest: 914d6606b938383710b923174e2505231fea144da555aa88119c7e3c88209c01 icon: https://raw.githubusercontent.com/rancher/prometheus-federator/main/assets/logos/prometheus-federator.svg name: prometheus-federator urls: From e1594ef27723be52045725802f1ec27132ba908b Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 13:58:43 -0400 Subject: [PATCH 37/54] Ensure rancher-project-monitoring chart is locked to working 0.3.4 version --- .github/workflows/e2e-ci.yaml | 2 +- package/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 7de21763..84e4df76 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -86,7 +86,7 @@ jobs: - name: Perform pre-e2e image build run: | - REPO=${REPO} TAG=${TAG} make build; + CHART_VERSION=0.3.4 REPO=${REPO} TAG=${TAG} make build; REPO=${REPO} TAG=${TAG} make package; - name : Install k3d diff --git a/package/Dockerfile b/package/Dockerfile index c33e79d7..3429db74 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -20,7 +20,7 @@ RUN xx-verify --static /helm/bin/helm FROM registry.suse.com/bci/golang:1.22 AS builder # Allow chart version config -ARG CHART_VERSION +ARG CHART_VERSION=0.3.4 ARG TAG='' ARG REPO='' ENV CHART_VERSION=$CHART_VERSION TAG=$TAG REPO=$REPO From 7da230bc7aedd5001877aceec115c92525dbad3a Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 15:06:50 -0400 Subject: [PATCH 38/54] Remove unnecessary wait --- .github/workflows/e2e-ci.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 84e4df76..d53d45f3 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -127,10 +127,6 @@ jobs: - name: Create Project Monitoring Stack via ProjectHelmChart CR run: DEFAULT_SLEEP_TIMEOUT_SECONDS=20 ./.github/workflows/e2e/scripts/create-projecthelmchart.sh; - - - name: Wait for a few minutes for chart installs - run: | - for i in {1..12}; do sleep 10; echo "Waited $((i*10)) seconds for metrics to be populated"...; done; - name: Check if the Project Prometheus Stack is up run: ./.github/workflows/e2e/scripts/validate-project-monitoring.sh; From 00170e1c96208bdb1708607d850653e0ce97e50a Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 15:53:26 -0400 Subject: [PATCH 39/54] reorg build script steps --- scripts/build | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/scripts/build b/scripts/build index be738bfd..eccfaa88 100755 --- a/scripts/build +++ b/scripts/build @@ -9,6 +9,14 @@ cd $(dirname $0)/.. echo "Starting \`prometheus-federator\` binary build:"; +mkdir -p bin +if [ "$(uname)" = "Linux" ]; then + OTHER_LINKFLAGS="-extldflags -static -s" +fi + +LINKFLAGS="-X github.com/rancher/prometheus-federator/pkg/version.Version=$VERSION" +LINKFLAGS="-X github.com/rancher/prometheus-federator/pkg/version.GitCommit=$COMMIT $LINKFLAGS" + ARCHES=( "$ARCH" ) # Set CROSS_ARCH to build for the other architecture if [ "$CROSS_ARCH" == "true" ]; then @@ -19,15 +27,8 @@ if [ "$CROSS_ARCH" == "true" ]; then esac ARCHES+=( "$XARCH" ) fi - echo "Building for Arch(s): ${ARCHES[*]}" -mkdir -p bin -if [ "$(uname)" = "Linux" ]; then - OTHER_LINKFLAGS="-extldflags -static -s" -fi -LINKFLAGS="-X github.com/rancher/prometheus-federator/pkg/version.Version=$VERSION" -LINKFLAGS="-X github.com/rancher/prometheus-federator/pkg/version.GitCommit=$COMMIT $LINKFLAGS" for A in "${ARCHES[@]}" ; do GOARCH="$A" CGO_ENABLED=0 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o "bin/prometheus-federator-$A" # Set CROSS to build for other OS'es From ffe1e8234a9946523956121f4bd8fe821aedba09 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 15:54:47 -0400 Subject: [PATCH 40/54] adjust version script --- scripts/version | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/scripts/version b/scripts/version index f36a6c68..2215508b 100755 --- a/scripts/version +++ b/scripts/version @@ -1,4 +1,5 @@ #!/bin/bash +set -x CHARTS_BUILD_SCRIPTS_REPO=https://github.com/rancher/charts-build-scripts.git CHARTS_BUILD_SCRIPT_VERSION=v0.9.2 @@ -21,10 +22,20 @@ if [ -z "$ARCH" ]; then ARCH=$(go env GOHOSTARCH) fi +SUFFIX="-${ARCH}" + TAG=${TAG:-${VERSION}} REPO=${REPO:-rancher} -if echo $TAG | grep -q dirty; then - TAG="dev-$COMMIT" +if echo "$TAG" | grep -q dirty; then + TAG="v0.0.0-dev.1-${COMMIT}" fi -IMAGE=${IMAGE:-$REPO/prometheus-federator:${TAG}} \ No newline at end of file +IMAGE=${IMAGE:-"$REPO/prometheus-federator:${TAG}"} + +function print_version_debug() { + echo "BUILD_TARGET: $BUILD_TARGET"; + echo "SUFFIX: $SUFFIX"; + echo "REPO: $REPO; TAG: $TAG"; + echo "IMAGE: $IMAGE"; +} +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then print_version_debug "$1"; fi \ No newline at end of file From 28cb7b6b2b57225848773c9735c029fffd4eaf3a Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 15:54:57 -0400 Subject: [PATCH 41/54] improve docker file --- package/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/package/Dockerfile b/package/Dockerfile index 3429db74..22540dd4 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -20,6 +20,7 @@ RUN xx-verify --static /helm/bin/helm FROM registry.suse.com/bci/golang:1.22 AS builder # Allow chart version config +ARG TARGETPLATFORM ARG CHART_VERSION=0.3.4 ARG TAG='' ARG REPO='' @@ -31,9 +32,9 @@ RUN zypper -n install git vim less file curl wget patch COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN make build +RUN ./scripts/build -FROM registry.suse.com/bci/bci-micro:15.6 +FROM registry.suse.com/bci/bci-micro:latest RUN echo 'prometheus:x:1000:1000::/home/prometheus:/bin/bash' >> /etc/passwd && \ echo 'prometheus:x:1000:' >> /etc/group && \ mkdir /home/prometheus && \ From b3a8fdd9d79f1139c7553e7936ef21f4b5491ddc Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 16:14:24 -0400 Subject: [PATCH 42/54] Only target transitional versions for disabling helm controller --- .github/workflows/e2e/scripts/install-federator.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e/scripts/install-federator.sh b/.github/workflows/e2e/scripts/install-federator.sh index 1c1aa44f..1a22fb9a 100755 --- a/.github/workflows/e2e/scripts/install-federator.sh +++ b/.github/workflows/e2e/scripts/install-federator.sh @@ -25,7 +25,7 @@ case "${KUBERNETES_DISTRIBUTION_TYPE}" in cluster_args="--set helmProjectOperator.helmController.enabled=false" fi ;; - *) + v1.25.*) embedded_helm_controller_fixed_version="v1.25.4" if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then cluster_args="--set helmProjectOperator.helmController.enabled=false" @@ -52,7 +52,7 @@ case "${KUBERNETES_DISTRIBUTION_TYPE}" in cluster_args="--set helmProjectOperator.helmController.enabled=false" fi ;; - *) + v1.25.*) embedded_helm_controller_fixed_version="v1.25.4" if [[ $(echo ${kubernetes_version} ${embedded_helm_controller_fixed_version} | tr " " "\n" | sort -rV | head -n 1 ) == "${embedded_helm_controller_fixed_version}" ]]; then cluster_args="--set helmProjectOperator.helmController.enabled=false" From d6a8e7fdb13903261f3e59f5783bd7e07c925d88 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 16:27:47 -0400 Subject: [PATCH 43/54] fix mixing artifact collection dir --- .github/workflows/e2e/scripts/generate-artifacts.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e/scripts/generate-artifacts.sh b/.github/workflows/e2e/scripts/generate-artifacts.sh index c1a13cb5..18c9b3a1 100755 --- a/.github/workflows/e2e/scripts/generate-artifacts.sh +++ b/.github/workflows/e2e/scripts/generate-artifacts.sh @@ -46,6 +46,7 @@ mkdir -p ${MANIFEST_DIRECTORY}/deployments mkdir -p ${MANIFEST_DIRECTORY}/jobs mkdir -p ${MANIFEST_DIRECTORY}/statefulsets mkdir -p ${MANIFEST_DIRECTORY}/pods +mkdir -p ${MANIFEST_DIRECTORY}/projecthelmcharts kubectl get namespaces -o yaml > ${MANIFEST_DIRECTORY}/namespaces.yaml || true kubectl get helmcharts -A > ${MANIFEST_DIRECTORY}/helmcharts-list.txt || true From 145580601a2aa2c0b7d6da172c1a0e6d01cff129 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 17:17:13 -0400 Subject: [PATCH 44/54] Adjust to use expected CI namespace --- .github/workflows/e2e/scripts/generate-artifacts.sh | 1 + .../workflows/e2e/scripts/validate-project-monitoring.sh | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e/scripts/generate-artifacts.sh b/.github/workflows/e2e/scripts/generate-artifacts.sh index 18c9b3a1..bae1d98d 100755 --- a/.github/workflows/e2e/scripts/generate-artifacts.sh +++ b/.github/workflows/e2e/scripts/generate-artifacts.sh @@ -61,6 +61,7 @@ kubectl get statefulset -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTO kubectl get pods -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-monitoring-system.yaml || true ## cattle-project-p-example ns manifests +kubectl get deployment -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/deployments/cattle-project-p-example.yaml || true kubectl get projecthelmchart -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/projecthelmcharts/cattle-project-p-example.yaml || true kubectl get statefulset -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/statefulsets/cattle-project-p-example.yaml || true kubectl get pods -n cattle-project-p-example -o yaml > ${MANIFEST_DIRECTORY}/pods/cattle-project-p-example.yaml || true diff --git a/.github/workflows/e2e/scripts/validate-project-monitoring.sh b/.github/workflows/e2e/scripts/validate-project-monitoring.sh index 44d6a518..63095e3b 100755 --- a/.github/workflows/e2e/scripts/validate-project-monitoring.sh +++ b/.github/workflows/e2e/scripts/validate-project-monitoring.sh @@ -6,17 +6,17 @@ source $(dirname $0)/entry cd $(dirname $0)/../../../.. -if ! kubectl -n cattle-project-p-example-monitoring rollout status statefulset alertmanager-cattle-project-p-example-m-alertmanager --timeout="${KUBECTL_WAIT_TIMEOUT}"; then +if ! kubectl -n cattle-project-p-example rollout status statefulset alertmanager-cattle-project-p-example-m-alertmanager --timeout="${KUBECTL_WAIT_TIMEOUT}"; then echo "ERROR: Project Alertmanager did not roll out" exit 1; fi -if ! kubectl -n cattle-project-p-example-monitoring rollout status statefulset prometheus-cattle-project-p-example-m-prometheus --timeout="${KUBECTL_WAIT_TIMEOUT}"; then +if ! kubectl -n cattle-project-p-example rollout status statefulset prometheus-cattle-project-p-example-m-prometheus --timeout="${KUBECTL_WAIT_TIMEOUT}"; then echo "ERROR: Project Prometheus did not roll out" exit 1; fi -if ! kubectl -n cattle-project-p-example-monitoring rollout status deployment cattle-project-p-example-monitoring-grafana --timeout="${KUBECTL_WAIT_TIMEOUT}"; then +if ! kubectl -n cattle-project-p-example rollout status deployment cattle-project-p-example-monitoring-grafana --timeout="${KUBECTL_WAIT_TIMEOUT}"; then echo "ERROR: Project Grafana did not roll out" exit 1 fi From 4a1494fa55e4d42389b6fd910ed5fc143ca6073a Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 17:32:19 -0400 Subject: [PATCH 45/54] Expand CI error --- .../e2e/scripts/validate-project-prometheus-targets.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh b/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh index 5335bdfe..b87e4e4a 100755 --- a/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh +++ b/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh @@ -25,7 +25,8 @@ yq '.data.activeTargets[] | {.labels.job: .health}' ${tmp_targets_yaml} > ${tmp_ echo "TARGETS:"; if [[ $(yq '. | length' ${tmp_targets_up_yaml}) != "4" ]]; then - echo "ERROR: Expected exacty 4 targets to be up in Project Prometheus: federate, cattle-project-p-example-m-alertmanager, cattle-project-p-example-m-prometheus, cattle-project-p-example-monitoring-grafana" + echo "ERROR: Expected exactly 4 targets but found $(yq '. | length' ${tmp_targets_up_yaml})." + echo "Expected Targets in Project Prometheus: federate, cattle-project-p-example-m-alertmanager, cattle-project-p-example-m-prometheus, cattle-project-p-example-monitoring-grafana" echo "TARGETS:" cat ${tmp_targets_up_yaml} exit 1 From 8bd057076ab8c310d714443ad927fa28308f4639 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 17:38:54 -0400 Subject: [PATCH 46/54] e2e: add sleep step back --- .github/workflows/e2e-ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index d53d45f3..9b9404a6 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -130,10 +130,10 @@ jobs: - name: Check if the Project Prometheus Stack is up run: ./.github/workflows/e2e/scripts/validate-project-monitoring.sh; - # - - # name: Wait for 8 minutes for enough scraping to be done to continue - # run: | - # for i in {1..48}; do sleep 10; echo "Waited $((i*10)) seconds for metrics to be populated"...; done; + - + name: Wait for 8 minutes for enough scraping to be done to continue + run: | + for i in {1..48}; do sleep 10; echo "Waited $((i*10)) seconds for metrics to be populated"...; done; - name: Validate Project Prometheus Targets run: ./.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh; From 9ff001049b3aa3c7d77516c04ff81853e4cf50b7 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 17:55:54 -0400 Subject: [PATCH 47/54] correct namespace in ci --- .../e2e/scripts/validate-project-prometheus-targets.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh b/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh index b87e4e4a..3e81a119 100755 --- a/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh +++ b/.github/workflows/e2e/scripts/validate-project-prometheus-targets.sh @@ -16,9 +16,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets | yq -P - > ${tmp_targets_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets | yq -P - > ${tmp_targets_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_targets_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/targets -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_targets_yaml} fi yq '.data.activeTargets[] | {.labels.job: .health}' ${tmp_targets_yaml} > ${tmp_targets_up_yaml}; From 2025425adf61cf8fa948e96f5989e4bbadfb1e2e Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 18:21:49 -0400 Subject: [PATCH 48/54] ci: grab services list for artifacts --- .github/workflows/e2e/scripts/generate-artifacts.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e/scripts/generate-artifacts.sh b/.github/workflows/e2e/scripts/generate-artifacts.sh index bae1d98d..cc44dabe 100755 --- a/.github/workflows/e2e/scripts/generate-artifacts.sh +++ b/.github/workflows/e2e/scripts/generate-artifacts.sh @@ -50,6 +50,7 @@ mkdir -p ${MANIFEST_DIRECTORY}/projecthelmcharts kubectl get namespaces -o yaml > ${MANIFEST_DIRECTORY}/namespaces.yaml || true kubectl get helmcharts -A > ${MANIFEST_DIRECTORY}/helmcharts-list.txt || true +kubectl get services -A > ${MANIFEST_DIRECTORY}/services-list.txt || true ## cattle-monitoring-system ns manifests kubectl get helmcharts -n cattle-monitoring-system -o yaml > ${MANIFEST_DIRECTORY}/helmcharts/cattle-monitoring-system.yaml || true From 6d86176458dd8d02306c6fdc37b1012c5ab34170 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 18:24:40 -0400 Subject: [PATCH 49/54] ci: correct e2e namespace issues --- .../e2e/scripts/validate-project-alertmanager.sh | 4 ++-- .../validate-project-grafana-dashboard-data.sh | 12 ++++++------ .../scripts/validate-project-grafana-dashboards.sh | 4 ++-- .../scripts/validate-project-grafana-datasource.sh | 8 ++++---- .../scripts/validate-project-prometheus-alerts.sh | 4 ++-- 5 files changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e/scripts/validate-project-alertmanager.sh b/.github/workflows/e2e/scripts/validate-project-alertmanager.sh index fdd7205f..8ec0454b 100755 --- a/.github/workflows/e2e/scripts/validate-project-alertmanager.sh +++ b/.github/workflows/e2e/scripts/validate-project-alertmanager.sh @@ -14,9 +14,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts | yq -P - > ${tmp_alerts_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts | yq -P - > ${tmp_alerts_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_alerts_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-alertmanager:9093/proxy/api/v2/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_alerts_yaml} fi if [[ $(yq '. | length' "${tmp_alerts_yaml}") != "1" ]]; then diff --git a/.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh b/.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh index 86b3d897..bdb7e697 100755 --- a/.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh +++ b/.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh @@ -16,9 +16,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml} fi dashboards=$(yq '.[].uri' ${tmp_dashboards_yaml}) @@ -27,9 +27,9 @@ dashboards=$(yq '.[].uri' ${tmp_dashboards_yaml}) for dashboard in ${dashboards[@]}; do dashboard_uid=$(yq ".[] | select(.uri==\"${dashboard}\") | .uid" ${tmp_dashboards_yaml}); if [[ -z "${RANCHER_TOKEN}" ]]; then - dashboard_json=$(curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/dashboards/uid/${dashboard_uid} | yq '.dashboard' -) + dashboard_json=$(curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/dashboards/uid/${dashboard_uid} | yq '.dashboard' -) else - dashboard_json=$(curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/dashboards/uid/${dashboard_uid} -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq '.dashboard' -) + dashboard_json=$(curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/dashboards/uid/${dashboard_uid} -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq '.dashboard' -) fi # TODO: Fix this to actually recursively utilize Grafana dashboard's yaml structure # Today, it just looks for .expr entries in .panels[], .panels[].panels[], and .rows[].panels[], which should cover all dashboards in Monitoring today @@ -147,9 +147,9 @@ for query_key in $(yq "keys" ${tmp_queries_yaml} | cut -d' ' -f2-); do EOF )" if [[ -z "${RANCHER_TOKEN}" ]]; then - query_response=$(curl -s "${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/ds/query" -H 'content-type: application/json' --data-raw "${query_body}") + query_response=$(curl -s "${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/ds/query" -H 'content-type: application/json' --data-raw "${query_body}") else - query_response=$(curl -s "${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/ds/query" -H 'content-type: application/json' --data-raw "${query_body}" -k -H "Authorization: Bearer ${RANCHER_TOKEN}") + query_response=$(curl -s "${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/ds/query" -H 'content-type: application/json' --data-raw "${query_body}" -k -H "Authorization: Bearer ${RANCHER_TOKEN}") fi if [[ "$(echo ${query_response} | yq '.message == "bad request data"')" == "true" ]]; then # echo "QUERY: ${query}" diff --git a/.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh b/.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh index 187678fe..fe1ff5ac 100755 --- a/.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh +++ b/.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh @@ -14,9 +14,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search | yq -P - > ${tmp_dashboards_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/search -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_dashboards_yaml} fi expected_dashboards=( diff --git a/.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh b/.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh index 078a8e5c..1cff7781 100755 --- a/.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh +++ b/.github/workflows/e2e/scripts/validate-project-grafana-datasource.sh @@ -14,9 +14,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/datasources | yq -P - > ${tmp_datasources_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/datasources | yq -P - > ${tmp_datasources_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/datasources -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_datasources_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-monitoring-grafana:80/proxy/api/datasources -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_datasources_yaml} fi if [[ $(yq '. | length' ${tmp_datasources_yaml}) != "1" ]]; then @@ -25,8 +25,8 @@ if [[ $(yq '. | length' ${tmp_datasources_yaml}) != "1" ]]; then exit 1 fi -if [[ $(yq '.[0].url' ${tmp_datasources_yaml}) != "http://cattle-project-p-example-m-prometheus.cattle-project-p-example-monitoring:9090/" ]]; then - echo "ERROR: Expected the only datasource to be configured to point to Project Prometheus at Kubernetes DNS http://cattle-project-p-example-m-prometheus.cattle-project-p-example-monitoring:9090/" +if [[ $(yq '.[0].url' ${tmp_datasources_yaml}) != "http://cattle-project-p-example-m-prometheus.cattle-project-p-example:9090/" ]]; then + echo "ERROR: Expected the only datasource to be configured to point to Project Prometheus at Kubernetes DNS http://cattle-project-p-example-m-prometheus.cattle-project-p-example:9090/" cat ${tmp_datasources_yaml} exit 1 fi diff --git a/.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh b/.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh index 1e8d99f9..0d9b3ceb 100755 --- a/.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh +++ b/.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh @@ -16,9 +16,9 @@ cleanup() { } if [[ -z "${RANCHER_TOKEN}" ]]; then - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts | yq -P - > ${tmp_rules_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts | yq -P - > ${tmp_rules_yaml} else - curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example-monitoring/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_rules_yaml} + curl -s ${API_SERVER_URL}/api/v1/namespaces/cattle-project-p-example/services/http:cattle-project-p-example-m-prometheus:9090/proxy/api/v1/alerts -k -H "Authorization: Bearer ${RANCHER_TOKEN}" | yq -P - > ${tmp_rules_yaml} fi yq '.data.alerts' ${tmp_rules_yaml} > ${tmp_alert_rules_yaml} From e33c05589dd71b3a43c624316fce2a5fa574d812 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 19:21:45 -0400 Subject: [PATCH 50/54] fix CI --- .github/workflows/e2e-ci.yaml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 9b9404a6..30b63ed2 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -143,9 +143,11 @@ jobs: - name: Validate Project Grafana Dashboards run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboards.sh; - - - name: Validate Project Grafana Dashboard Data - run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh; + # Re-disable this as it's been broken since Jun 28, 2023 + # More context: https://github.com/rancher/prometheus-federator/pull/73 + # - + # name: Validate Project Grafana Dashboard Data + # run: ./.github/workflows/e2e/scripts/validate-project-grafana-dashboard-data.sh; - name: Validate Project Prometheus Alerts run: ./.github/workflows/e2e/scripts/validate-project-prometheus-alerts.sh; From 9dabd5bef77f5d0e304b49b6353b3fdbc80425ec Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Mon, 7 Oct 2024 19:38:27 -0400 Subject: [PATCH 51/54] fix projecthelmchart deletion script --- .github/workflows/e2e/scripts/delete-projecthelmchart.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e/scripts/delete-projecthelmchart.sh b/.github/workflows/e2e/scripts/delete-projecthelmchart.sh index 20754a6c..f4862a42 100755 --- a/.github/workflows/e2e/scripts/delete-projecthelmchart.sh +++ b/.github/workflows/e2e/scripts/delete-projecthelmchart.sh @@ -6,7 +6,11 @@ source $(dirname $0)/entry cd $(dirname $0)/../../../.. -kubectl delete -f ./examples/ci-example.yaml +if [[ "${E2E_CI}" == "true" ]]; then + kubectl delete -f ./examples/ci/project-helm-chart.yaml +else + kubectl delete -f ./examples/project-helm-chart.yaml +fi if kubectl get -n cattle-monitoring-system job/helm-delete-cattle-project-p-example-monitoring --ignore-not-found; then if ! kubectl wait --for=condition=complete --timeout="${KUBECTL_WAIT_TIMEOUT}" -n cattle-monitoring-system job/helm-delete-cattle-project-p-example-monitoring; then echo "ERROR: Helm Uninstall Job for Project Monitoring Stack never completed after ${KUBECTL_WAIT_TIMEOUT}" From 48f69c0754fd2270763da6fe4527a98fdac2be63 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Tue, 8 Oct 2024 13:34:52 -0400 Subject: [PATCH 52/54] Adjust the method to check for Not Found response --- scripts/pull-scripts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/pull-scripts b/scripts/pull-scripts index 5cb48d29..9a8464d5 100755 --- a/scripts/pull-scripts +++ b/scripts/pull-scripts @@ -26,10 +26,10 @@ if [[ "$OS" == "windows" ]]; then else BINARY_NAME="charts-build-scripts_${OS}_${ARCH}" fi -curl -s -L ${CHARTS_BUILD_SCRIPTS_REPO%.git}/releases/download/${CHARTS_BUILD_SCRIPT_VERSION}/${BINARY_NAME} --output bin/charts-build-scripts +response_code=$(curl -s -o bin/charts-build-scripts -w "%{http_code}" -L "${CHARTS_BUILD_SCRIPTS_REPO%.git}/releases/download/${CHARTS_BUILD_SCRIPT_VERSION}/${BINARY_NAME}") # Fall back to binary name format from old release scheme -if ! [[ -f bin/charts-build-scripts ]] || [[ $(cat bin/charts-build-scripts) == "Not Found" ]]; then +if ! [[ -f bin/charts-build-scripts ]] || [[ "$response_code" == "404" ]]; then echo "Falling back to old binary name format..." rm bin/charts-build-scripts; if [[ ${OS} == "linux" ]]; then @@ -37,11 +37,11 @@ if ! [[ -f bin/charts-build-scripts ]] || [[ $(cat bin/charts-build-scripts) == else BINARY_NAME=charts-build-scripts-${OS} fi - curl -s -L ${CHARTS_BUILD_SCRIPTS_REPO%.git}/releases/download/${CHARTS_BUILD_SCRIPT_VERSION}/${BINARY_NAME} --output bin/charts-build-scripts + response_code=$(curl -s -o bin/charts-build-scripts -w "%{http_code}" -L "${CHARTS_BUILD_SCRIPTS_REPO%.git}/releases/download/${CHARTS_BUILD_SCRIPT_VERSION}/${BINARY_NAME}") fi # If falling back to old binary name format did not work, fail -if ! [[ -f bin/charts-build-scripts ]] || [[ $(cat bin/charts-build-scripts) == "Not Found" ]]; then +if ! [[ -f bin/charts-build-scripts ]] || [[ "$response_code" == "404" ]]; then echo "Failed to find charts-build-scripts binary" rm bin/charts-build-scripts; exit 1 From 6d4b0edcef5d760ed0ae75f6fd8229104e4a88a0 Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Tue, 8 Oct 2024 15:03:27 -0400 Subject: [PATCH 53/54] Rename build variable to be more clear --- .github/workflows/e2e-ci.yaml | 2 +- package/Dockerfile | 4 ++-- scripts/build-chart | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/e2e-ci.yaml b/.github/workflows/e2e-ci.yaml index 30b63ed2..32b165c9 100644 --- a/.github/workflows/e2e-ci.yaml +++ b/.github/workflows/e2e-ci.yaml @@ -86,7 +86,7 @@ jobs: - name: Perform pre-e2e image build run: | - CHART_VERSION=0.3.4 REPO=${REPO} TAG=${TAG} make build; + EMBEDED_CHART_VERSION=0.3.4 REPO=${REPO} TAG=${TAG} make build; REPO=${REPO} TAG=${TAG} make package; - name : Install k3d diff --git a/package/Dockerfile b/package/Dockerfile index 22540dd4..1ae16f4d 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -21,10 +21,10 @@ FROM registry.suse.com/bci/golang:1.22 AS builder # Allow chart version config ARG TARGETPLATFORM -ARG CHART_VERSION=0.3.4 +ARG EMBEDED_CHART_VERSION=0.3.4 ARG TAG='' ARG REPO='' -ENV CHART_VERSION=$CHART_VERSION TAG=$TAG REPO=$REPO +ENV EMBEDED_CHART_VERSION=$EMBEDED_CHART_VERSION TAG=$TAG REPO=$REPO WORKDIR /usr/src/app COPY --from=helm ./helm/bin/helm /usr/local/bin/ diff --git a/scripts/build-chart b/scripts/build-chart index 27bc7e84..600d06fd 100755 --- a/scripts/build-chart +++ b/scripts/build-chart @@ -6,7 +6,7 @@ source $(dirname $0)/version cd $(dirname $0)/.. CHART=${CHART:-rancher-project-monitoring} -VERSION=${CHART_VERSION:-$(find ./charts/${CHART} -maxdepth 1 -mindepth 1 -type d | tr - \~ | sort -rV | tr \~ - | head -n1 | cut -d'/' -f4)} +VERSION=${EMBEDED_CHART_VERSION:-$(find ./charts/${CHART} -maxdepth 1 -mindepth 1 -type d | tr - \~ | sort -rV | tr \~ - | head -n1 | cut -d'/' -f4)} helm package charts/${CHART}/${VERSION} --destination bin/${CHART} base64 -i bin/${CHART}/${CHART}-${VERSION}.tgz > bin/${CHART}/${CHART}.tgz.base64 From a97f78f5762dae374380eab6edfb3a175a9c6f9b Mon Sep 17 00:00:00 2001 From: Dan Pock Date: Tue, 8 Oct 2024 15:04:26 -0400 Subject: [PATCH 54/54] Adjust build paths to not overlap --- .gitignore | 1 + docs/developing.md | 2 +- main.go | 2 +- package/Dockerfile | 2 +- scripts/build | 10 +++++----- scripts/build-chart | 8 +++++--- scripts/package | 2 +- 7 files changed, 15 insertions(+), 12 deletions(-) diff --git a/.gitignore b/.gitignore index 650bf217..bfae014b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ /.cache /bin +/build /dist *.swp .idea diff --git a/docs/developing.md b/docs/developing.md index 0800b763..04d743cf 100644 --- a/docs/developing.md +++ b/docs/developing.md @@ -102,7 +102,7 @@ If you don't want to run all the steps in CI every time you make a change, you c REPO= TAG= -./scripts/build-chart && GOOS=linux CGO_ENABLED=0 go build -ldflags "-extldflags -static -s" -o bin/prometheus-federator && REPO=${REPO} TAG=${TAG} make package +./scripts/build-chart && GOOS=linux CGO_ENABLED=0 go build -ldflags "-extldflags -static -s" -o build/bin/prometheus-federator && REPO=${REPO} TAG=${TAG} make package ``` Once the image is successfully packaged, simply run `docker push ${REPO}/prometheus-federator:${TAG}` to push your image to your Docker repository. diff --git a/main.go b/main.go index a17a0e21..81265b34 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,7 @@ var ( // SystemNamespaces is the system namespaces scoped for the embedded monitoring chart (rancher-project-monitoring) SystemNamespaces = []string{"kube-system", "cattle-monitoring-system", "cattle-dashboards"} - //go:embed bin/rancher-project-monitoring/rancher-project-monitoring.tgz.base64 + //go:embed build/chart/rancher-project-monitoring.tgz.base64 base64TgzChart string debugConfig command.DebugConfig diff --git a/package/Dockerfile b/package/Dockerfile index 1ae16f4d..46fbc9c7 100644 --- a/package/Dockerfile +++ b/package/Dockerfile @@ -40,6 +40,6 @@ RUN echo 'prometheus:x:1000:1000::/home/prometheus:/bin/bash' >> /etc/passwd && mkdir /home/prometheus && \ chown -R prometheus:prometheus /home/prometheus COPY --from=helm ./helm/bin/helm /usr/local/bin/ -COPY --from=builder /usr/src/app/bin/prometheus-federator /usr/bin/ +COPY --from=builder /usr/src/app/build/bin/prometheus-federator /usr/bin/ USER prometheus CMD ["prometheus-federator"] diff --git a/scripts/build b/scripts/build index eccfaa88..f3f04ce6 100755 --- a/scripts/build +++ b/scripts/build @@ -9,7 +9,7 @@ cd $(dirname $0)/.. echo "Starting \`prometheus-federator\` binary build:"; -mkdir -p bin +mkdir -p build/bin if [ "$(uname)" = "Linux" ]; then OTHER_LINKFLAGS="-extldflags -static -s" fi @@ -30,18 +30,18 @@ fi echo "Building for Arch(s): ${ARCHES[*]}" for A in "${ARCHES[@]}" ; do - GOARCH="$A" CGO_ENABLED=0 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o "bin/prometheus-federator-$A" + GOARCH="$A" CGO_ENABLED=0 go build -ldflags "$LINKFLAGS $OTHER_LINKFLAGS" -o "build/bin/prometheus-federator-$A" # Set CROSS to build for other OS'es if [ "$CROSS" = "true" ]; then for OS in darwin windows ; do - GOARCH="$A" GOOS=$OS go build -ldflags "$LINKFLAGS" -o "bin/prometheus-federator-$OS-$A" + GOARCH="$A" GOOS=$OS go build -ldflags "$LINKFLAGS" -o "build/bin/prometheus-federator-$OS-$A" echo "Built \`prometheus-federator-$OS-$A\`" done fi done -cd bin +cd build/bin ln -sf "./prometheus-federator-$ARCH" "./prometheus-federator" -cd .. +cd ../.. echo "Completed \`prometheus-federator\` binary build." \ No newline at end of file diff --git a/scripts/build-chart b/scripts/build-chart index 600d06fd..4196c4ef 100755 --- a/scripts/build-chart +++ b/scripts/build-chart @@ -8,8 +8,10 @@ cd $(dirname $0)/.. CHART=${CHART:-rancher-project-monitoring} VERSION=${EMBEDED_CHART_VERSION:-$(find ./charts/${CHART} -maxdepth 1 -mindepth 1 -type d | tr - \~ | sort -rV | tr \~ - | head -n1 | cut -d'/' -f4)} -helm package charts/${CHART}/${VERSION} --destination bin/${CHART} -base64 -i bin/${CHART}/${CHART}-${VERSION}.tgz > bin/${CHART}/${CHART}.tgz.base64 -rm bin/${CHART}/${CHART}-${VERSION}.tgz +mkdir -p build/bin + +helm package charts/${CHART}/${VERSION} --destination build/chart +base64 -i build/chart/${CHART}-${VERSION}.tgz > build/chart/${CHART}.tgz.base64 +rm build/chart/${CHART}-${VERSION}.tgz echo "Completed ${CHART} (${VERSION}) build process." \ No newline at end of file diff --git a/scripts/package b/scripts/package index b28ce5cd..9f7a9160 100755 --- a/scripts/package +++ b/scripts/package @@ -8,7 +8,7 @@ cd $(dirname $0)/.. echo "Starting \`prometheus-federator\` packaging:"; mkdir -p dist/artifacts -cp bin/prometheus-federator dist/artifacts/prometheus-federator${SUFFIX} +cp build/bin/prometheus-federator dist/artifacts/prometheus-federator${SUFFIX} IMAGE=${REPO}/prometheus-federator:${TAG} DOCKERFILE=package/Dockerfile