From 6f2ee02343d0cdf7cdf9d1318c7d28521f83af08 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 20 Sep 2024 13:05:54 -0700 Subject: [PATCH 01/10] feat: run blob reaper every 24h --- .github/workflows/main.yaml | 11 ++++++- kustomize/overlays/dev/registry.yaml | 45 +++++++--------------------- okteto.yml | 8 +++++ registry/garbage-collect.sh | 2 +- 4 files changed, 29 insertions(+), 37 deletions(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 4b97aa0..db51b67 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -11,7 +11,7 @@ jobs: env: HEROKU_API_KEY: ${{secrets.HEROKU_API_KEY}} steps: - - name: Checkout + - name: checkout uses: actions/checkout@v2 - name: heroku registry login @@ -52,3 +52,12 @@ jobs: - name: release reaper run: heroku container:release -a ttlsh-hooks reap + + - name: build blob reaper + run: docker build -f Dockerfile.blob-reap -t registry.heroku.com/ttlsh-hooks/blob-reap registry + + - name: push blob reaper + run: docker push registry.heroku.com/ttlsh-hooks/blob-reap + + - name: release blob reaper + run: heroku container:release -a ttlsh-hooks blob-reap diff --git a/kustomize/overlays/dev/registry.yaml b/kustomize/overlays/dev/registry.yaml index 8d8aded..4d3245b 100644 --- a/kustomize/overlays/dev/registry.yaml +++ b/kustomize/overlays/dev/registry.yaml @@ -16,19 +16,13 @@ spec: spec: containers: - name: ttl-registry - image: registry:2 + image: ttl-registry command: ["/entrypoint.sh"] ports: - containerPort: 5000 volumeMounts: - name: registry-data mountPath: /var/lib/registry - - name: registry-scripts - mountPath: /entrypoint.sh - subPath: entrypoint.sh - - name: registry-scripts - mountPath: /garbage-collect.sh - subPath: garbage-collect.sh - name: registry-config mountPath: /etc/docker/registry/config.yml subPath: config.yml @@ -42,40 +36,21 @@ spec: path: / port: 5000 scheme: HTTP + - name: ttl-blob-reap + image: ttl-blob-reap + command: ["/garbage-collect.sh"] + volumeMounts: + - name: registry-data + mountPath: /var/lib/registry + - name: registry-config + mountPath: /etc/docker/registry/config.yml + subPath: config.yml volumes: - name: registry-data emptyDir: {} - name: registry-config configMap: name: ttl-registry-config - - name: registry-scripts - configMap: - name: ttl-registry-scripts - defaultMode: 0755 ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: ttl-registry-scripts - labels: - app: ttl-registry -data: - entrypoint.sh: | - #!/bin/sh - set -e - # /garbage-collect.sh & - registry serve /etc/docker/registry/config.yml - garbage-collect.sh: | - #!/bin/sh - while true; do - sleep 1m - if [ ! -d "/var/lib/registry/docker" ]; then - echo "No registry data found, skipping garbage collection" - else - echo "Starting garbage collection..." - registry garbage-collect /etc/docker/registry/config.yml || true - fi - done --- apiVersion: v1 kind: ConfigMap diff --git a/okteto.yml b/okteto.yml index 2a8d697..88410a1 100644 --- a/okteto.yml +++ b/okteto.yml @@ -1,4 +1,10 @@ build: + ttl-registry: + context: ./registry + dockerfile: ./registry/Dockerfile + ttl-blob-reap: + context: ./registry + dockerfile: ./registry/Dockerfile.blob-reap ttl-hooks: context: ./hooks dockerfile: ./hooks/Dockerfile.hooks @@ -7,6 +13,8 @@ build: dockerfile: ./hooks/Dockerfile.reap deploy: + - cd kustomize/overlays/dev && kustomize edit set image ttl-registry=${OKTETO_BUILD_REGISTRY_IMAGE} + - cd kustomize/overlays/dev && kustomize edit set image ttl-blob-reap=${OKTETO_BUILD_TTL_BLOB_REAP_IMAGE} - cd kustomize/overlays/dev && kustomize edit set image ttl-hooks=${OKTETO_BUILD_TTL_HOOKS_IMAGE} - cd kustomize/overlays/dev && kustomize edit set image ttl-reaper=${OKTETO_BUILD_TTL_REAPER_IMAGE} diff --git a/registry/garbage-collect.sh b/registry/garbage-collect.sh index 93c424a..aeb7543 100755 --- a/registry/garbage-collect.sh +++ b/registry/garbage-collect.sh @@ -1,7 +1,7 @@ #!/bin/sh while true; do - sleep 1m + sleep 24h echo "Starting garbage collection..." registry garbage-collect /etc/docker/registry/config.yml || true done From d0692e66a88c0ddc7c816c0e2e8e84c9ffacc195 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 20 Sep 2024 13:07:09 -0700 Subject: [PATCH 02/10] f --- registry/Dockerfile.blob-reap | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 registry/Dockerfile.blob-reap diff --git a/registry/Dockerfile.blob-reap b/registry/Dockerfile.blob-reap new file mode 100644 index 0000000..a499bd2 --- /dev/null +++ b/registry/Dockerfile.blob-reap @@ -0,0 +1,9 @@ +# built from https://github.com/sgalsaleh/distribution/tree/custom-garbage-collect +FROM replicated/registry-blob-reap:latest + +ADD ./entrypoint.sh /entrypoint.sh +ADD ./config.yml /etc/docker/registry/config.yml +ADD ./garbage-collect.sh /garbage-collect.sh + +ENTRYPOINT ["/garbage-collect.sh"] +CMD ["/etc/docker/registry/config.yml"] From c88543583389a3c0287b590364729077d33b899b Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 20 Sep 2024 13:21:16 -0700 Subject: [PATCH 03/10] f --- okteto.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/okteto.yml b/okteto.yml index 88410a1..ed270e9 100644 --- a/okteto.yml +++ b/okteto.yml @@ -13,7 +13,7 @@ build: dockerfile: ./hooks/Dockerfile.reap deploy: - - cd kustomize/overlays/dev && kustomize edit set image ttl-registry=${OKTETO_BUILD_REGISTRY_IMAGE} + - cd kustomize/overlays/dev && kustomize edit set image ttl-registry=${OKTETO_BUILD_TTL_REGISTRY_IMAGE} - cd kustomize/overlays/dev && kustomize edit set image ttl-blob-reap=${OKTETO_BUILD_TTL_BLOB_REAP_IMAGE} - cd kustomize/overlays/dev && kustomize edit set image ttl-hooks=${OKTETO_BUILD_TTL_HOOKS_IMAGE} - cd kustomize/overlays/dev && kustomize edit set image ttl-reaper=${OKTETO_BUILD_TTL_REAPER_IMAGE} From 0516e6f704cf630b9a4a57ed5fa7df9366b529e8 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 20 Sep 2024 13:29:41 -0700 Subject: [PATCH 04/10] f --- registry/entrypoint.sh | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/registry/entrypoint.sh b/registry/entrypoint.sh index fb74653..cf9edee 100755 --- a/registry/entrypoint.sh +++ b/registry/entrypoint.sh @@ -2,11 +2,12 @@ set -e -sed -i "s/__PORT__/$PORT/g" /etc/docker/registry/config.yml -sed -i "s/__HOOK_TOKEN__/$HOOK_TOKEN/g" /etc/docker/registry/config.yml -sed -i "s/__HOOK_URI__/$HOOK_URI/g" /etc/docker/registry/config.yml -sed -i "s/__REPLREG_HOST__/$REPLREG_HOST/g" /etc/docker/registry/config.yml -sed -i "s/__REPLREG_SECRET__/$REPLREG_SECRET/g" /etc/docker/registry/config.yml +cp /etc/docker/registry/config.yml /etc/docker/registry/config.yml.bak +sed -i "s/__PORT__/$PORT/g" /etc/docker/registry/config.yml.bak +sed -i "s/__HOOK_TOKEN__/$HOOK_TOKEN/g" /etc/docker/registry/config.yml.bak +sed -i "s/__HOOK_URI__/$HOOK_URI/g" /etc/docker/registry/config.yml.bak +sed -i "s/__REPLREG_HOST__/$REPLREG_HOST/g" /etc/docker/registry/config.yml.bak +sed -i "s/__REPLREG_SECRET__/$REPLREG_SECRET/g" /etc/docker/registry/config.yml.bak if [[ -z "${GCS_KEY_ENCODED}" ]]; then echo "Set GCS_KEY_ENCODED variable" @@ -19,8 +20,8 @@ fi # /garbage-collect.sh & case "$1" in - *.yaml|*.yml) set -- registry serve "$@" ;; - serve|garbage-collect|help|-*) set -- registry "$@" ;; + *.yaml|*.yml) set -- registry serve /etc/docker/registry/config.yml.bak ;; + serve|garbage-collect|help|-*) set -- registry "$1" /etc/docker/registry/config.yml.bak ;; esac exec "$@" From 59a13add25157c858549ff8344f7effc6e8b3600 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 20 Sep 2024 13:36:26 -0700 Subject: [PATCH 05/10] f --- registry/Dockerfile | 5 ++++- registry/Dockerfile.blob-reap | 5 ++++- registry/entrypoint.sh | 27 +++++++++++---------------- 3 files changed, 19 insertions(+), 18 deletions(-) diff --git a/registry/Dockerfile b/registry/Dockerfile index ed08947..36a14d2 100644 --- a/registry/Dockerfile +++ b/registry/Dockerfile @@ -1,8 +1,11 @@ FROM registry:2.7.1 ADD ./entrypoint.sh /entrypoint.sh -ADD ./config.yml /etc/docker/registry/config.yml +ADD ./config.yml /etc/docker/registry/config.tmpl.yml ADD ./garbage-collect.sh /garbage-collect.sh +# entrypoint.sh will copy config.tmpl.yml to config.yml and replace variables +RUN rm -f /etc/docker/registry/config.yml + ENTRYPOINT ["/entrypoint.sh"] CMD ["/etc/docker/registry/config.yml"] diff --git a/registry/Dockerfile.blob-reap b/registry/Dockerfile.blob-reap index a499bd2..858590b 100644 --- a/registry/Dockerfile.blob-reap +++ b/registry/Dockerfile.blob-reap @@ -2,8 +2,11 @@ FROM replicated/registry-blob-reap:latest ADD ./entrypoint.sh /entrypoint.sh -ADD ./config.yml /etc/docker/registry/config.yml +ADD ./config.yml /etc/docker/registry/config.tmpl.yml ADD ./garbage-collect.sh /garbage-collect.sh +# entrypoint.sh will copy config.tmpl.yml to config.yml and replace variables +RUN rm -f /etc/docker/registry/config.yml + ENTRYPOINT ["/garbage-collect.sh"] CMD ["/etc/docker/registry/config.yml"] diff --git a/registry/entrypoint.sh b/registry/entrypoint.sh index cf9edee..fcdd0fe 100755 --- a/registry/entrypoint.sh +++ b/registry/entrypoint.sh @@ -2,26 +2,21 @@ set -e -cp /etc/docker/registry/config.yml /etc/docker/registry/config.yml.bak -sed -i "s/__PORT__/$PORT/g" /etc/docker/registry/config.yml.bak -sed -i "s/__HOOK_TOKEN__/$HOOK_TOKEN/g" /etc/docker/registry/config.yml.bak -sed -i "s/__HOOK_URI__/$HOOK_URI/g" /etc/docker/registry/config.yml.bak -sed -i "s/__REPLREG_HOST__/$REPLREG_HOST/g" /etc/docker/registry/config.yml.bak -sed -i "s/__REPLREG_SECRET__/$REPLREG_SECRET/g" /etc/docker/registry/config.yml.bak +# in dev we mount this file into the container, so we can ignore the template +if [ ! -f /etc/docker/registry/config.yml ]; then + cp /etc/docker/registry/config.tmpl.yml /etc/docker/registry/config.yml + sed -i "s/__PORT__/$PORT/g" /etc/docker/registry/config.yml + sed -i "s/__HOOK_TOKEN__/$HOOK_TOKEN/g" /etc/docker/registry/config.yml + sed -i "s/__HOOK_URI__/$HOOK_URI/g" /etc/docker/registry/config.yml + sed -i "s/__REPLREG_HOST__/$REPLREG_HOST/g" /etc/docker/registry/config.yml + sed -i "s/__REPLREG_SECRET__/$REPLREG_SECRET/g" /etc/docker/registry/config.yml +fi -if [[ -z "${GCS_KEY_ENCODED}" ]]; then +if [ -z "${GCS_KEY_ENCODED}" ]; then echo "Set GCS_KEY_ENCODED variable" else - echo ${GCS_KEY_ENCODED} | base64 -d > /etc/gcs.json + echo "${GCS_KEY_ENCODED}" | base64 -d > /etc/gcs.json chmod 0400 /etc/gcs.json fi -# Run garbage collection job in background -# /garbage-collect.sh & - -case "$1" in - *.yaml|*.yml) set -- registry serve /etc/docker/registry/config.yml.bak ;; - serve|garbage-collect|help|-*) set -- registry "$1" /etc/docker/registry/config.yml.bak ;; -esac - exec "$@" From b391cbb1f30a55e81fcb93fb5a0880882ab2f2a6 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 20 Sep 2024 13:39:50 -0700 Subject: [PATCH 06/10] f --- kustomize/overlays/dev/registry.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kustomize/overlays/dev/registry.yaml b/kustomize/overlays/dev/registry.yaml index 4d3245b..27beef5 100644 --- a/kustomize/overlays/dev/registry.yaml +++ b/kustomize/overlays/dev/registry.yaml @@ -17,7 +17,7 @@ spec: containers: - name: ttl-registry image: ttl-registry - command: ["/entrypoint.sh"] + command: ["registry", "serve", "/etc/docker/registry/config.yml"] ports: - containerPort: 5000 volumeMounts: From 969d795d2d8d330306ec5006fb03997fd99719f2 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 20 Sep 2024 13:41:22 -0700 Subject: [PATCH 07/10] f --- registry/garbage-collect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry/garbage-collect.sh b/registry/garbage-collect.sh index aeb7543..d9865bc 100755 --- a/registry/garbage-collect.sh +++ b/registry/garbage-collect.sh @@ -1,7 +1,7 @@ #!/bin/sh while true; do - sleep 24h echo "Starting garbage collection..." registry garbage-collect /etc/docker/registry/config.yml || true + sleep 24h done From 1c64ce8dec6568d4333edbbbb46f58f8879d4183 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 20 Sep 2024 13:53:23 -0700 Subject: [PATCH 08/10] f --- registry/garbage-collect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry/garbage-collect.sh b/registry/garbage-collect.sh index d9865bc..aeb7543 100755 --- a/registry/garbage-collect.sh +++ b/registry/garbage-collect.sh @@ -1,7 +1,7 @@ #!/bin/sh while true; do + sleep 24h echo "Starting garbage collection..." registry garbage-collect /etc/docker/registry/config.yml || true - sleep 24h done From be168f863b7da9ed69ca729e7dc5b73055e12583 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 20 Sep 2024 13:54:00 -0700 Subject: [PATCH 09/10] f --- registry/garbage-collect.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry/garbage-collect.sh b/registry/garbage-collect.sh index aeb7543..ea62ca3 100755 --- a/registry/garbage-collect.sh +++ b/registry/garbage-collect.sh @@ -1,7 +1,7 @@ #!/bin/sh while true; do - sleep 24h + sleep 72h # every 3 days echo "Starting garbage collection..." registry garbage-collect /etc/docker/registry/config.yml || true done From bd98dc45ce5b46fe1b4b9998283ae378d4731268 Mon Sep 17 00:00:00 2001 From: Ethan Mosbaugh Date: Fri, 20 Sep 2024 13:55:16 -0700 Subject: [PATCH 10/10] f --- .github/workflows/main.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index db51b67..cca93a8 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -54,7 +54,7 @@ jobs: run: heroku container:release -a ttlsh-hooks reap - name: build blob reaper - run: docker build -f Dockerfile.blob-reap -t registry.heroku.com/ttlsh-hooks/blob-reap registry + run: docker build -f registry/Dockerfile.blob-reap -t registry.heroku.com/ttlsh-hooks/blob-reap registry - name: push blob reaper run: docker push registry.heroku.com/ttlsh-hooks/blob-reap