Skip to content

Commit 9a915e1

Browse files
author
Mathis Marcotte
committed
fix workflow
1 parent 84f323b commit 9a915e1

File tree

2 files changed

+286
-1
lines changed

2 files changed

+286
-1
lines changed

‎.github/workflows/build_push.yaml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ jobs:
281281
- name: Pull built base jupyterlab image
282282
run: |
283283
docker images
284-
docker pull $REGISTRY/jupyterlab:$(GIT_SHA)
284+
docker pull $REGISTRY/jupyterlab:$CI_COMMIT_REF_NAME
285285
docker tag ${{ needs.build-jupyter.outputs.jupyter-image-name }} zone-jupyterlab
286286
287287
# make build emits full_image_name, image_tag, and image_repo outputs

‎.github/workflows/build_push2.yaml

Lines changed: 285 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,285 @@
1+
# This workflow:
2+
# * Builds, tests, and scans all images
3+
# * (optionally) pushes the images to ACR
4+
#
5+
# This workflow triggers on:
6+
# * a push to master
7+
# * any create/synchronize to a PR (eg: any time you push an update to a PR).
8+
#
9+
# Image build/test/scan will run on any of the above events.
10+
# Image push will run only if:
11+
# * this is a push to master
12+
# * if the PR triggering this event has the label 'auto-deploy'
13+
#
14+
# To configure this workflow:
15+
#
16+
# 1. Set up the following secrets in your workspace:
17+
# a. REGISTRY_USERNAME with ACR username
18+
# b. REGISTRY_PASSWORD with ACR Password
19+
# c. AZURE_CREDENTIALS with the output of `az ad sp create-for-rbac --sdk-auth`
20+
# d. DEV_REGISTRY_USERNAME with the DEV ACR username
21+
# e. DEV_REGISTRY_PASSWORD with the DEV ACR Password
22+
#
23+
# 2. Change the values for the REGISTRY_NAME, CLUSTER_NAME, CLUSTER_RESOURCE_GROUP and NAMESPACE environment variables (below in build-push).
24+
name: build_and_push2
25+
on:
26+
schedule:
27+
# Execute at 2am EST every day
28+
- cron: '0 21 * * *'
29+
push:
30+
branches:
31+
- 'master-2.0'
32+
pull_request:
33+
types:
34+
- 'opened'
35+
- 'synchronize'
36+
- 'reopened'
37+
env:
38+
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
39+
40+
jobs:
41+
# Any checks that run pre-build
42+
# pre-build-checks:
43+
# runs-on: ubuntu-latest
44+
# steps:
45+
# - uses: actions/checkout@master
46+
47+
# - name: Assert committed ./output folder matches `make generate-dockerfiles` output
48+
# run: |
49+
# sudo apt-get install --yes make
50+
# make clean
51+
# make generate-dockerfiles
52+
# if ! git diff --quiet output/; then
53+
# echo 'output folder and docker-bits/resources out of sync!'
54+
# exit 1
55+
# fi
56+
57+
build-jupyter2:
58+
env:
59+
REGISTRY_NAME: k8scc01covidacr
60+
DEV_REGISTRY_NAME: k8scc01covidacrdev
61+
CLUSTER_NAME: k8s-cancentral-01-covid-aks
62+
CLUSTER_RESOURCE_GROUP: k8s-cancentral-01-covid-aks
63+
LOCAL_REPO: localhost:5000
64+
TRIVY_VERSION: "v0.31.3"
65+
HADOLINT_VERSION: "2.12.0"
66+
strategy:
67+
fail-fast: false
68+
# needs: pre-build-checks
69+
runs-on: ubuntu-latest
70+
services:
71+
registry:
72+
image: registry:2
73+
ports:
74+
- 5000:5000
75+
outputs:
76+
jupyter-image-name: ${{ steps.build-image.outputs.full_image_name }}
77+
steps:
78+
- name: Set ENV variables for a PR containing the auto-deploy tag
79+
if: github.event_name == 'pull_request' && contains( github.event.pull_request.labels.*.name, 'auto-deploy')
80+
run: |
81+
echo "REGISTRY=k8scc01covidacrdev.azurecr.io" >> "$GITHUB_ENV"
82+
echo "IMAGE_VERSION=dev" >> "$GITHUB_ENV"
83+
84+
- name: Set ENV variables for pushes to master
85+
if: github.event_name == 'push' && github.ref == 'refs/heads/master-2.0'
86+
run: |
87+
echo "REGISTRY=k8scc01covidacr.azurecr.io" >> "$GITHUB_ENV"
88+
echo "IMAGE_VERSION=v2" >> "$GITHUB_ENV"
89+
echo "IS_LATEST=true" >> "$GITHUB_ENV"
90+
91+
- uses: actions/checkout@master
92+
93+
- name: Echo disk usage before clean up
94+
run: ./.github/scripts/echo_usage.sh
95+
96+
- name: Free up all available disk space before building
97+
run: ./.github/scripts/cleanup_runner.sh
98+
99+
- name: Echo disk usage before build start
100+
run: ./.github/scripts/echo_usage.sh
101+
102+
- name: Get current notebook name
103+
id: notebook-name
104+
shell: bash
105+
run: |
106+
echo NOTEBOOK_NAME=jupyterlab >> $GITHUB_OUTPUT
107+
108+
# Connect to Azure Container registry (ACR)
109+
- uses: azure/docker-login@v1
110+
with:
111+
login-server: ${{ env.REGISTRY_NAME }}.azurecr.io
112+
username: ${{ secrets.REGISTRY_USERNAME }}
113+
password: ${{ secrets.REGISTRY_PASSWORD }}
114+
115+
# Connect to Azure DEV Container registry (ACR)
116+
- uses: azure/docker-login@v1
117+
with:
118+
login-server: ${{ env.DEV_REGISTRY_NAME }}.azurecr.io
119+
username: ${{ secrets.DEV_REGISTRY_USERNAME }}
120+
password: ${{ secrets.DEV_REGISTRY_PASSWORD }}
121+
122+
- name: Run Hadolint
123+
run: |
124+
sudo curl -L https://github.com/hadolint/hadolint/releases/download/v${{ env.HADOLINT_VERSION }}/hadolint-Linux-x86_64 --output hadolint
125+
sudo chmod +x hadolint
126+
./hadolint dockerfiles/jupyterlab/Dockerfile --no-fail
127+
128+
# make build emits full_image_name, image_tag, and image_repo outputs
129+
- name: Build image
130+
id: build-image
131+
run: make build/jupyterlab REPO=${{ env.LOCAL_REPO }}
132+
133+
- name: Echo disk usage after build completion
134+
run: ./.github/scripts/echo_usage.sh
135+
136+
- name: Add standard tag names (short sha, sha, and branch) and any other post-build activity
137+
run: make post-build/jupyterlab REPO=${{ env.LOCAL_REPO }}
138+
139+
- name: Push image to local registry (default pushes all tags)
140+
run: make push/jupyterlab REPO=${{ env.LOCAL_REPO }}
141+
# Image testing
142+
143+
- name: Set Up Python for Test Suite
144+
uses: actions/setup-python@v4
145+
with:
146+
python-version: '3.10'
147+
148+
- name: Set up venv for Test Suite
149+
run: |
150+
python -m pip install --upgrade pip
151+
make install-python-dev-venv
152+
153+
- name: Test image
154+
run: make test/jupyterlab REPO=${{ env.LOCAL_REPO }}
155+
156+
# Free up space from build process (containerscan action will run out of space if we don't)
157+
- run: ./.github/scripts/cleanup_runner.sh
158+
159+
# Scan image for vulnerabilities
160+
- name: Aqua Security Trivy image scan
161+
# see https://github.com/StatCan/aaw-private/issues/11 -- should be re-enabled
162+
if: steps.notebook-name.outputs.NOTEBOOK_NAME != 'sas'
163+
run: |
164+
printf ${{ secrets.CVE_ALLOWLIST }} > .trivyignore
165+
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin ${{ env.TRIVY_VERSION }}
166+
trivy image ${{ steps.build-image.outputs.full_image_name }} --exit-code 1 --timeout=20m --security-checks vuln --severity CRITICAL
167+
168+
# Push image to ACR
169+
# Pushes if this is a push to master or an update to a PR that has auto-deploy label
170+
- name: Test if we should push to ACR
171+
id: should-i-push
172+
if: |
173+
github.event_name == 'push' ||
174+
(
175+
github.event_name == 'pull_request' &&
176+
contains( github.event.pull_request.labels.*.name, 'auto-deploy')
177+
)
178+
run: echo 'boolean=true' >> $GITHUB_OUTPUT
179+
180+
# Pull the local image back, then "build" it (will just tag the pulled image)
181+
- name: Pull image back from local repo
182+
if: steps.should-i-push.outputs.boolean == 'true'
183+
run: docker pull ${{ steps.build-image.outputs.full_image_name }}
184+
185+
# Rename the localhost:5000/imagename:tag built above to use the real repo
186+
# (get above's name from build-image's output)
187+
- name: Tag images with real repository
188+
if: steps.should-i-push.outputs.boolean == 'true'
189+
run: >
190+
make post-build/jupyterlab DEFAULT_REPO=$REGISTRY IS_LATEST=$IS_LATEST
191+
IMAGE_VERSION=$IMAGE_VERSION SOURCE_FULL_IMAGE_NAME=${{ steps.build-image.outputs.full_image_name }}
192+
193+
- name: Push image to registry
194+
if: steps.should-i-push.outputs.boolean == 'true'
195+
run: |
196+
make push/jupyterlab DEFAULT_REPO=$REGISTRY
197+
198+
## SAS BUILD
199+
- name: Get current notebook name
200+
id: notebook-name
201+
shell: bash
202+
run: |
203+
echo NOTEBOOK_NAME=sas >> $GITHUB_OUTPUT
204+
205+
- name: Run Hadolint
206+
run: |
207+
sudo curl -L https://github.com/hadolint/hadolint/releases/download/v${{ env.HADOLINT_VERSION }}/hadolint-Linux-x86_64 --output hadolint
208+
sudo chmod +x hadolint
209+
./hadolint dockerfiles/sas/Dockerfile --no-fail
210+
211+
- name: Build image
212+
id: build-image
213+
run: make build/sas REPO=${{ env.LOCAL_REPO }}
214+
215+
- name: Echo disk usage after build completion
216+
run: ./.github/scripts/echo_usage.sh
217+
218+
- name: Add standard tag names (short sha, sha, and branch) and any other post-build activity
219+
run: make post-build/sas REPO=${{ env.LOCAL_REPO }}
220+
221+
- name: Push image to local registry (default pushes all tags)
222+
run: make push/sas REPO=${{ env.LOCAL_REPO }}
223+
# Image testing
224+
225+
- name: Set Up Python for Test Suite
226+
uses: actions/setup-python@v4
227+
with:
228+
python-version: '3.10'
229+
230+
- name: Set up venv for Test Suite
231+
run: |
232+
python -m pip install --upgrade pip
233+
make install-python-dev-venv
234+
235+
- name: Test image
236+
run: make test/sas REPO=${{ env.LOCAL_REPO }}
237+
238+
# Free up space from build process (containerscan action will run out of space if we don't)
239+
- run: ./.github/scripts/cleanup_runner.sh
240+
241+
# Scan image for vulnerabilities
242+
- name: Aqua Security Trivy image scan
243+
# see https://github.com/StatCan/aaw-private/issues/11 -- should be re-enabled
244+
if: steps.notebook-name.outputs.NOTEBOOK_NAME != 'sas'
245+
run: |
246+
printf ${{ secrets.CVE_ALLOWLIST }} > .trivyignore
247+
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin ${{ env.TRIVY_VERSION }}
248+
trivy image ${{ steps.build-image.outputs.full_image_name }} --exit-code 1 --timeout=20m --security-checks vuln --severity CRITICAL
249+
250+
# Push image to ACR
251+
# Pushes if this is a push to master or an update to a PR that has auto-deploy label
252+
- name: Test if we should push to ACR
253+
id: should-i-push
254+
if: |
255+
github.event_name == 'push' ||
256+
(
257+
github.event_name == 'pull_request' &&
258+
contains( github.event.pull_request.labels.*.name, 'auto-deploy')
259+
)
260+
run: echo 'boolean=true' >> $GITHUB_OUTPUT
261+
262+
# Pull the local image back, then "build" it (will just tag the pulled image)
263+
- name: Pull image back from local repo
264+
if: steps.should-i-push.outputs.boolean == 'true'
265+
run: docker pull ${{ steps.build-image.outputs.full_image_name }}
266+
267+
# Rename the localhost:5000/imagename:tag built above to use the real repo
268+
# (get above's name from build-image's output)
269+
- name: Tag images with real repository
270+
if: steps.should-i-push.outputs.boolean == 'true'
271+
run: >
272+
make post-build/sas DEFAULT_REPO=$REGISTRY IS_LATEST=$IS_LATEST
273+
IMAGE_VERSION=$IMAGE_VERSION SOURCE_FULL_IMAGE_NAME=${{ steps.build-image.outputs.full_image_name }}
274+
275+
- name: Push image to registry
276+
if: steps.should-i-push.outputs.boolean == 'true'
277+
run: |
278+
make push/sas DEFAULT_REPO=$REGISTRY
279+
280+
- name: Slack Notification
281+
if: failure() && github.event_name=='schedule'
282+
uses: act10ns/slack@v1
283+
with:
284+
status: failure
285+
message: Build failed. https://github.com/StatCan/aaw-kubeflow-containers/actions/runs/${{github.run_id}}

0 commit comments

Comments
 (0)