tasks: Add integration test for image-refresh via issue-scan

This covers the real thing in between getting the "issue" webhook data
and uploading the image to the S3 server, and hence `issue-scan`,
`image-refresh`, `image-create`, S3 image credentials, etc.

Extend mock-github for the operations that `image-refresh` does, like
converting an issue to a pull, posting test statuses, etc. These aren't
asserted: it's  possible in principle with keeping state in the mock,
but these aren't the parts that are sensitive to us changing our bots
infrastructure. Also, these are better done in bots' unittests than this
integration test.

tasks/mock-github

@@ -1,5 +1,5 @@
 #!/usr/bin/env python3
-# Mock GitHub API server for testing an opened PR
+# Mock GitHub API server for testing an opened PR or an issue for an image-refresh
 # You can run this manually in `tasks/run-local.sh -i` with `podman cp` and running
 # cd bots
 # PYTHONPATH=. ./mock-github cockpit-project/bots $(git rev-parse HEAD) &
@@ -27,6 +27,12 @@ class Handler(MockHandler):
             self.replyJson(self.server.data[self.path])
         elif self.path.startswith(f'/repos/{repo}/pulls?'):
             self.replyJson([self.server.data[f'/repos/{repo}/pulls/1']])
+        elif self.path == f'/repos/{repo}/pulls/2':
+            # image-refresh issue converted into PR
+            self.replyJson({
+                **self.server.data[f'/repos/{repo}/issues/2'],
+                "head": {"sha": "a1b2c3"},
+            })
         elif self.path == f'/{repo}/{sha}/.cockpit-ci/container':
             self.replyData('quay.io/cockpit/tasks')
         else:
@@ -35,6 +41,18 @@ class Handler(MockHandler):
     def do_POST(self):
         if self.path.startswith(f'/repos/{repo}/statuses/{sha}'):
             self.replyJson({})
+        # new SHA from mock-pushed PR #2 for image-refresh
+        elif self.path.startswith(f'/repos/{repo}/statuses/a1b2c3'):
+            self.replyJson({})
+        elif self.path.startswith(f'/repos/{repo}/issues/2'):
+            # updates the issue to "in progress", sets label, adds comment etc.; maybe keep state and assert?
+            self.replyJson({})
+        elif self.path == f'/repos/{repo}/pulls':
+            # image-refresh creates a PR for a refresh isssue
+            self.replyJson({
+                **GITHUB_DATA[f'/repos/{repo}/issues/2'],
+                "head": {"sha": "987654"},
+            })
         else:
             self.send_error(405, 'Method not allowed: ' + self.path)
 
@@ -43,6 +61,8 @@ argparser = argparse.ArgumentParser()
 argparser.add_argument('--port', type=int, default=8443, help="Port to listen on (default: %(default)s)")
 argparser.add_argument('--print-pr-event', action='store_true',
                        help="Print GitHub webhook pull_request event and exit")
+argparser.add_argument('--print-image-refresh-event', action='store_true',
+                       help="Print GitHub webhook issue event for an image-refresh and exit")
 argparser.add_argument('repo', metavar='USER/PROJECT', help="GitHub user/org and project name")
 argparser.add_argument('sha', help="SHA to test in repo for the mock PR")
 args = argparser.parse_args()
@@ -52,6 +72,9 @@ sha = args.sha
 ADDRESS = ('127.0.0.7', args.port)
 
 GITHUB_DATA = {
+    f'/repos/{repo}': {
+        "default_branch": "main",
+    },
     f'/repos/{repo}/pulls/1': {
         'title': 'mock PR',
         'number': 1,
@@ -67,6 +90,18 @@ GITHUB_DATA = {
         'statuses': [],
         'sha': sha,
     },
+    f'/repos/{repo}/issues/2': {
+        'title': 'Refresh foonux image',
+        'number': 2,
+        'body': "blabla\n - [ ] image-refresh foonux\n",
+        # is in our allowlist
+        'user': {"login": "cockpit-project"},
+        'labels': [{"name": "bot"}],
+        'url': f'http://{ADDRESS[0]}/{repo}/issues/2',
+    },
+    f'/repos/{repo}/git/ref/heads/main': {
+        'object': {'sha': sha},
+    },
 }
 
 if args.print_pr_event:
@@ -79,6 +114,17 @@ if args.print_pr_event:
     }, indent=4))
     exit(0)
 
+if args.print_image_refresh_event:
+    print(json.dumps({
+        'event': 'issues',
+        'request': {
+            'action': 'opened',
+            'issue': GITHUB_DATA[f'/repos/{repo}/issues/2'],
+            'repository': {'full_name': repo},
+        }
+    }, indent=4))
+    exit(0)
+
 temp = tempfile.TemporaryDirectory()
 cache_dir = os.path.join(temp.name, 'cache')
 os.environ['XDG_CACHE_HOME'] = cache_dir

tasks/run-local.sh

@@ -176,6 +176,7 @@ EOF
 
     # Run tasks container in the background
     # use bash as pid 1 to mop up zombies
+    # we always want to upload images to our local S3 store
     podman run -d -it --name cockpituous-tasks --pod=cockpituous \
         --security-opt=label=disable \
         -v "$SECRETS"/tasks:/run/secrets/tasks:ro \
@@ -189,6 +190,7 @@ EOF
         --env=AMQP_SERVER=$AMQP_POD \
         --env=S3_LOGS_URL=$S3_URL_POD/logs/ \
         --env=COCKPIT_S3_KEY_DIR=/run/secrets/tasks/s3-keys \
+        --env=COCKPIT_IMAGE_UPLOAD_STORE=$S3_URL_POD/images/ \
         --env=SKIP_STATIC_CHECK=1 \
         quay.io/cockpit/tasks:${TASKS_TAG:-latest} bash
 
@@ -340,6 +342,76 @@ test_pr() {
     fi
 }
 
+test_mock_image_refresh() {
+    podman cp "$MYDIR/mock-github" cockpituous-tasks:/work/bots/mock-github
+
+    # the last step of an image refresh is to push the branch back to origin; we can't nor
+    # want to do that here, so divert "git push" to a log file and check that
+    cat <<EOF | podman exec -i -u root cockpituous-tasks sh -euxc "cat > /usr/local/bin/git; chmod +x /usr/local/bin/git"
+#!/bin/sh
+if [ "\$1" = push ]; then
+    echo "\$@" >> /work/git-push.log
+    exit 0
+fi
+exec /usr/bin/git "\$@"
+EOF
+
+    podman exec -i cockpituous-tasks sh -euxc "
+        cd bots
+        # start mock GH server; use a mock SHA, only used for posting statuses
+        SHA=123abc
+        PYTHONPATH=. ./mock-github cockpit-project/bots \$SHA &
+        GH_MOCK_PID=\$!
+        export GITHUB_API=http://127.0.0.7:8443
+        until curl --silent \$GITHUB_API; do sleep 0.1; done
+
+        # simulate GitHub webhook event, put that into the webhook queue
+        PYTHONPATH=. ./mock-github --print-image-refresh-event cockpit-project/bots \$SHA | \
+            ./publish-queue --amqp $AMQP_POD --create --queue webhook
+
+        ./inspect-queue --amqp $AMQP_POD
+
+        # first run-queue processes webhook → issue-scan → public queue
+        ./run-queue --amqp $AMQP_POD
+        ./inspect-queue --amqp $AMQP_POD
+
+        # second run-queue actually runs the image refresh
+        ./run-queue --amqp $AMQP_POD
+
+        kill \$GH_MOCK_PID
+    "
+
+    # successful refresh log
+    LOGS_URL="$S3_URL_HOST/logs/"
+    CURL="curl --cacert $SECRETS/ca.pem --silent --fail --show-error"
+    LOG_MATCH="$($CURL $LOGS_URL| grep -o "image-refresh-foonux-[[:alnum:]-]*/log<")"
+    LOG="$($CURL "${LOGS_URL}${LOG_MATCH%<}")"
+    echo "--------------- mock image-refresh test log -----------------"
+    echo  "$LOG"
+    echo "--------------- mock image-refresh test log end -------------"
+    assert_in 'Running on:.*cockpituous' "$LOG"
+    assert_in './image-create.*foonux' "$LOG"
+    assert_in "Uploading to $S3_URL_POD/images/foonux.*qcow2" "$LOG"
+    assert_in 'Success.' "$LOG"
+
+    # branch was (mock) pushed
+    PUSH_LOG="$(podman exec -i cockpituous-tasks cat /work/git-push.log)"
+    assert_in 'push origin +HEAD:refs/heads/image-refresh-foonux-' "$PUSH_LOG"
+    podman exec -i cockpituous-tasks rm /work/git-push.log
+    podman exec -i -u root cockpituous-tasks rm /usr/local/bin/git
+
+    podman exec -i cockpituous-tasks sh -euxc '
+        # validate image contents
+        qemu-img convert /cache/images/foonux-*.qcow2 /tmp/foonux.raw
+        grep "^fakeimage" /tmp/foonux.raw
+        rm /tmp/foonux.raw
+
+        # image is on the S3 server
+        cd bots
+        python3 -m lib.s3 ls '$S3_URL_POD'/images/ | grep "foonux.*qcow"
+    '
+}
+
 test_queue() {
     # tasks can connect to queue
     OUT=$(podman exec -i cockpituous-tasks bots/inspect-queue --amqp $AMQP_POD)
@@ -376,6 +448,8 @@ else
     test_podman
     # "almost" end-to-end, starting with GitHub webhook JSON payload injection; fully localy, no privs
     test_mock_pr
+    # similar structure for issue-scan for an image refresh
+    test_mock_image_refresh
     # if we have a PR number, run a unit test inside local deployment, and update PR status
     [ -z "$PR" ] || test_pr
 fi