Merge pull request Kaggle#394 from Kaggle/package-diff-ci

Show package versions diff in the docker build pipeline

Author: rosbo

Jenkinsfile

@@ -96,6 +96,28 @@ pipeline {
         '''
       }
     }
+
+    stage('Package Versions') {
+      parallel {
+        stage('CPU Diff') {
+          steps {
+            slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} diff CPU image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
+            sh '''#!/bin/bash
+            ./diff
+          '''
+          }
+        }
+        stage('GPU Diff') {
+          agent { label 'ephemeral-linux-gpu' }
+          steps {
+            slackSend color: 'none', message: "*<${env.BUILD_URL}console|${JOB_NAME} diff GPU image>* ${GIT_COMMIT_SUMMARY}", channel: env.SLACK_CHANNEL
+            sh '''#!/bin/bash
+            ./diff --gpu
+          '''
+          }
+        }
+      }
+    }
   }
 
   post {

diff

@@ -0,0 +1,55 @@
+#!/bin/bash
+set -e
+
+usage() {
+cat << EOF
+Usage: $0 [OPTIONS]
+Compare the locally built Docker image package versions against the prod image.
+
+Options:
+    -g, --gpu       Compare GPU images.
+EOF
+}
+
+
+BASE_IMAGE_TAG="gcr.io/kaggle-images/python:latest"
+TARGET_IMAGE_TAG="kaggle/python-build"
+
+while :; do
+    case "$1" in 
+        -h|--help)
+            usage
+            exit
+            ;;
+        -g|--gpu)
+            BASE_IMAGE_TAG="gcr.io/kaggle-private-byod/python:latest"
+            TARGET_IMAGE_TAG="kaggle/python-gpu-build"
+            ;;
+        -?*)
+            usage
+            printf 'ERROR: Unknown option: %s\n' "$1" >&2
+            exit
+            ;;
+        *)            
+            break
+    esac
+
+    shift
+done
+
+readonly BASE_IMAGE_TAG
+readonly TARGET_IMAGE_TAG
+
+echo "Base: $BASE_IMAGE_TAG"
+echo "Target: $TARGET_IMAGE_TAG"
+
+docker pull "$BASE_IMAGE_TAG"
+
+CMDS=('dpkg-query --show -f "${Package}==${Version}\n"' 'pip freeze')
+for cmd in "${CMDS[@]}"; do
+    echo "== Comparing $cmd =="
+    diff --suppress-common-lines --side-by-side \
+        <(docker run --rm "$BASE_IMAGE_TAG" $cmd) \
+        <(docker run --rm "$TARGET_IMAGE_TAG" $cmd) \
+        && echo "No diff" || true
+done