[Enhance] Add CI for turbo and enhance other tests

.github/workflows/ci-basic.yml

@@ -5,14 +5,17 @@ on:
       image-tag:
         required: true
         type: string
+      github-repository:
+        required: true
+        type: string
 
 jobs:
   run-ci-basic:
     name: Run CI | Basic
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     container:
-      image: ghcr.io/${{ github.repository }}/overlaybd-ci-images:${{ inputs.image-tag }}
+      image: ghcr.io/${{ inputs.github-repository }}/overlaybd-ci-images:${{ inputs.image-tag }}
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-build-image.yml

@@ -8,6 +8,9 @@ on:
       image-tag:
         required: true
         type: string
+      github-repository:
+        required: true
+        type: string
 
 env:
   GO_VERSION: "1.19"
@@ -51,7 +54,7 @@ jobs:
       - name: Build and Push
         uses: docker/build-push-action@v4
         with:
-          tags: ghcr.io/${{ github.repository }}/overlaybd-ci-images:${{ inputs.image-tag }}
+          tags: ghcr.io/${{ inputs.github-repository }}/overlaybd-ci-images:${{ inputs.image-tag }}
           push: true
           context: .
           platforms: linux/amd64

.github/workflows/ci-e2e.yml

@@ -8,6 +8,9 @@ on:
       image-tag:
         required: true
         type: string
+      github-repository:
+        required: true
+        type: string
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -19,13 +22,15 @@ jobs:
     with:
       commit-hash: ${{ inputs.commit-hash }}
       image-tag: ${{ inputs.image-tag }}
+      github-repository: ${{ inputs.github-repository }}
     secrets: inherit
 
   run-ci-basic:
     needs: build-test-image-amd64
     uses: ./.github/workflows/ci-basic.yml
     with:
       image-tag: ${{ inputs.image-tag }}
+      github-repository: ${{ inputs.github-repository }}
     secrets: inherit
 
   run-ci-userspace-convertor:
@@ -34,4 +39,5 @@ jobs:
     with:
       commit-hash: ${{ inputs.commit-hash }}
       image-tag: ${{ inputs.image-tag }}
+      github-repository: ${{ inputs.github-repository }}
     secrets: inherit

.github/workflows/ci-userspace-convertor.yml

@@ -8,14 +8,17 @@ on:
       image-tag:
         required: true
         type: string
+      github-repository:
+        required: true
+        type: string
 
 jobs:
   run-ci-userspace-convertor:
     name: Run CI | Userspace Convertor
     runs-on: ubuntu-22.04
     timeout-minutes: 10
     container:
-      image: ghcr.io/${{ github.repository }}/overlaybd-ci-images:${{ inputs.image-tag }}
+      image: ghcr.io/${{ inputs.github-repository }}/overlaybd-ci-images:${{ inputs.image-tag }}
       credentials:
         username: ${{ github.actor }}
         password: ${{ secrets.GITHUB_TOKEN }}
@@ -25,6 +28,8 @@ jobs:
         - /sys/kernel/config:/sys/kernel/config
         - /mnt:/var/lib/containerd
       options: --privileged
+      env:
+        DOCKER_HOST: "unix:///app/dockerd/docker.sock"
 
     steps:
       - name: Checkout
@@ -40,8 +45,27 @@ jobs:
           bash start_services.sh
           sleep 5s
 
+      - name: Prepare Local Registry
+        working-directory: ci/scripts
+        shell: bash
+        run: |
+          bash new_registry.sh
+          bash prepare_image.sh registry.hub.docker.com/overlaybd/centos:centos7.9.2009 localhost:5000/centos:centos7.9.2009 && \
+          bash prepare_image.sh registry.hub.docker.com/overlaybd/ubuntu:22.04          localhost:5000/ubuntu:22.04 && \
+          bash prepare_image.sh registry.hub.docker.com/overlaybd/redis:7.2.3           localhost:5000/redis:7.2.3 && \
+          bash prepare_image.sh registry.hub.docker.com/overlaybd/wordpress:6.4.2       localhost:5000/wordpress:6.4.2 && \
+          bash prepare_image.sh registry.hub.docker.com/overlaybd/nginx:1.25.3          localhost:5000/nginx:1.25.3
+
       - name: CI - uconv reproduce
         working-directory: ci/uconv_reproduce
         shell: bash
         run: |
           bash ci-uconv-reproduce.sh
+
+      - name: CI - uconv E2E
+        working-directory: ci/scripts
+        shell: bash
+        run: |
+          /opt/overlaybd/snapshotter/convertor -r localhost:5000/redis -i 7.2.3 --overlaybd 7.2.3_overlaybd --turboOCI 7.2.3_turbo
+          bash run_container.sh localhost:5000/redis:7.2.3_overlaybd
+          bash run_container.sh localhost:5000/redis:7.2.3_turbo

.github/workflows/ci.yml

@@ -45,11 +45,23 @@ jobs:
         run: |
           sudo GO_TESTFLAGS=-v make test
 
+  lowercase-repo:
+    name: Lowercase Repo
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    outputs:
+      repository: ${{ steps.lowercase_repository.outputs.repository }}
+    steps:
+    - id: lowercase_repository
+      run: echo "repository=${GITHUB_REPOSITORY,,}" >> "$GITHUB_OUTPUT"
+
   e2e-test:
+    needs: lowercase-repo
     name: E2E Test
     if: ${{ contains(github.event.pull_request.labels.*.name, 'ok-to-test') || github.event_name == 'push' }}
     uses: ./.github/workflows/ci-e2e.yml
     with:
       commit-hash: ${{ github.event.pull_request.head.sha || github.sha }}
       image-tag: ${{ github.event.pull_request.number || 'default' }}
+      github-repository: ${{ needs.lowercase-repo.outputs.repository }}
     secrets: inherit

ci/build_image/Dockerfile

@@ -41,8 +41,15 @@ SHELL ["/bin/bash", "-c"]
 WORKDIR /app
 COPY --from=builder /tmp/overlaybd-snapshotter_${RELEASE_VERSION}_amd64.deb .
 COPY ./ci/build_image/start_services.sh .
-RUN \
-    apt update && apt install -y containerd libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \
+
+RUN apt-get update && apt-get install -y apt-transport-https ca-certificates curl gnupg lsb-release software-properties-common && \
+    curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg && \
+    echo \
+      "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \
+      $(lsb_release -cs) stable" | tee /etc/apt/sources.list.d/docker.list > /dev/null && \
+    apt-get update && apt-get install -y docker-ce docker-ce-cli containerd.io && \
+    apt-get install -y libnl-3-200 libnl-genl-3-200 libcurl4-openssl-dev libaio-dev wget less kmod && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* && \
     wget https://github.com/containerd/overlaybd/releases/download/v${OBD_VERSION}/overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \
     dpkg -i overlaybd-${OBD_VERSION}-0ubuntu1.22.04.x86_64.deb && \
     dpkg -i overlaybd-snapshotter_${RELEASE_VERSION}_amd64.deb && \

ci/scripts/new_registry.sh

@@ -0,0 +1,85 @@
+#!/bin/bash
+#
+# run a HTTPS registry
+
+set -x
+
+mkdir -p /app/dockerd/
+echo '{ "hosts": ["unix:///app/dockerd/docker.sock"] }' > /etc/docker/daemon.json
+dockerd &>/var/log/dockerd.log &
+
+sleep 3
+
+rm -rf /etc/registry/
+mkdir -p /etc/registry/certs/
+mkdir -p /etc/registry/config/
+
+# generate server certifications
+cat << EOF > /etc/registry/openssl.cnf
+[req]
+distinguished_name = req_distinguished_name
+x509_extensions = v3_req
+prompt = no
+
+[req_distinguished_name]
+C = CN
+ST = Beijing
+L = Beijing City
+O = Alibaba
+CN = localhost
+
+[v3_req]
+basicConstraints = CA:FALSE
+keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+subjectAltName = @alt_names
+
+[alt_names]
+DNS.1 = localhost
+IP.1 = 127.0.0.1
+EOF
+
+openssl req -new -x509 -newkey rsa:2048 -sha256 -nodes -config /etc/registry/openssl.cnf \
+  -days 365 -out /etc/registry/certs/server.crt -keyout /etc/registry/certs/server.key
+
+ls /etc/registry/certs/
+cp /etc/registry/certs/server.crt /usr/local/share/ca-certificates/registry.crt
+update-ca-certificates
+
+# start registry
+cat << EOF > /etc/registry/config/config.yml
+version: 0.1
+log:
+  fields:
+    service: registry
+storage:
+  cache:
+    blobdescriptor: inmemory
+  filesystem:
+    rootdirectory: /var/lib/registry
+http:
+  addr: :5000
+  headers:
+    X-Content-Type-Options: [nosniff]
+  tls:
+    certificate: /certs/server.crt
+    key: /certs/server.key
+health:
+  storagedriver:
+    enabled: true
+    interval: 10s
+    threshold: 3
+EOF
+
+docker run -d --restart=always --name registry -p 5000:5000 \
+  -v /etc/registry/certs:/certs \
+  -v /etc/registry/config:/etc/docker/registry/ \
+  registry:2
+
+sleep 5s
+
+docker ps -a
+apt-get update && apt-get install -y lsof
+lsof -i :5000
+curl http://localhost:5000/v2/_catalog
+lsof -i :5000
+curl https://localhost:5000/v2/_catalog

ci/scripts/prepare_image.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+from=${1:?}
+to=${2:?}
+
+set -x
+
+ctr i pull "${from}"
+ctr i tag "${from}" "${to}"
+ctr i push "${to}"
+ctr i rm "${from}" "${to}"

ci/scripts/run_container.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# rpull and run on-demand
+
+image=$1
+container_name=${2:-test}
+
+exit_code=0
+
+/opt/overlaybd/snapshotter/ctr rpull "${image}"
+if ! ctr run -d --net-host --snapshotter=overlaybd "${image}" "${container_name}"; then
+  exit_code=1
+fi
+if ! ctr t ls | grep "${container_name}"; then
+  exit_code=1
+fi
+ctr t kill -s 9 "${container_name}" && sleep 5s && ctr t ls
+ctr c rm "${container_name}" && ctr c ls
+ctr i rm "${image}"
+
+if [[ ${exit_code} -ne 0 ]]; then
+  cat /var/log/overlaybd.log
+fi
+
+exit ${exit_code}

ci/uconv_reproduce/centos/config-turbo.json

@@ -0,0 +1 @@
+{"created":"2021-09-15T18:20:23.99863383Z","architecture":"amd64","os":"linux","config":{"Env":["PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"],"Cmd":["/bin/bash"],"Labels":{"org.label-schema.build-date":"20201113","org.label-schema.license":"GPLv2","org.label-schema.name":"CentOS Base Image","org.label-schema.schema-version":"1.0","org.label-schema.vendor":"CentOS","org.opencontainers.image.created":"2020-11-13 00:00:00+00:00","org.opencontainers.image.licenses":"GPL-2.0-only","org.opencontainers.image.title":"CentOS Base Image","org.opencontainers.image.vendor":"CentOS"}},"rootfs":{"type":"layers","diff_ids":["sha256:a5cdf182bbffc6992b556633bbb275b559056b6e953133772a80fa261886c07b"]},"history":[{"created":"2021-09-15T18:20:23.417639551Z","created_by":"/bin/sh -c #(nop) ADD file:b3ebbe8bd304723d43b7b44a6d990cd657b63d93d6a2a9293983a30bfc1dfa53 in / "},{"created":"2021-09-15T18:20:23.819893035Z","created_by":"/bin/sh -c #(nop)  LABEL org.label-schema.schema-version=1.0 org.label-schema.name=CentOS Base Image org.label-schema.vendor=CentOS org.label-schema.license=GPLv2 org.label-schema.build-date=20201113 org.opencontainers.image.title=CentOS Base Image org.opencontainers.image.vendor=CentOS org.opencontainers.image.licenses=GPL-2.0-only org.opencontainers.image.created=2020-11-13 00:00:00+00:00","empty_layer":true},{"created":"2021-09-15T18:20:23.99863383Z","created_by":"/bin/sh -c #(nop)  CMD [\"/bin/bash\"]","empty_layer":true}]}

ci/uconv_reproduce/centos/manifest-turbo.json

@@ -0,0 +1 @@
+{"schemaVersion":2,"mediaType":"application/vnd.docker.distribution.manifest.v2+json","config":{"mediaType":"application/vnd.docker.container.image.v1+json","digest":"sha256:bdeebc045cc88055079dd5ab899cf594614b2757d5b572ec7acc6f4a24031fb1","size":1476},"layers":[{"mediaType":"application/vnd.docker.image.rootfs.diff.tar.gzip","digest":"sha256:e91a425b0b5001fa96ae2909db392034ee9eccfdf9347410a1343719db83054e","size":2717102,"annotations":{"containerd.io/snapshot/overlaybd/blob-digest":"sha256:e91a425b0b5001fa96ae2909db392034ee9eccfdf9347410a1343719db83054e","containerd.io/snapshot/overlaybd/blob-size":"2717102","containerd.io/snapshot/overlaybd/turbo-oci/target-digest":"sha256:2d473b07cdd5f0912cd6f1a703352c82b512407db6b05b43f2553732b55df3bc","containerd.io/snapshot/overlaybd/turbo-oci/target-media-type":"application/vnd.docker.image.rootfs.diff.tar.gzip","containerd.io/snapshot/overlaybd/version":"0.1.0-turbo.ociv1"}}]}

ci/uconv_reproduce/ci-uconv-reproduce.sh

@@ -1,52 +1,70 @@
 #!/bin/bash
 
-apt install -y python3
+apt update && apt install -y python3 jq
 
 convertor="/opt/overlaybd/snapshotter/convertor"
 images=("centos:centos7.9.2009" "ubuntu:22.04" "redis:7.2.3" "wordpress:6.4.2" "nginx:1.25.3")
-repo="registry.hub.docker.com/overlaybd"
+registry="localhost:5000"
 ci_base=$(pwd)
 
 result=0
 
-for image in ${images[@]}
+for image in "${images[@]}"
 do
     img=${image%%":"*}
     tag=${image##*":"}
-    echo ${img} ${tag}
+    echo "${img} ${tag}"
 
-    o_tag="${tag}_obd"
-    tmp_dir="${ci_base}/tmp_conv_${image/:/_}"
+    workspace="${ci_base}/workspace_${image/:/_}"
 
-    rm -rf ${tmp_dir}
-    mkdir -p ${tmp_dir}
+    rm -rf "${workspace}"
+    mkdir -p "${workspace}"
 
-    ${convertor} -r ${repo}/${img}  \
-        --reserve --no-upload --dump-manifest \
-        -i ${tag} -o ${o_tag} -d ${tmp_dir} &>${tmp_dir}/convert.out
+    tag_obd="${tag}_overlaybd"
+    tag_turbo="${tag}_turbo"
+    manifest_obd="${workspace}/manifest.json"
+    manifest_turbo="${workspace}/manifest-turbo.json"
+    config_obd="${workspace}/config.json"
+    config_turbo="${workspace}/config-turbo.json"
+    output_obd="${workspace}/convert.overlaybd.out"
+    output_turbo="${workspace}/convert.turbo.out"
+
+    ${convertor} -r "${registry}/${img}" -i "${tag}" --overlaybd "${tag_obd}" -d "${workspace}/overlaybd_tmp_conv" &> "${output_obd}"
+    curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_obd}" "https://${registry}/v2/${img}/manifests/${tag_obd}" &> /dev/null
+    configDigest=$(jq '.config.digest' "${manifest_obd}")
+    configDigest=${configDigest//\"/}
+    curl -o "${config_obd}" "https://${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null
+
+    ${convertor} -r "${registry}/${img}" -i "${tag}" --turboOCI "${tag_turbo}" -d "${workspace}/turbo_tmp_conv" &> "${output_turbo}"
+    curl -H "Accept: application/vnd.docker.distribution.manifest.v2+json,application/vnd.oci.image.manifest.v1+json" -o "${manifest_turbo}" "https://${registry}/v2/${img}/manifests/${tag_turbo}" &> /dev/null
+    configDigest=$(jq '.config.digest' "${manifest_turbo}")
+    configDigest=${configDigest//\"/}
+    curl -o "${config_turbo}" "https://${registry}/v2/${img}/blobs/${configDigest}" &> /dev/null
 
     prefix=$(date +%Y%m%d%H%M%S)
-    files=("manifest" "config")
+
+    mode=("manifest" "config" "manifest" "config")
+    actual=("${manifest_obd}" "${config_obd}" "${manifest_turbo}" "${config_turbo}")
+    expected=("${ci_base}/${img}/manifest.json" "${ci_base}/${img}/config.json" "${ci_base}/${img}/manifest-turbo.json" "${ci_base}/${img}/config-turbo.json")
+
     conv_res=0
-    for file in ${files[@]}
-    do
-        fn="${file}.json"
-        # diff ${tmp_dir}/${fn} ${ci_base}/${img}/${fn}
-        python3 compare_layers.py ${file} ${tmp_dir}/${fn} ${ci_base}/${img}/${fn}
+    n=${#mode[@]}
+    for ((i=0; i<n; i++)); do
+        python3 compare_layers.py ${mode[$i]} ${actual[$i]} ${expected[$i]}
         ret=$?
         if [[ ${ret} -eq 0 ]]; then
-            echo "${prefix} ${img} ${file} consistent"
+            echo "${prefix} ${img} ${expected[$i]} consistent"
         else
-            echo "${prefix} ${img} ${file} diff"
+            echo "${prefix} ${img} ${expected[$i]} diff"
             conv_res=1
         fi
     done
 
     if [[ ${conv_res} -eq 1 ]]; then
-        cat ${tmp_dir}/convert.out | sed 's/\\n/\n/g'
+        sed 's/\\n/\n/g' < "${output_obd}"
+        sed 's/\\n/\n/g' < "${output_turbo}"
         result=1
     fi
-    rm -rf ${tmp_dir}
 done
 
 exit ${result}