Calico CI

.cloudtest.yaml

@@ -1,7 +1,7 @@
 ---
 version: 1.0
 root: "./.tests/cloud_test/"
-timeout: 7200  # 2 hour total total timeout
+timeout: 10800  # 3 hour total total timeout
 shuffle-enabled: true
 statistics:
   enabled: true

.cloudtest_calico.yaml

@@ -0,0 +1,31 @@
+---
+version: 1.0
+root: "./.tests/cloud_test_calico/"
+timeout: 10800  # 3 hour total total timeout
+shuffle-enabled: true
+statistics:
+  enabled: true
+  interval: 60  # 60 seconds for statistics
+import:
+  - cloudtest/packet.yaml
+  - cloudtest/tests.yaml
+
+retest:  # Allow to do test re-run if some kind of failures are detected, line CNI network plugin errors.
+  count: 1  # Allow 5 times to do restart
+  warmup-time: 15  # Put 15 seconds warmup for cluster instance to be used again.
+  allowed-retests: 2  # If cluster instance have few attempts with retest requests one after another, we need to restart cluster.
+  pattern:
+    - "NetworkPlugin cni failed to set up pod"    # Error in AWS dur to leak of IPs or not ability to assign them.
+    - "etcdserver: request timed out"  # Error in any could, reason unknown.
+    - "unable to establish connection to VPP (VPP API socket file /run/vpp/api.sock does not exist)"  # a VPP is not started, it will be re-started in general, but will cause test fail.
+    # Sometimes (rarely) docker registry is unavailable for a moment
+    - "Error response from daemon: Get https://.*docker.io/.*: dial tcp: lookup registry"
+    - "Error response from daemon: Get https://.*docker.io/.*: net/http: request canceled while waiting for connection"
+    - "Failed create pod sandbox"
+reporting:
+  junit-report: "results/junit.xml"
+health-check:
+  - message: "Branch is not up to date"
+    interval: 60  # 1 minute
+    run: |
+      echo "Health check!"

.github/workflows/ci.yaml

@@ -66,8 +66,12 @@ jobs:
           git diff --name-only --exit-code go.sum || ( echo "Run go tidy" && false )
 
   packet:
-    name: packet
+    name: packet (Calico ${{ matrix.calico }})
     runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        calico: ["off", "on"]
     steps:
       - name: Set up /bin permissions   # 1. Set up /bin permissions
         run: |
@@ -79,6 +83,7 @@ jobs:
       - name: Install cloudtest         # 3. Install cloudtest
         run: |
           go get github.com/networkservicemesh/cloudtest@master
+#          GOPROXY=direct go get github.com/Mixaster995/cloudtest@no-cleanup
         env:
           GO111MODULE: on
           GOBIN: /bin
@@ -87,31 +92,42 @@ jobs:
         with:
           repository: networkservicemesh/deployments-k8s
           path: networkservicemesh/deployments-k8s
-      - name: Checkout files            # 5. Checkout files
+      - name: Compute suffix            # 5. Compute suffix for cloudtest input and output paths
+        id: suffix
+        run: |
+          if [[ "${CALICO}" == "on" ]]; then
+            echo ::set-output name=val::_calico
+          fi
+        env:
+          CALICO: ${{ matrix.calico }}
+      - name: Checkout files            # 6. Checkout files
         uses: actions/checkout@v2
         with:
           path: ${{ github.repository }}
-      - name: Run tests with cloudtest  # 6. Run tests with cloudtest
+      - name: Run tests with cloudtest  # 7. Run tests with cloudtest
         working-directory: ${{ github.repository }}
         run: |
-          cloudtest
+          cloudtest --config=.cloudtest${suffix}.yaml
         env:
           PACKET_AUTH_TOKEN: ${{ secrets.PACKET_AUTH_TOKEN }}
           PACKET_PROJECT_ID: 383890d0-f5d1-4de1-881a-4d1ede549d18
           KUBERNETES_VERSION: ${{ secrets.NSM_KUBERNETES_VERSION }}
-      - name: Publish test report       # 7. Publish test report
+          CALICO: ${{ matrix.calico }}
+          suffix: ${{ steps.suffix.outputs.val }}
+      - name: Publish test report       # 8. Publish test report
         uses: mikepenz/[email protected]
         if: ${{ always() }}
         with:
-          report_paths: "**/cloud_test/results/junit.xml"
+          report_paths: "**/cloud_test${{ steps.suffix.outputs.val }}/results/junit.xml"
           suite_regex: "Test*"
           github_token: ${{ secrets.GITHUB_TOKEN }}
-      - name: Upload logs               # 8. Upload logs
+          check_name: "JUnit Test Report (Calico ${{ matrix.calico }})"
+      - name: Upload logs               # 9. Upload logs
         uses: actions/upload-artifact@v2
         if: ${{ always() }}
         with:
           name: logs-${{ github.run_number }}
-          path: ${{ github.repository }}/.tests/cloud_test/
+          path: ${{ github.repository }}/.tests/
 
   packet-cleanup:
     name: packet cleanup

cloudtest/packet.yaml

@@ -25,13 +25,15 @@ providers:
           os: "ubuntu_20_04"
           billing-cycle: "hourly"
           port-vlans:
-            eth3: 1044
+            eth1: 3000  # calico VLAN
+            eth3: 1044  # SR-IOV VLAN
         - name: "Worker"
           host-name: "SR-IOV-Worker-${CLUSTER_NAME}"
           os: "ubuntu_20_04"
           billing-cycle: "hourly"
           port-vlans:
-            eth3: 1044
+            eth1: 3000  # calico VLAN
+            eth3: 1044  # SR-IOV VLAN
       hardware-reservations:
         - 2cf78481-53b0-46c8-a084-6e9815acdb0b
         - 2361d3c2-f694-4fa7-a683-a9f69e2abe7c

go.mod

@@ -1,10 +1,10 @@
-module github.com/networkservicemesh/integration-k8s-packet
+module github.com/networkservicemesh/integratiohuhgyn-k8s-packet
 
 go 1.16
 
 require (
 	github.com/googleapis/gnostic v0.5.1 // indirect
-	github.com/networkservicemesh/integration-tests v0.0.0-20220118134157-d3e1be7ce81a
+	github.com/networkservicemesh/integration-tests v0.0.0-20220120091746-3e71046e4a08
 	github.com/stretchr/testify v1.7.0
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -158,8 +158,8 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m
 github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
 github.com/networkservicemesh/gotestmd v0.0.0-20211116145945-871d2aaf07ab h1:/dIr8Nky77grI3s9Rc78eFH9M1Svobyj2XJBaKm27ts=
 github.com/networkservicemesh/gotestmd v0.0.0-20211116145945-871d2aaf07ab/go.mod h1:8EWnekTRNX+NxBdTFE24WqUoM7SgJHbiafDBrIIdOmQ=
-github.com/networkservicemesh/integration-tests v0.0.0-20220118134157-d3e1be7ce81a h1:BpCtVcuuc5nGN0ihT3IC6XRVSWLfwwjwBVwHZkH8aX0=
-github.com/networkservicemesh/integration-tests v0.0.0-20220118134157-d3e1be7ce81a/go.mod h1:0o7WrzxlHEwnDSuZPEM1BnKd4hr7+akKgymoAoTTbv8=
+github.com/networkservicemesh/integration-tests v0.0.0-20220120091746-3e71046e4a08 h1:CSMRodhf4Lz8o52yO3iX/u0bTAJkt3Bje0NG6GH2jAU=
+github.com/networkservicemesh/integration-tests v0.0.0-20220120091746-3e71046e4a08/go.mod h1:0o7WrzxlHEwnDSuZPEM1BnKd4hr7+akKgymoAoTTbv8=
 github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=

main_test.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2020-2021 Doc.ai and/or its affiliates.
+// Copyright (c) 2020-2022 Doc.ai and/or its affiliates.
 //
 // SPDX-License-Identifier: Apache-2.0
 //
@@ -17,28 +17,53 @@
 package main_test
 
 import (
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/suite"
 
+	"github.com/networkservicemesh/integration-tests/suites/calico"
 	"github.com/networkservicemesh/integration-tests/suites/heal"
 	"github.com/networkservicemesh/integration-tests/suites/memory"
 	"github.com/networkservicemesh/integration-tests/suites/multiforwarder"
 	"github.com/networkservicemesh/integration-tests/suites/sriov"
 )
 
+func isCalico() bool {
+	return os.Getenv("CALICO") == "on"
+}
+
 func TestMemory(t *testing.T) {
+	if isCalico() {
+		t.Skip("not available with Calico")
+	}
 	suite.Run(t, new(memory.Suite))
 }
 
 func TestSRIOV(t *testing.T) {
+	if isCalico() {
+		t.Skip("not available with Calico")
+	}
 	suite.Run(t, new(sriov.Suite))
 }
 
 func TestMultiForwarder(t *testing.T) {
+	if isCalico() {
+		t.Skip("not available with Calico")
+	}
 	suite.Run(t, new(multiforwarder.Suite))
 }
 
 func TestHeal(t *testing.T) {
+	if isCalico() {
+		t.Skip("not available with Calico")
+	}
 	suite.Run(t, new(heal.Suite))
 }
+
+func TestCalico(t *testing.T) {
+	if !isCalico() {
+		t.Skip("not available without Calico")
+	}
+	suite.Run(t, new(calico.Suite))
+}

scripts/calico/deploy-calico.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+function on_error() {
+  kubectl describe pods --all-namespaces
+  exit 1
+}
+trap 'on_error' ERR
+
+kubectl apply -k scripts/calico
+
+kubectl -n calico-vpp-dataplane rollout status daemonset/calico-vpp-node --timeout=5m
+kubectl -n kube-system rollout status deployment/calico-kube-controllers --timeout=5m

scripts/calico/kustomization.yaml

@@ -0,0 +1,9 @@
+---
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+resources:
+  - https://raw.githubusercontent.com/projectcalico/vpp-dataplane/v0.17.0-calicov3.20.2/yaml/generated/calico-vpp-nohuge.yaml
+
+patchesStrategicMerge:
+  - patch.yaml

scripts/calico/patch.yaml

@@ -0,0 +1,8 @@
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: calico-vpp-config
+  namespace: calico-vpp-dataplane
+data:
+  vpp_dataplane_interface: eno2

scripts/calico/setup-interfaces.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+ip="$1"
+
+ip addr add "${ip}" dev eno2
+ip link set up dev eno2

scripts/calico/setup-node-ip.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+set -e
+
+ip="$1"
+
+sed -Ei "s/(.*)\"/\1 --node-ip=${ip}\"/g" /var/lib/kubelet/kubeadm-flags.env
+systemctl restart kubelet