Correct path to test-upgrade.py (#3411)

correct path to the test-upgrade.py file
retries on the test--cluster.py
rework of the test-cluster.py to have retries and timeouts so it is more reliable when run from our CI

Author: ktsakalozos

tests/test-cluster.py

@@ -1,9 +1,9 @@
 import string
 import random
 import time
-
 import pytest
 import os
+import signal
 import subprocess
 from os import path
 
@@ -79,10 +79,19 @@ def _setup_lxc(self, channel_or_snap):
             if channel_or_snap.startswith("/"):
                 self._transfer_install_local_snap_lxc(channel_or_snap)
             else:
-                cmd_prefix = "/snap/bin/lxc exec {}  -- script -e -c".format(self.vm_name).split()
-                cmd = ["snap install microk8s --classic --channel {}".format(channel_or_snap)]
+                cmd = "snap install microk8s --classic --channel {}".format(channel_or_snap)
                 time.sleep(20)
-                subprocess.check_output(cmd_prefix + cmd)
+                print("About to run {}".format(cmd))
+                output = ""
+                attempt = 0
+                while attempt < 3:
+                    try:
+                        output = self.run(cmd)
+                        break
+                    except ChildProcessError:
+                        time.sleep(10)
+                        attempt += 1
+                print(output.decode())
         else:
             if channel_or_snap.startswith("/"):
                 self._transfer_install_local_snap_lxc(channel_or_snap)
@@ -153,8 +162,18 @@ def run(self, cmd):
             )
             return output
         elif self.backend == "lxc":
-            cmd_prefix = "/snap/bin/lxc exec {}  -- script -e -c ".format(self.vm_name).split()
-            output = subprocess.check_output(cmd_prefix + [cmd])
+            cmd_prefix = "/snap/bin/lxc exec {}  -- ".format(self.vm_name)
+            with subprocess.Popen(
+                cmd_prefix + cmd, shell=True, stdout=subprocess.PIPE, preexec_fn=os.setsid
+            ) as process:
+                try:
+                    output = process.communicate(timeout=300)[0]
+                    if process.returncode != 0:
+                        raise ChildProcessError("Failed to run command")
+                except subprocess.TimeoutExpired:
+                    os.killpg(process.pid, signal.SIGKILL)  # send signal to the process group
+                    print("Process timed out")
+                    output = process.communicate()[0]
             return output
         else:
             raise Exception("Not implemented for backend {}".format(self.backend))
@@ -211,11 +230,20 @@ def setup_cluster(self):
 
             # Wait for nodes to be ready
             print("Waiting for nodes to register")
-            connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no")
-            while "NotReady" in connected_nodes.decode():
-                time.sleep(5)
-                connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no")
-            print(connected_nodes.decode())
+            attempt = 0
+            while attempt < 10:
+                try:
+                    connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no")
+                    if "NotReady" in connected_nodes.decode():
+                        time.sleep(5)
+                    connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no")
+                    print(connected_nodes.decode())
+                    break
+                except ChildProcessError:
+                    time.sleep(10)
+                    attempt += 1
+                    if attempt == 10:
+                        raise
 
             # Wait for CNI pods
             print("Waiting for cni")
@@ -339,10 +367,20 @@ def test_nodes_in_ha(self):
         self.VM[0].run("/snap/bin/microk8s.join {}".format(endpoint[0]))
 
         print("Waiting for nodes to be ready")
-        connected_nodes = self.VM[0].run("/snap/bin/microk8s.kubectl get no")
-        while "NotReady" in connected_nodes.decode():
-            time.sleep(5)
-            connected_nodes = self.VM[0].run("/snap/bin/microk8s.kubectl get no")
+        attempt = 0
+        while attempt < 10:
+            try:
+                connected_nodes = self.VM[0].run("/snap/bin/microk8s.kubectl get no")
+                if "NotReady" in connected_nodes.decode():
+                    time.sleep(5)
+                    continue
+                print(connected_nodes.decode())
+                break
+            except ChildProcessError:
+                time.sleep(10)
+                attempt += 1
+                if attempt == 10:
+                    raise
 
         attempt = 100
         while True:
@@ -375,11 +413,20 @@ def test_worker_noode(self):
 
         # Wait for nodes to be ready
         print("Waiting for node to register")
-        connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no")
-        while "NotReady" in connected_nodes.decode():
-            time.sleep(5)
-            connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no")
-        print(connected_nodes.decode())
+        attempt = 0
+        while attempt < 10:
+            try:
+                connected_nodes = vm_master.run("/snap/bin/microk8s.kubectl get no")
+                if "NotReady" in connected_nodes.decode():
+                    time.sleep(5)
+                    continue
+                print(connected_nodes.decode())
+                break
+            except ChildProcessError:
+                time.sleep(10)
+                attempt += 1
+                if attempt == 10:
+                    raise
 
         # Check that kubelet talks to the control plane node via the local proxy
         print("Checking the worker's configuration")

tests/test-distro.sh

@@ -57,14 +57,27 @@ fi
 # therefore we do not need to run it inside a VM/container
 apt-get install python3-pip -y
 pip3 install -U pytest requests pyyaml sh
-LXC_PROFILE="tests/lxc/microk8s.profile" BACKEND="lxc" CHANNEL_TO_TEST=${TO_CHANNEL} pytest -s tests/test-cluster.py
+export LXC_PROFILE="tests/lxc/microk8s.profile"
+export BACKEND="lxc"
+export CHANNEL_TO_TEST=${TO_CHANNEL}
+TRY_ATTEMPT=0
+while ! (timeout 3600 pytest -s tests/test-cluster.py) &&
+      ! [ ${TRY_ATTEMPT} -eq 3 ]
+do
+  TRY_ATTEMPT=$((TRY_ATTEMPT+1))
+  sleep 1
+done
+if [ ${TRY_ATTEMPT} -eq 3 ]
+then
+  echo "Test clusterring took longer than expected"
+  exit 1
+fi
 
 # Test addons upgrade
-
 NAME=machine-$RANDOM
 create_machine $NAME $PROXY
 # use 'script' for required tty: https://github.com/lxc/lxd/issues/1724#issuecomment-194416774
-lxc exec $NAME -- script -e -c "UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=${TO_CHANNEL} pytest -s /var/snap/microk8s/common/addons/core/tests/test-upgrade.py"
+lxc exec $NAME -- script -e -c "UPGRADE_MICROK8S_FROM=${FROM_CHANNEL} UPGRADE_MICROK8S_TO=${TO_CHANNEL} pytest -s /var/tmp/tests/test-upgrade.py"
 lxc delete $NAME --force
 
 # Test upgrade-path