VM hot plugging and unplugging (#11149)

Signed-off-by: AYUSH-D-PATNI <[email protected]>

ocs_ci/helpers/cnv_helpers.py

@@ -5,10 +5,12 @@
 import os
 import base64
 import logging
+import re
 
 from ocs_ci.helpers.helpers import create_unique_resource_name, create_resource
 from ocs_ci.ocs import constants
 from ocs_ci.ocs.exceptions import CommandFailed
+from ocs_ci.ocs.ocp import OCP
 from ocs_ci.utility import templating
 from ocs_ci.helpers.helpers import (
     create_ocs_object_from_kind_and_name,
@@ -365,3 +367,76 @@ def run_dd_io(vm_obj, file_path, size="10240", username=None, verify=False):
             file_path=file_path,
             username=username,
         )
+
+
+def verifyvolume(vm_name, volume_name, namespace):
+    """
+    Verify a volume in VM.
+
+    Args:
+        vm_name (str): Name of the virtual machine
+        volume_name (str): Name of the volume (PVC) to verify
+        namespace (str): Virtual Machine Namespace
+
+    Returns:
+        bool: True if the volume (PVC) is found, False otherwise
+
+    """
+    cmd = (
+        f"get vm {vm_name} -n {namespace} -o "
+        + "jsonpath='{.spec.template.spec.volumes}'"
+    )
+    try:
+        output = OCP().exec_oc_cmd(command=cmd)
+        logger.info(f"Output of the command '{cmd}': {output}")
+        for volume in output:
+            if volume.get("persistentVolumeClaim", {}).get("claimName") == volume_name:
+                logger.info(
+                    f"Hotpluggable PVC {volume_name} is visible inside the VM {vm_name}"
+                )
+                return True
+        logger.warning(f"PVC {volume_name} not found inside the VM {vm_name}")
+        return False
+    except Exception as e:
+        logger.error(f"Error executing command '{cmd}': {e}")
+        return False
+
+
+def verify_hotplug(vm_obj, disks_before_hotplug):
+    """
+    Verifies if a disk has been hot-plugged into/removed from a VM.
+
+    Args:
+        disks_before_hotplug (str): Set of disk information before hot-plug or add.
+        vm_obj (VM object): The virtual machine object to check.
+
+    Returns:
+        bool: True if a hot-plugged disk is detected, False otherwise.
+
+    """
+    try:
+        disks_after_hotplug_raw = vm_obj.run_ssh_cmd("lsblk -o NAME,SIZE,MOUNTPOINT -P")
+        disks_after_hotplug = set(
+            re.findall(r'NAME="([^"]+)"', disks_after_hotplug_raw)
+        )
+        disks_before_hotplug = set(re.findall(r'NAME="([^"]+)"', disks_before_hotplug))
+
+        logger.info(f"Disks before hotplug:\n{disks_before_hotplug}")
+        logger.info(f"Disks found after hotplug:\n{disks_after_hotplug}")
+
+        added_disks = disks_after_hotplug - disks_before_hotplug
+        removed_disks = disks_before_hotplug - disks_after_hotplug
+
+        if added_disks or removed_disks:
+            logger.info(
+                f"Hotplug difference detected: Added: {added_disks}, "
+                f"Removed: {removed_disks}"
+            )
+            return True
+        logger.info(f"No hotplug difference detected in VM {vm_obj.name}")
+        return False
+    except Exception as error:
+        logger.error(
+            f"Error occurred while verifying hotplug in VM {vm_obj.name}: {str(error)}"
+        )
+        return False

ocs_ci/ocs/cnv/virtual_machine.py

@@ -11,6 +11,7 @@
     create_vm_secret,
     create_dv,
     clone_dv,
+    verifyvolume,
 )
 
 from ocs_ci.helpers.helpers import (
@@ -432,11 +433,12 @@ def stop(self, force=False, wait=True):
             self.wait_for_vm_status(status=constants.CNV_VM_STOPPED)
             logger.info(f"VM: {self._vm_name} reached Stopped state")
 
-    def restart(self, wait=True):
+    def restart(self, wait=True, verify=True):
         """
         Restart the VirtualMachine.
 
         Args:
+            verify(bool): True to wait for VM ssh up after restart
             wait (bool): True to wait for the VirtualMachine to reach the "Running" status.
 
         """
@@ -448,15 +450,19 @@ def restart(self, wait=True):
             logger.info(
                 f"VM: {self._vm_name} reached Running state state after restart operation"
             )
+        if verify:
+            self.verify_vm(verify_ssh=True)
+            logger.info(f"VM: {self._vm_name} ssh working successfully!")
 
-    def addvolme(self, volume_name, persist=True, serial=None):
+    def addvolume(self, volume_name, persist=True, serial=None, verify=True):
         """
         Add a volume to a VM
 
         Args:
             volume_name (str): Name of the volume/PVC to add.
             persist (bool): True to persist the volume.
             serial (str): Serial number for the volume.
+            verify (bool): If true, checks volume_name present in vm yaml.
 
         Returns:
              str: stdout of command
@@ -469,13 +475,23 @@ def addvolme(self, volume_name, persist=True, serial=None):
             persist=persist,
             serial=serial,
         )
-        logger.info(f"Successfully HotPlugged disk {volume_name} to {self._vm_name}")
+        if verify:
+            sample = TimeoutSampler(
+                timeout=600,
+                sleep=15,
+                func=verifyvolume,
+                vm_name=self._vm_name,
+                volume_name=volume_name,
+                namespace=self.namespace,
+            )
+            sample.wait_for_func_value(value=True)
 
-    def removevolume(self, volume_name):
+    def removevolume(self, volume_name, verify=True):
         """
         Remove a volume from a VM
 
         Args:
+            verify: If true, checks volume_name not present in vm yaml
             volume_name (str): Name of the volume to remove.
 
         Returns:
@@ -484,9 +500,16 @@ def removevolume(self, volume_name):
         """
         logger.info(f"Removing {volume_name} from {self._vm_name}")
         self.remove_volume(vm_name=self._vm_name, volume_name=volume_name)
-        logger.info(
-            f"Successfully HotUnplugged disk {volume_name} from {self._vm_name}"
-        )
+        if verify:
+            sample = TimeoutSampler(
+                timeout=600,
+                sleep=15,
+                func=verifyvolume,
+                vm_name=self._vm_name,
+                volume_name=volume_name,
+                namespace=self.namespace,
+            )
+            sample.wait_for_func_value(value=False)
 
     def scp_to_vm(
         self,

ocs_ci/templates/cnv-vm-workload/vm.yaml

@@ -17,7 +17,7 @@ spec:
       architecture: amd64
       domain:
         cpu:
-          cores: 2
+          cores: 1
           sockets: 1
           threads: 1
         devices:
@@ -37,7 +37,7 @@ spec:
           networkInterfaceMultiqueue: true
           rng: {}
         memory:
-          guest: 6Gi
+          guest: 4Gi
         resources: {}
       evictionStrategy: LiveMigrate
       networks:

tests/functional/workloads/cnv/test_vm_hotplug_unplug.py

@@ -0,0 +1,135 @@
+import logging
+import pytest
+
+from ocs_ci.framework.pytest_customization.marks import magenta_squad, workloads
+from ocs_ci.framework.testlib import E2ETest
+from ocs_ci.helpers.cnv_helpers import (
+    cal_md5sum_vm,
+    run_dd_io,
+    verifyvolume,
+    verify_hotplug,
+)
+from ocs_ci.ocs import constants
+from ocs_ci.utility.utils import TimeoutSampler
+
+log = logging.getLogger(__name__)
+
+
+@magenta_squad
+@workloads
+@pytest.mark.polarion_id("OCS-6322")
+class TestVmHotPlugUnplug(E2ETest):
+    """
+    Test case for VM hot plugging and unplugging of PVC disks.
+    This test ensures that PVC disks can be hotplugged into a running VM
+    and that data written to the disk is persisted after reboot.
+    """
+
+    def test_vm_hot_plugging_unplugging(
+        self,
+        setup_cnv,
+        project_factory,
+        multi_cnv_workload,
+        pvc_factory,
+    ):
+        """
+        Test the hot plugging and unplugging of a PVC into/from a VM.
+
+        The test involves:
+        1. Hotplugging a disk into a running VM based on PVC.
+        2. Verifying the disk is attached to the VM.
+        3. Writing data to the disk and rebooting the VM to test persistence.
+        4. Hotplugging another disk without the --persist flag and verifying it is detached correctly.
+        """
+
+        proj_obj = project_factory()
+        file_paths = ["/file.txt", "/new_file.txt"]
+        vm_objs_def, vm_objs_aggr, sc_objs_def, sc_objs_aggr = multi_cnv_workload(
+            namespace=proj_obj.namespace
+        )
+        vm_list = vm_objs_def + vm_objs_aggr
+        log.info(f"Total VMs to process: {len(vm_list)}")
+
+        for vm_obj in vm_list:
+            sc_obj = sc_objs_def if vm_obj in vm_objs_def else sc_objs_aggr
+            before_disks = vm_obj.run_ssh_cmd("lsblk -o NAME,SIZE,MOUNTPOINT -P")
+            log.info(f"Disks before hotplug:\n{before_disks}")
+
+            pvc_obj = pvc_factory(
+                project=proj_obj,
+                storageclass=sc_obj,
+                size=20,
+                access_mode=constants.ACCESS_MODE_RWX,
+                volume_mode=constants.VOLUME_MODE_BLOCK,
+            )
+            log.info(f"PVC {pvc_obj.name} created successfully")
+
+            vm_obj.addvolume(volume_name=pvc_obj.name)
+            log.info(f"Hotplugged PVC {pvc_obj.name} to VM {vm_obj.name}")
+
+            sample = TimeoutSampler(
+                timeout=600,
+                sleep=5,
+                func=verify_hotplug,
+                vm_obj=vm_obj,
+                disks_before_hotplug=before_disks,
+            )
+            sample.wait_for_func_value(value=True)
+
+            log.info(f"Running I/O operation on VM {vm_obj.name}")
+            source_csum = run_dd_io(vm_obj=vm_obj, file_path=file_paths[0], verify=True)
+
+            log.info(f"Rebooting VM {vm_obj.name}")
+            vm_obj.restart(wait=True, verify=True)
+            log.info(f"Reboot Success for VM: {vm_obj.name}")
+
+            assert verifyvolume(
+                vm_obj.name, volume_name=pvc_obj.name, namespace=vm_obj.namespace
+            ), f"Unable to find volume {pvc_obj.name} mounted on VM: {vm_obj.name}"
+
+            new_csum = cal_md5sum_vm(vm_obj=vm_obj, file_path=file_paths[0])
+            assert (
+                source_csum == new_csum
+            ), f"MD5 mismatch after reboot for VM {vm_obj.name}"
+
+            pvc_obj_wout = pvc_factory(
+                project=proj_obj,
+                storageclass=sc_obj,
+                size=20,
+                access_mode=constants.ACCESS_MODE_RWX,
+                volume_mode=constants.VOLUME_MODE_BLOCK,
+            )
+            log.info(f"PVC {pvc_obj_wout.name} created successfully")
+            before_disks_wout = vm_obj.run_ssh_cmd(
+                "lsblk -o NAME,SIZE," "MOUNTPOINT -P"
+            )
+            log.info(f"Disks before hotplug (without persist):\n{before_disks_wout}")
+
+            vm_obj.addvolume(volume_name=pvc_obj_wout.name, persist=False, verify=False)
+
+            sample = TimeoutSampler(
+                timeout=600,
+                sleep=5,
+                func=verify_hotplug,
+                vm_obj=vm_obj,
+                disks_before_hotplug=before_disks_wout,
+            )
+            sample.wait_for_func_value(value=True)
+
+            log.info(f"Running I/O operation on VM {vm_obj.name}")
+            run_dd_io(vm_obj=vm_obj, file_path=file_paths[1])
+
+            before_disks_wout_rm = vm_obj.run_ssh_cmd(
+                "lsblk -o NAME,SIZE," "MOUNTPOINT -P"
+            )
+            vm_obj.removevolume(volume_name=pvc_obj_wout.name)
+
+            sample = TimeoutSampler(
+                timeout=600,
+                sleep=5,
+                func=verify_hotplug,
+                vm_obj=vm_obj,
+                disks_before_hotplug=before_disks_wout_rm,
+            )
+            sample.wait_for_func_value(value=True)
+            vm_obj.stop()