Dpdk: 32bit test

Allow building rdma-core and dpdk as 32bit applications before test.
Add the 32bit test.

This is a beefy commit because we must extend the Installer class
to support 32bit builds. This means adding invalid arch checks,
adding arch checks to the DependencyInstaller, adding
changes to PKG_CONFIG_PATH and allowing updating environment variables
everywhere.

The result is that we can install DPDK and RDMA core and run their
32bit versions to test basic send/receive stuff.

Author: mcgov

lisa/tools/meson.py

@@ -2,7 +2,7 @@
 # Licensed under the MIT license.
 
 from pathlib import PurePath
-from typing import cast
+from typing import Dict, Optional, cast
 
 from semver import VersionInfo
 
@@ -50,15 +50,22 @@ def _install(self) -> bool:
             )
         return self._check_exists()
 
-    def setup(self, args: str, cwd: PurePath, build_dir: str = "build") -> PurePath:
+    def setup(
+        self,
+        args: str,
+        cwd: PurePath,
+        build_dir: str = "build",
+        update_envs: Optional[Dict[str, str]] = None,
+    ) -> PurePath:
         self.run(
-            f"{args} {build_dir}",
+            parameters=f"{args} {build_dir}",
             force_run=True,
             shell=True,
             cwd=cwd,
             expected_exit_code=0,
             expected_exit_code_failure_message=(
                 f"Could not configure {str(cwd)} with meson using args {args}"
             ),
+            update_envs=update_envs,
         )
         return cwd.joinpath(build_dir)

microsoft/testsuites/dpdk/common.py

@@ -11,15 +11,22 @@
 from lisa import Node
 from lisa.executable import Tool
 from lisa.operating_system import Debian, Fedora, Oracle, Posix, Redhat, Suse, Ubuntu
-from lisa.tools import Git, Tar, Wget
-from lisa.util import UnsupportedDistroException
+from lisa.tools import Git, Lscpu, Tar, Wget
+from lisa.tools.lscpu import CpuArchitecture
+from lisa.util import UnsupportedCpuArchitectureException, UnsupportedDistroException
 
 DPDK_STABLE_GIT_REPO = "https://dpdk.org/git/dpdk-stable"
 
 # azure routing table magic subnet prefix
 # signals 'route all traffic on this subnet'
 AZ_ROUTE_ALL_TRAFFIC = "0.0.0.0/0"
 
+ARCH_COMPATIBILITY_MATRIX = {
+    CpuArchitecture.X64: [CpuArchitecture.X64],
+    CpuArchitecture.I386: [CpuArchitecture.I386, CpuArchitecture.X64],
+    CpuArchitecture.ARM64: [CpuArchitecture.ARM64],
+}
+
 
 # Attempt to clean up the DPDK package dependency mess
 # Make a Installer class that implements the common steps
@@ -34,7 +41,7 @@ class OsPackageDependencies:
     # the packages to install on that OS.
     def __init__(
         self,
-        matcher: Callable[[Posix], bool],
+        matcher: Callable[[Posix, Optional[CpuArchitecture]], bool],
         packages: Optional[Sequence[Union[str, Tool, Type[Tool]]]] = None,
         stop_on_match: bool = False,
     ) -> None:
@@ -45,14 +52,21 @@ def __init__(
 
 class DependencyInstaller:
     # provide a list of OsPackageDependencies for a project
-    def __init__(self, requirements: List[OsPackageDependencies]) -> None:
+    def __init__(
+        self,
+        requirements: List[OsPackageDependencies],
+        arch: Optional[CpuArchitecture] = None,
+    ) -> None:
         self.requirements = requirements
+        self._arch = arch
 
     # evaluate the list of package dependencies,
     def install_required_packages(
-        self, node: Node, extra_args: Union[List[str], None]
+        self,
+        os: Posix,
+        extra_args: Union[List[str], None],
+        arch: Optional[CpuArchitecture] = None,
     ) -> None:
-        os = node.os
         assert isinstance(os, Posix), (
             "DependencyInstaller is not compatible with this OS: "
             f"{os.information.vendor} {os.information.release}"
@@ -61,13 +75,16 @@ def install_required_packages(
         # stop on list end or if exclusive_match parameter is true.
         packages: List[Union[str, Tool, Type[Tool]]] = []
         for requirement in self.requirements:
-            if requirement.matcher(os) and requirement.packages:
-                packages += requirement.packages
+            if requirement.matcher(os, arch):
+                if requirement.packages is not None and len(requirement.packages) > 0:
+                    packages += requirement.packages
                 if requirement.stop_on_match:
                     break
+
         os.install_packages(packages=packages, extra_args=extra_args)
 
         # NOTE: It is up to the caller to raise an exception on an invalid OS
+        # see unsupported_os_thrower as a catch-all 'list end' function
 
 
 class Downloader:
@@ -200,23 +217,43 @@ def _uninstall(self) -> None:
     def _install_dependencies(self) -> None:
         if self._os_dependencies is not None:
             self._os_dependencies.install_required_packages(
-                self._node, extra_args=self._package_manager_extra_args
+                self._os, extra_args=self._package_manager_extra_args, arch=self._arch
             )
 
     # define how to check the installed version
     def get_installed_version(self) -> VersionInfo:
         raise NotImplementedError(f"get_installed_version {self._err_msg}")
 
-    def _should_install(self, required_version: Optional[VersionInfo] = None) -> bool:
-        return (not self._check_if_installed()) or (
-            required_version is not None
-            and required_version > self.get_installed_version()
+    def _should_install(
+        self,
+        required_version: Optional[VersionInfo] = None,
+        required_arch: Optional[CpuArchitecture] = None,
+    ) -> bool:
+        return (
+            (not self._check_if_installed())
+            # NOTE: Taking advantage of longer-than-expected lifetimes here.
+            # If the tool still exists we ~should~ be able to check the old version
+            # from a previous test environment.
+            # At the moment, we use create() to force re-initialization.
+            # If we ever fix things so that we use .get,
+            # we will need this check. So add it now.q
+            or (required_arch != self._arch)
+            or (
+                required_version is not None
+                and required_version > self.get_installed_version()
+            )
         )
 
     # run the defined setup and installation steps.
-    def do_installation(self, required_version: Optional[VersionInfo] = None) -> None:
+    def do_installation(
+        self,
+        required_version: Optional[VersionInfo] = None,
+        required_arch: Optional[CpuArchitecture] = None,
+    ) -> None:
         self._setup_node()
-        if self._should_install():
+        if self._should_install(
+            required_version=required_version, required_arch=required_arch
+        ):
             self._uninstall()
             self._install_dependencies()
             self._install()
@@ -226,6 +263,7 @@ def __init__(
         node: Node,
         os_dependencies: Optional[DependencyInstaller] = None,
         downloader: Optional[Downloader] = None,
+        arch: Optional[CpuArchitecture] = None,
     ) -> None:
         self._node = node
         if not isinstance(self._node.os, Posix):
@@ -236,20 +274,34 @@ def __init__(
         self._package_manager_extra_args: List[str] = []
         self._os_dependencies = os_dependencies
         self._downloader = downloader
+        self._arch = arch
+        if self._arch:
+            # avoid building/running arm64 on i386, etc
+            system_arch = self._node.tools[Lscpu].get_architecture()
+            if system_arch not in ARCH_COMPATIBILITY_MATRIX[self._arch]:
+                raise UnsupportedCpuArchitectureException(system_arch)
 
 
 # Base class for package manager installation
 class PackageManagerInstall(Installer):
-    def __init__(self, node: Node, os_dependencies: DependencyInstaller) -> None:
+    def __init__(
+        self,
+        node: Node,
+        os_dependencies: DependencyInstaller,
+        arch: Optional[CpuArchitecture],
+    ) -> None:
         super().__init__(node, os_dependencies)
 
     # uninstall from the package manager
     def _uninstall(self) -> None:
         if not (isinstance(self._os, Posix) and self._check_if_installed()):
             return
         if self._os_dependencies is not None:
-            for os_package_check in self._os_dependencies.requirements:
-                if os_package_check.matcher(self._os) and os_package_check.packages:
+            for os_package_check in self._os_dependencies:
+                if (
+                    os_package_check.matcher(self._os, self._arch)
+                    and os_package_check.packages
+                ):
                     self._os.uninstall_packages(os_package_check.packages)
                     if os_package_check.stop_on_match:
                         break
@@ -261,7 +313,7 @@ def _check_if_installed(self) -> bool:
         # For dpdk, pkg-manager install is only for 'dpdk' and 'dpdk-dev'
         # This will take too long if it's more than a few packages.
         if self._os_dependencies is not None:
-            for os_package_check in self._os_dependencies.requirements:
+            for os_package_check in self._os_dependencies:
                 if os_package_check.matcher(self._os) and os_package_check.packages:
                     for pkg in os_package_check.packages:
                         if not self._os.package_exists(pkg):
@@ -271,9 +323,13 @@ def _check_if_installed(self) -> bool:
         return True
 
 
-def force_dpdk_default_source(variables: Dict[str, Any]) -> None:
+def force_dpdk_default_source(
+    variables: Dict[str, Any], build_arch: Optional[CpuArchitecture] = None
+) -> None:
     if not variables.get("dpdk_source", None):
         variables["dpdk_source"] = DPDK_STABLE_GIT_REPO
+    if build_arch:
+        variables["build_arch"] = build_arch
 
 
 _UBUNTU_LTS_VERSIONS = ["24.4.0", "22.4.0", "20.4.0", "18.4.0"]
@@ -368,13 +424,27 @@ def is_url_for_git_repo(url: str) -> bool:
     return scheme == "git" or check_for_git_https
 
 
-def unsupported_os_thrower(os: Posix) -> bool:
+# utility matcher to throw an OS error type for a match.
+def unsupported_os_thrower(os: Posix, arch: Optional[CpuArchitecture]) -> bool:
+    if arch:
+        message_suffix = f"OS and Architecture ({arch})"
+    else:
+        message_suffix = "OS"
     raise UnsupportedDistroException(
         os,
-        message=("Installer did not define dependencies for this os."),
+        message=f"Installer did not define dependencies for this {message_suffix}",
     )
 
 
+# utility matcher to throw an OS error when i386 is not supported
+def i386_not_implemented_thrower(os: Posix, arch: Optional[CpuArchitecture]) -> bool:
+    if arch and arch == CpuArchitecture.I386:
+        raise NotImplementedError(
+            "i386 is not implemented for this (installer,OS) combo."
+        )
+    return False
+
+
 def get_debian_backport_repo_args(os: Debian) -> List[str]:
     # ex: 'bionic-backports' or 'buster-backports'
     # these backport repos are available for the older OS's

microsoft/testsuites/dpdk/dpdksuite.py

@@ -25,6 +25,7 @@
 from lisa.testsuite import simple_requirement
 from lisa.tools import Echo, Git, Hugepages, Ip, Kill, Lsmod, Make, Modprobe
 from lisa.tools.hugepages import HugePageSize
+from lisa.tools.lscpu import CpuArchitecture
 from lisa.util.constants import SIGINT
 from microsoft.testsuites.dpdk.common import (
     DPDK_STABLE_GIT_REPO,
@@ -95,6 +96,29 @@ def verify_dpdk_build_netvsc(
     ) -> None:
         verify_dpdk_build(node, log, variables, "netvsc", HugePageSize.HUGE_2MB)
 
+    @TestCaseMetadata(
+        description="""
+            netvsc pmd version.
+            This test case checks DPDK can be built and installed correctly.
+            Prerequisites, accelerated networking must be enabled.
+            The VM should have at least two network interfaces,
+             with one interface for management.
+            More details refer https://docs.microsoft.com/en-us/azure/virtual-network/setup-dpdk#prerequisites # noqa: E501
+        """,
+        priority=2,
+        requirement=simple_requirement(
+            min_core_count=8,
+            min_nic_count=2,
+            network_interface=Sriov(),
+            unsupported_features=[Gpu, Infiniband],
+        ),
+    )
+    def verify_dpdk_build_netvsc_32bit(
+        self, node: Node, log: Logger, variables: Dict[str, Any]
+    ) -> None:
+        force_dpdk_default_source(variables, build_arch=CpuArchitecture.I386)
+        verify_dpdk_build(node, log, variables, "netvsc", HugePageSize.HUGE_2MB)
+
     @TestCaseMetadata(
         description="""
             netvsc pmd version with 1GiB hugepages