From e909ffb09fc1f2262f00d504127cc7571f3a18de Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Thu, 17 Oct 2024 23:46:18 +1300 Subject: [PATCH 01/14] Removed mixing, use a simple regex instead. --- source/fab/tools/__init__.py | 5 +- source/fab/tools/compiler.py | 133 ++++++++---------------- source/fab/tools/compiler_wrapper.py | 5 +- tests/conftest.py | 4 +- tests/unit_tests/tools/test_compiler.py | 23 ++-- tests/unit_tests/tools/test_tool_box.py | 6 +- 6 files changed, 61 insertions(+), 115 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index 45eb666f..baa06c01 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -10,8 +10,7 @@ from fab.tools.ar import Ar from fab.tools.category import Category from fab.tools.compiler import (CCompiler, Compiler, FortranCompiler, Gcc, - Gfortran, GnuVersionHandling, Icc, Ifort, - IntelVersionHandling) + Gfortran, Icc, Ifort) from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90 from fab.tools.flags import Flags from fab.tools.linker import Linker @@ -39,10 +38,8 @@ "Gcc", "Gfortran", "Git", - "GnuVersionHandling", "Icc", "Ifort", - "IntelVersionHandling", "Linker", "Mpif90", "Mpicc", diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index e1f87271..8c4f43ee 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -30,6 +30,9 @@ class Compiler(CompilerSuiteTool): :param name: name of the compiler. :param exec_name: name of the executable to start. :param suite: name of the compiler suite this tool belongs to. + :param version_regex: A regular expression that allows extraction of + the version number from the version output of the compiler. The + version is taken from the first group of a match. :param category: the Category (C_COMPILER or FORTRAN_COMPILER). :param mpi: whether the compiler or linker support MPI. :param compile_flag: the compilation flag to use when only requesting @@ -47,6 +50,7 @@ class Compiler(CompilerSuiteTool): def __init__(self, name: str, exec_name: Union[str, Path], suite: str, + version_regex: str, category: Category, mpi: bool = False, compile_flag: Optional[str] = None, @@ -61,6 +65,7 @@ def __init__(self, name: str, self._output_flag = output_flag if output_flag else "-o" self._openmp_flag = openmp_flag if openmp_flag else "" self.flags.extend(os.getenv("FFLAGS", "").split()) + self._version_regex = version_regex @property def mpi(self) -> bool: @@ -156,7 +161,14 @@ def get_version(self) -> Tuple[int, ...]: # Run the compiler to get the version and parse the output # The implementations depend on vendor output = self.run_version_command() - version_string = self.parse_version_output(self.category, output) + + # Multiline is required in case that the version number is the end + # of the string, otherwise the $ would not match the end of line + matches = re.search(self._version_regex, output, re.MULTILINE) + if not matches: + raise RuntimeError(f"Unexpected version output format for " + f"compiler '{self.name}': {output}") + version_string = matches.groups()[0] # Expect the version to be dot-separated integers. try: @@ -195,15 +207,6 @@ def run_version_command( raise RuntimeError(f"Error asking for version of compiler " f"'{self.name}'") from err - def parse_version_output(self, category: Category, - version_output: str) -> str: - ''' - Extract the numerical part from the version output. - Implemented in specific compilers. - ''' - raise NotImplementedError("The method `parse_version_output` must be " - "provided using a mixin.") - def get_version_string(self) -> str: """ Get a string representing the version of the given compiler. @@ -226,6 +229,8 @@ class CCompiler(Compiler): :param name: name of the compiler. :param exec_name: name of the executable to start. :param suite: name of the compiler suite. + :param version_regex: A regular expression that allows extraction of + the version number from the version output of the compiler. :param mpi: whether the compiler or linker support MPI. :param compile_flag: the compilation flag to use when only requesting compilation (not linking). @@ -236,6 +241,7 @@ class CCompiler(Compiler): # pylint: disable=too-many-arguments def __init__(self, name: str, exec_name: str, suite: str, + version_regex: str, mpi: bool = False, compile_flag: Optional[str] = None, output_flag: Optional[str] = None, @@ -243,7 +249,8 @@ def __init__(self, name: str, exec_name: str, suite: str, super().__init__(name, exec_name, suite, category=Category.C_COMPILER, mpi=mpi, compile_flag=compile_flag, output_flag=output_flag, - openmp_flag=openmp_flag) + openmp_flag=openmp_flag, + version_regex=version_regex) # ============================================================================ @@ -255,6 +262,8 @@ class FortranCompiler(Compiler): :param name: name of the compiler. :param exec_name: name of the executable to start. :param suite: name of the compiler suite. + :param version_regex: A regular expression that allows extraction of + the version number from the version output of the compiler. :param mpi: whether MPI is supported by this compiler or not. :param compile_flag: the compilation flag to use when only requesting compilation (not linking). @@ -269,6 +278,7 @@ class FortranCompiler(Compiler): # pylint: disable=too-many-arguments def __init__(self, name: str, exec_name: str, suite: str, + version_regex: str, mpi: bool = False, compile_flag: Optional[str] = None, output_flag: Optional[str] = None, @@ -280,7 +290,8 @@ def __init__(self, name: str, exec_name: str, suite: str, super().__init__(name=name, exec_name=exec_name, suite=suite, category=Category.FORTRAN_COMPILER, mpi=mpi, compile_flag=compile_flag, - output_flag=output_flag, openmp_flag=openmp_flag) + output_flag=output_flag, openmp_flag=openmp_flag, + version_regex=version_regex) self._module_folder_flag = (module_folder_flag if module_folder_flag else "") self._syntax_only_flag = syntax_only_flag @@ -334,45 +345,7 @@ def compile_file(self, input_file: Path, # ============================================================================ -class GnuVersionHandling(): - '''Mixin to handle version information from GNU compilers''' - - def parse_version_output(self, category: Category, - version_output: str) -> str: - ''' - Extract the numerical part from a GNU compiler's version output - - :param name: the compiler's name - :param category: the compiler's Category - :param version_output: the full version output from the compiler - :returns: the actual version as a string - - :raises RuntimeError: if the output is not in an expected format. - ''' - - # Expect the version to appear after some in parentheses, e.g. - # "GNU Fortran (...) n.n[.n, ...]" or # "gcc (...) n.n[.n, ...]" - if category is Category.FORTRAN_COMPILER: - name = "GNU Fortran" - else: - name = "gcc" - # A version number is a digit, followed by a sequence of digits and - # '.'', ending with a digit. It must then be followed by either the - # end of the string, or a space (e.g. "... 5.6 123456"). We can't use - # \b to determine the end, since then "1.2." would be matched - # excluding the dot (so it would become a valid 1.2) - exp = name + r" \(.*?\) (\d[\d\.]+\d)(?:$| )" - # Multiline is required in case that the version number is the - # end of the string, otherwise the $ would not match the end of line - matches = re.search(exp, version_output, re.MULTILINE) - if not matches: - raise RuntimeError(f"Unexpected version output format for " - f"compiler '{name}': {version_output}") - return matches.groups()[0] - - -# ============================================================================ -class Gcc(GnuVersionHandling, CCompiler): +class Gcc(CCompiler): '''Class for GNU's gcc compiler. :param name: name of this compiler. @@ -383,12 +356,18 @@ def __init__(self, name: str = "gcc", exec_name: str = "gcc", mpi: bool = False): + # A version number is a digit, followed by a sequence of digits and + # '.'', ending with a digit. It must then be followed by either the + # end of the string, or a space (e.g. "... 5.6 123456"). We can't use + # \b to determine the end, since then "1.2." would be matched + # excluding the dot (so it would become a valid 1.2) super().__init__(name, exec_name, suite="gnu", mpi=mpi, - openmp_flag="-fopenmp") + openmp_flag="-fopenmp", + version_regex=r"gcc \(.*?\) (\d[\d\.]+\d)(?:$| )") # ============================================================================ -class Gfortran(GnuVersionHandling, FortranCompiler): +class Gfortran(FortranCompiler): '''Class for GNU's gfortran compiler. :param name: name of this compiler. @@ -401,45 +380,13 @@ def __init__(self, name: str = "gfortran", super().__init__(name, exec_name, suite="gnu", openmp_flag="-fopenmp", module_folder_flag="-J", - syntax_only_flag="-fsyntax-only") + syntax_only_flag="-fsyntax-only", + version_regex=(r"GNU Fortran \(.*?\) " + r"(\d[\d\.]+\d)(?:$| )")) # ============================================================================ -class IntelVersionHandling(): - '''Mixin to handle version information from Intel compilers''' - - def parse_version_output(self, category: Category, - version_output: str) -> str: - ''' - Extract the numerical part from an Intel compiler's version output - - :param name: the compiler's name - :param version_output: the full version output from the compiler - :returns: the actual version as a string - - :raises RuntimeError: if the output is not in an expected format. - ''' - - # Expect the version to appear after some in parentheses, e.g. - # "icc (...) n.n[.n, ...]" or "ifort (...) n.n[.n, ...]" - if category == Category.C_COMPILER: - name = "icc" - else: - name = "ifort" - - # A version number is a digit, followed by a sequence of digits and - # '.'', ending with a digit. It must then be followed by a space. - exp = name + r" \(.*?\) (\d[\d\.]+\d) " - matches = re.search(exp, version_output) - - if not matches: - raise RuntimeError(f"Unexpected version output format for " - f"compiler '{name}': {version_output}") - return matches.groups()[0] - - -# ============================================================================ -class Icc(IntelVersionHandling, CCompiler): +class Icc(CCompiler): '''Class for the Intel's icc compiler. :param name: name of this compiler. @@ -449,11 +396,12 @@ class Icc(IntelVersionHandling, CCompiler): def __init__(self, name: str = "icc", exec_name: str = "icc"): super().__init__(name, exec_name, suite="intel-classic", - openmp_flag="-qopenmp") + openmp_flag="-qopenmp", + version_regex=r"icc \(ICC\) (\d[\d\.]+\d) ") # ============================================================================ -class Ifort(IntelVersionHandling, FortranCompiler): +class Ifort(FortranCompiler): '''Class for Intel's ifort compiler. :param name: name of this compiler. @@ -465,4 +413,5 @@ def __init__(self, name: str = "ifort", exec_name: str = "ifort"): super().__init__(name, exec_name, suite="intel-classic", module_folder_flag="-module", openmp_flag="-qopenmp", - syntax_only_flag="-syntax-only") + syntax_only_flag="-syntax-only", + version_regex=r"ifort \(IFORT\) (\d[\d\.]+\d) ") diff --git a/source/fab/tools/compiler_wrapper.py b/source/fab/tools/compiler_wrapper.py index ad76bfad..4bd68ffc 100644 --- a/source/fab/tools/compiler_wrapper.py +++ b/source/fab/tools/compiler_wrapper.py @@ -36,12 +36,9 @@ def __init__(self, name: str, exec_name: str, name=name, exec_name=exec_name, category=self._compiler.category, suite=self._compiler.suite, + version_regex=self._compiler._version_regex, mpi=mpi, availability_option=self._compiler.availability_option) - # We need to have the right version to parse the version output - # So we set this function based on the function that the - # wrapped compiler uses: - setattr(self, "parse_version_output", compiler.parse_version_output) def __str__(self): return f"{type(self).__name__}({self._compiler.name})" diff --git a/tests/conftest.py b/tests/conftest.py index 559d4f3b..86de6476 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -18,7 +18,8 @@ @pytest.fixture(name="mock_c_compiler") def fixture_mock_c_compiler(): '''Provides a mock C-compiler.''' - mock_compiler = CCompiler("mock_c_compiler", "mock_exec", "suite") + mock_compiler = CCompiler("mock_c_compiler", "mock_exec", "suite", + version_regex="something") mock_compiler.run = mock.Mock() mock_compiler._version = (1, 2, 3) mock_compiler._name = "mock_c_compiler" @@ -32,6 +33,7 @@ def fixture_mock_fortran_compiler(): '''Provides a mock Fortran-compiler.''' mock_compiler = FortranCompiler("mock_fortran_compiler", "mock_exec", "suite", module_folder_flag="", + version_regex="something", syntax_only_flag=None, compile_flag=None, output_flag=None, openmp_flag=None) mock_compiler.run = mock.Mock() diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index c31e480e..ca15e279 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -20,7 +20,8 @@ def test_compiler(): '''Test the compiler constructor.''' - cc = Compiler("gcc", "gcc", "gnu", category=Category.C_COMPILER, openmp_flag="-fopenmp") + cc = Compiler("gcc", "gcc", "gnu", version_regex="some_regex", + category=Category.C_COMPILER, openmp_flag="-fopenmp") assert cc.category == Category.C_COMPILER assert cc._compile_flag == "-c" assert cc._output_flag == "-o" @@ -29,13 +30,9 @@ def test_compiler(): assert cc.suite == "gnu" assert not cc.mpi assert cc.openmp_flag == "-fopenmp" - with pytest.raises(NotImplementedError) as err: - cc.parse_version_output(Category.FORTRAN_COMPILER, "NOT NEEDED") - assert ("The method `parse_version_output` must be provided using a mixin." - in str(err.value)) fc = FortranCompiler("gfortran", "gfortran", "gnu", openmp_flag="-fopenmp", - module_folder_flag="-J") + version_regex="something", module_folder_flag="-J") assert fc._compile_flag == "-c" assert fc._output_flag == "-o" assert fc.category == Category.FORTRAN_COMPILER @@ -44,10 +41,6 @@ def test_compiler(): assert fc.flags == [] assert not fc.mpi assert fc.openmp_flag == "-fopenmp" - with pytest.raises(NotImplementedError) as err: - fc.parse_version_output(Category.FORTRAN_COMPILER, "NOT NEEDED") - assert ("The method `parse_version_output` must be provided using a mixin." - in str(err.value)) def test_compiler_openmp(): @@ -148,16 +141,19 @@ def test_compiler_with_env_fflags(): def test_compiler_syntax_only(): '''Tests handling of syntax only flags.''' fc = FortranCompiler("gfortran", "gfortran", "gnu", + version_regex="something", openmp_flag="-fopenmp", module_folder_flag="-J") # Empty since no flag is defined assert not fc.has_syntax_only fc = FortranCompiler("gfortran", "gfortran", "gnu", openmp_flag="-fopenmp", - module_folder_flag="-J", syntax_only_flag=None) + version_regex="something", module_folder_flag="-J", + syntax_only_flag=None) # Empty since no flag is defined assert not fc.has_syntax_only fc = FortranCompiler("gfortran", "gfortran", "gnu", + version_regex="something", openmp_flag="-fopenmp", module_folder_flag="-J", syntax_only_flag="-fsyntax-only") @@ -168,6 +164,7 @@ def test_compiler_syntax_only(): def test_compiler_without_openmp(): '''Tests that the openmp flag is not used when openmp is not enabled. ''' fc = FortranCompiler("gfortran", "gfortran", "gnu", + version_regex="something", openmp_flag="-fopenmp", module_folder_flag="-J", syntax_only_flag="-fsyntax-only") @@ -184,6 +181,7 @@ def test_compiler_with_openmp(): '''Tests that the openmp flag is used as expected if openmp is enabled. ''' fc = FortranCompiler("gfortran", "gfortran", "gnu", + version_regex="something", openmp_flag="-fopenmp", module_folder_flag="-J", syntax_only_flag="-fsyntax-only") @@ -199,7 +197,7 @@ def test_compiler_with_openmp(): def test_compiler_module_output(): '''Tests handling of module output_flags.''' fc = FortranCompiler("gfortran", "gfortran", suite="gnu", - module_folder_flag="-J") + version_regex="something", module_folder_flag="-J") fc.set_module_output_path("/module_out") assert fc._module_output_path == "/module_out" fc.run = mock.MagicMock() @@ -212,6 +210,7 @@ def test_compiler_module_output(): def test_compiler_with_add_args(): '''Tests that additional arguments are handled as expected.''' fc = FortranCompiler("gfortran", "gfortran", suite="gnu", + version_regex="something", openmp_flag="-fopenmp", module_folder_flag="-J") fc.set_module_output_path("/module_out") diff --git a/tests/unit_tests/tools/test_tool_box.py b/tests/unit_tests/tools/test_tool_box.py index 29bedf30..2e886ea5 100644 --- a/tests/unit_tests/tools/test_tool_box.py +++ b/tests/unit_tests/tools/test_tool_box.py @@ -44,9 +44,11 @@ def test_tool_box_add_tool_replacement(): warning can be disabled.''' tb = ToolBox() - mock_compiler1 = CCompiler("mock_c_compiler1", "mock_exec1", "suite") + mock_compiler1 = CCompiler("mock_c_compiler1", "mock_exec1", "suite", + version_regex="something") mock_compiler1._is_available = True - mock_compiler2 = CCompiler("mock_c_compiler2", "mock_exec2", "suite") + mock_compiler2 = CCompiler("mock_c_compiler2", "mock_exec2", "suite", + version_regex="something") mock_compiler2._is_available = True tb.add_tool(mock_compiler1) From 0557b03abe7e59a303b9e6870cf2c3557ac4bccb Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 18 Oct 2024 15:07:11 +1300 Subject: [PATCH 02/14] Added support for ifx/icx compiler as intel-llvm class. --- source/fab/tools/__init__.py | 4 +- source/fab/tools/compiler.py | 30 ++++++++++ tests/unit_tests/tools/test_compiler.py | 77 ++++++++++++++++++++++++- 3 files changed, 109 insertions(+), 2 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index baa06c01..42357f2a 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -10,7 +10,7 @@ from fab.tools.ar import Ar from fab.tools.category import Category from fab.tools.compiler import (CCompiler, Compiler, FortranCompiler, Gcc, - Gfortran, Icc, Ifort) + Gfortran, Icc, Icx, Ifort, Ifx) from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90 from fab.tools.flags import Flags from fab.tools.linker import Linker @@ -39,7 +39,9 @@ "Gfortran", "Git", "Icc", + "Icx", "Ifort", + "Ifx", "Linker", "Mpif90", "Mpicc", diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index 8c4f43ee..d2604a3b 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -400,6 +400,20 @@ def __init__(self, name: str = "icc", exec_name: str = "icc"): version_regex=r"icc \(ICC\) (\d[\d\.]+\d) ") +# ============================================================================ +class Icx(CCompiler): + '''Class for the Intel's new llvm based icx compiler. + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + def __init__(self, name: str = "icx", exec_name: str = "icx"): + super().__init__(name, exec_name, suite="intel-llvm", + openmp_flag="-qopenmp", + version_regex=(r"Intel\(R\) oneAPI DPC\+\+/C\+\+ " + r"Compiler (\d[\d\.]+\d) ")) + + # ============================================================================ class Ifort(FortranCompiler): '''Class for Intel's ifort compiler. @@ -415,3 +429,19 @@ def __init__(self, name: str = "ifort", exec_name: str = "ifort"): openmp_flag="-qopenmp", syntax_only_flag="-syntax-only", version_regex=r"ifort \(IFORT\) (\d[\d\.]+\d) ") + + +# ============================================================================ +class Ifx(FortranCompiler): + '''Class for Intel's new ifx compiler. + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + + def __init__(self, name: str = "ifx", exec_name: str = "ifx"): + super().__init__(name, exec_name, suite="intel-llvm", + module_folder_flag="-module", + openmp_flag="-qopenmp", + syntax_only_flag="-syntax-only", + version_regex=r"ifx \(IFORT\) (\d[\d\.]+\d) ") diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index ca15e279..5acb1cf4 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -15,7 +15,7 @@ import pytest from fab.tools import (Category, CCompiler, Compiler, FortranCompiler, - Gcc, Gfortran, Icc, Ifort) + Gcc, Gfortran, Icc, Icx, Ifort, Ifx) def test_compiler(): @@ -653,3 +653,78 @@ def test_ifort_get_version_invalid_version(version): with pytest.raises(RuntimeError) as err: ifort.get_version() assert "Unexpected version output format for compiler" in str(err.value) + + +# ============================================================================ +def test_icx(): + '''Tests the icx class.''' + icx = Icx() + assert icx.name == "icx" + assert isinstance(icx, CCompiler) + assert icx.category == Category.C_COMPILER + assert not icx.mpi + + +def test_icx_get_version_2023(): + '''Test icx 2023.0.0 version detection.''' + full_output = dedent(""" +Intel(R) oneAPI DPC++/C++ Compiler 2023.0.0 (2023.0.0.20221201) +Target: x86_64-unknown-linux-gnu +Thread model: posix +InstalledDir: /opt/intel/oneapi/compiler/2023.0.0/linux/bin-llvm +Configuration file: /opt/intel/oneapi/compiler/2023.0.0/linux/bin-llvm/../bin/icx.cfg + + """) + icx = Icx() + with mock.patch.object(icx, "run", mock.Mock(return_value=full_output)): + assert icx.get_version() == (2023, 0, 0) + + +def test_icx_get_version_with_icc_string(): + '''Tests the icx class with an icc version output.''' + full_output = dedent(""" + icc (ICC) 2021.10.0 20230609 + Copyright (C) 1985-2023 Intel Corporation. All rights reserved. + + """) + icx = Icx() + with mock.patch.object(icx, "run", mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + icx.get_version() + assert "Unexpected version output format for compiler" in str(err.value) + + +# ============================================================================ +def test_ifx(): + '''Tests the ifx class.''' + ifx = Ifx() + assert ifx.name == "ifx" + assert isinstance(ifx, FortranCompiler) + assert ifx.category == Category.FORTRAN_COMPILER + assert not ifx.mpi + + +def test_ifx_get_version_2023(): + '''Test ifx 2023.0.0 version detection.''' + full_output = dedent(""" +ifx (IFORT) 2023.0.0 20221201 +Copyright (C) 1985-2022 Intel Corporation. All rights reserved. + + """) + ifx = Ifx() + with mock.patch.object(ifx, "run", mock.Mock(return_value=full_output)): + assert ifx.get_version() == (2023, 0, 0) + + +def test_ifx_get_version_with_icc_string(): + '''Tests the ifx class with an icc version output.''' + full_output = dedent(""" + icc (ICC) 2021.10.0 20230609 + Copyright (C) 1985-2023 Intel Corporation. All rights reserved. + + """) + ifx = Ifx() + with mock.patch.object(ifx, "run", mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + ifx.get_version() + assert "Unexpected version output format for compiler" in str(err.value) From d75ea1668eed3c403073252662c7903e10412a3f Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 18 Oct 2024 16:40:06 +1300 Subject: [PATCH 03/14] Added support for nvidia compiler. --- source/fab/tools/__init__.py | 4 +- source/fab/tools/compiler.py | 94 ++++++++++++++--- tests/unit_tests/tools/test_compiler.py | 132 +++++++++++++++++++++--- 3 files changed, 203 insertions(+), 27 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index 42357f2a..ec2a6e23 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -10,7 +10,7 @@ from fab.tools.ar import Ar from fab.tools.category import Category from fab.tools.compiler import (CCompiler, Compiler, FortranCompiler, Gcc, - Gfortran, Icc, Icx, Ifort, Ifx) + Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90 from fab.tools.flags import Flags from fab.tools.linker import Linker @@ -45,6 +45,8 @@ "Linker", "Mpif90", "Mpicc", + "Nvc", + "Nvfortran", "Preprocessor", "Psyclone", "Rsync", diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index d2604a3b..7f66eddf 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -386,6 +386,8 @@ def __init__(self, name: str = "gfortran", # ============================================================================ +# intel-classic +# class Icc(CCompiler): '''Class for the Intel's icc compiler. @@ -401,6 +403,25 @@ def __init__(self, name: str = "icc", exec_name: str = "icc"): # ============================================================================ +class Ifort(FortranCompiler): + '''Class for Intel's ifort compiler. + + :param name: name of this compiler. + :param exec_name: name of the executable. + :param mpi: whether the compiler supports MPI. + ''' + + def __init__(self, name: str = "ifort", exec_name: str = "ifort"): + super().__init__(name, exec_name, suite="intel-classic", + module_folder_flag="-module", + openmp_flag="-qopenmp", + syntax_only_flag="-syntax-only", + version_regex=r"ifort \(IFORT\) (\d[\d\.]+\d) ") + + +# ============================================================================ +# intel-llvm +# class Icx(CCompiler): '''Class for the Intel's new llvm based icx compiler. @@ -415,33 +436,80 @@ def __init__(self, name: str = "icx", exec_name: str = "icx"): # ============================================================================ -class Ifort(FortranCompiler): - '''Class for Intel's ifort compiler. +class Ifx(FortranCompiler): + '''Class for Intel's new ifx compiler. :param name: name of this compiler. :param exec_name: name of the executable. - :param mpi: whether the compiler supports MPI. ''' - def __init__(self, name: str = "ifort", exec_name: str = "ifort"): - super().__init__(name, exec_name, suite="intel-classic", + def __init__(self, name: str = "ifx", exec_name: str = "ifx"): + super().__init__(name, exec_name, suite="intel-llvm", module_folder_flag="-module", openmp_flag="-qopenmp", syntax_only_flag="-syntax-only", - version_regex=r"ifort \(IFORT\) (\d[\d\.]+\d) ") + version_regex=r"ifx \(IFORT\) (\d[\d\.]+\d) ") # ============================================================================ -class Ifx(FortranCompiler): - '''Class for Intel's new ifx compiler. +# nvidia +# +class Nvc(CCompiler): + '''Class for Nvidia's nvc compiler. Nvc has a '-' in the + version number. In order to get this, we overwrite run_version_command + and replace any '-' with a '.' :param name: name of this compiler. :param exec_name: name of the executable. ''' - def __init__(self, name: str = "ifx", exec_name: str = "ifx"): - super().__init__(name, exec_name, suite="intel-llvm", + def __init__(self, name: str = "nvc", exec_name: str = "nvc"): + super().__init__(name, exec_name, suite="nvidia", + openmp_flag="-mp", + version_regex=r"nvc (\d[\d\.-]+\d)") + + def run_version_command( + self, version_command: Optional[str] = '--version') -> str: + '''Run the compiler's command to get its version. This implementation + runs the function in the base class, and changes any '-' into a + '.' to support nvidia version numbers which have dashes, e.g. 23.5-0. + + :param version_command: The compiler argument used to get version info. + + :returns: The output from the version command, with any '-' replaced + with '.' + ''' + version_string = super().run_version_command() + return version_string.replace("-", ".") + + +# ============================================================================ +class Nvfortran(FortranCompiler): + '''Class for Nvidia's nvfortran compiler. Nvfortran has a '-' in the + version number. In order to get this, we overwrite run_version_command + and replace any '-' with a '.' + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + + def __init__(self, name: str = "nvfortran", exec_name: str = "nvfortran"): + super().__init__(name, exec_name, suite="nvidia", module_folder_flag="-module", - openmp_flag="-qopenmp", - syntax_only_flag="-syntax-only", - version_regex=r"ifx \(IFORT\) (\d[\d\.]+\d) ") + openmp_flag="-mp", + syntax_only_flag="-Msyntax-only", + version_regex=r"nvfortran (\d[\d\.-]+\d)") + + def run_version_command( + self, version_command: Optional[str] = '--version') -> str: + '''Run the compiler's command to get its version. This implementation + runs the function in the base class, and changes any '-' into a + '.' to support nvidia version numbers which have dashes, e.g. 23.5-0. + + :param version_command: The compiler argument used to get version info. + + :returns: The output from the version command, with any '-' replaced + with '.' + ''' + version_string = super().run_version_command() + return version_string.replace("-", ".") diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index 5acb1cf4..43b8606b 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -15,7 +15,7 @@ import pytest from fab.tools import (Category, CCompiler, Compiler, FortranCompiler, - Gcc, Gfortran, Icc, Icx, Ifort, Ifx) + Gcc, Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) def test_compiler(): @@ -232,6 +232,9 @@ def test_compiler_with_add_args(): openmp=True, syntax_only=True) +# ============================================================================ +# Test version number handling +# ============================================================================ def test_get_version_string(): '''Tests the get_version_string() method. ''' @@ -392,6 +395,8 @@ def test_get_version_bad_result_is_not_cached(): assert c.run.called +# ============================================================================ +# gcc # ============================================================================ def test_gcc(): '''Tests the gcc class.''' @@ -427,6 +432,8 @@ def test_gcc_get_version_with_icc_string(): assert "Unexpected version output format for compiler" in str(err.value) +# ============================================================================ +# gfortran # ============================================================================ def test_gfortran(): '''Tests the gfortran class.''' @@ -511,7 +518,8 @@ def test_gfortran_get_version_12(): """) gfortran = Gfortran() - with mock.patch.object(gfortran, "run", mock.Mock(return_value=full_output)): + with mock.patch.object(gfortran, "run", + mock.Mock(return_value=full_output)): assert gfortran.get_version() == (12, 1, 0) @@ -523,12 +531,16 @@ def test_gfortran_get_version_with_ifort_string(): """) gfortran = Gfortran() - with mock.patch.object(gfortran, "run", mock.Mock(return_value=full_output)): + with mock.patch.object(gfortran, "run", + mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: gfortran.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) +# ============================================================================ +# icc # ============================================================================ def test_icc(): '''Tests the icc class.''' @@ -561,9 +573,12 @@ def test_icc_get_version_with_gcc_string(): with mock.patch.object(icc, "run", mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: icc.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) +# ============================================================================ +# ifort # ============================================================================ def test_ifort(): '''Tests the ifort class.''' @@ -633,7 +648,8 @@ def test_ifort_get_version_with_icc_string(): with mock.patch.object(ifort, "run", mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: ifort.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) @pytest.mark.parametrize("version", ["5.15f.2", @@ -652,9 +668,12 @@ def test_ifort_get_version_invalid_version(version): with mock.patch.object(ifort, "run", mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: ifort.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) +# ============================================================================ +# icx # ============================================================================ def test_icx(): '''Tests the icx class.''' @@ -672,7 +691,8 @@ def test_icx_get_version_2023(): Target: x86_64-unknown-linux-gnu Thread model: posix InstalledDir: /opt/intel/oneapi/compiler/2023.0.0/linux/bin-llvm -Configuration file: /opt/intel/oneapi/compiler/2023.0.0/linux/bin-llvm/../bin/icx.cfg +Configuration file: /opt/intel/oneapi/compiler/2023.0.0/linux/bin-llvm/""" + """../bin/icx.cfg """) icx = Icx() @@ -691,9 +711,12 @@ def test_icx_get_version_with_icc_string(): with mock.patch.object(icx, "run", mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: icx.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) +# ============================================================================ +# ifx # ============================================================================ def test_ifx(): '''Tests the ifx class.''' @@ -716,15 +739,98 @@ def test_ifx_get_version_2023(): assert ifx.get_version() == (2023, 0, 0) -def test_ifx_get_version_with_icc_string(): +def test_ifx_get_version_with_ifort_string(): '''Tests the ifx class with an icc version output.''' full_output = dedent(""" - icc (ICC) 2021.10.0 20230609 - Copyright (C) 1985-2023 Intel Corporation. All rights reserved. + ifort (IFORT) 19.0.0.117 20180804 + Copyright (C) 1985-2018 Intel Corporation. All rights reserved. """) ifx = Ifx() with mock.patch.object(ifx, "run", mock.Mock(return_value=full_output)): with pytest.raises(RuntimeError) as err: ifx.get_version() - assert "Unexpected version output format for compiler" in str(err.value) + assert ("Unexpected version output format for compiler" + in str(err.value)) + + +# ============================================================================ +# nvc +# ============================================================================ +def test_nvc(): + '''Tests the nvc class.''' + nvc = Nvc() + assert nvc.name == "nvc" + assert isinstance(nvc, CCompiler) + assert nvc.category == Category.C_COMPILER + assert not nvc.mpi + + +def test_nvc_get_version_2023(): + '''Test nvc .23.5 version detection.''' + full_output = dedent(""" + +nvc 23.5-0 64-bit target on x86-64 Linux -tp icelake-server +NVIDIA Compilers and Tools +Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + """) + nvc = Nvc() + with mock.patch.object(nvc, "run", mock.Mock(return_value=full_output)): + assert nvc.get_version() == (23, 5, 0) + + +def test_nvc_get_version_with_icc_string(): + '''Tests the nvc class with an icc version output.''' + full_output = dedent(""" + icc (ICC) 2021.10.0 20230609 + Copyright (C) 1985-2023 Intel Corporation. All rights reserved. + + """) + nvc = Nvc() + with mock.patch.object(nvc, "run", mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + nvc.get_version() + assert ("Unexpected version output format for compiler" + in str(err.value)) + + +# ============================================================================ +# nvfortran +# ============================================================================ +def test_nvfortran(): + '''Tests the nvfortran class.''' + nvfortran = Nvfortran() + assert nvfortran.name == "nvfortran" + assert isinstance(nvfortran, FortranCompiler) + assert nvfortran.category == Category.FORTRAN_COMPILER + assert not nvfortran.mpi + + +def test_nvfortran_get_version_2023(): + '''Test nvfortran .23.5 version detection.''' + full_output = dedent(""" + +nvfortran 23.5-0 64-bit target on x86-64 Linux -tp icelake-server +NVIDIA Compilers and Tools +Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. All rights reserved. + """) + nvfortran = Nvfortran() + with mock.patch.object(nvfortran, "run", + mock.Mock(return_value=full_output)): + assert nvfortran.get_version() == (23, 5, 0) + + +def test_nvfortran_get_version_with_ifort_string(): + '''Tests the nvfortran class with an icc version output.''' + full_output = dedent(""" + ifort (IFORT) 19.0.0.117 20180804 + Copyright (C) 1985-2018 Intel Corporation. All rights reserved. + + """) + nvfortran = Nvfortran() + with mock.patch.object(nvfortran, "run", + mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + nvfortran.get_version() + assert ("Unexpected version output format for compiler" + in str(err.value)) From 9dca741d147790ae4028f411d67550655f1058bd Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 18 Oct 2024 18:18:03 +1300 Subject: [PATCH 04/14] Add preliminary support for Cray compiler. --- source/fab/tools/__init__.py | 7 +- source/fab/tools/compiler.py | 50 ++++++++- tests/unit_tests/tools/test_compiler.py | 129 ++++++++++++++++++++++-- 3 files changed, 174 insertions(+), 12 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index ec2a6e23..4dc59d14 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -9,8 +9,9 @@ from fab.tools.ar import Ar from fab.tools.category import Category -from fab.tools.compiler import (CCompiler, Compiler, FortranCompiler, Gcc, - Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) +from fab.tools.compiler import (CCompiler, Compiler, Craycc, Crayftn, + FortranCompiler, Gcc, Gfortran, Icc, + Icx, Ifort, Ifx, Nvc, Nvfortran) from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90 from fab.tools.flags import Flags from fab.tools.linker import Linker @@ -31,6 +32,8 @@ "CompilerWrapper", "Cpp", "CppFortran", + "Craycc", + "Crayftn", "Fcm", "Flags", "FortranCompiler", diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index 7f66eddf..d2db3f58 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -165,11 +165,12 @@ def get_version(self) -> Tuple[int, ...]: # Multiline is required in case that the version number is the end # of the string, otherwise the $ would not match the end of line matches = re.search(self._version_regex, output, re.MULTILINE) + print("XXX", output, matches) if not matches: raise RuntimeError(f"Unexpected version output format for " f"compiler '{self.name}': {output}") version_string = matches.groups()[0] - + print("YYY", matches.groups(), version_string) # Expect the version to be dot-separated integers. try: version = tuple(int(x) for x in version_string.split('.')) @@ -344,6 +345,8 @@ def compile_file(self, input_file: Path, add_flags=params) +# ============================================================================ +# Gnu # ============================================================================ class Gcc(CCompiler): '''Class for GNU's gcc compiler. @@ -387,7 +390,7 @@ def __init__(self, name: str = "gfortran", # ============================================================================ # intel-classic -# +# ============================================================================ class Icc(CCompiler): '''Class for the Intel's icc compiler. @@ -421,7 +424,7 @@ def __init__(self, name: str = "ifort", exec_name: str = "ifort"): # ============================================================================ # intel-llvm -# +# ============================================================================ class Icx(CCompiler): '''Class for the Intel's new llvm based icx compiler. @@ -453,7 +456,7 @@ def __init__(self, name: str = "ifx", exec_name: str = "ifx"): # ============================================================================ # nvidia -# +# ============================================================================ class Nvc(CCompiler): '''Class for Nvidia's nvc compiler. Nvc has a '-' in the version number. In order to get this, we overwrite run_version_command @@ -513,3 +516,42 @@ def run_version_command( ''' version_string = super().run_version_command() return version_string.replace("-", ".") + + +# ============================================================================ +# Cray compiler +# ============================================================================ +class Craycc(CCompiler): + '''Class for the native Cray C compiler. Cray has two different compilers. + Older ones have as version number: + Cray C : Version 8.7.0 Tue Jul 23, 2024 07:39:46 + Newer compiler (several lines, the important one): + Cray clang version 15.0.1 (66f7391d6a03cf932f321b9f6b1d8612ef5f362c) + We use the beginning ("cray c") to identify the compiler, which works for + both cray c and cray clang. Then we ignore non-numbers, to reach the + version number which is then extracted. + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + def __init__(self, name: str = "craycc", exec_name: str = "craycc"): + super().__init__(name, exec_name, suite="cray", mpi=True, + openmp_flag="-qopenmp", + version_regex=r"Cray [Cc][^\d]* (\d[\d\.]+\d) ") + + +# ============================================================================ +class Crayftn(FortranCompiler): + '''Class for the native Cray Fortran compiler. + + :param name: name of this compiler. + :param exec_name: name of the executable. + ''' + + def __init__(self, name: str = "crayftn", exec_name: str = "crayftn"): + super().__init__(name, exec_name, suite="cray", mpi=True, + module_folder_flag="-module", + openmp_flag="-qopenmp", + syntax_only_flag="-syntax-only", + version_regex=(r"Cray Fortran : Version " + r"(\d[\d\.]+\d) ")) diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index 43b8606b..f95026ef 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -14,8 +14,9 @@ import pytest -from fab.tools import (Category, CCompiler, Compiler, FortranCompiler, - Gcc, Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) +from fab.tools import (Category, CCompiler, Compiler, Craycc, Crayftn, + FortranCompiler, Gcc, Gfortran, Icc, Icx, Ifort, Ifx, + Nvc, Nvfortran) def test_compiler(): @@ -766,8 +767,8 @@ def test_nvc(): assert not nvc.mpi -def test_nvc_get_version_2023(): - '''Test nvc .23.5 version detection.''' +def test_nvc_get_version_23_5_0(): + '''Test nvc 23.5.0 version detection.''' full_output = dedent(""" nvc 23.5-0 64-bit target on x86-64 Linux -tp icelake-server @@ -806,8 +807,8 @@ def test_nvfortran(): assert not nvfortran.mpi -def test_nvfortran_get_version_2023(): - '''Test nvfortran .23.5 version detection.''' +def test_nvfortran_get_version_23_5_0(): + '''Test nvfortran 23.5 version detection.''' full_output = dedent(""" nvfortran 23.5-0 64-bit target on x86-64 Linux -tp icelake-server @@ -834,3 +835,119 @@ def test_nvfortran_get_version_with_ifort_string(): nvfortran.get_version() assert ("Unexpected version output format for compiler" in str(err.value)) + + +# ============================================================================ +# Craycc +# ============================================================================ +def test_craycc(): + '''Tests the Craycc class.''' + craycc = Craycc() + assert craycc.name == "craycc" + assert isinstance(craycc, CCompiler) + assert craycc.category == Category.C_COMPILER + assert craycc.mpi + + +def test_craycc_get_version_8_7_0(): + '''Test craycc .23.5 version detection.''' + full_output = dedent(""" +Cray C : Version 8.7.0 Tue Jul 23, 2024 07:39:46 + + """) + craycc = Craycc() + with mock.patch.object(craycc, "run", mock.Mock(return_value=full_output)): + assert craycc.get_version() == (8, 7, 0) + + +def test_craycc_get_version_2023(): + '''Test craycc .23.5 version detection.''' + full_output = dedent(""" +Cray clang version 15.0.1 (66f7391d6a03cf932f321b9f6b1d8612ef5f362c) + +Target: x86_64-unknown-linux-gnu + +Thread model: posix + +InstalledDir: /opt/cray/pe/cce/15.0.1/cce-clang/x86_64/share/../bin + +Found candidate GCC installation: /opt/gcc/10.3.0/snos/lib/gcc/x86_64-""" + """suse-linux/10.3.0 + +Selected GCC installation: /opt/gcc/10.3.0/snos/lib/gcc/x86_64-suse-""" + """linux/10.3.0 + +Candidate multilib: .;@m64 + +Selected multilib: .;@m64 + +OFFICIAL + """) + craycc = Craycc() + with mock.patch.object(craycc, "run", mock.Mock(return_value=full_output)): + assert craycc.get_version() == (15, 0, 1) + + +def test_craycc_get_version_with_icc_string(): + '''Tests the Craycc class with an icc version output.''' + full_output = dedent(""" + icc (ICC) 2021.10.0 20230609 + Copyright (C) 1985-2023 Intel Corporation. All rights reserved. + + """) + craycc = Craycc() + with mock.patch.object(craycc, "run", mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + craycc.get_version() + assert ("Unexpected version output format for compiler" + in str(err.value)) + + +# ============================================================================ +# Crayftn +# ============================================================================ +def test_crayftn(): + '''Tests the Crayftn class.''' + crayftn = Crayftn() + assert crayftn.name == "crayftn" + assert isinstance(crayftn, FortranCompiler) + assert crayftn.category == Category.FORTRAN_COMPILER + assert crayftn.mpi + + +def test_crayftn_get_version_8_7_0(): + '''Test crayftn .23.5 version detection.''' + full_output = dedent(""" +Cray Fortran : Version 8.7.0 Tue Jul 23, 2024 07:39:25 + """) + crayftn = Crayftn() + with mock.patch.object(crayftn, "run", + mock.Mock(return_value=full_output)): + assert crayftn.get_version() == (8, 7, 0) + + +def test_crayftn_get_version_15_0_1(): + '''Test Crayftn 15.0.1 version detection.''' + full_output = dedent(""" +Cray Fortran : Version 15.0.1 Tue Jul 23, 2024 07:39:25 + """) + crayftn = Crayftn() + with mock.patch.object(crayftn, "run", + mock.Mock(return_value=full_output)): + assert crayftn.get_version() == (15, 0, 1) + + +def test_crayftn_get_version_with_ifort_string(): + '''Tests the crayftn class with an icc version output.''' + full_output = dedent(""" + ifort (IFORT) 19.0.0.117 20180804 + Copyright (C) 1985-2018 Intel Corporation. All rights reserved. + + """) + crayftn = Crayftn() + with mock.patch.object(crayftn, "run", + mock.Mock(return_value=full_output)): + with pytest.raises(RuntimeError) as err: + crayftn.get_version() + assert ("Unexpected version output format for compiler" + in str(err.value)) From 3d3a2c8ced2f44faaaa5334ee2456f8a9a789e3f Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 18 Oct 2024 19:02:58 +1300 Subject: [PATCH 05/14] Added Cray compiler wrapper ftn and cc. --- source/fab/tools/__init__.py | 5 +- source/fab/tools/compiler.py | 2 - source/fab/tools/compiler_wrapper.py | 24 +++++++++ source/fab/tools/tool_repository.py | 31 ++++++++---- .../unit_tests/tools/test_compiler_wrapper.py | 49 ++++++++++++++++++- 5 files changed, 97 insertions(+), 14 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index 4dc59d14..4ee807da 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -12,7 +12,8 @@ from fab.tools.compiler import (CCompiler, Compiler, Craycc, Crayftn, FortranCompiler, Gcc, Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) -from fab.tools.compiler_wrapper import CompilerWrapper, Mpicc, Mpif90 +from fab.tools.compiler_wrapper import (CompilerWrapper, CrayCc, CrayFtn, + Mpicc, Mpif90) from fab.tools.flags import Flags from fab.tools.linker import Linker from fab.tools.psyclone import Psyclone @@ -33,7 +34,9 @@ "Cpp", "CppFortran", "Craycc", + "CrayCc", "Crayftn", + "CrayFtn", "Fcm", "Flags", "FortranCompiler", diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index d2db3f58..1ce603fd 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -165,12 +165,10 @@ def get_version(self) -> Tuple[int, ...]: # Multiline is required in case that the version number is the end # of the string, otherwise the $ would not match the end of line matches = re.search(self._version_regex, output, re.MULTILINE) - print("XXX", output, matches) if not matches: raise RuntimeError(f"Unexpected version output format for " f"compiler '{self.name}': {output}") version_string = matches.groups()[0] - print("YYY", matches.groups(), version_string) # Expect the version to be dot-separated integers. try: version = tuple(int(x) for x in version_string.split('.')) diff --git a/source/fab/tools/compiler_wrapper.py b/source/fab/tools/compiler_wrapper.py index 4bd68ffc..1c7ea666 100644 --- a/source/fab/tools/compiler_wrapper.py +++ b/source/fab/tools/compiler_wrapper.py @@ -201,3 +201,27 @@ class Mpicc(CompilerWrapper): def __init__(self, compiler: Compiler): super().__init__(name=f"mpicc-{compiler.name}", exec_name="mpicc", compiler=compiler, mpi=True) + + +# ============================================================================ +class CrayFtn(CompilerWrapper): + '''Class for the Cray Fortran compiler wrapper. + + :param compiler: the compiler that the ftn wrapper will use. + ''' + + def __init__(self, compiler: Compiler): + super().__init__(name=f"crayftn-{compiler.name}", + exec_name="ftn", compiler=compiler, mpi=True) + + +# ============================================================================ +class CrayCc(CompilerWrapper): + '''Class for the Cray C compiler wrapper + + :param compiler: the compiler that the mpicc wrapper will use. + ''' + + def __init__(self, compiler: Compiler): + super().__init__(name=f"craycc-{compiler.name}", + exec_name="cc", compiler=compiler, mpi=True) diff --git a/source/fab/tools/tool_repository.py b/source/fab/tools/tool_repository.py index 6a077b67..427f770a 100644 --- a/source/fab/tools/tool_repository.py +++ b/source/fab/tools/tool_repository.py @@ -17,8 +17,12 @@ from fab.tools.tool import Tool from fab.tools.category import Category from fab.tools.compiler import Compiler +from fab.tools.compiler_wrapper import CrayCc, CrayFtn, Mpif90, Mpicc from fab.tools.linker import Linker from fab.tools.versioning import Fcm, Git, Subversion +from fab.tools import (Ar, Cpp, CppFortran, Craycc, Crayftn, + Gcc, Gfortran, Icc, Icx, Ifort, Ifx, + Nvc, Nvfortran, Psyclone, Rsync) class ToolRepository(dict): @@ -57,26 +61,35 @@ def __init__(self): # Add the FAB default tools: # TODO: sort the defaults so that they actually work (since not all - # tools FAB knows about are available). For now, disable Fpp: - # We get circular dependencies if imported at top of the file: - # pylint: disable=import-outside-toplevel - from fab.tools import (Ar, Cpp, CppFortran, Gcc, Gfortran, - Icc, Ifort, Psyclone, Rsync) - - for cls in [Gcc, Icc, Gfortran, Ifort, Cpp, CppFortran, - Fcm, Git, Subversion, Ar, Psyclone, Rsync]: + # tools FAB knows about are available). For now, disable Fpp (by not + # adding it). IF someone actually uses it it can added. + for cls in [Craycc, Crayftn, + Gcc, Gfortran, + Icc, Icx, Ifort, Ifx, + Nvc, Nvfortran, + Cpp, CppFortran, + Ar, Fcm, Git, Psyclone, Rsync, Subversion]: self.add_tool(cls()) - from fab.tools.compiler_wrapper import Mpif90, Mpicc + # Now create the potential mpif90 and Cray ftn wrapper all_fc = self[Category.FORTRAN_COMPILER][:] for fc in all_fc: mpif90 = Mpif90(fc) self.add_tool(mpif90) + # I assume cray has (besides cray) only support for gfortran/ifort + if fc.name in ["gfortran", "ifort"]: + crayftn = CrayFtn(fc) + self.add_tool(crayftn) + # Now create the potential mpicc and Cray cc wrapper all_cc = self[Category.C_COMPILER][:] for cc in all_cc: mpicc = Mpicc(cc) self.add_tool(mpicc) + # I assume cray has (besides cray) only support for gfortran/ifort + if cc.name in ["gcc", "icc"]: + craycc = CrayCc(cc) + self.add_tool(craycc) def add_tool(self, tool: Tool): '''Creates an instance of the specified class and adds it diff --git a/tests/unit_tests/tools/test_compiler_wrapper.py b/tests/unit_tests/tools/test_compiler_wrapper.py index 11fdde57..42ee31ab 100644 --- a/tests/unit_tests/tools/test_compiler_wrapper.py +++ b/tests/unit_tests/tools/test_compiler_wrapper.py @@ -12,8 +12,9 @@ import pytest -from fab.tools import (Category, CompilerWrapper, Gcc, Gfortran, Icc, Ifort, - Mpicc, Mpif90, ToolRepository) +from fab.tools import (Category, CompilerWrapper, CrayCc, CrayFtn, + Gcc, Gfortran, Icc, Ifort, Mpicc, Mpif90, + ToolRepository) def test_compiler_wrapper_compiler_getter(): @@ -346,3 +347,47 @@ def test_compiler_wrapper_mpi_ifort(): assert mpi_ifort.category == Category.FORTRAN_COMPILER assert mpi_ifort.mpi assert mpi_ifort.suite == "intel-classic" + + +def test_compiler_wrapper_cray_icc(): + '''Tests the Cray wrapper for icc.''' + craycc = CrayCc(Icc()) + assert craycc.name == "craycc-icc" + assert str(craycc) == "CrayCc(icc)" + assert isinstance(craycc, CompilerWrapper) + assert craycc.category == Category.C_COMPILER + assert craycc.mpi + assert craycc.suite == "intel-classic" + + +def test_compiler_wrapper_cray_ifort(): + '''Tests the Cray wrapper for ifort.''' + crayftn = CrayFtn(Ifort()) + assert crayftn.name == "crayftn-ifort" + assert str(crayftn) == "CrayFtn(ifort)" + assert isinstance(crayftn, CompilerWrapper) + assert crayftn.category == Category.FORTRAN_COMPILER + assert crayftn.mpi + assert crayftn.suite == "intel-classic" + + +def test_compiler_wrapper_cray_gcc(): + '''Tests the Cray wrapper for gcc.''' + craycc = CrayCc(Gcc()) + assert craycc.name == "craycc-gcc" + assert str(craycc) == "CrayCc(gcc)" + assert isinstance(craycc, CompilerWrapper) + assert craycc.category == Category.C_COMPILER + assert craycc.mpi + assert craycc.suite == "gnu" + + +def test_compiler_wrapper_cray_gfortran(): + '''Tests the Cray wrapper for gfortran.''' + crayftn = CrayFtn(Gfortran()) + assert crayftn.name == "crayftn-gfortran" + assert str(crayftn) == "CrayFtn(gfortran)" + assert isinstance(crayftn, CompilerWrapper) + assert crayftn.category == Category.FORTRAN_COMPILER + assert crayftn.mpi + assert crayftn.suite == "gnu" From 6685ac15de4950ee4227c4c68e088498b8db7b1d Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 22 Oct 2024 14:06:54 +1100 Subject: [PATCH 06/14] Follow a more consistent naming scheme for crays, even though the native compiler names are longer (crayftn-cray, craycc-cray). --- source/fab/tools/compiler.py | 21 +++++++++++++-------- tests/unit_tests/tools/test_compiler.py | 4 ++-- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index 1ce603fd..6194afb4 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -520,8 +520,11 @@ def run_version_command( # Cray compiler # ============================================================================ class Craycc(CCompiler): - '''Class for the native Cray C compiler. Cray has two different compilers. - Older ones have as version number: + '''Class for the native Cray C compiler. Since cc is actually a compiler + wrapper, follow the naming scheme of a compiler wrapper and call it: + craycc-cray. + + Cray has two different compilers. Older ones have as version number: Cray C : Version 8.7.0 Tue Jul 23, 2024 07:39:46 Newer compiler (several lines, the important one): Cray clang version 15.0.1 (66f7391d6a03cf932f321b9f6b1d8612ef5f362c) @@ -532,24 +535,26 @@ class Craycc(CCompiler): :param name: name of this compiler. :param exec_name: name of the executable. ''' - def __init__(self, name: str = "craycc", exec_name: str = "craycc"): + def __init__(self, name: str = "craycc-cray", exec_name: str = "cc"): super().__init__(name, exec_name, suite="cray", mpi=True, - openmp_flag="-qopenmp", + openmp_flag="-homp", version_regex=r"Cray [Cc][^\d]* (\d[\d\.]+\d) ") # ============================================================================ class Crayftn(FortranCompiler): - '''Class for the native Cray Fortran compiler. + '''Class for the native Cray Fortran compiler. Since ftn is actually a + compiler wrapper, follow the naming scheme of Cray compiler wrapper + and call it crayftn-cray. :param name: name of this compiler. :param exec_name: name of the executable. ''' - def __init__(self, name: str = "crayftn", exec_name: str = "crayftn"): + def __init__(self, name: str = "crayftn-cray", exec_name: str = "ftn"): super().__init__(name, exec_name, suite="cray", mpi=True, - module_folder_flag="-module", - openmp_flag="-qopenmp", + module_folder_flag="-J", + openmp_flag="-homp", syntax_only_flag="-syntax-only", version_regex=(r"Cray Fortran : Version " r"(\d[\d\.]+\d) ")) diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index f95026ef..67efaacd 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -843,7 +843,7 @@ def test_nvfortran_get_version_with_ifort_string(): def test_craycc(): '''Tests the Craycc class.''' craycc = Craycc() - assert craycc.name == "craycc" + assert craycc.name == "craycc-cray" assert isinstance(craycc, CCompiler) assert craycc.category == Category.C_COMPILER assert craycc.mpi @@ -909,7 +909,7 @@ def test_craycc_get_version_with_icc_string(): def test_crayftn(): '''Tests the Crayftn class.''' crayftn = Crayftn() - assert crayftn.name == "crayftn" + assert crayftn.name == "crayftn-cray" assert isinstance(crayftn, FortranCompiler) assert crayftn.category == Category.FORTRAN_COMPILER assert crayftn.mpi From 8b39f17e31dbeb0494736dbbbc28d2c9aa4bb1a3 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Tue, 22 Oct 2024 14:17:59 +1100 Subject: [PATCH 07/14] Changed names again. --- source/fab/tools/compiler.py | 4 ++-- tests/unit_tests/tools/test_compiler.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index 6194afb4..bcabd894 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -535,7 +535,7 @@ class Craycc(CCompiler): :param name: name of this compiler. :param exec_name: name of the executable. ''' - def __init__(self, name: str = "craycc-cray", exec_name: str = "cc"): + def __init__(self, name: str = "craycc-cc", exec_name: str = "cc"): super().__init__(name, exec_name, suite="cray", mpi=True, openmp_flag="-homp", version_regex=r"Cray [Cc][^\d]* (\d[\d\.]+\d) ") @@ -551,7 +551,7 @@ class Crayftn(FortranCompiler): :param exec_name: name of the executable. ''' - def __init__(self, name: str = "crayftn-cray", exec_name: str = "ftn"): + def __init__(self, name: str = "crayftn-ftn", exec_name: str = "ftn"): super().__init__(name, exec_name, suite="cray", mpi=True, module_folder_flag="-J", openmp_flag="-homp", diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index 67efaacd..70c16da0 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -843,7 +843,7 @@ def test_nvfortran_get_version_with_ifort_string(): def test_craycc(): '''Tests the Craycc class.''' craycc = Craycc() - assert craycc.name == "craycc-cray" + assert craycc.name == "craycc-cc" assert isinstance(craycc, CCompiler) assert craycc.category == Category.C_COMPILER assert craycc.mpi @@ -909,7 +909,7 @@ def test_craycc_get_version_with_icc_string(): def test_crayftn(): '''Tests the Crayftn class.''' crayftn = Crayftn() - assert crayftn.name == "crayftn-cray" + assert crayftn.name == "crayftn-ftn" assert isinstance(crayftn, FortranCompiler) assert crayftn.category == Category.FORTRAN_COMPILER assert crayftn.mpi From be1a426a5edef50bdc29abbd82faa7280b399019 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 11 Nov 2024 11:00:26 +1100 Subject: [PATCH 08/14] Renamed cray compiler wrapper to be CrayCcWrapper and CrayFtnWrapper, to avoid confusion with Craycc. --- source/fab/tools/__init__.py | 8 +++---- source/fab/tools/compiler_wrapper.py | 10 +++++---- source/fab/tools/tool_repository.py | 12 +++++----- .../unit_tests/tools/test_compiler_wrapper.py | 22 +++++++++---------- 4 files changed, 28 insertions(+), 24 deletions(-) diff --git a/source/fab/tools/__init__.py b/source/fab/tools/__init__.py index 4ee807da..bc7430b7 100644 --- a/source/fab/tools/__init__.py +++ b/source/fab/tools/__init__.py @@ -12,8 +12,8 @@ from fab.tools.compiler import (CCompiler, Compiler, Craycc, Crayftn, FortranCompiler, Gcc, Gfortran, Icc, Icx, Ifort, Ifx, Nvc, Nvfortran) -from fab.tools.compiler_wrapper import (CompilerWrapper, CrayCc, CrayFtn, - Mpicc, Mpif90) +from fab.tools.compiler_wrapper import (CompilerWrapper, CrayCcWrapper, + CrayFtnWrapper, Mpicc, Mpif90) from fab.tools.flags import Flags from fab.tools.linker import Linker from fab.tools.psyclone import Psyclone @@ -34,9 +34,9 @@ "Cpp", "CppFortran", "Craycc", - "CrayCc", + "CrayCcWrapper", "Crayftn", - "CrayFtn", + "CrayFtnWrapper", "Fcm", "Flags", "FortranCompiler", diff --git a/source/fab/tools/compiler_wrapper.py b/source/fab/tools/compiler_wrapper.py index 1c7ea666..09ce5015 100644 --- a/source/fab/tools/compiler_wrapper.py +++ b/source/fab/tools/compiler_wrapper.py @@ -204,8 +204,9 @@ def __init__(self, compiler: Compiler): # ============================================================================ -class CrayFtn(CompilerWrapper): - '''Class for the Cray Fortran compiler wrapper. +class CrayFtnWrapper(CompilerWrapper): + '''Class for the Cray Fortran compiler wrapper. We add 'wrapper' to the + class name to make this class distinct from the Crayftn compiler class. :param compiler: the compiler that the ftn wrapper will use. ''' @@ -216,8 +217,9 @@ def __init__(self, compiler: Compiler): # ============================================================================ -class CrayCc(CompilerWrapper): - '''Class for the Cray C compiler wrapper +class CrayCcWrapper(CompilerWrapper): + '''Class for the Cray C compiler wrapper. We add 'wrapper' to the class + name to make this class distinct from the Craycc compiler class :param compiler: the compiler that the mpicc wrapper will use. ''' diff --git a/source/fab/tools/tool_repository.py b/source/fab/tools/tool_repository.py index 427f770a..d699574a 100644 --- a/source/fab/tools/tool_repository.py +++ b/source/fab/tools/tool_repository.py @@ -17,7 +17,8 @@ from fab.tools.tool import Tool from fab.tools.category import Category from fab.tools.compiler import Compiler -from fab.tools.compiler_wrapper import CrayCc, CrayFtn, Mpif90, Mpicc +from fab.tools.compiler_wrapper import (CrayCcWrapper, CrayFtnWrapper, + Mpif90, Mpicc) from fab.tools.linker import Linker from fab.tools.versioning import Fcm, Git, Subversion from fab.tools import (Ar, Cpp, CppFortran, Craycc, Crayftn, @@ -76,9 +77,10 @@ def __init__(self): for fc in all_fc: mpif90 = Mpif90(fc) self.add_tool(mpif90) - # I assume cray has (besides cray) only support for gfortran/ifort + # I assume cray has (besides cray) only support for Intel and GNU if fc.name in ["gfortran", "ifort"]: - crayftn = CrayFtn(fc) + crayftn = CrayFtnWrapper(fc) + print("NEW NAME", crayftn, crayftn.name) self.add_tool(crayftn) # Now create the potential mpicc and Cray cc wrapper @@ -86,9 +88,9 @@ def __init__(self): for cc in all_cc: mpicc = Mpicc(cc) self.add_tool(mpicc) - # I assume cray has (besides cray) only support for gfortran/ifort + # I assume cray has (besides cray) only support for Intel and GNU if cc.name in ["gcc", "icc"]: - craycc = CrayCc(cc) + craycc = CrayCcWrapper(cc) self.add_tool(craycc) def add_tool(self, tool: Tool): diff --git a/tests/unit_tests/tools/test_compiler_wrapper.py b/tests/unit_tests/tools/test_compiler_wrapper.py index 42ee31ab..07f9a08b 100644 --- a/tests/unit_tests/tools/test_compiler_wrapper.py +++ b/tests/unit_tests/tools/test_compiler_wrapper.py @@ -12,9 +12,9 @@ import pytest -from fab.tools import (Category, CompilerWrapper, CrayCc, CrayFtn, - Gcc, Gfortran, Icc, Ifort, Mpicc, Mpif90, - ToolRepository) +from fab.tools import (Category, CompilerWrapper, CrayCcWrapper, + CrayFtnWrapper, Gcc, Gfortran, Icc, Ifort, + Mpicc, Mpif90, ToolRepository) def test_compiler_wrapper_compiler_getter(): @@ -351,9 +351,9 @@ def test_compiler_wrapper_mpi_ifort(): def test_compiler_wrapper_cray_icc(): '''Tests the Cray wrapper for icc.''' - craycc = CrayCc(Icc()) + craycc = CrayCcWrapper(Icc()) assert craycc.name == "craycc-icc" - assert str(craycc) == "CrayCc(icc)" + assert str(craycc) == "CrayCcWrapper(icc)" assert isinstance(craycc, CompilerWrapper) assert craycc.category == Category.C_COMPILER assert craycc.mpi @@ -362,9 +362,9 @@ def test_compiler_wrapper_cray_icc(): def test_compiler_wrapper_cray_ifort(): '''Tests the Cray wrapper for ifort.''' - crayftn = CrayFtn(Ifort()) + crayftn = CrayFtnWrapper(Ifort()) assert crayftn.name == "crayftn-ifort" - assert str(crayftn) == "CrayFtn(ifort)" + assert str(crayftn) == "CrayFtnWrapper(ifort)" assert isinstance(crayftn, CompilerWrapper) assert crayftn.category == Category.FORTRAN_COMPILER assert crayftn.mpi @@ -373,9 +373,9 @@ def test_compiler_wrapper_cray_ifort(): def test_compiler_wrapper_cray_gcc(): '''Tests the Cray wrapper for gcc.''' - craycc = CrayCc(Gcc()) + craycc = CrayCcWrapper(Gcc()) assert craycc.name == "craycc-gcc" - assert str(craycc) == "CrayCc(gcc)" + assert str(craycc) == "CrayCcWrapper(gcc)" assert isinstance(craycc, CompilerWrapper) assert craycc.category == Category.C_COMPILER assert craycc.mpi @@ -384,9 +384,9 @@ def test_compiler_wrapper_cray_gcc(): def test_compiler_wrapper_cray_gfortran(): '''Tests the Cray wrapper for gfortran.''' - crayftn = CrayFtn(Gfortran()) + crayftn = CrayFtnWrapper(Gfortran()) assert crayftn.name == "crayftn-gfortran" - assert str(crayftn) == "CrayFtn(gfortran)" + assert str(crayftn) == "CrayFtnWrapper(gfortran)" assert isinstance(crayftn, CompilerWrapper) assert crayftn.category == Category.FORTRAN_COMPILER assert crayftn.mpi From f8b6e2781a2c29b7287b5b6c79488d921846adbc Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 11 Nov 2024 12:47:25 +1100 Subject: [PATCH 09/14] Fixed incorrect name in comments. --- source/fab/tools/compiler.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/source/fab/tools/compiler.py b/source/fab/tools/compiler.py index bcabd894..3f8c31e1 100644 --- a/source/fab/tools/compiler.py +++ b/source/fab/tools/compiler.py @@ -522,7 +522,7 @@ def run_version_command( class Craycc(CCompiler): '''Class for the native Cray C compiler. Since cc is actually a compiler wrapper, follow the naming scheme of a compiler wrapper and call it: - craycc-cray. + craycc-cc. Cray has two different compilers. Older ones have as version number: Cray C : Version 8.7.0 Tue Jul 23, 2024 07:39:46 @@ -545,7 +545,7 @@ def __init__(self, name: str = "craycc-cc", exec_name: str = "cc"): class Crayftn(FortranCompiler): '''Class for the native Cray Fortran compiler. Since ftn is actually a compiler wrapper, follow the naming scheme of Cray compiler wrapper - and call it crayftn-cray. + and call it crayftn-ftn. :param name: name of this compiler. :param exec_name: name of the executable. From e7c2c8378c936ba594b21154588d075816a59273 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 27 Sep 2024 01:55:23 +1000 Subject: [PATCH 10/14] Support new and old style of PSyclone command line (no more nemo api etc) --- source/fab/tools/psyclone.py | 129 +++++++++++++- .../psyclone/test_psyclone_system_test.py | 6 +- tests/unit_tests/tools/test_psyclone.py | 163 +++++++++++------- 3 files changed, 229 insertions(+), 69 deletions(-) diff --git a/source/fab/tools/psyclone.py b/source/fab/tools/psyclone.py index 1a2b3b40..1d7fa255 100644 --- a/source/fab/tools/psyclone.py +++ b/source/fab/tools/psyclone.py @@ -8,7 +8,9 @@ """ from pathlib import Path +import re from typing import Callable, List, Optional, TYPE_CHECKING, Union +import warnings from fab.tools.category import Category from fab.tools.tool import Tool @@ -24,15 +26,75 @@ class Psyclone(Tool): '''This is the base class for `PSyclone`. ''' - def __init__(self, api: Optional[str] = None): + def __init__(self): super().__init__("psyclone", "psyclone", Category.PSYCLONE) - self._api = api + self._version = None + + def check_available(self) -> bool: + '''This function determines if PSyclone is available. Additionally, + it established the version, since command line option changes + significantly from python 2.5.0 to the next release. + ''' + + # First get the version (and confirm that PSyclone is installed): + try: + # Older versions of PSyclone (2.3.1 and earlier) expect a filename + # even when --version is used, and won't produce version info + # without this. So provide a dummy file (which does not need to + # exist), and check the error for details to see if PSyclone does + # not exist, or if the error is because of the non-existing file + version_output = self.run(["--version", "does_not_exist"], + capture_output=True) + except RuntimeError as err: + # If the command is not found, the error contains the following: + if "could not be executed" in str(err): + return False + # Otherwise, psyclone likely complained about the not existing + # file. Continue and try to find version information in the output: + version_output = str(err) + + # Search for the version info: + exp = r"PSyclone version: (\d[\d.]+\d)" + print("VERSION [", version_output, "]") + matches = re.search(exp, version_output) + if not matches: + warnings.warn(f"Unexpected version information for PSyclone: " + f"'{version_output}'.") + # If we don't recognise the version number, something is wrong + return False + + # Now convert the version info to integer. The regular expression + # match guarantees that we have integer numbers now: + version = tuple(int(x) for x in matches.groups()[0].split('.')) + + if version == (2, 5, 0): + # The behaviour of PSyclone changes from 2.5.0 to the next + # release. But since head-of-trunk still reports 2.5.0, we + # need to run additional tests to see if we have the official + # 2.5.0 release, or current trunk (which already has the new + # command line options). PSyclone needs an existing file + # in order to work, so use __file__ to present this file. + # PSyclone will obviously abort since this is not a Fortran + # file, but we only need to check the error message to + # see if the domain name is incorrect (--> current trunk) + # or not (2.5.0 release) + try: + self.run(["-api", "nemo", __file__], capture_output=True) + except RuntimeError as err: + if "Unsupported PSyKAL DSL / API 'nemo' specified" in str(err): + # It is current development. Just give it a version number + # greater than 2.5.0 + version = (2, 5, 0, 1) + + self._version = version + return True def process(self, config: "BuildConfig", x90_file: Path, - psy_file: Path, - alg_file: Union[Path, str], + psy_file: Optional[Path] = None, + alg_file: Optional[Union[Path, str]] = None, + transformed_file: Optional[Path] = None, transformation_script: Optional[Callable[[Path, "BuildConfig"], Path]] = None, additional_parameters: Optional[List[str]] = None, @@ -40,29 +102,78 @@ def process(self, api: Optional[str] = None, ): # pylint: disable=too-many-arguments - '''Run PSyclone with the specified parameters. + '''Run PSyclone with the specified parameters. If PSyclone is used to + transform existing Fortran files, `api` must be None, and the output + file name is `transformed_file`. If PSyclone is using its DSL + features, api must be a valid PSyclone API, and the two output + filenames are `psy_file` and `alg_file`. :param api: the PSyclone API. :param x90_file: the input file for PSyclone :param psy_file: the output PSy-layer file. :param alg_file: the output modified algorithm file. + :param transformed_file: the output filename if PSyclone is called + as transformation tool. :param transformation_script: an optional transformation script :param additional_parameters: optional additional parameters for PSyclone :param kernel_roots: optional directories with kernels. ''' + if not self.is_available: + raise RuntimeError("PSyclone is not available.") + + if api: + # API specified, we need both psy- and alg-file, but not + # transformed file. + if not psy_file: + raise RuntimeError(f"PSyclone called with api '{api}', but " + f"no psy_file is specified.") + if not alg_file: + raise RuntimeError(f"PSyclone called with api '{api}', but " + f"no alg_file is specified.") + if transformed_file: + raise RuntimeError(f"PSyclone called with api '{api}' and " + f"transformed_file.") + else: + if psy_file: + raise RuntimeError("PSyclone called without api, but " + "psy_file is specified.") + if alg_file: + raise RuntimeError("PSyclone called without api, but " + "alg_file is specified.") + if not transformed_file: + raise RuntimeError("PSyclone called without api, but " + "transformed_file it not specified.") + parameters: List[Union[str, Path]] = [] # If an api is defined in this call (or in the constructor) add it # as parameter. No API is required if PSyclone works as # transformation tool only, so calling PSyclone without api is # actually valid. + print("API", api, self._version) if api: - parameters.extend(["-api", api]) - elif self._api: - parameters.extend(["-api", self._api]) + if self._version > (2, 5, 0): + api_param = "--psykal-dsl" + # Mapping from old names to new names: + mapping = {"dynamo0.3": "lfric", + "gocean1.0": "gocean"} + else: + api_param = "-api" + # Mapping from new names to old names: + mapping = {"lfric": "dynamo0.3", + "gocean": "gocean1.0"} - parameters.extend(["-l", "all", "-opsy", psy_file, "-oalg", alg_file]) + parameters.extend([api_param, mapping.get(api, api), + "-opsy", psy_file, "-oalg", alg_file]) + else: # no api + if self._version > (2, 5, 0): + # New version: no API, parameter, but -o for output name: + parameters.extend(["-o", transformed_file]) + else: + # 2.5.0 or earlier: needs api nemo, output name is -oalg + parameters.extend(["-api", "nemo", "-opsy", transformed_file]) + parameters.extend(["-l", "all"]) if transformation_script: transformation_script_return_path = \ diff --git a/tests/system_tests/psyclone/test_psyclone_system_test.py b/tests/system_tests/psyclone/test_psyclone_system_test.py index cf3c80d0..3c16fd4a 100644 --- a/tests/system_tests/psyclone/test_psyclone_system_test.py +++ b/tests/system_tests/psyclone/test_psyclone_system_test.py @@ -199,6 +199,9 @@ class TestTransformationScript: """ def test_transformation_script(self, psyclone_lfric_api): psyclone_tool = Psyclone() + psyclone_tool._version = (2, 4, 0) + psyclone_tool._is_available = True + mock_transformation_script = mock.Mock(return_value=__file__) with mock.patch('fab.tools.psyclone.Psyclone.run') as mock_run_command: mock_transformation_script.return_value = Path(__file__) @@ -216,8 +219,9 @@ def test_transformation_script(self, psyclone_lfric_api): mock_transformation_script.assert_called_once_with(Path(__file__), None) # check transformation_script is passed to psyclone command with '-s' mock_run_command.assert_called_with( - additional_parameters=['-api', psyclone_lfric_api, '-l', 'all', + additional_parameters=['-api', psyclone_lfric_api, '-opsy', Path(__file__), '-oalg', Path(__file__), + '-l', 'all', '-s', Path(__file__), __file__]) diff --git a/tests/unit_tests/tools/test_psyclone.py b/tests/unit_tests/tools/test_psyclone.py index 7efc60ec..619c0260 100644 --- a/tests/unit_tests/tools/test_psyclone.py +++ b/tests/unit_tests/tools/test_psyclone.py @@ -10,9 +10,26 @@ from importlib import reload from unittest import mock +import pytest + from fab.tools import (Category, Psyclone) +def get_mock_result(version_info: str) -> mock.Mock: + '''Returns a mock PSyclone object that will return + the specified str as version info. + + :param version_info: the simulated output of psyclone --version + The leading "PSyclone version: " will be added automatically. + ''' + # The return of subprocess run has an attribute 'stdout', + # that returns the stdout when its `decode` method is called. + # So we mock stdout, then put this mock_stdout into the mock result: + mock_stdout = mock.Mock(decode=lambda: f"PSyclone version: {version_info}") + mock_result = mock.Mock(stdout=mock_stdout, returncode=0) + return mock_result + + def test_psyclone_constructor(): '''Test the PSyclone constructor.''' psyclone = Psyclone() @@ -20,45 +37,102 @@ def test_psyclone_constructor(): assert psyclone.name == "psyclone" assert psyclone.exec_name == "psyclone" assert psyclone.flags == [] - assert psyclone._api is None - psyclone = Psyclone(api="gocean1.0") - assert psyclone.category == Category.PSYCLONE - assert psyclone.name == "psyclone" - assert psyclone.exec_name == "psyclone" - assert psyclone.flags == [] - assert psyclone._api == "gocean1.0" - -def test_psyclone_check_available(): - '''Tests the is_available functionality.''' +def test_psyclone_check_available_2_4_0(): + '''Tests the is_available functionality with version 2.4.0. + We get only one call. + ''' psyclone = Psyclone() - mock_result = mock.Mock(returncode=0) + + mock_result = get_mock_result("2.4.0") with mock.patch('fab.tools.tool.subprocess.run', return_value=mock_result) as tool_run: assert psyclone.check_available() tool_run.assert_called_once_with( - ["psyclone", "--version"], capture_output=True, env=None, - cwd=None, check=False) + ["psyclone", "--version", mock.ANY], capture_output=True, + env=None, cwd=None, check=False) + + +def test_psyclone_check_available_2_5_0(): + '''Tests the is_available functionality with PSyclone 2.5.0. + We get two calls. First version, then check if nemo API exists + ''' + psyclone = Psyclone() + + mock_result = get_mock_result("2.5.0") + with mock.patch('fab.tools.tool.subprocess.run', + return_value=mock_result) as tool_run: + assert psyclone.check_available() + tool_run.assert_any_call( + ["psyclone", "--version", mock.ANY], capture_output=True, + env=None, cwd=None, check=False) + tool_run.assert_any_call( + ["psyclone", "-api", "nemo", mock.ANY], capture_output=True, + env=None, cwd=None, check=False) # Test behaviour if a runtime error happens: with mock.patch("fab.tools.tool.Tool.run", side_effect=RuntimeError("")) as tool_run: + with pytest.warns(UserWarning, + match="Unexpected version information " + "for PSyclone: ''."): + assert not psyclone.check_available() + + +def test_psyclone_check_available_after_2_5_0(): + '''Tests the is_available functionality with releases after 2.5.0. + We get two calls. First version, then check if nemo API exists + ''' + psyclone = Psyclone() + + # We detect the dummy version '2.5.0.1' if psyclone reports 2.5.0 + # but the command line option "-api nemo" is not accepted. + # So we need to return two results from our mock objects: first + # success for version 2.5.0, then a failure with an appropriate + # error message: + mock_result1 = get_mock_result("2.5.0") + mock_result2 = get_mock_result("Unsupported PSyKAL DSL / " + "API 'nemo' specified") + mock_result2.returncode = 1 + + # "Unsupported PSyKAL DSL / API 'nemo' specified" + with mock.patch('fab.tools.tool.subprocess.run', + return_value=mock_result1) as tool_run: + tool_run.side_effect = [mock_result1, mock_result2] + assert psyclone.check_available() + assert psyclone._version == (2, 5, 0, 1) + + +def test_psyclone_check_available_errors(): + '''Test various errors that can happen in check_available. + ''' + psyclone = Psyclone() + with mock.patch('fab.tools.tool.subprocess.run', + side_effect=FileNotFoundError("ERR")): assert not psyclone.check_available() + psyclone = Psyclone() + mock_result = get_mock_result("NOT_A_NUMBER.4.0") + with mock.patch('fab.tools.tool.subprocess.run', + return_value=mock_result): + with pytest.warns(UserWarning, + match="Unexpected version information for PSyclone: " + "'PSyclone version: NOT_A_NUMBER.4.0'"): + assert not psyclone.check_available() -def test_psyclone_process(psyclone_lfric_api): + +@pytest.mark.parametrize("api", ["dynamo0.3", "lfric"]) +def test_psyclone_process_api_2_4_0(api): '''Test running PSyclone.''' psyclone = Psyclone() - mock_result = mock.Mock(returncode=0) - # Create a mock function that returns a 'transformation script' - # called `script_called`: + mock_result = get_mock_result("2.4.0") transformation_function = mock.Mock(return_value="script_called") config = mock.Mock() with mock.patch('fab.tools.tool.subprocess.run', return_value=mock_result) as tool_run: psyclone.process(config=config, - api=psyclone_lfric_api, + api=api, x90_file="x90_file", psy_file="psy_file", alg_file="alg_file", @@ -66,60 +140,31 @@ def test_psyclone_process(psyclone_lfric_api): kernel_roots=["root1", "root2"], additional_parameters=["-c", "psyclone.cfg"]) tool_run.assert_called_with( - ['psyclone', '-api', psyclone_lfric_api, '-l', 'all', '-opsy', - 'psy_file', '-oalg', 'alg_file', '-s', 'script_called', '-c', + ['psyclone', '-api', 'dynamo0.3', '-opsy', 'psy_file', + '-oalg', 'alg_file', '-l', 'all', '-s', 'script_called', '-c', 'psyclone.cfg', '-d', 'root1', '-d', 'root2', 'x90_file'], capture_output=True, env=None, cwd=None, check=False) - # Don't specify an API: - with mock.patch('fab.tools.tool.subprocess.run', - return_value=mock_result) as tool_run: - psyclone.process(config=config, - x90_file="x90_file", - psy_file="psy_file", - alg_file="alg_file", - transformation_script=transformation_function, - kernel_roots=["root1", "root2"], - additional_parameters=["-c", "psyclone.cfg"]) - tool_run.assert_called_with( - ['psyclone', '-l', 'all', '-opsy', 'psy_file', '-oalg', 'alg_file', - '-s', 'script_called', '-c', - 'psyclone.cfg', '-d', 'root1', '-d', 'root2', 'x90_file'], - capture_output=True, env=None, cwd=None, check=False) - # Don't specify an API, but define an API on the PSyclone tool: - psyclone = Psyclone(api="gocean1.0") - with mock.patch('fab.tools.tool.subprocess.run', - return_value=mock_result) as tool_run: - psyclone.process(config=config, - x90_file="x90_file", - psy_file="psy_file", - alg_file="alg_file", - transformation_script=transformation_function, - kernel_roots=["root1", "root2"], - additional_parameters=["-c", "psyclone.cfg"]) - tool_run.assert_called_with( - ['psyclone', '-api', 'gocean1.0', '-l', 'all', '-opsy', 'psy_file', - '-oalg', 'alg_file', '-s', 'script_called', '-c', - 'psyclone.cfg', '-d', 'root1', '-d', 'root2', 'x90_file'], - capture_output=True, env=None, cwd=None, check=False) +def test_psyclone_process_no_api_2_4_0(): + '''Test running PSyclone.''' + psyclone = Psyclone() + mock_result = get_mock_result("2.4.0") + transformation_function = mock.Mock(return_value="script_called") + config = mock.Mock() - # Have both a default and a command line option - the latter - # must take precedence: - psyclone = Psyclone(api="gocean1.0") with mock.patch('fab.tools.tool.subprocess.run', return_value=mock_result) as tool_run: psyclone.process(config=config, + api="", x90_file="x90_file", - psy_file="psy_file", - alg_file="alg_file", - api=psyclone_lfric_api, + transformed_file="psy_file", transformation_script=transformation_function, kernel_roots=["root1", "root2"], additional_parameters=["-c", "psyclone.cfg"]) tool_run.assert_called_with( - ['psyclone', '-api', psyclone_lfric_api, '-l', 'all', '-opsy', - 'psy_file', '-oalg', 'alg_file', '-s', 'script_called', '-c', + ['psyclone', '-api', 'nemo', '-opsy', 'psy_file', '-l', 'all', + '-s', 'script_called', '-c', 'psyclone.cfg', '-d', 'root1', '-d', 'root2', 'x90_file'], capture_output=True, env=None, cwd=None, check=False) From e2051f2ba058f85be480f19dd2fc40262930df73 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 27 Sep 2024 02:00:29 +1000 Subject: [PATCH 11/14] Fix mypy errors. --- source/fab/tools/psyclone.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/source/fab/tools/psyclone.py b/source/fab/tools/psyclone.py index 1d7fa255..424b3907 100644 --- a/source/fab/tools/psyclone.py +++ b/source/fab/tools/psyclone.py @@ -55,7 +55,6 @@ def check_available(self) -> bool: # Search for the version info: exp = r"PSyclone version: (\d[\d.]+\d)" - print("VERSION [", version_output, "]") matches = re.search(exp, version_output) if not matches: warnings.warn(f"Unexpected version information for PSyclone: " @@ -151,7 +150,6 @@ def process(self, # as parameter. No API is required if PSyclone works as # transformation tool only, so calling PSyclone without api is # actually valid. - print("API", api, self._version) if api: if self._version > (2, 5, 0): api_param = "--psykal-dsl" @@ -163,10 +161,16 @@ def process(self, # Mapping from new names to old names: mapping = {"lfric": "dynamo0.3", "gocean": "gocean1.0"} - + # Make mypy happy - we tested above that these variables + # are defined + assert psy_file + assert alg_file parameters.extend([api_param, mapping.get(api, api), "-opsy", psy_file, "-oalg", alg_file]) else: # no api + # Make mypy happy - we tested above that transformed_file is + # specified when no api is specified. + assert transformed_file if self._version > (2, 5, 0): # New version: no API, parameter, but -o for output name: parameters.extend(["-o", transformed_file]) From 0ad85ee1401c78e5ee2347065dfb337bce3676f7 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 30 Sep 2024 12:02:07 +1000 Subject: [PATCH 12/14] Added missing tests for calling psyclone, and converting old style to new stle arguments and vice versa. --- source/fab/tools/psyclone.py | 8 +- tests/unit_tests/tools/test_psyclone.py | 198 +++++++++++++++++++++++- 2 files changed, 196 insertions(+), 10 deletions(-) diff --git a/source/fab/tools/psyclone.py b/source/fab/tools/psyclone.py index 424b3907..fa508d7a 100644 --- a/source/fab/tools/psyclone.py +++ b/source/fab/tools/psyclone.py @@ -100,7 +100,7 @@ def process(self, kernel_roots: Optional[List[Union[str, Path]]] = None, api: Optional[str] = None, ): - # pylint: disable=too-many-arguments + # pylint: disable=too-many-arguments, too-many-branches '''Run PSyclone with the specified parameters. If PSyclone is used to transform existing Fortran files, `api` must be None, and the output file name is `transformed_file`. If PSyclone is using its DSL @@ -122,6 +122,10 @@ def process(self, if not self.is_available: raise RuntimeError("PSyclone is not available.") + # Convert the old style API nemo to be empty + if api and api.lower() == "nemo": + api = "" + if api: # API specified, we need both psy- and alg-file, but not # transformed file. @@ -143,7 +147,7 @@ def process(self, "alg_file is specified.") if not transformed_file: raise RuntimeError("PSyclone called without api, but " - "transformed_file it not specified.") + "transformed_file is not specified.") parameters: List[Union[str, Path]] = [] # If an api is defined in this call (or in the constructor) add it diff --git a/tests/unit_tests/tools/test_psyclone.py b/tests/unit_tests/tools/test_psyclone.py index 619c0260..5586c485 100644 --- a/tests/unit_tests/tools/test_psyclone.py +++ b/tests/unit_tests/tools/test_psyclone.py @@ -36,6 +36,7 @@ def test_psyclone_constructor(): assert psyclone.category == Category.PSYCLONE assert psyclone.name == "psyclone" assert psyclone.exec_name == "psyclone" + # pylint: disable=use-implicit-booleaness-not-comparison assert psyclone.flags == [] @@ -120,19 +121,82 @@ def test_psyclone_check_available_errors(): match="Unexpected version information for PSyclone: " "'PSyclone version: NOT_A_NUMBER.4.0'"): assert not psyclone.check_available() + # Also check that we can't call process if PSyclone is not available. + psyclone._is_available = False + config = mock.Mock() + with pytest.raises(RuntimeError) as err: + psyclone.process(config, "x90file") + assert "PSyclone is not available" in str(err.value) + + +def test_psyclone_processing_errors_without_api(): + '''Test all processing errors in PSyclone if no API is specified.''' + + psyclone = Psyclone() + psyclone._is_available = True + config = mock.Mock() + + # No API --> we need transformed file, but not psy or alg: + with pytest.raises(RuntimeError) as err: + psyclone.process(config, "x90file", api=None, psy_file="psy_file") + assert ("PSyclone called without api, but psy_file is specified" + in str(err.value)) + with pytest.raises(RuntimeError) as err: + psyclone.process(config, "x90file", api=None, alg_file="alg_file") + assert ("PSyclone called without api, but alg_file is specified" + in str(err.value)) + with pytest.raises(RuntimeError) as err: + psyclone.process(config, "x90file", api=None) + assert ("PSyclone called without api, but transformed_file is not " + "specified" in str(err.value)) @pytest.mark.parametrize("api", ["dynamo0.3", "lfric"]) -def test_psyclone_process_api_2_4_0(api): - '''Test running PSyclone.''' +def test_psyclone_processing_errors_with_api(api): + '''Test all processing errors in PSyclone if an API is specified.''' + psyclone = Psyclone() - mock_result = get_mock_result("2.4.0") + psyclone._is_available = True + config = mock.Mock() + + # No API --> we need transformed file, but not psy or alg: + with pytest.raises(RuntimeError) as err: + psyclone.process(config, "x90file", api=api, psy_file="psy_file") + assert (f"PSyclone called with api '{api}', but no alg_file is specified" + in str(err.value)) + with pytest.raises(RuntimeError) as err: + psyclone.process(config, "x90file", api=api, alg_file="alg_file") + assert (f"PSyclone called with api '{api}', but no psy_file is specified" + in str(err.value)) + with pytest.raises(RuntimeError) as err: + psyclone.process(config, "x90file", api=api, + psy_file="psy_file", alg_file="alg_file", + transformed_file="transformed_file") + assert (f"PSyclone called with api '{api}' and transformed_file" + in str(err.value)) + + +@pytest.mark.parametrize("version", ["2.4.0", "2.5.0"]) +@pytest.mark.parametrize("api", [("dynamo0.3", "dynamo0.3"), + ("lfric", "dynamo0.3"), + ("gocean1.0", "gocean1.0"), + ("gocean", "gocean1.0") + ]) +def test_psyclone_process_api_old_psyclone(api, version): + '''Test running 'old style' PSyclone (2.5.0 and earlier) with the old API + names (dynamo0.3 and gocean1.0). Also check that the new API names will + be accepted, but are mapped to the old style names. The 'api' parameter + contains the input api, and expected output API. + ''' + api_in, api_out = api + psyclone = Psyclone() + mock_result = get_mock_result(version) transformation_function = mock.Mock(return_value="script_called") config = mock.Mock() with mock.patch('fab.tools.tool.subprocess.run', return_value=mock_result) as tool_run: psyclone.process(config=config, - api=api, + api=api_in, x90_file="x90_file", psy_file="psy_file", alg_file="alg_file", @@ -140,16 +204,20 @@ def test_psyclone_process_api_2_4_0(api): kernel_roots=["root1", "root2"], additional_parameters=["-c", "psyclone.cfg"]) tool_run.assert_called_with( - ['psyclone', '-api', 'dynamo0.3', '-opsy', 'psy_file', + ['psyclone', '-api', api_out, '-opsy', 'psy_file', '-oalg', 'alg_file', '-l', 'all', '-s', 'script_called', '-c', 'psyclone.cfg', '-d', 'root1', '-d', 'root2', 'x90_file'], capture_output=True, env=None, cwd=None, check=False) -def test_psyclone_process_no_api_2_4_0(): - '''Test running PSyclone.''' +@pytest.mark.parametrize("version", ["2.4.0", "2.5.0"]) +def test_psyclone_process_no_api_old_psyclone(version): + '''Test running old-style PSyclone (2.5.0 and earlier) when requesting + to transform existing files by not specifying an API. We need to add + the flags `-api nemo` in this case for older PSyclone versions. + ''' psyclone = Psyclone() - mock_result = get_mock_result("2.4.0") + mock_result = get_mock_result(version) transformation_function = mock.Mock(return_value="script_called") config = mock.Mock() @@ -169,6 +237,119 @@ def test_psyclone_process_no_api_2_4_0(): capture_output=True, env=None, cwd=None, check=False) +@pytest.mark.parametrize("version", ["2.4.0", "2.5.0"]) +def test_psyclone_process_nemo_api_old_psyclone(version): + '''Test running old-style PSyclone (2.5.0 and earlier) when requesting + to transform existing files by specifying the nemo api. + ''' + + psyclone = Psyclone() + mock_result = get_mock_result(version) + transformation_function = mock.Mock(return_value="script_called") + config = mock.Mock() + + with mock.patch('fab.tools.tool.subprocess.run', + return_value=mock_result) as tool_run: + psyclone.process(config=config, + api="nemo", + x90_file="x90_file", + transformed_file="psy_file", + transformation_script=transformation_function, + kernel_roots=["root1", "root2"], + additional_parameters=["-c", "psyclone.cfg"]) + tool_run.assert_called_with( + ['psyclone', '-api', 'nemo', '-opsy', 'psy_file', '-l', 'all', + '-s', 'script_called', '-c', + 'psyclone.cfg', '-d', 'root1', '-d', 'root2', 'x90_file'], + capture_output=True, env=None, cwd=None, check=False) + + +@pytest.mark.parametrize("api", [("dynamo0.3", "lfric"), + ("lfric", "lfric"), + ("gocean1.0", "gocean"), + ("gocean", "gocean") + ]) +def test_psyclone_process_api_new__psyclone(api): + '''Test running the new PSyclone version. Since this version is not + yet released, we use the Fab internal version number 2.5.0.1 for + now. It uses new API names, and we need to check that the old style + names are converted to the new names. + ''' + api_in, api_out = api + psyclone = Psyclone() + mock_result = get_mock_result("2.5.0.1") + transformation_function = mock.Mock(return_value="script_called") + config = mock.Mock() + with mock.patch('fab.tools.tool.subprocess.run', + return_value=mock_result) as tool_run: + psyclone.process(config=config, + api=api_in, + x90_file="x90_file", + psy_file="psy_file", + alg_file="alg_file", + transformation_script=transformation_function, + kernel_roots=["root1", "root2"], + additional_parameters=["-c", "psyclone.cfg"]) + tool_run.assert_called_with( + ['psyclone', '--psykal-dsl', api_out, '-opsy', 'psy_file', + '-oalg', 'alg_file', '-l', 'all', '-s', 'script_called', '-c', + 'psyclone.cfg', '-d', 'root1', '-d', 'root2', 'x90_file'], + capture_output=True, env=None, cwd=None, check=False) + + +def test_psyclone_process_no_api_new_psyclone(): + '''Test running the new PSyclone version without an API. Since this + version is not yet released, we use the Fab internal version number + 2.5.0.1 for now. + ''' + psyclone = Psyclone() + mock_result = get_mock_result("2.5.0.1") + transformation_function = mock.Mock(return_value="script_called") + config = mock.Mock() + + with mock.patch('fab.tools.tool.subprocess.run', + return_value=mock_result) as tool_run: + psyclone.process(config=config, + api="", + x90_file="x90_file", + transformed_file="psy_file", + transformation_script=transformation_function, + kernel_roots=["root1", "root2"], + additional_parameters=["-c", "psyclone.cfg"]) + tool_run.assert_called_with( + ['psyclone', '-o', 'psy_file', '-l', 'all', + '-s', 'script_called', '-c', + 'psyclone.cfg', '-d', 'root1', '-d', 'root2', 'x90_file'], + capture_output=True, env=None, cwd=None, check=False) + + +def test_psyclone_process_nemo_api_new_psyclone(): + '''Test running PSyclone. Since this version is not yet released, we use + the Fab internal version number 2.5.0.1 for now. This tests that + backwards compatibility of using the nemo api works, i.e. '-api nemo' is + just removed. + ''' + psyclone = Psyclone() + mock_result = get_mock_result("2.5.0.1") + transformation_function = mock.Mock(return_value="script_called") + config = mock.Mock() + + with mock.patch('fab.tools.tool.subprocess.run', + return_value=mock_result) as tool_run: + psyclone.process(config=config, + api="nemo", + x90_file="x90_file", + transformed_file="psy_file", + transformation_script=transformation_function, + kernel_roots=["root1", "root2"], + additional_parameters=["-c", "psyclone.cfg"]) + tool_run.assert_called_with( + ['psyclone', '-o', 'psy_file', '-l', 'all', + '-s', 'script_called', '-c', + 'psyclone.cfg', '-d', 'root1', '-d', 'root2', 'x90_file'], + capture_output=True, env=None, cwd=None, check=False) + + def test_type_checking_import(): '''PSyclone contains an import of TYPE_CHECKING to break a circular dependency. In order to reach 100% coverage of PSyclone, we set @@ -178,5 +359,6 @@ def test_type_checking_import(): with mock.patch('typing.TYPE_CHECKING', True): # This import will not actually re-import, since the module # is already imported. But we need this in order to call reload: + # pylint: disable=import-outside-toplevel import fab.tools.psyclone reload(fab.tools.psyclone) From 890b50dde7e9e3b6154137f5e98abffaa7ee5765 Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Mon, 30 Sep 2024 13:52:05 +1000 Subject: [PATCH 13/14] Updated comment. --- source/fab/tools/psyclone.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/source/fab/tools/psyclone.py b/source/fab/tools/psyclone.py index fa508d7a..cbf12a9f 100644 --- a/source/fab/tools/psyclone.py +++ b/source/fab/tools/psyclone.py @@ -82,7 +82,8 @@ def check_available(self) -> bool: except RuntimeError as err: if "Unsupported PSyKAL DSL / API 'nemo' specified" in str(err): # It is current development. Just give it a version number - # greater than 2.5.0 + # greater than 2.5.0 for now, till the official release + # is done. version = (2, 5, 0, 1) self._version = version From 032ab2651337d6456ba84ae9e7f92b670001c46d Mon Sep 17 00:00:00 2001 From: Joerg Henrichs Date: Fri, 22 Nov 2024 00:42:48 +1100 Subject: [PATCH 14/14] Fixed failing tests. --- tests/unit_tests/tools/test_compiler.py | 13 +++++++------ tests/unit_tests/tools/test_tool_repository.py | 2 +- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/tests/unit_tests/tools/test_compiler.py b/tests/unit_tests/tools/test_compiler.py index 70c16da0..f6c7c158 100644 --- a/tests/unit_tests/tools/test_compiler.py +++ b/tests/unit_tests/tools/test_compiler.py @@ -47,26 +47,27 @@ def test_compiler(): def test_compiler_openmp(): '''Test that the openmp flag is correctly reflected in the test if a compiler supports OpenMP or not.''' - cc = CCompiler("gcc", "gcc", "gnu", openmp_flag="-fopenmp") + cc = CCompiler("gcc", "gcc", "gnu", openmp_flag="-fopenmp", + version_regex=None) assert cc.openmp_flag == "-fopenmp" assert cc.openmp - cc = CCompiler("gcc", "gcc", "gnu", openmp_flag=None) + cc = CCompiler("gcc", "gcc", "gnu", openmp_flag=None, version_regex=None) assert cc.openmp_flag == "" assert not cc.openmp - cc = CCompiler("gcc", "gcc", "gnu") + cc = CCompiler("gcc", "gcc", "gnu", version_regex=None) assert cc.openmp_flag == "" assert not cc.openmp fc = FortranCompiler("gfortran", "gfortran", "gnu", openmp_flag="-fopenmp", - module_folder_flag="-J") + module_folder_flag="-J", version_regex=None) assert fc.openmp_flag == "-fopenmp" assert fc.openmp fc = FortranCompiler("gfortran", "gfortran", "gnu", openmp_flag=None, - module_folder_flag="-J") + module_folder_flag="-J", version_regex=None) assert fc.openmp_flag == "" assert not fc.openmp fc = FortranCompiler("gfortran", "gfortran", "gnu", - module_folder_flag="-J") + module_folder_flag="-J", version_regex=None) assert fc.openmp_flag == "" assert not fc.openmp diff --git a/tests/unit_tests/tools/test_tool_repository.py b/tests/unit_tests/tools/test_tool_repository.py index 8369668e..e9bfb0c1 100644 --- a/tests/unit_tests/tools/test_tool_repository.py +++ b/tests/unit_tests/tools/test_tool_repository.py @@ -137,7 +137,7 @@ def test_tool_repository_get_default_error_missing_openmp_compiler(): ToolRepository.''' tr = ToolRepository() fc = FortranCompiler("gfortran", "gfortran", "gnu", openmp_flag=None, - module_folder_flag="-J") + module_folder_flag="-J", version_regex=None) with mock.patch.dict(tr, {Category.FORTRAN_COMPILER: [fc]}), \ pytest.raises(RuntimeError) as err: