Add shell tool clean

source/fab/tools/__init__.py

@@ -19,6 +19,7 @@
 from fab.tools.psyclone import Psyclone
 from fab.tools.rsync import Rsync
 from fab.tools.preprocessor import Cpp, CppFortran, Fpp, Preprocessor
+from fab.tools.shell import Shell
 from fab.tools.tool import Tool, CompilerSuiteTool
 # Order here is important to avoid a circular import
 from fab.tools.tool_repository import ToolRepository
@@ -56,6 +57,7 @@
            "Preprocessor",
            "Psyclone",
            "Rsync",
+           "Shell",
            "Subversion",
            "Tool",
            "ToolBox",

source/fab/tools/category.py

@@ -25,6 +25,7 @@ class Category(Enum):
     SUBVERSION = auto()
     AR = auto()
     RSYNC = auto()
+    SHELL = auto()
     MISC = auto()
 
     def __str__(self):

source/fab/tools/compiler.py

@@ -56,7 +56,7 @@ def __init__(self, name: str,
                  compile_flag: Optional[str] = None,
                  output_flag: Optional[str] = None,
                  openmp_flag: Optional[str] = None,
-                 availability_option: Optional[str] = None):
+                 availability_option: Optional[Union[str, List[str]]] = None):
         super().__init__(name, exec_name, suite, category=category,
                          availability_option=availability_option)
         self._version: Union[Tuple[int, ...], None] = None

source/fab/tools/psyclone.py

@@ -180,7 +180,7 @@ def process(self,
                 # New version: no API, parameter, but -o for output name:
                 parameters.extend(["-o", transformed_file])
             else:
-                # 2.5.0 or earlier: needs api nemo, output name is -oalg
+                # 2.5.0 or earlier: needs api nemo, output name is -opsy
                 parameters.extend(["-api", "nemo", "-opsy", transformed_file])
         parameters.extend(["-l", "all"])
 

source/fab/tools/shell.py

@@ -0,0 +1,46 @@
+##############################################################################
+# (c) Crown copyright Met Office. All rights reserved.
+# For further details please refer to the file COPYRIGHT
+# which you should have received as part of this distribution
+##############################################################################
+
+"""This file contains a base class for shells. This can be used to execute
+other scripts.
+"""
+
+from pathlib import Path
+from typing import List, Union
+
+from fab.tools.category import Category
+from fab.tools.tool import Tool
+
+
+class Shell(Tool):
+    '''A simple wrapper that runs a shell script. There seems to be no
+    consistent way to simply check if a shell is working - not only support
+    a version command (e.g. sh and dash don't). Instead, availability
+    is tested by running a simple 'echo' command.
+
+    :name: the path to the script to run.
+    '''
+    def __init__(self, name: str):
+        super().__init__(name=name, exec_name=name,
+                         availability_option=["-c", "echo hello"],
+                         category=Category.SHELL)
+
+    def exec(self, command: Union[str, List[Union[Path, str]]]) -> str:
+        '''Executes the specified command.
+
+        :param command: the command and potential parameters to execute.
+
+        :returns: stdout of the result.
+        '''
+        # Make mypy happy:
+        params: List[Union[str, Path]]
+        if isinstance(command, str):
+            params = ["-c", command]
+        else:
+            params = ["-c"]
+            params.extend(command)
+        return super().run(additional_parameters=params,
+                           capture_output=True)

source/fab/tools/tool.py

@@ -16,7 +16,7 @@
 import logging
 from pathlib import Path
 import subprocess
-from typing import Dict, List, Optional, Union
+from typing import Dict, List, Optional, Sequence, Union
 
 from fab.tools.category import Category
 from fab.tools.flags import Flags
@@ -36,7 +36,7 @@ class Tool:
 
     def __init__(self, name: str, exec_name: Union[str, Path],
                  category: Category = Category.MISC,
-                 availability_option: Optional[str] = None):
+                 availability_option: Optional[Union[str, List[str]]] = None):
         self._logger = logging.getLogger(__name__)
         self._name = name
         self._exec_name = str(exec_name)
@@ -63,7 +63,8 @@ def check_available(self) -> bool:
         :returns: whether the tool is working (True) or not.
         '''
         try:
-            self.run(self._availability_option)
+            op = self._availability_option
+            self.run(op)
         except (RuntimeError, FileNotFoundError):
             return False
         return True
@@ -107,7 +108,7 @@ def name(self) -> str:
         return self._name
 
     @property
-    def availability_option(self) -> str:
+    def availability_option(self) -> Union[str, List[str]]:
         ''':returns: the option to use to check if the tool is available.'''
         return self._availability_option
 
@@ -139,7 +140,7 @@ def __str__(self):
 
     def run(self,
             additional_parameters: Optional[
-                Union[str, List[Union[Path, str]]]] = None,
+                Union[str, Sequence[Union[Path, str]]]] = None,
             env: Optional[Dict[str, str]] = None,
             cwd: Optional[Union[Path, str]] = None,
             capture_output=True) -> str:
@@ -210,7 +211,7 @@ class CompilerSuiteTool(Tool):
     '''
     def __init__(self, name: str, exec_name: Union[str, Path], suite: str,
                  category: Category,
-                 availability_option: Optional[str] = None):
+                 availability_option: Optional[Union[str, List[str]]] = None):
         super().__init__(name, exec_name, category,
                          availability_option=availability_option)
         self._suite = suite

source/fab/tools/tool_repository.py

@@ -23,7 +23,7 @@
 from fab.tools.versioning import Fcm, Git, Subversion
 from fab.tools import (Ar, Cpp, CppFortran, Craycc, Crayftn,
                        Gcc, Gfortran, Icc, Icx, Ifort, Ifx,
-                       Nvc, Nvfortran, Psyclone, Rsync)
+                       Nvc, Nvfortran, Psyclone, Rsync, Shell)
 
 
 class ToolRepository(dict):
@@ -63,7 +63,7 @@ def __init__(self):
         # Add the FAB default tools:
         # TODO: sort the defaults so that they actually work (since not all
         # tools FAB knows about are available). For now, disable Fpp (by not
-        # adding it). IF someone actually uses it it can added.
+        # adding it). If someone actually uses it it can added.
         for cls in [Craycc, Crayftn,
                     Gcc, Gfortran,
                     Icc, Icx, Ifort, Ifx,
@@ -72,6 +72,12 @@ def __init__(self):
                     Ar, Fcm, Git, Psyclone, Rsync, Subversion]:
             self.add_tool(cls())
 
+        # Add the common shells. While Fab itself does not need this,
+        # it is a very convenient tool for user configuration (e.g. to
+        # query nc-config etc)
+        for shell_name in ["bash", "sh", "ksh", "dash"]:
+            self.add_tool(Shell(shell_name))
+
         # Now create the potential mpif90 and Cray ftn wrapper
         all_fc = self[Category.FORTRAN_COMPILER][:]
         for fc in all_fc:
@@ -95,9 +101,10 @@ def __init__(self):
 
     def add_tool(self, tool: Tool):
         '''Creates an instance of the specified class and adds it
-        to the tool repository.
+        to the tool repository. If the tool is a compiler, it automatically
+        adds the compiler as a linker as well (named "linker-{tool.name}").
 
-        :param cls: the tool to instantiate.
+        :param tool: the tool to add.
         '''
 
         # We do not test if a tool is actually available. The ToolRepository
@@ -155,15 +162,16 @@ def set_default_compiler_suite(self, suite: str):
     def get_default(self, category: Category,
                     mpi: Optional[bool] = None,
                     openmp: Optional[bool] = None):
-        '''Returns the default tool for a given category. For most tools
-        that will be the first entry in the list of tools. The exception
-        are compilers and linker: in this case it must be specified if
-        MPI support is required or not. And the default return will be
+        '''Returns the default tool for a given category that is available.
+        For most tools that will be the first entry in the list of tools. The
+        exception are compilers and linker: in this case it must be specified
+        if MPI support is required or not. And the default return will be
         the first tool that either supports MPI or not.
 
         :param category: the category for which to return the default tool.
         :param mpi: if a compiler or linker is required that supports MPI.
-        :param open: if a compiler or linker is required that supports OpenMP.
+        :param openmp: if a compiler or linker is required that supports
+            OpenMP.
 
         :raises KeyError: if the category does not exist.
         :raises RuntimeError: if no compiler/linker is found with the
@@ -176,7 +184,12 @@ def get_default(self, category: Category,
 
         # If not a compiler or linker, return the first tool
         if not category.is_compiler and category != Category.LINKER:
-            return self[category][0]
+            for tool in self[category]:
+                if tool.is_available:
+                    return tool
+            tool_names = ",".join(i.name for i in self[category])
+            raise RuntimeError(f"Can't find available '{category}' tool. "
+                               f"Tools are '{tool_names}'.")
 
         if not isinstance(mpi, bool):
             raise RuntimeError(f"Invalid or missing mpi specification "
@@ -191,8 +204,8 @@ def get_default(self, category: Category,
             # ignore it.
             if openmp and not tool.openmp:
                 continue
-            # If the tool supports/does not support MPI, return it.
-            if mpi == tool.mpi:
+            # If the tool supports/does not support MPI, return the first one
+            if tool.is_available and mpi == tool.mpi:
                 return tool
 
         # Don't bother returning an MPI enabled tool if no-MPI is requested -

tests/unit_tests/steps/test_archive_objects.py

@@ -84,7 +84,6 @@ def test_for_library(self):
     def test_incorrect_tool(self, tool_box):
         '''Test that an incorrect archive tool is detected
         '''
-
         config = BuildConfig('proj', tool_box)
         cc = tool_box.get_tool(Category.C_COMPILER, config.mpi, config.openmp)
         # And set its category to be AR

tests/unit_tests/steps/test_grab.py

@@ -3,6 +3,10 @@
 # For further details please refer to the file COPYRIGHT
 # which you should have received as part of this distribution
 ##############################################################################
+
+'''Test various grab implementation - folders and fcm.
+'''
+
 from pathlib import Path
 from types import SimpleNamespace
 from unittest import mock
@@ -15,24 +19,37 @@
 
 
 class TestGrabFolder:
+    '''Test grab folder functionality.'''
 
     def test_trailing_slash(self):
-        with pytest.warns(UserWarning, match="_metric_send_conn not set, cannot send metrics"):
-            self._common(grab_src='/grab/source/', expect_grab_src='/grab/source/')
+        '''Test folder grabbing with a trailing slash.'''
+        with pytest.warns(UserWarning, match="_metric_send_conn not set, "
+                                             "cannot send metrics"):
+            self._common(grab_src='/grab/source/',
+                         expect_grab_src='/grab/source/')
 
     def test_no_trailing_slash(self):
-        with pytest.warns(UserWarning, match="_metric_send_conn not set, cannot send metrics"):
-            self._common(grab_src='/grab/source', expect_grab_src='/grab/source/')
+        '''Test folder grabbing without a trailing slash.'''
+        with pytest.warns(UserWarning, match="_metric_send_conn not set, "
+                                             "cannot send metrics"):
+            self._common(grab_src='/grab/source',
+                         expect_grab_src='/grab/source/')
 
     def _common(self, grab_src, expect_grab_src):
         source_root = Path('/workspace/source')
         dst = 'bar'
 
         mock_config = SimpleNamespace(source_root=source_root,
                                       tool_box=ToolBox())
+        # Since is_available calls run, in order to test a single run call,
+        # we patch is_available to be always true.
         with mock.patch('pathlib.Path.mkdir'):
             with mock.patch('fab.tools.tool.Tool.run') as mock_run:
-                grab_folder(mock_config, src=grab_src, dst_label=dst)
+                with mock.patch(
+                        'fab.tools.tool.Tool.is_available',
+                        new_callable=mock.PropertyMock) as is_available:
+                    is_available.return_value = True
+                    grab_folder(mock_config, src=grab_src, dst_label=dst)
 
         expect_dst = mock_config.source_root / dst
         mock_run.assert_called_once_with(
@@ -41,24 +58,34 @@ def _common(self, grab_src, expect_grab_src):
 
 
 class TestGrabFcm:
+    '''Test FCM functionality.'''
 
     def test_no_revision(self):
+        '''Test FCM without specifying a revision.'''
         source_root = Path('/workspace/source')
         source_url = '/www.example.com/bar'
         dst_label = 'bar'
 
         mock_config = SimpleNamespace(source_root=source_root,
                                       tool_box=ToolBox())
         with mock.patch('pathlib.Path.mkdir'):
-            with mock.patch('fab.tools.tool.Tool.run') as mock_run, \
-                 pytest.warns(UserWarning, match="_metric_send_conn not set, cannot send metrics"):
-                fcm_export(config=mock_config, src=source_url, dst_label=dst_label)
+            with mock.patch('fab.tools.tool.Tool.is_available',
+                            new_callable=mock.PropertyMock) as is_available:
+                is_available.return_value = True
+                with mock.patch('fab.tools.tool.Tool.run') as mock_run, \
+                    pytest.warns(UserWarning,
+                                 match="_metric_send_conn not "
+                                       "set, cannot send metrics"):
+                    fcm_export(config=mock_config, src=source_url,
+                               dst_label=dst_label)
 
         mock_run.assert_called_once_with(['export', '--force', source_url,
                                           str(source_root / dst_label)],
-                                         env=None, cwd=None, capture_output=True)
+                                         env=None, cwd=None,
+                                         capture_output=True)
 
     def test_revision(self):
+        '''Test that the revision is passed on correctly in fcm export.'''
         source_root = Path('/workspace/source')
         source_url = '/www.example.com/bar'
         dst_label = 'bar'
@@ -67,12 +94,19 @@ def test_revision(self):
         mock_config = SimpleNamespace(source_root=source_root,
                                       tool_box=ToolBox())
         with mock.patch('pathlib.Path.mkdir'):
-            with mock.patch('fab.tools.tool.Tool.run') as mock_run, \
-                 pytest.warns(UserWarning, match="_metric_send_conn not set, cannot send metrics"):
-                fcm_export(mock_config, src=source_url, dst_label=dst_label, revision=revision)
+            with mock.patch('fab.tools.tool.Tool.is_available',
+                            new_callable=mock.PropertyMock) as is_available:
+                is_available.return_value = True
+                with mock.patch('fab.tools.tool.Tool.run') as mock_run, \
+                    pytest.warns(
+                        UserWarning, match="_metric_send_conn not set, "
+                                           "cannot send metrics"):
+                    fcm_export(mock_config, src=source_url,
+                               dst_label=dst_label, revision=revision)
 
         mock_run.assert_called_once_with(
-            ['export', '--force', '--revision', '42', f'{source_url}', str(source_root / dst_label)],
+            ['export', '--force', '--revision', '42', f'{source_url}',
+             str(source_root / dst_label)],
             env=None, cwd=None, capture_output=True)
 
     # todo: test missing repo