More refactors

Author: perseoGI

conan/internal/runner/docker.py

@@ -4,7 +4,7 @@
 import json
 import platform
 import shutil
-from typing import Optional, NamedTuple
+from typing import Optional, NamedTuple, Dict, List
 import yaml
 from conan.api.conan_api import ConanAPI
 from conan.api.model import ListPattern
@@ -16,45 +16,40 @@
 from conans.model.version import Version
 from pathlib import Path
 
+class _ContainerConfig(NamedTuple):
+    class Build(NamedTuple):
+        dockerfile: Optional[str] = None
+        build_context: Optional[str] = None
+        build_args: Optional[Dict[str, str]] = None
+        cache_from: Optional[List[str]] = None
+
+    class Run(NamedTuple):
+        name: Optional[str] = None
+        environment: Optional[Dict[str, str]] = None
+        user: Optional[str] = None
+        privileged: Optional[bool] = None
+        cap_add: Optional[List[str]] = None
+        security_opt: Optional[List[str]] = None
+        volumes: Optional[Dict[str, str]] = None
+        network: Optional[str] = None
+
+    image: Optional[str] = None
+    build: Build = Build()
+    run: Run = Run()
+
 
 class DockerRunner:
     def __init__(self, conan_api: ConanAPI, command: str, host_profile: Profile, build_profile: Profile, args: Namespace, raw_args: list[str]):
-        import docker
-        import docker.api.build
-        try:
-            docker_base_urls = [
-                None, # Default docker configuration, let the python library detect the socket
-                os.environ.get('DOCKER_HOST'), # Connect to socket defined in DOCKER_HOST
-                'unix:///var/run/docker.sock', # Default linux socket
-                f'unix://{os.path.expanduser("~")}/.rd/docker.sock' # Rancher linux socket
-            ]
-            for base_url in docker_base_urls:
-                try:
-                    ConanOutput().verbose(f'Trying to connect to docker "{base_url or "default"}" socket')
-                    self.docker_client = docker.DockerClient(base_url=base_url, version='auto')
-                    ConanOutput().verbose(f'Connected to docker "{base_url or "default"}" socket')
-                    break
-                except:
-                    continue
-            self.docker_api = self.docker_client.api
-            docker.api.build.process_dockerfile = lambda dockerfile, path: ('Dockerfile', dockerfile)
-        except:
-            raise ConanException("Docker Client failed to initialize."
-                                 "\n - Check if docker is installed and running"
-                                 "\n - Run 'pip install conan[runners]'")
+        self.docker_client = self._initialize_docker_client()
+        self.docker_api = self.docker_client.api
         self.conan_api = conan_api
         self.build_profile = build_profile
+        self.abs_host_path = self._get_abs_host_path(args.path)
         self.args = args
-        abs_path = make_abs_path(args.path)
-        if abs_path is None:
-            raise ConanException("Could not determine absolute path")
-        self.abs_host_path = Path(abs_path)
         if args.format:
             raise ConanException("format argument is forbidden if running in a docker runner")
 
-        # Container config
-        # https://containers.dev/implementors/json_reference/
-        self.configfile = self.config_parser(host_profile.runner.get('configfile'))
+        self.configfile = self._load_config(host_profile.runner.get('configfile'))
         self.dockerfile = host_profile.runner.get('dockerfile') or self.configfile.build.dockerfile
         self.docker_build_context = host_profile.runner.get('build_context') or self.configfile.build.build_context
         self.image = host_profile.runner.get('image') or self.configfile.image
@@ -64,28 +59,30 @@ def __init__(self, conan_api: ConanAPI, command: str, host_profile: Profile, bui
         self.name = self.configfile.run.name or f'conan-runner-{host_profile.runner.get("suffix", "docker")}'
         self.remove = str(host_profile.runner.get('remove', 'false')).lower() == 'true'
         self.cache = str(host_profile.runner.get('cache', 'clean'))
+        if self.cache not in ['clean', 'copy', 'shared']:
+            raise ConanException(f'Invalid cache value: "{self.cache}". Valid values are: clean, copy, shared')
         self.container = None
 
-        # Runner config>
+        # Runner config
         self.abs_runner_home_path = self.abs_host_path / '.conanrunner'
         self.docker_user_name = self.configfile.run.user or 'root'
         self.abs_docker_path = os.path.join(f'/{self.docker_user_name}/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/")
 
         # Update conan command and some paths to run inside the container
         raw_args[raw_args.index(args.path)] = self.abs_docker_path
         self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json'])
-        self.logger = DockerOutput(self.name)
+        self.logger = DockerRunner.Output(self.name)
 
     def run(self) -> None:
         """
         Run conan inside a Docker container
         """
-        self.build_image()
-        self.start_container()
+        self._build_image()
+        self._start_container()
         try:
-            self.init_container()
-            self.run_command(self.command)
-            self.update_local_cache()
+            self._init_container()
+            self._run_command(self.command)
+            self._update_local_cache()
         except RunnerException as e:
             raise ConanException(f'"{e.command}" inside docker fail'
                                  f'\n\nLast command output: {str(e.stdout_log)}')
@@ -99,28 +96,56 @@ def run(self) -> None:
                     log('Removing container')
                     self.container.remove()
 
-    Build = NamedTuple('Build', [('dockerfile', Optional[str]), ('build_context', Optional[str]), ('build_args', Optional[dict]), ('cache_from', Optional[list])])
-    Run = NamedTuple('Run', [('name', Optional[str]), ('environment', Optional[dict]), ('user', Optional[str]), ('privileged', Optional[bool]), ('cap_add', Optional[list]), ('security_opt', Optional[list]), ('volumes', Optional[dict]), ('network', Optional[str])])
-    Conf = NamedTuple('Conf', [('image', Optional[str]), ('build', Build), ('run', Run)])
+    def _initialize_docker_client(self):
+        import docker
+        import docker.api.build
+
+        docker_base_urls = [
+            None,  # Default docker configuration
+            os.environ.get('DOCKER_HOST'),  # From DOCKER_HOST environment variable
+            'unix:///var/run/docker.sock',  # Default Linux socket
+            f'unix://{os.path.expanduser("~")}/.rd/docker.sock'  # Rancher Linux socket
+        ]
+
+        for base_url in docker_base_urls:
+            try:
+                ConanOutput().verbose(f'Trying to connect to Docker: "{base_url or "default"}"')
+                client = docker.DockerClient(base_url=base_url, version='auto')
+                ConanOutput().verbose(f'Connected to Docker: "{base_url or "default"}"')
+                docker.api.build.process_dockerfile = lambda dockerfile, path: ('Dockerfile', dockerfile)
+                return client
+            except Exception:
+                continue
+        raise ConanException("Docker Client failed to initialize."
+                             "\n - Check if docker is installed and running"
+                             "\n - Run 'pip install conan[runners]'")
+
+    def _get_abs_host_path(self, path: str) -> Path:
+        abs_path = make_abs_path(path)
+        if abs_path is None:
+            raise ConanException("Could not determine the absolute path.")
+        return Path(abs_path)
 
     @staticmethod
-    def config_parser(file_path: str) -> Conf:
+    def _load_config(file_path: Optional[str]) -> _ContainerConfig:
+        # Container config
+        # https://containers.dev/implementors/json_reference/
         if file_path:
             def _instans_or_error(value, obj):
                 if value and (not isinstance(value, obj)):
                     raise ConanException(f"docker runner configfile syntax error: {value} must be a {obj.__name__}")
                 return value
             with open(file_path, 'r') as f:
                 runnerfile = yaml.safe_load(f)
-            return DockerRunner.Conf(
+            return _ContainerConfig(
                 image=_instans_or_error(runnerfile.get('image'), str),
-                build=DockerRunner.Build(
+                build=_ContainerConfig.Build(
                     dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str),
                     build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str),
                     build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict),
                     cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list),
                 ),
-                run=DockerRunner.Run(
+                run=_ContainerConfig.Run(
                     name=_instans_or_error(runnerfile.get('run', {}).get('name'), str),
                     environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict),
                     user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str),
@@ -132,14 +157,9 @@ def _instans_or_error(value, obj):
                 )
             )
         else:
-            return DockerRunner.Conf(
-                image=None,
-                build=DockerRunner.Build(dockerfile=None, build_context=None, build_args=None, cache_from=None),
-                run=DockerRunner.Run(name=None, environment=None, user=None, privileged=None, cap_add=None,
-                        security_opt=None, volumes=None, network=None)
-            )
+            return _ContainerConfig()
 
-    def build_image(self) -> None:
+    def _build_image(self) -> None:
         if not self.dockerfile:
             return
         self.logger.status(f'Building the Docker image: {self.image}')
@@ -164,8 +184,8 @@ def build_image(self) -> None:
                     if stream:
                         ConanOutput().status(stream.strip())
 
-    def start_container(self) -> None:
-        volumes, environment = self.create_runner_environment()
+    def _start_container(self) -> None:
+        volumes, environment = self._create_runner_environment()
         try:
             if self.docker_client.containers.list(all=True, filters={'name': self.name}):
                 self.logger.status('Starting the docker container', fg=Color.BRIGHT_MAGENTA)
@@ -195,7 +215,7 @@ def start_container(self) -> None:
             raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"'
                                  f'\n\n{str(e)}')
 
-    def run_command(self, command: str, workdir: Optional[str] = None, log: bool = True) -> tuple[str, str]:
+    def _run_command(self, command: str, workdir: Optional[str] = None, log: bool = True) -> tuple[str, str]:
         workdir = workdir or self.abs_docker_path
         if log:
             self.logger.status(f'Running in container: "{command}"')
@@ -224,7 +244,7 @@ def run_command(self, command: str, workdir: Optional[str] = None, log: bool = T
             raise RunnerException(command=command, stdout_log=stdout_log, stderr_log=stderr_log)
         return stdout_log, stderr_log
 
-    def create_runner_environment(self) -> tuple[dict, dict]:
+    def _create_runner_environment(self) -> tuple[dict, dict]:
         shutil.rmtree(self.abs_runner_home_path, ignore_errors=True)
         volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}}
         environment = {'CONAN_RUNNER_ENVIRONMENT': '1'}
@@ -250,35 +270,36 @@ def create_runner_environment(self) -> tuple[dict, dict]:
                 self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path)
         return volumes, environment
 
-    def init_container(self) -> None:
+    def _init_container(self) -> None:
         min_conan_version = '2.1'
-        stdout, _ = self.run_command('conan --version', log=True)
+        stdout, _ = self._run_command('conan --version', log=True)
         docker_conan_version = str(stdout.split('Conan version ')[1].replace('\n', '').replace('\r', '')) # Remove all characters and color
         if Version(docker_conan_version) <= Version(min_conan_version):
             raise ConanException(f'conan version inside the container must be greater than {min_conan_version}')
         if self.cache != 'shared':
-            self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False)
-            self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False)
+            self._run_command('mkdir -p ${HOME}/.conan2/profiles', log=False)
+            self._run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False)
+
             for file_name in ['global.conf', 'settings.yml', 'remotes.json']:
                 if os.path.exists( os.path.join(self.abs_runner_home_path, file_name)):
-                    self.run_command('cp "'+self.abs_docker_path+'/.conanrunner/'+file_name+'" ${HOME}/.conan2/'+file_name, log=False)
+                    self._run_command('cp "'+self.abs_docker_path+'/.conanrunner/'+file_name+'" ${HOME}/.conan2/'+file_name, log=False)
             if self.cache in ['copy']:
-                self.run_command('conan cache restore "'+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz"')
+                self._run_command('conan cache restore "'+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz"')
 
-    def update_local_cache(self) -> None:
+    def _update_local_cache(self) -> None:
         if self.cache != 'shared':
-            self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False)
-            self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz')
+            self._run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False)
+            self._run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz')
             tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz')
             self.logger.status(f'Restore host cache from: {tgz_path}')
             self.conan_api.cache.restore(tgz_path)
 
-class DockerOutput(ConanOutput):
-    def __init__(self, image: str):
-        super().__init__()
-        self.image = image
+    class Output(ConanOutput):
+        def __init__(self, image: str):
+            super().__init__()
+            self.image = image
 
-    def _write_message(self, msg, fg=None, bg=None, newline=True):
-        super()._write_message(f"===> Docker Runner ({self.image}): ", Color.BLACK,
-                               Color.BRIGHT_YELLOW, newline=False)
-        super()._write_message(msg, fg, bg, newline)
+        def _write_message(self, msg, fg=None, bg=None, newline=True):
+            super()._write_message(f"===> Docker Runner ({self.image}): ", Color.BLACK,
+                                   Color.BRIGHT_YELLOW, newline=False)
+            super()._write_message(msg, fg, bg, newline)

conans/model/conf.py

@@ -131,6 +131,13 @@
     "tools.build:linker_scripts": "List of linker script files to pass to the linker used by different toolchains like CMakeToolchain, AutotoolsToolchain, and MesonToolchain",
     # Package ID composition
     "tools.info.package_id:confs": "List of existing configuration to be part of the package ID",
+    # Runners
+    "runner.type": "Type of runner to use. Possible values: 'docker'",
+    "runner.dockerfile": "(Docker) Path to the Dockerfile to use in case of building a docker image",
+    "runner.image": "(Docker) Image name to download from registry or the name of the built image in case of defining dockerfile path",
+    "runner.cache": "(Docker) Host's conan cache behavior. Possible values: 'clean' (use empty cache), 'copy' (copy whole cache) or 'shared' (mount chache as shared volume)",
+    "runner.remove": "(Docker) (boolean) Remove the container after running the Conan command",
+    "runner.configfile": "(Docker) Path to a configuration file with extra parameters (https://containers.dev/implementors/json_reference/#image-specific)",
 }
 
 BUILT_IN_CONFS = {key: value for key, value in sorted(BUILT_IN_CONFS.items())}