Add support for checking out GitHub repositories

Fixes #183

Author: SeanBryan51

.conda/benchcab-dev.yaml

@@ -9,4 +9,5 @@ dependencies:
   - pytest-cov
   - pyyaml
   - flatdict
-  - cerberus>=1.3.5
+  - cerberus>=1.3.5
+  - gitpython

.conda/meta.yaml

@@ -27,3 +27,4 @@ requirements:
         - f90nml
         - flatdict
         - cerberus >=1.3.5
+        - gitpython

benchcab/benchcab.py

@@ -22,6 +22,7 @@
 from benchcab.model import Model
 from benchcab.utils.fs import mkdir, next_path
 from benchcab.utils.pbs import render_job_script
+from benchcab.utils.repo import SVNRepo, create_repo
 from benchcab.utils.subprocess import SubprocessWrapper, SubprocessWrapperInterface
 from benchcab.workdir import setup_fluxsite_directory_tree
 
@@ -42,7 +43,7 @@ def __init__(
         self.validate_env = validate_env
 
         self._config: Optional[dict] = None
-        self._repos: list[Model] = []
+        self._models: list[Model] = []
         self.tasks: list[Task] = []  # initialise fluxsite tasks lazily
 
     def _validate_environment(self, project: str, modules: list):
@@ -99,18 +100,21 @@ def _get_config(self, config_path: str) -> dict:
             self._config = read_config(config_path)
         return self._config
 
-    def _get_repos(self, config: dict) -> list[Model]:
-        if not self._repos:
-            self._repos = [
-                Model(**repo_config, repo_id=id)
-                for id, repo_config in enumerate(config["realisations"])
-            ]
-        return self._repos
+    def _get_models(self, config: dict) -> list[Model]:
+        if not self._models:
+            for id, sub_config in enumerate(config["realisations"]):
+                name = sub_config.get("name")
+                repo = create_repo(
+                    spec=sub_config.pop("repo"),
+                    path=internal.SRC_DIR / name if name else internal.SRC_DIR,
+                )
+                self._models.append(Model(repo=repo, model_id=id, **sub_config))
+        return self._models
 
     def _initialise_tasks(self, config: dict) -> list[Task]:
         """A helper method that initialises and returns the `tasks` attribute."""
         self.tasks = get_fluxsite_tasks(
-            repos=self._get_repos(config),
+            models=self._get_models(config),
             science_configurations=config.get(
                 "science_configurations", internal.DEFAULT_SCIENCE_CONFIGURATIONS
             ),
@@ -181,15 +185,16 @@ def checkout(self, config_path: str, verbose: bool):
 
         print("Checking out repositories...")
         rev_number_log = ""
-        for repo in self._get_repos(config):
-            repo.checkout(verbose=verbose)
-            rev_number_log += (
-                f"{repo.name} last changed revision: "
-                f"{repo.svn_info_show_item('last-changed-revision')}\n"
-            )
+        for model in self._get_models(config):
+            model.repo.checkout(verbose=verbose)
+            rev_number_log += f"{model.name}: {model.repo.get_revision()}\n"
 
         # TODO(Sean) we should archive revision numbers for CABLE-AUX
-        cable_aux_repo = Model(path=internal.CABLE_AUX_RELATIVE_SVN_PATH)
+        cable_aux_repo = SVNRepo(
+            svn_root=internal.CABLE_SVN_ROOT,
+            branch_path=internal.CABLE_AUX_RELATIVE_SVN_PATH,
+            path=internal.SRC_DIR / "CABLE-AUX",
+        )
         cable_aux_repo.checkout(verbose=verbose)
 
         rev_number_log_path = self.root_dir / next_path(
@@ -208,7 +213,7 @@ def build(self, config_path: str, verbose: bool):
         config = self._get_config(config_path)
         self._validate_environment(project=config["project"], modules=config["modules"])
 
-        for repo in self._get_repos(config):
+        for repo in self._get_models(config):
             if repo.build_script:
                 print(
                     "Compiling CABLE using custom build script for "

benchcab/data/config-schema.yml

@@ -29,17 +29,38 @@ realisations:
   schema:
     type: "dict"
     schema:
-      path:
-        type: "string"
+      repo:
+        type: "dict"
+        required: true
+        schema:
+          git:
+            type: "dict"
+            excludes: "svn"
+            schema:
+              branch:
+                type: "string"
+                required: true
+              commit:
+                type: "string"
+                required: false
+              url:
+                type: "string"
+                required: false
+          svn:
+            type: "dict"
+            excludes: "git"
+            schema:
+              branch_path:
+                type: "string"
+                required: true
+              revision:
+                type: "integer"
       name:
         type: "string"
         required: false
       build_script:
         type: "string"
         required: false
-      revision:
-        type: "string"
-        required: false
       patch:
         type: "dict"
         required: false

benchcab/data/test/config-valid.yml

@@ -19,14 +19,13 @@ project: w97
 
 experiment: five-site-test
 
-realisations: [
-  {
-    path: "trunk",
-  },
-  {
-    path: "branches/Users/ccc561/v3.0-YP-changes",
-  }
-]
+realisations:
+  - repo:
+      svn:
+        branch_path: trunk
+  - repo:
+      svn:
+        branch_path: branches/Users/ccc561/v3.0-YP-changes
 
 modules: [
   intel-compiler/2021.1.1,

benchcab/data/test/integration.sh

@@ -20,8 +20,12 @@ project: $PROJECT
 experiment: AU-Tum
 
 realisations:
-  - path: trunk
-  - path: branches/Users/sb8430/test-branch
+  - repo:
+      svn:
+        branch_path: trunk
+  - repo:
+      svn:
+        branch_path: branches/Users/sb8430/test-branch
 
 modules: [
   intel-compiler/2021.1.1,

benchcab/fluxsite.py

@@ -95,20 +95,20 @@ class Task:
 
     def __init__(
         self,
-        repo: Model,
+        model: Model,
         met_forcing_file: str,
         sci_conf_id: int,
         sci_config: dict,
     ) -> None:
-        self.repo = repo
+        self.model = model
         self.met_forcing_file = met_forcing_file
         self.sci_conf_id = sci_conf_id
         self.sci_config = sci_config
 
     def get_task_name(self) -> str:
         """Returns the file name convention used for this task."""
         met_forcing_base_filename = self.met_forcing_file.split(".")[0]
-        return f"{met_forcing_base_filename}_R{self.repo.repo_id}_S{self.sci_conf_id}"
+        return f"{met_forcing_base_filename}_R{self.model.model_id}_S{self.sci_conf_id}"
 
     def get_output_filename(self) -> str:
         """Returns the file name convention used for the netcdf output file."""
@@ -188,19 +188,19 @@ def setup_task(self, verbose=False):
             print(f"  Adding science configurations to CABLE namelist file {nml_path}")
         patch_namelist(nml_path, self.sci_config)
 
-        if self.repo.patch:
+        if self.model.patch:
             if verbose:
                 print(
                     f"  Adding branch specific configurations to CABLE namelist file {nml_path}"
                 )
-            patch_namelist(nml_path, self.repo.patch)
+            patch_namelist(nml_path, self.model.patch)
 
-        if self.repo.patch_remove:
+        if self.model.patch_remove:
             if verbose:
                 print(
                     f"  Removing branch specific configurations from CABLE namelist file {nml_path}"
                 )
-            patch_remove_namelist(nml_path, self.repo.patch_remove)
+            patch_remove_namelist(nml_path, self.model.patch_remove)
 
     def clean_task(self, verbose=False):
         """Cleans output files, namelist files, log files and cable executables if they exist."""
@@ -264,7 +264,7 @@ def fetch_files(self, verbose=False):
             self.root_dir / internal.NAMELIST_DIR, task_dir, dirs_exist_ok=True
         )
 
-        exe_src = self.repo.get_exe_path()
+        exe_src = self.model.get_exe_path()
         exe_dest = task_dir / internal.CABLE_EXE
 
         if verbose:
@@ -343,28 +343,28 @@ def add_provenance_info(self, verbose=False):
                         ).items()
                     },
                     **{
-                        "cable_branch": self.repo.svn_info_show_item("url"),
-                        "svn_revision_number": self.repo.svn_info_show_item("revision"),
+                        "cable_branch": self.model.repo.get_branch_name(),
+                        "svn_revision_number": self.model.repo.get_revision(),
                         "benchcab_version": __version__,
                     },
                 }
             )
 
 
 def get_fluxsite_tasks(
-    repos: list[Model],
+    models: list[Model],
     science_configurations: list[dict],
     fluxsite_forcing_file_names: list[str],
 ) -> list[Task]:
     """Returns a list of fluxsite tasks to run."""
     tasks = [
         Task(
-            repo=repo,
+            model=model,
             met_forcing_file=file_name,
             sci_conf_id=sci_conf_id,
             sci_config=sci_config,
         )
-        for repo in repos
+        for model in models
         for file_name in fluxsite_forcing_file_names
         for sci_conf_id, sci_config in enumerate(science_configurations)
     ]
@@ -403,14 +403,14 @@ def get_fluxsite_comparisons(
                 output_dir / task_b.get_output_filename(),
             ),
             task_name=get_comparison_name(
-                task_a.repo, task_b.repo, task_a.met_forcing_file, task_a.sci_conf_id
+                task_a.model, task_b.model, task_a.met_forcing_file, task_a.sci_conf_id
             ),
         )
         for task_a in tasks
         for task_b in tasks
         if task_a.met_forcing_file == task_b.met_forcing_file
         and task_a.sci_conf_id == task_b.sci_conf_id
-        and task_a.repo.repo_id < task_b.repo.repo_id
+        and task_a.model.model_id < task_b.model.model_id
         # TODO(Sean): Review later - the following code avoids using a double
         # for loop to generate pair wise combinations, however we would have
         # to re-initialize task instances to get access to the output file path
@@ -425,14 +425,14 @@ def get_fluxsite_comparisons(
 
 
 def get_comparison_name(
-    repo_a: Model,
-    repo_b: Model,
+    model_a: Model,
+    model_b: Model,
     met_forcing_file: str,
     sci_conf_id: int,
 ) -> str:
     """Returns the naming convention used for bitwise comparisons."""
     met_forcing_base_filename = met_forcing_file.split(".")[0]
     return (
         f"{met_forcing_base_filename}_S{sci_conf_id}"
-        f"_R{repo_a.repo_id}_R{repo_b.repo_id}"
+        f"_R{model_a.model_id}_R{model_b.model_id}"
     )

benchcab/internal.py

@@ -75,16 +75,17 @@
 FLUXSITE_DIRS["ANALYSIS"] = FLUXSITE_DIRS["RUN"] / "analysis"
 
 # Relative path to directory that stores bitwise comparison results
-FLUXSITE_DIRS["BITWISE_CMP"] = (
-    FLUXSITE_DIRS["ANALYSIS"] / "bitwise-comparisons"
-)
+FLUXSITE_DIRS["BITWISE_CMP"] = FLUXSITE_DIRS["ANALYSIS"] / "bitwise-comparisons"
 
 # Path to met files:
 MET_DIR = Path("/g/data/ks32/CLEX_Data/PLUMBER2/v1-0/Met/")
 
 # CABLE SVN root url:
 CABLE_SVN_ROOT = "https://trac.nci.org.au/svn/cable"
 
+# CABLE GitHub URL:
+CABLE_GIT_URL = "https://github.com/CABLE-LSM/CABLE.git"
+
 # CABLE executable file name:
 CABLE_EXE = "cable-mpi" if MPI else "cable"