diff --git a/README.md b/README.md index 0f62d4d..341870d 100644 --- a/README.md +++ b/README.md @@ -301,14 +301,34 @@ params: **Note**: If duplicate parameters are provided, the parameters provided as key-value pairs inside the `params` nested dictionary of the YAML file will take precedence **over** values in the provided `params-file`. -### 2. `overwrite` Functionality +### 2. `on_exists` Functionality -For every entity defined in your YAML file, you can specify `overwrite: True` to overwrite any existing entities in Seqera Platform of the same name. +For every entity defined in your YAML file, you can specify how to handle cases where the entity already exists using the `on_exists` parameter with one of three options: -`seqerakit` will first check to see if the name of the entity exists, if so, it will invoke a `tw delete` command before attempting to create it based on the options defined in the YAML file. +- `fail` (default): Raise an error if the entity already exists +- `ignore`: Skip creation if the entity already exists +- `overwrite`: Delete the existing entity and create a new one based on the YAML configuration + +Example usage in YAML: + +```yaml +workspaces: + - name: 'showcase' + organization: 'seqerakit_automation' + on_exists: 'overwrite' # Will delete and recreate if exists +``` + +```yaml +credentials: + - name: 'github_credentials' + workspace: 'seqerakit_automation/showcase' + on_exists: 'ignore' # Will skip if already exists +``` + +When using the `overwrite` option, `seqerakit` will first check if the entity exists, and if so, it will invoke a `tw delete` command before attempting to create it based on the options defined in the YAML file: ```console -DEBUG:root: Overwrite is set to 'True' for organizations +DEBUG:root: on_exists is set to 'overwrite' for organizations DEBUG:root: Running command: tw -o json organizations list DEBUG:root: The attempted organizations resource already exists. Overwriting. @@ -317,6 +337,8 @@ DEBUG:root: Running command: tw organizations delete --name $SEQERA_ORGANIZATION DEBUG:root: Running command: tw organizations add --name $SEQERA_ORGANIZATION_NAME --full-name $SEQERA_ORGANIZATION_NAME --description 'Example of an organization' ``` +> **Note**: For backward compatibility, the `overwrite: True|False` parameter is still supported but deprecated. It will be mapped to `on_exists: 'overwrite'|'fail'` respectively. + ### 3. Specifying JSON configuration files with `file-path` The Seqera Platform CLI allows export and import of entities through JSON configuration files for pipelines and compute environments. To use these files to add a pipeline or compute environment to a workspace, use the `file-path` key to specify a path to a JSON configuration file. diff --git a/seqerakit/cli.py b/seqerakit/cli.py index f125fe8..bded549 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -21,7 +21,7 @@ import logging import sys import os -import yaml +import yaml # type: ignore from pathlib import Path @@ -32,13 +32,14 @@ CommandError, ) from seqerakit import __version__ +from seqerakit.on_exists import OnExists logger = logging.getLogger(__name__) def parse_args(args=None): parser = argparse.ArgumentParser( - description="Seqerakit: Python wrapper for the Seqera Platform CLI" + description="Build a Seqera Platform instance from a YAML configuration file." ) # General options general = parser.add_argument_group("General Options") @@ -106,10 +107,20 @@ def parse_args(args=None): type=str, help="Path to a YAML file containing environment variables for configuration.", ) + yaml_processing.add_argument( + "--on-exists", + dest="on_exists", + type=str, + help="Globally specifies the action to take if a resource already exists.", + choices=[e.name.lower() for e in OnExists], + ) yaml_processing.add_argument( "--overwrite", action="store_true", - help="Globally enable overwrite for all resources defined in YAML input(s).", + help=""" + Globally enable overwrite for all resources defined in YAML input(s). + "Deprecated: Please use '--on-exists=overwrite' instead.""", + deprecated=True, ) return parser.parse_args(args) @@ -147,7 +158,7 @@ def handle_block(self, block, args, destroy=False, dryrun=False): if destroy: logging.debug(" The '--delete' flag has been specified.\n") self.overwrite_method.handle_overwrite( - block, args["cmd_args"], overwrite=False, destroy=True + block, args["cmd_args"], on_exists=OnExists.FAIL, destroy=True ) return @@ -162,16 +173,46 @@ def handle_block(self, block, args, destroy=False, dryrun=False): ), } - # Check if overwrite is set to True or globally, and call overwrite handler - overwrite_option = args.get("overwrite", False) or getattr( - self.sp, "overwrite", False - ) - if overwrite_option and not dryrun: - logging.debug(f" Overwrite is set to 'True' for {block}\n") - self.overwrite_method.handle_overwrite( - block, args["cmd_args"], overwrite_option + # Determine the on_exists behavior (default to FAIL) + on_exists = OnExists.FAIL + + # Check for global settings (they override block-level settings) + if ( + hasattr(self.sp, "global_on_exists") + and self.sp.global_on_exists is not None + ): + on_exists = self.sp.global_on_exists + elif getattr(self.sp, "overwrite", False): + logging.warning( + "The '--overwrite' flag is deprecated. " + "Please use '--on-exists=overwrite' instead." + ) + on_exists = OnExists.OVERWRITE + + # If no global setting, use block-level setting if provided + elif "on_exists" in args: + on_exists_value = args["on_exists"] + if isinstance(on_exists_value, str): + try: + on_exists = OnExists[on_exists_value.upper()] + except KeyError as err: + logging.error(f"Invalid on_exists option: {on_exists_value}") + raise err + + if not dryrun: + # Use on_exists.name.lower() only if it's an enum, otherwise use the string + on_exists_str = ( + on_exists.name.lower() if hasattr(on_exists, "name") else on_exists + ) + logging.debug(f" on_exists is set to '{on_exists_str}' for {block}\n") + should_continue = self.overwrite_method.handle_overwrite( + block, args["cmd_args"], on_exists=on_exists ) + # If on_exists is "ignore" and resource exists, skip creation + if not should_continue: + return + if block in self.list_for_add_method: helper.handle_generic_block(self.sp, block, args["cmd_args"]) elif block in block_handler_map: @@ -248,6 +289,16 @@ def main(args=None): ) sp.overwrite = options.overwrite # If global overwrite is set + # Set global on_exists parameter if provided + if options.on_exists: + try: + sp.global_on_exists = OnExists[options.on_exists.upper()] + except KeyError: + logging.error(f"Invalid on_exists option: {options.on_exists}") + raise + else: + sp.global_on_exists = None + # If the info flag is set, run 'tw info' try: if options.info: diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 31c3654..d417310 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -17,10 +17,11 @@ Including handling methods for each block in the YAML file, and parsing methods for each block in the YAML file. """ -import yaml +import yaml # type: ignore from seqerakit import utils import sys import json +from seqerakit.on_exists import OnExists def parse_yaml_block(yaml_data, block_name): @@ -163,11 +164,31 @@ def parse_block(block_name, item): } # Use the generic block function as a default. parse_fn = block_to_function.get(block_name, parse_generic_block) - overwrite = item.pop("overwrite", False) - # Call the appropriate function and return its result along with overwrite value. + # Get on_exists setting with backward compatibility for overwrite + overwrite = item.pop("overwrite", None) + on_exists_str = item.pop("on_exists", "fail") + + # Determine final on_exists value + if overwrite is not None: + # overwrite takes precedence for backward compatibility + on_exists = OnExists.OVERWRITE if overwrite else OnExists.FAIL + elif isinstance(on_exists_str, str): + try: + on_exists = OnExists[on_exists_str.upper()] + except KeyError: + raise ValueError( + f"Invalid on_exists option: '{on_exists_str}'. " + f"Valid options are: " + f"{', '.join(behaviour.name.lower() for behaviour in OnExists)}" + ) + else: + # Use directly if already an enum + on_exists = on_exists_str + + # Parse the block and return with on_exists value cmd_args = parse_fn(item) - return {"cmd_args": cmd_args, "overwrite": overwrite} + return {"cmd_args": cmd_args, "on_exists": on_exists.name.lower()} # Parsers for certain blocks of yaml that require handling diff --git a/seqerakit/on_exists.py b/seqerakit/on_exists.py new file mode 100644 index 0000000..d89ceee --- /dev/null +++ b/seqerakit/on_exists.py @@ -0,0 +1,9 @@ +from enum import Enum, auto + + +class OnExists(Enum): + """Enum defining behavior when a resource already exists.""" + + FAIL = auto() + IGNORE = auto() + OVERWRITE = auto() diff --git a/seqerakit/overwrite.py b/seqerakit/overwrite.py index 7cab3d6..48af5b7 100644 --- a/seqerakit/overwrite.py +++ b/seqerakit/overwrite.py @@ -15,6 +15,7 @@ import json from seqerakit import utils from seqerakit.seqeraplatform import ResourceExistsError +from seqerakit.on_exists import OnExists import logging @@ -37,6 +38,9 @@ class Overwrite: "studios", ] + # Define valid on_exists options as enum values + VALID_ON_EXISTS_OPTIONS = [e.name.lower() for e in OnExists] + def __init__(self, sp): """ Initializes an Overwrite instance. @@ -95,11 +99,36 @@ def __init__(self, sp): }, } - def handle_overwrite(self, block, args, overwrite=False, destroy=False): + def handle_overwrite( + self, block, args, on_exists=OnExists.FAIL, destroy=False, overwrite=None + ): """ - Handles overwrite functionality for Seqera Platform resources and - calling the 'tw delete' method with the correct args. + Handles resource existence behavior for Seqera Platform resources. + + Args: + block: The resource block type (e.g., "organizations", "teams") + args: Command line arguments for the resource + on_exists: How to handle existing resources. Options: + - OnExists.FAIL (default): Raise an error if resource exists + - OnExists.IGNORE: Skip creation if resource exists + - OnExists.OVERWRITE: Delete existing resource and create new one + destroy: Whether to delete the resource + overwrite: Legacy parameter for backward compatibility """ + # Convert string to enum if needed + if isinstance(on_exists, str): + try: + on_exists = OnExists[on_exists.upper()] + except KeyError: + raise ValueError( + f"Invalid on_exists option: {on_exists}. " + f"Valid options are: {', '.join(self.VALID_ON_EXISTS_OPTIONS)}" + ) + + # For backward compatibility + if overwrite is not None: + on_exists = OnExists.OVERWRITE if overwrite else OnExists.FAIL + if block in Overwrite.generic_deletion: self.block_operations[block] = { "keys": ["name", "workspace"], @@ -122,23 +151,32 @@ def handle_overwrite(self, block, args, overwrite=False, destroy=False): elif block == "members": # Rename the user key to name to correctly index JSON data sp_args["name"] = sp_args.pop("user") + if self.check_resource_exists(operation["name_key"], sp_args): - # if resource exists and overwrite is true, delete - if overwrite: + # Handle based on on_exists parameter + if on_exists == OnExists.OVERWRITE: logging.info( f" The attempted {block} resource already exists." " Overwriting.\n" ) self.delete_resource(block, operation, sp_args) + elif on_exists == OnExists.IGNORE: + logging.info( + f" The {block} resource already exists." " Skipping creation.\n" + ) + return False elif destroy: logging.info(f" Deleting the {block} resource.") self.delete_resource(block, operation, sp_args) - else: # return an error if resource exists, overwrite=False + else: # fail raise ResourceExistsError( - f" The {block} resource already exists and" - " will not be created. Please set 'overwrite: True'" - " in your config file.\n" + f"The {block} resource already exists and " + "will not be created. Please set 'on_exists: overwrite' " + "to replace the resource or set 'on_exists: ignore' to " + "ignore this error.\n" ) + return True + return True def _get_organization_args(self, args): """ diff --git a/templates/teams.yml b/templates/teams.yml index e6f921b..14da104 100644 --- a/templates/teams.yml +++ b/templates/teams.yml @@ -5,4 +5,4 @@ teams: description: 'My test team' # optional members: # optional - 'my_team_member@gmail.com' - overwrite: True # optional \ No newline at end of file + on_exists: 'overwrite' # optional - values: 'fail' (default), 'ignore', 'overwrite' diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 4247b0d..40b4c19 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -28,7 +28,7 @@ def test_create_mock_organization_yaml(mock_yaml_file): "description": "My test organization 1", "location": "Global", "url": "https://example.com", - "overwrite": True, + "on_exists": "overwrite", } ] } @@ -46,7 +46,7 @@ def test_create_mock_organization_yaml(mock_yaml_file): "--url", "https://example.com", ], - "overwrite": True, + "on_exists": "overwrite", } ] file_path = mock_yaml_file(test_data) @@ -66,7 +66,7 @@ def test_create_mock_workspace_yaml(mock_yaml_file): "organization": "my_organization", "description": "My test workspace 1", "visibility": "PRIVATE", - "overwrite": True, + "on_exists": "overwrite", } ] } @@ -84,7 +84,7 @@ def test_create_mock_workspace_yaml(mock_yaml_file): "--visibility", "PRIVATE", ], - "overwrite": True, + "on_exists": "overwrite", } ] @@ -104,7 +104,7 @@ def test_create_mock_dataset_yaml(mock_yaml_file): "workspace": "my_organization/my_workspace", "header": True, "file-path": "./examples/yaml/datasets/samples.csv", - "overwrite": True, + "on_exists": "overwrite", } ] } @@ -120,7 +120,7 @@ def test_create_mock_dataset_yaml(mock_yaml_file): "My test dataset 1", "--header", ], - "overwrite": True, + "on_exists": "overwrite", } ] @@ -142,7 +142,7 @@ def test_create_mock_computeevs_source_yaml(mock_yaml_file): "wait": "AVAILABLE", "fusion-v2": True, "fargate": False, - "overwrite": True, + "on_exists": "overwrite", } ], } @@ -161,7 +161,7 @@ def test_create_mock_computeevs_source_yaml(mock_yaml_file): "--workspace", "my_organization/my_workspace", ], - "overwrite": True, + "on_exists": "overwrite", } ] @@ -200,7 +200,7 @@ def test_create_mock_computeevs_cli_yaml(mock_yaml_file): "--workspace", "my_organization/my_workspace", ], - "overwrite": False, + "on_exists": "fail", } ] file_path = mock_yaml_file(test_data) @@ -224,7 +224,7 @@ def test_create_mock_pipeline_add_yaml(mock_yaml_file): "config": "./examples/yaml/pipelines/test_pipeline1/config.txt", "pre-run": "./examples/yaml/pipelines/test_pipeline1/pre_run.sh", "revision": "master", - "overwrite": True, + "on_exists": "overwrite", "stub-run": True, } ] @@ -256,7 +256,7 @@ def test_create_mock_pipeline_add_yaml(mock_yaml_file): "--params-file", "./examples/yaml/pipelines/test_pipeline1/params.yaml", ], - "overwrite": True, + "on_exists": "overwrite", } ] @@ -275,7 +275,7 @@ def test_create_mock_teams_yaml(mock_yaml_file): "organization": "my_organization", "description": "My test team 1", "members": ["user1@org.io"], - "overwrite": True, + "on_exists": "overwrite", }, ] } @@ -302,7 +302,7 @@ def test_create_mock_teams_yaml(mock_yaml_file): ] ], ), - "overwrite": True, + "on_exists": "overwrite", } ] @@ -323,7 +323,7 @@ def test_create_mock_members_yaml(mock_yaml_file): "--user", "bob@myorg.io", ], - "overwrite": False, + "on_exists": "fail", } ] file_path = mock_yaml_file(test_data) @@ -344,7 +344,7 @@ def test_create_mock_studios_yaml(mock_yaml_file): "cpu": 2, "memory": 4096, "autoStart": False, - "overwrite": True, + "on_exists": "overwrite", "mount-data-ids": "v1-user-bf73f9d33997f93a20ee3e6911779951", } ] @@ -368,7 +368,7 @@ def test_create_mock_studios_yaml(mock_yaml_file): "--workspace", "my_organization/my_workspace", ], - "overwrite": True, + "on_exists": "overwrite", } ] @@ -388,7 +388,7 @@ def test_create_mock_data_links_yaml(mock_yaml_file): "provider": "aws", "credentials": "my_credentials", "uri": "s3://scidev-playground-eu-west-2/esha/nf-core-scrnaseq/", - "overwrite": True, + "on_exists": "overwrite", } ] } @@ -406,7 +406,7 @@ def test_create_mock_data_links_yaml(mock_yaml_file): "--workspace", "my_organization/my_workspace", ], - "overwrite": True, + "on_exists": "overwrite", } ] file_path = mock_yaml_file(test_data) @@ -466,7 +466,7 @@ def test_stdin_yaml_file(): "--wait", "AVAILABLE", ], - "overwrite": False, + "on_exists": "fail", } ] assert "compute-envs" in result @@ -549,7 +549,7 @@ def test_targets_specified(): expected_organizations_output = [ { "cmd_args": ["--name", "org1", "--description", "Organization 1"], - "overwrite": False, + "on_exists": "fail", } ] expected_workspaces_output = [ @@ -562,7 +562,7 @@ def test_targets_specified(): "--description", "Workspace 1", ], - "overwrite": False, + "on_exists": "fail", } ] # Check that only 'organizations' and 'workspaces' are in the result diff --git a/tests/unit/test_overwrite.py b/tests/unit/test_overwrite.py index 15e83ea..e4abd60 100644 --- a/tests/unit/test_overwrite.py +++ b/tests/unit/test_overwrite.py @@ -3,6 +3,7 @@ import json from seqerakit.overwrite import Overwrite from seqerakit.seqeraplatform import ResourceExistsError +from seqerakit.on_exists import OnExists class TestOverwrite(unittest.TestCase): @@ -43,7 +44,9 @@ def test_handle_overwrite_generic_deletion(self): {"name": "test-resource"} ) - self.overwrite.handle_overwrite("credentials", args, overwrite=True) + self.overwrite.handle_overwrite( + "credentials", args, on_exists=OnExists.OVERWRITE + ) self.mock_sp.credentials.assert_called_with( "delete", "--name", "test-resource", "--workspace", "test-workspace" @@ -58,7 +61,9 @@ def test_handle_overwrite_resource_exists_no_overwrite(self): ) with self.assertRaises(ResourceExistsError): - self.overwrite.handle_overwrite("credentials", args, overwrite=False) + self.overwrite.handle_overwrite( + "credentials", args, on_exists=OnExists.FAIL + ) def test_team_deletion(self): args = {"name": "test-team", "organization": "test-org"} @@ -92,7 +97,9 @@ def test_workspace_deletion(self): self.mock_sp.__getattr__("-o json").return_value = self.sample_workspace_json - self.overwrite.handle_overwrite("workspaces", args, overwrite=True) + self.overwrite.handle_overwrite( + "workspaces", args, on_exists=OnExists.OVERWRITE + ) self.mock_sp.workspaces.assert_called_with("delete", "--id", "456") @@ -108,7 +115,7 @@ def test_label_deletion(self): self.mock_sp.__getattr__("-o json").return_value = self.sample_labels_json - self.overwrite.handle_overwrite("labels", args, overwrite=True) + self.overwrite.handle_overwrite("labels", args, on_exists=OnExists.OVERWRITE) self.mock_sp.labels.assert_called_with( "delete", "--id", "789", "-w", "test-workspace" @@ -128,7 +135,9 @@ def test_participant_deletion(self): {"teamName": "test-team"} ) - self.overwrite.handle_overwrite("participants", team_args, overwrite=True) + self.overwrite.handle_overwrite( + "participants", team_args, on_exists=OnExists.OVERWRITE + ) self.mock_sp.participants.assert_called_with( "delete", @@ -159,12 +168,98 @@ def test_organization_deletion_with_env_var(self, mock_resolve_env_var): self.mock_sp.configure_mock(**{"-o json": json_method_mock}) - self.overwrite.handle_overwrite("organizations", args, overwrite=True) + self.overwrite.handle_overwrite( + "organizations", args, on_exists=OnExists.OVERWRITE + ) mock_resolve_env_var.assert_any_call("${ORG_NAME}") self.mock_sp.organizations.assert_called_with("delete", "--name", "${ORG_NAME}") + def test_handle_on_exists_overwrite(self): + # Test for credentials, secrets, compute-envs, datasets, actions, pipelines + args = ["--name", "test-resource", "--workspace", "test-workspace"] + + self.mock_sp.__getattr__("-o json").return_value = json.dumps( + {"name": "test-resource"} + ) + + # Test with on_exists='overwrite' + self.overwrite.handle_overwrite("credentials", args, on_exists="overwrite") + + self.mock_sp.credentials.assert_called_with( + "delete", "--name", "test-resource", "--workspace", "test-workspace" + ) + + def test_handle_on_exists_ignore(self): + # Test for credentials with on_exists='ignore' + args = ["--name", "test-resource", "--workspace", "test-workspace"] + + self.mock_sp.__getattr__("-o json").return_value = json.dumps( + {"name": "test-resource"} + ) + + # Test with on_exists='ignore' + result = self.overwrite.handle_overwrite( + "credentials", args, on_exists="ignore" + ) + + # Should return False to indicate creation should be skipped + self.assertFalse(result) + + # Should not call delete + self.mock_sp.credentials.assert_not_called() + + def test_handle_on_exists_fail(self): + args = ["--name", "test-resource", "--workspace", "test-workspace"] + + # Mock JSON response indicating resource exists + self.mock_sp.__getattr__("-o json").return_value = json.dumps( + {"name": "test-resource"} + ) + + # Test with on_exists='fail' (default) + with self.assertRaises(ResourceExistsError): + self.overwrite.handle_overwrite("credentials", args, on_exists="fail") + + # Should not call delete + self.mock_sp.credentials.assert_not_called() + + def test_handle_invalid_on_exists(self): + args = ["--name", "test-resource", "--workspace", "test-workspace"] + + # Test with invalid on_exists value + with self.assertRaises(ValueError): + self.overwrite.handle_overwrite( + "credentials", args, on_exists="invalid_option" + ) + + def test_backward_compatibility_overwrite(self): + args = ["--name", "test-resource", "--workspace", "test-workspace"] + + self.mock_sp.__getattr__("-o json").return_value = json.dumps( + {"name": "test-resource"} + ) + + # Test with legacy overwrite=True parameter + self.overwrite.handle_overwrite("credentials", args, overwrite=True) + + self.mock_sp.credentials.assert_called_with( + "delete", "--name", "test-resource", "--workspace", "test-workspace" + ) + + def test_backward_compatibility_no_overwrite(self): + args = ["--name", "test-resource", "--workspace", "test-workspace"] + + # Mock JSON response indicating resource exists + self.mock_sp.__getattr__("-o json").return_value = json.dumps( + {"name": "test-resource"} + ) + + # Test with legacy overwrite=False parameter + with self.assertRaises(ResourceExistsError): + self.overwrite.handle_overwrite("credentials", args, overwrite=False) + # TODO: tests for destroy and JSON caching