From 6e8a3841c19ce287c98f75fe2cb2c2cd3a7ce217 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Tue, 21 Nov 2023 00:12:35 -0500 Subject: [PATCH 01/28] feat: add info flag and support for tw info --- seqerakit/cli.py | 19 ++++++++++++++++++- seqerakit/seqeraplatform.py | 10 +++++++++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/seqerakit/cli.py b/seqerakit/cli.py index e75aeec..7c87322 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -39,6 +39,11 @@ def parse_args(args=None): help="The desired log level (default: INFO).", type=str.upper, ) + parser.add_argument( + "--info", + action="store_true", + help="Display information about the Seqera Platform and exit", + ) parser.add_argument( "--dryrun", action="store_true", @@ -47,7 +52,7 @@ def parse_args(args=None): parser.add_argument( "yaml", type=Path, - nargs="+", # allow multiple YAML paths + nargs="*", # allow multiple YAML paths help="One or more YAML files with Seqera Platform resources to create", ) parser.add_argument( @@ -127,6 +132,18 @@ def main(args=None): options = parse_args(args if args is not None else sys.argv[1:]) logging.basicConfig(level=options.log_level) + # If the info flag is set, run 'tw info' + if options.info: + sp = seqeraplatform.SeqeraPlatform() + print(sp.info()) + return + + if not options.yaml: + logging.error( + " No YAML(s) provided. Please provide atleast one YAML configuration file." + ) + sys.exit(1) + # Parse CLI arguments into a list cli_args_list = options.cli_args.split() if options.cli_args else [] diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py index be34dd1..6381e10 100644 --- a/seqerakit/seqeraplatform.py +++ b/seqerakit/seqeraplatform.py @@ -101,6 +101,11 @@ def _execute_command(self, full_cmd, to_json=False): return json.loads(stdout) if to_json else stdout + def _execute_info_command(self): + # Directly execute 'tw info' command + command = "tw info" + return self._execute_command(command) + def _handle_command_errors(self, stdout): logging.error(stdout) @@ -126,7 +131,10 @@ def _tw_run(self, cmd, *args, **kwargs): # Allow any 'tw' subcommand to be called as a method. def __getattr__(self, cmd): - return self.TwCommand(self, cmd.replace("_", "-")) + if cmd == "info": + return self._execute_info_command + else: + return self.TwCommand(self, cmd.replace("_", "-")) class ResourceExistsError(Exception): From 5d7deb5fa563b83f0fe30d542addce1cba482510 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Tue, 21 Nov 2023 17:32:02 -0500 Subject: [PATCH 02/28] feat: support params and params-file keys --- seqerakit/helper.py | 17 ++++++++++++----- seqerakit/utils.py | 19 +++++++++++++++++-- 2 files changed, 29 insertions(+), 7 deletions(-) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 9751cd7..f72401b 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -223,8 +223,9 @@ def parse_pipelines_block(item): if key == "url": repo_args.extend([str(value)]) elif key == "params": - temp_file_name = utils.create_temp_yaml(value) - params_args.extend(["--params-file", temp_file_name]) + params_dict = value + elif key == "params-file": + params_file_path = str(value) elif key == "file-path": repo_args.extend([str(value)]) elif isinstance(value, bool) and value: @@ -232,9 +233,15 @@ def parse_pipelines_block(item): else: cmd_args.extend([f"--{key}", str(value)]) - # First append the url related arguments then append the remaining ones - cmd_args = repo_args + params_args + cmd_args - return cmd_args + # Create the temporary YAML file after processing all items + if "params" in item: + temp_file_name = utils.create_temp_yaml( + params_dict, params_file=params_file_path + ) + params_args.extend(["--params-file", temp_file_name]) + + combined_args = cmd_args + repo_args + params_args + return combined_args def parse_launch_block(item): diff --git a/seqerakit/utils.py b/seqerakit/utils.py index 6048c1a..e33fa5e 100644 --- a/seqerakit/utils.py +++ b/seqerakit/utils.py @@ -111,11 +111,19 @@ def is_url(s): return False -def create_temp_yaml(params_dict): +def create_temp_yaml(params_dict, params_file=None): """ Create a generic temporary yaml file given a dictionary + Optionally combine with contents from a JSON or YAML file if provided. """ + def read_file(file_path): + with open(file_path, "r") as file: + if file_path.endswith(".json"): + return json.load(file) + else: + return yaml.safe_load(file) + class quoted_str(str): pass @@ -123,6 +131,13 @@ def quoted_str_representer(dumper, data): return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"') yaml.add_representer(quoted_str, quoted_str_representer) + + # If a params_file is provided, combine its contents with params_dict + if params_file: + file_params = read_file(params_file) + params_dict.update(file_params) + + # Expand environment variables and quote strings params_dict = { k: quoted_str(os.path.expandvars(v)) if isinstance(v, str) else v for k, v in params_dict.items() @@ -131,7 +146,7 @@ def quoted_str_representer(dumper, data): with tempfile.NamedTemporaryFile( mode="w", delete=False, suffix=".yaml" ) as temp_file: - yaml.dump(params_dict, temp_file) + yaml.dump(params_dict, temp_file, default_flow_style=False) return temp_file.name From e7561e9aa6283bd402d9ae65a773648f98bb5e95 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Tue, 21 Nov 2023 18:43:15 -0500 Subject: [PATCH 03/28] feat: support params and params-file for launch --- seqerakit/helper.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index f72401b..28b54bb 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -247,16 +247,24 @@ def parse_pipelines_block(item): def parse_launch_block(item): repo_args = [] cmd_args = [] + params_args = [] for key, value in item.items(): if key == "pipeline" or key == "url": repo_args.extend([str(value)]) elif key == "params": - temp_file_name = utils.create_temp_yaml(value) - cmd_args.extend(["--params-file", temp_file_name]) + params_dict = value + elif key == "params-file": + params_file_path = str(value) else: cmd_args.extend([f"--{key}", str(value)]) - cmd_args = repo_args + cmd_args + if "params" in item: + temp_file_name = utils.create_temp_yaml( + params_dict, params_file=params_file_path + ) + params_args.extend(["--params-file", temp_file_name]) + + cmd_args = cmd_args + repo_args + params_args return cmd_args From 3e3fa4fbc9b0857c3b4cda607c56b16d7117727b Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 23 Nov 2023 20:39:16 -0500 Subject: [PATCH 04/28] fix: set params-file to default none --- seqerakit/helper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 28b54bb..1e95e27 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -218,6 +218,7 @@ def parse_pipelines_block(item): cmd_args = [] repo_args = [] params_args = [] + params_file_path = None for key, value in item.items(): if key == "url": @@ -248,6 +249,8 @@ def parse_launch_block(item): repo_args = [] cmd_args = [] params_args = [] + params_file_path = None + for key, value in item.items(): if key == "pipeline" or key == "url": repo_args.extend([str(value)]) From e57d241c81af22061a36469cbd44df5ff401cb5d Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 23 Nov 2023 20:39:56 -0500 Subject: [PATCH 05/28] test: add tests for parsing yaml to cmd args --- tests/unit/test_helper.py | 245 +++++++++++++++++++++++++++++++++----- 1 file changed, 215 insertions(+), 30 deletions(-) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index ac9f459..7ba4b5c 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -1,42 +1,227 @@ -import unittest -from unittest.mock import mock_open, patch +from unittest.mock import mock_open from seqerakit import helper +import yaml +import pytest -mocked_yaml = """ -datasets: - - name: 'test_dataset1' - description: 'My test dataset 1' - header: true - workspace: 'my_organization/my_workspace' - file-path: './examples/yaml/datasets/samples.csv' - overwrite: True -""" +# Fixture to mock a YAML file +@pytest.fixture +def mock_yaml_file(mocker): + def _mock_yaml_file(test_data, file_name="mock_file.yaml"): + # Convert test data to YAML format + yaml_content = yaml.dump(test_data, default_flow_style=False) + mock_file = mock_open(read_data=yaml_content) + mocker.patch("builtins.open", mock_file) -mocked_file = mock_open(read_data=mocked_yaml) + return file_name + return _mock_yaml_file -class TestYamlParserFunctions(unittest.TestCase): - @patch("builtins.open", mocked_file) - def test_parse_datasets_yaml(self): - result = helper.parse_all_yaml(["test_path.yaml"]) - expected_block_output = [ + +def test_create_mock_organization_yaml(mock_yaml_file): + + test_data = { + "organizations": [ + { + "name": "test_organization1", + "full-name": "My test organization 1", + "description": "My test organization 1", + "location": "Global", + "url": "https://example.com", + "overwrite": True, + } + ] + } + expected_block_output = [ + { + "cmd_args": [ + "--description", + "My test organization 1", + "--full-name", + "My test organization 1", + "--location", + "Global", + "--name", + "test_organization1", + "--url", + "https://example.com", + ], + "overwrite": True, + } + ] + + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "organizations" in result + assert result["organizations"] == expected_block_output + + +def test_create_mock_workspace_yaml(mock_yaml_file): + + test_data = { + "workspaces": [ + { + "name": "test_workspace1", + "full-name": "My test workspace 1", + "organization": "my_organization", + "description": "My test workspace 1", + "visibility": "PRIVATE", + "overwrite": True, + } + ] + } + expected_block_output = [ + { + "cmd_args": [ + "--description", + "My test workspace 1", + "--full-name", + "My test workspace 1", + "--name", + "test_workspace1", + "--organization", + "my_organization", + "--visibility", + "PRIVATE", + ], + "overwrite": True, + } + ] + + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "workspaces" in result + assert result["workspaces"] == expected_block_output + + +def test_create_mock_dataset_yaml(mock_yaml_file): + + test_data = { + "datasets": [ + { + "name": "test_dataset1", + "description": "My test dataset 1", + "workspace": "my_organization/my_workspace", + "header": True, + "file-path": "./examples/yaml/datasets/samples.csv", + "overwrite": True, + } + ] + } + expected_block_output = [ + { + "cmd_args": [ + "./examples/yaml/datasets/samples.csv", + "--name", + "test_dataset1", + "--workspace", + "my_organization/my_workspace", + "--description", + "My test dataset 1", + "--header", + ], + "overwrite": True, + } + ] + + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "datasets" in result + assert result["datasets"] == expected_block_output + + +def test_create_mock_computeevs_yaml(mock_yaml_file): + + test_data = { + "compute-envs": [ + { + "name": "test_computeenv", + "workspace": "my_organization/my_workspace", + "credentials": "my_credentials", + "file-path": "./examples/yaml/computeenvs/computeenvs.yaml", + "wait": "AVAILABLE", + "overwrite": True, + } + ], + } + + expected_block_output = [ + { + "cmd_args": [ + "--credentials", + "my_credentials", + "./examples/yaml/computeenvs/computeenvs.yaml", + "--name", + "test_computeenv", + "--wait", + "AVAILABLE", + "--workspace", + "my_organization/my_workspace", + ], + "overwrite": True, + } + ] + + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "compute-envs" in result + assert result["compute-envs"] == expected_block_output + + +def test_create_mock_pipeline_add_yaml(mock_yaml_file): + + test_data = { + "pipelines": [ { - "cmd_args": [ - "--header", - "./examples/yaml/datasets/samples.csv", - "--name", - "test_dataset1", - "--workspace", - "my_organization/my_workspace", - "--description", - "My test dataset 1", - ], + "name": "test_pipeline1", + "url": "https://github.com/nf-core/test_pipeline1", + "workspace": "my_organization/my_workspace", + "description": "My test pipeline 1", + "compute-env": "my_computeenv", + "work-dir": "s3://work", + "profile": "test", + "params-file": "./examples/yaml/pipelines/test_pipeline1/params.yaml", + "config": "./examples/yaml/pipelines/test_pipeline1/config.txt", + "pre-run": "./examples/yaml/pipelines/test_pipeline1/pre_run.sh", + "revision": "master", "overwrite": True, } ] + } - self.assertIn("datasets", result) - self.assertEqual(result["datasets"], expected_block_output) + # params file cmds parsed separately + expected_block_output = [ + { + "cmd_args": [ + "--compute-env", + "my_computeenv", + "--config", + "./examples/yaml/pipelines/test_pipeline1/config.txt", + "--description", + "My test pipeline 1", + "--name", + "test_pipeline1", + "--pre-run", + "./examples/yaml/pipelines/test_pipeline1/pre_run.sh", + "--profile", + "test", + "--revision", + "master", + "--work-dir", + "s3://work", + "--workspace", + "my_organization/my_workspace", + "https://github.com/nf-core/test_pipeline1", + ], + "overwrite": True, + } + ] + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) -# TODO: add more tests for other functions in helper.py + assert "pipelines" in result + assert result["pipelines"] == expected_block_output From 94206fe31d1452afcbf393d1656101fa2396c6d2 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 23 Nov 2023 20:54:45 -0500 Subject: [PATCH 06/28] docs: add yaml config options to README --- README.md | 70 ++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 51 insertions(+), 19 deletions(-) diff --git a/README.md b/README.md index 1521fe6..75ebd80 100644 --- a/README.md +++ b/README.md @@ -44,14 +44,13 @@ conda activate seqerakit If you already have [Seqera Platform CLI](https://github.com/seqeralabs/tower-cli#1-installation) and Python installed on your system, you can install `seqerakit` directly from [PyPI](https://pypi.org/project/seqerakit/): - -``` +```console pip install seqerakit ``` You can force overwrite the installation to use the latest changes with the command below: -``` +```console pip install --upgrade --force-reinstall seqerakit ``` @@ -69,7 +68,7 @@ export TOWER_ACCESS_TOKEN= Use the -h or --help parameter to list the available commands and their associated options: -``` +```bash seqerakit -h ``` @@ -77,7 +76,7 @@ seqerakit -h To print the commands that would executed with `tw` when using a YAML file, you can run `seqerakit` with the `--dryrun` flag: -``` +```bash seqerakit file.yaml --dryrun ``` @@ -85,7 +84,7 @@ seqerakit file.yaml --dryrun Instead of adding or creating resources, you can recursively delete resources in your YAML file by specifying the `--delete` flag: -``` +```bash seqerakit file.yaml --delete ``` @@ -95,7 +94,7 @@ For example, if you have a YAML file that defines an Organization -> Workspace - `tw` specific CLI options can be specified with the `--cli=` flag: -``` +```bash seqerakit file.yaml --cli="--arg1 --arg2" ``` @@ -107,7 +106,7 @@ To use `tw` specific CLI options such as `--insecure`, use the `--cli=` flag, fo For example: -``` +```bash seqerakit file.yaml --cli="--insecure" ``` @@ -115,12 +114,45 @@ To use an SSL certificate that is not accepted by the default Java certificate a For example: -``` +```bash seqerakit hello-world-config.yml --cli="-Djavax.net.ssl.trustStore=/absolute/path/to/cacerts" ``` Note: Use of `--verbose` option for the `tw` CLI is currently not supported by `seqerakit`. Supplying `--cli="--verbose"` will raise an error. +## YAML Configuration Options + +There are several options that can be provided in your YAML configuration file, that are handled specially by seqerakit and not `tw` specific CLI options. + +### 1. Pipeline parameters using `params` and `params-file` + +To specify pipeline parameters, you may either use `params:` to specify a list of parameters, or use `params-file:` to point to a parameters file. + +For example, to specify pipeline parameters within your YAML: + +```yaml +params: + outdir: 's3://path/to/outdir' + fasta: 's3://path/to/reference.fasta' +``` + +Alternatively, to specify a file containing pipeline parameters: + +```yaml +params-file: '/path/to/my/parameters.yaml' +``` + +Optionally, you may provide both: + +```yaml +params-file: '/path/to/my/parameters.yaml' +params: + outdir: 's3://path/to/outdir' + fasta: 's3://path/to/reference.fasta' +``` + +If duplicate parameters are provided, the parameters in `params-file` will take precedence. + ## Quick start You must provide a YAML file that defines the options for each of the entities you would like to create in Seqera Platform. @@ -131,18 +163,18 @@ You will need to have an account on Seqera Platform (see [Plans and pricing](htt 1. Create a YAML file called `hello-world-config.yml` with the contents below, and customise the `` and `` entries as required: - ``` + ```yaml launch: - - name: 'hello-world' # Workflow name - workspace: '' # Workspace name - compute-env: '' # Compute environment - revision: 'master' # Pipeline revision - pipeline: 'https://github.com/nextflow-io/hello' # Pipeline URL + - name: 'hello-world' # Workflow name + workspace: '' # Workspace name + compute-env: '' # Compute environment + revision: 'master' # Pipeline revision + pipeline: 'https://github.com/nextflow-io/hello' # Pipeline URL ``` 2. Launch the pipeline with `seqerakit`: - ``` + ```bash seqerakit hello-world-config.yml ``` @@ -156,9 +188,9 @@ You can also launch the same pipeline via a Python script. This will essentially 2. Launch the pipeline with `seqerakit`: - ``` +```bash python launch_hello_world.py - ``` +``` 3. Login to your Seqera Platform instance and check the Runs page in the appropriate Workspace for the pipeline you just launched! @@ -168,7 +200,7 @@ Please see [`seqerakit-e2e.yml`](https://github.com/seqeralabs/seqera-kit/blob/m You can modify this YAML to similarly create Seqera Platform resources end-to-end for your setup. This YAML encodes environment variables to protect sensitive keys, usernames, and passwords that are required to create or add certain resources (i.e. credentials, compute environments). Prior to running it with `seqerakit examples/yaml/seqerakit-e2e.yml`, you will have to set the following environment variables: -``` +```console $TOWER_GITHUB_PASSWORD $DOCKERHUB_PASSWORD $AWS_ACCESS_KEY_ID From f1d271a35e85267f57a53bb44bfe08f4b59d9210 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Fri, 24 Nov 2023 17:16:07 -0500 Subject: [PATCH 07/28] fix: params to take precedence over params-file --- README.md | 2 +- seqerakit/utils.py | 19 +++++++++++-------- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 75ebd80..765d629 100644 --- a/README.md +++ b/README.md @@ -151,7 +151,7 @@ params: fasta: 's3://path/to/reference.fasta' ``` -If duplicate parameters are provided, the parameters in `params-file` will take precedence. +**Note**: If duplicate parameters are provided, the parameters provided as key-value pairs inside the `params` nested dictionary of the YAML file will take precedence. ## Quick start diff --git a/seqerakit/utils.py b/seqerakit/utils.py index e33fa5e..dd20d8d 100644 --- a/seqerakit/utils.py +++ b/seqerakit/utils.py @@ -116,6 +116,7 @@ def create_temp_yaml(params_dict, params_file=None): Create a generic temporary yaml file given a dictionary Optionally combine with contents from a JSON or YAML file if provided. """ + combined_params = {} def read_file(file_path): with open(file_path, "r") as file: @@ -124,6 +125,13 @@ def read_file(file_path): else: return yaml.safe_load(file) + # If a params_file is provided, update the dict + if params_file: + file_params = read_file(params_file) + combined_params.update(file_params) + + combined_params.update(params_dict) + class quoted_str(str): pass @@ -132,21 +140,16 @@ def quoted_str_representer(dumper, data): yaml.add_representer(quoted_str, quoted_str_representer) - # If a params_file is provided, combine its contents with params_dict - if params_file: - file_params = read_file(params_file) - params_dict.update(file_params) - - # Expand environment variables and quote strings + # # Expand environment variables and quote strings params_dict = { k: quoted_str(os.path.expandvars(v)) if isinstance(v, str) else v - for k, v in params_dict.items() + for k, v in combined_params.items() } with tempfile.NamedTemporaryFile( mode="w", delete=False, suffix=".yaml" ) as temp_file: - yaml.dump(params_dict, temp_file, default_flow_style=False) + yaml.dump(combined_params, temp_file, default_flow_style=False) return temp_file.name From ef1ece01d92c4928f602df86b02d9e80330ccfbb Mon Sep 17 00:00:00 2001 From: ejseqera Date: Fri, 24 Nov 2023 17:22:17 -0500 Subject: [PATCH 08/28] docs: update docs with precedence info for params --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index 765d629..8882292 100644 --- a/README.md +++ b/README.md @@ -163,12 +163,12 @@ You will need to have an account on Seqera Platform (see [Plans and pricing](htt 1. Create a YAML file called `hello-world-config.yml` with the contents below, and customise the `` and `` entries as required: - ```yaml + ```yaml # noqa launch: - - name: 'hello-world' # Workflow name - workspace: '' # Workspace name - compute-env: '' # Compute environment - revision: 'master' # Pipeline revision + - name: 'hello-world' # Workflow name + workspace: '' # Workspace name + compute-env: '' # Compute environment + revision: 'master' # Pipeline revision pipeline: 'https://github.com/nextflow-io/hello' # Pipeline URL ``` From 966d8a96a7a9a5308005706dd29e9ece8d7e9d94 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Fri, 24 Nov 2023 17:25:38 -0500 Subject: [PATCH 09/28] ci: add pytest-mock to dependencies --- .github/workflows/python-testing.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index 98a2a59..2ce0e91 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -22,6 +22,7 @@ jobs: run: | python -m pip install --upgrade pip pip install pytest + pip install pytest-mock pip install -e . - name: Run tests run: | From 700d6c9f01a66009e845087139674e2e8f72a190 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Fri, 24 Nov 2023 17:26:01 -0500 Subject: [PATCH 10/28] chore: linting errors in tests --- tests/unit/test_helper.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 7ba4b5c..8800bf3 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -3,6 +3,7 @@ import yaml import pytest + # Fixture to mock a YAML file @pytest.fixture def mock_yaml_file(mocker): @@ -18,7 +19,6 @@ def _mock_yaml_file(test_data, file_name="mock_file.yaml"): def test_create_mock_organization_yaml(mock_yaml_file): - test_data = { "organizations": [ { @@ -57,7 +57,6 @@ def test_create_mock_organization_yaml(mock_yaml_file): def test_create_mock_workspace_yaml(mock_yaml_file): - test_data = { "workspaces": [ { @@ -96,7 +95,6 @@ def test_create_mock_workspace_yaml(mock_yaml_file): def test_create_mock_dataset_yaml(mock_yaml_file): - test_data = { "datasets": [ { @@ -133,7 +131,6 @@ def test_create_mock_dataset_yaml(mock_yaml_file): def test_create_mock_computeevs_yaml(mock_yaml_file): - test_data = { "compute-envs": [ { @@ -172,7 +169,6 @@ def test_create_mock_computeevs_yaml(mock_yaml_file): def test_create_mock_pipeline_add_yaml(mock_yaml_file): - test_data = { "pipelines": [ { From 4a2d88830d41d89b024967af692c710f65fd4a66 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Sat, 25 Nov 2023 22:06:53 -0500 Subject: [PATCH 11/28] docs: add details on yaml config options, new info arg --- README.md | 48 ++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 8882292..e35d4a9 100644 --- a/README.md +++ b/README.md @@ -66,10 +66,16 @@ export TOWER_ACCESS_TOKEN= ## Usage -Use the -h or --help parameter to list the available commands and their associated options: +To confirm the installation of `seqerakit`, configuration of the Seqera Platform CLI and connection is working as expected: ```bash -seqerakit -h +seqerakit --info +``` + +Use the `-h` or `--help `parameter to list the available commands and their associated options: + +```bash +seqerakit --help ``` ### Dryrun @@ -100,7 +106,7 @@ seqerakit file.yaml --cli="--arg1 --arg2" You can find the full list of options by running `tw -h`. -The Seqera Platform CLI expects to connect to a Seqera Platform instance that is secured by a TLS certificate. If your Seqera Platform instance does not present a certificate, you will need to qualify and run your `tw` commands with the `--insecure` flag. +The Seqera Platform CLI expects to connect to a Seqera Platform instance that is secured by a TLS certificate. If your Seqera Platform Enterprise instance does not present a certificate, you will need to qualify and run your `tw` commands with the `--insecure` flag. To use `tw` specific CLI options such as `--insecure`, use the `--cli=` flag, followed by the options you would like to use enclosed in double quotes. @@ -110,7 +116,7 @@ For example: seqerakit file.yaml --cli="--insecure" ``` -To use an SSL certificate that is not accepted by the default Java certificate authorities and specify a custom `cacerts` store as accepted by the `tw` CLI, you can specify the `-Djavax.net.ssl.trustStore=/absolute/path/to/cacerts` option enclosed in double quotes to `seqerakit` as you would to `tw`, preceded by `--cli=`. +For Seqera Platform Enterprise, to use an SSL certificate that is not accepted by the default Java certificate authorities and specify a custom `cacerts` store as accepted by the `tw` CLI, you can specify the `-Djavax.net.ssl.trustStore=/absolute/path/to/cacerts` option enclosed in double quotes to `seqerakit` as you would to `tw`, preceded by `--cli=`. For example: @@ -122,7 +128,7 @@ seqerakit hello-world-config.yml --cli="-Djavax.net.ssl.trustStore=/absolute/pat ## YAML Configuration Options -There are several options that can be provided in your YAML configuration file, that are handled specially by seqerakit and not `tw` specific CLI options. +There are several options that can be provided in your YAML configuration file, that are handled specially by seqerakit and/or are not expoed as `tw` CLI options. ### 1. Pipeline parameters using `params` and `params-file` @@ -151,7 +157,37 @@ params: fasta: 's3://path/to/reference.fasta' ``` -**Note**: If duplicate parameters are provided, the parameters provided as key-value pairs inside the `params` nested dictionary of the YAML file will take precedence. +**Note**: If duplicate parameters are provided, the parameters provided as key-value pairs inside the `params` nested dictionary of the YAML file will take precedence **over** values in the provided `params-file`. + +### 2. `overwrite` Functionality +For every entity defined in your YAML file, you can specify `overwrite: True` to overwrite any existing entities in Seqera Platform of the same name. + +`seqerakit` will first check to see if the name of the entity exists, if so, it will invoke a `tw delete` command before attempting to create it based on the options defined in the YAML file. + +```console +DEBUG:root: Overwrite is set to 'True' for organizations + +DEBUG:root: Running command: tw -o json organizations list +DEBUG:root: The attempted organizations resource already exists. Overwriting. + +DEBUG:root: Running command: tw organizations delete --name $SEQERA_ORGANIZATION_NAME +DEBUG:root: Running command: tw organizations add --name $SEQERA_ORGANIZATION_NAME --full-name $SEQERA_ORGANIZATION_NAME --description 'Example of an organization' +``` +### 3. Specifying JSON configuration files with `file-path` +The Seqera Platform CLI allows export and import of entities through JSON configuration files for pipelines and compute environments. To use these files to add a pipeline or compute environment to a workspace, use the `file-path` key to specify a path to a JSON configuration file. + +An example of the `file-path` option is provided in the [compute-envs.yml](templates/compute-envs.yml) template: + +```yaml +compute-envs: + - name: 'my_aws_compute_environment' # required + workspace: 'my_organization/my_workspace' # required + credentials: 'my_aws_credentials' # required + wait: 'AVAILABLE' # optional + file-path: './compute-envs/my_aws_compute_environment.json' # required + overwrite: True +``` + ## Quick start From b4436823276bf54b2942fb77b315bbb1a901aaaf Mon Sep 17 00:00:00 2001 From: ejseqera Date: Sat, 25 Nov 2023 22:53:46 -0500 Subject: [PATCH 12/28] docs: fix alignment on templates --- templates/actions.yml | 40 +++++++++++------------ templates/compute-envs.yml | 36 ++++++++++----------- templates/credentials.yml | 66 +++++++++++++++++++------------------- templates/launch.yml | 28 ++++++++-------- templates/pipelines.yml | 36 ++++++++++----------- 5 files changed, 103 insertions(+), 103 deletions(-) diff --git a/templates/actions.yml b/templates/actions.yml index 9d0bf16..d243b9a 100644 --- a/templates/actions.yml +++ b/templates/actions.yml @@ -1,24 +1,24 @@ ## To see the full list of options available run: "tw actions add" actions: - - type: 'github' # required - name: 'my_github_action' # required - pipeline: 'https://github.com/my_username/my_repo' # required - workspace: 'my_organization/my_workspace' # required - compute-env: 'my_aws_compute_environment' # required - work-dir: 's3://my_bucket' # required - revision: 'main' # required - profile: 'test' # optional - params: # optional + - type: 'github' # required + name: 'my_github_action' # required + pipeline: 'https://github.com/my_username/my_repo' # required + workspace: 'my_organization/my_workspace' # required + compute-env: 'my_aws_compute_environment' # required + work-dir: 's3://my_bucket' # required + revision: 'main' # required + profile: 'test' # optional + params: # optional outdir: 's3://my_bucket/my_results' - overwrite: True # optional - - type: 'tower' # required # TODO - name: 'my_tower_action' # required - pipeline: 'https://github.com/my_username/my_repo' # required - workspace: 'my_organization/my_workspace' # required - compute-env: 'my_aws_compute_environment' # required - work-dir: 's3://my_bucket' # required - revision: 'main' # required - profile: 'test' # optional - params: # optional + overwrite: True # optional + - type: 'tower' # required + name: 'my_tower_action' # required + pipeline: 'https://github.com/my_username/my_repo' # required + workspace: 'my_organization/my_workspace' # required + compute-env: 'my_aws_compute_environment' # required + work-dir: 's3://my_bucket' # required + revision: 'main' # required + profile: 'test' # optional + params: # optional outdir: 's3://my_bucket/my_results' - overwrite: True # optional + overwrite: True # optional diff --git a/templates/compute-envs.yml b/templates/compute-envs.yml index e8f3c33..f3e3d08 100644 --- a/templates/compute-envs.yml +++ b/templates/compute-envs.yml @@ -3,21 +3,21 @@ ## 1. Explicitly in this file ## 2. Via a JSON file exported from Seqera Platform with the "tw compute-envs export" command compute-envs: - - name: 'my_aws_compute_environment' # required - workspace: 'my_organization/my_workspace' # required - credentials: 'my_aws_credentials' # required - wait: 'AVAILABLE' # optional - file-path: './compute-envs/my_aws_compute_environment.json' # required - overwrite: True # optional - - name: 'my_azure_compute_environment' # required - workspace: 'my_organization/my_workspace' # required - credentials: 'my_azure_credentials' # required - wait: 'AVAILABLE' # optional - file-path: './compute-envs/my_azure_compute_environment.json' # required - overwrite: True # optional - - name: 'my_google_compute_environment' # required - workspace: 'my_organization/my_workspace' # required - credentials: 'my_google_credentials' # required - wait: 'AVAILABLE' # optional - file-path: './compute-envs/my_google_compute_environment.json' # required - overwrite: True # optional + - name: 'my_aws_compute_environment' # required + workspace: 'my_organization/my_workspace' # required + credentials: 'my_aws_credentials' # required + wait: 'AVAILABLE' # optional + file-path: './compute-envs/my_aws_compute_environment.json' # required + overwrite: True # optional + - name: 'my_azure_compute_environment' # required + workspace: 'my_organization/my_workspace' # required + credentials: 'my_azure_credentials' # required + wait: 'AVAILABLE' # optional + file-path: './compute-envs/my_azure_compute_environment.json' # required + overwrite: True # optional + - name: 'my_google_compute_environment' # required + workspace: 'my_organization/my_workspace' # required + credentials: 'my_google_credentials' # required + wait: 'AVAILABLE' # optional + file-path: './compute-envs/my_google_compute_environment.json' # required + overwrite: True # optional diff --git a/templates/credentials.yml b/templates/credentials.yml index 490d5d5..d60dbf5 100644 --- a/templates/credentials.yml +++ b/templates/credentials.yml @@ -2,36 +2,36 @@ ## To avoid exposing sensitive information about your credentials, ## use environment variables to supply passwords and secret keys credentials: - - type: 'github' # required - name: 'my_github_credentials' # required - workspace: 'my_organization/my_workspace' # required - username: 'my_username' # required - password: '$SEQPLATFORM_GITHUB_PASSWORD' # required - overwrite: True # optional - - type: 'container-reg' # required - name: 'my_dockerhub_credentials' # required - workspace: 'my_organization/my_workspace' # required - username: 'my_username' # required - password: '$DOCKERHUB_PASSWORD' # required - registry: 'docker.io' # required - overwrite: True # optional - - type: 'google' # required - name: 'my_google_credentials' # required - workspace: 'my_organization/my_workspace' # required - key: '$GOOGLE_KEY' # required - overwrite: True # optional - - type: 'aws' # required - name: 'my_aws_credentials' # required - workspace: 'my_organization/my_workspace' # required - access-key: '$AWS_ACCESS_KEY_ID' # required - secret-key: '$AWS_SECRET_ACCESS_KEY' # required - assume-role-arn: '$AWS_ASSUME_ROLE_ARN' # required - overwrite: True # optional - - type: 'azure' # required - name: 'my_azure_credentials' # required - workspace: 'my_organization/my_workspace' # required - batch-key: '$AZURE_BATCH_KEY' # required - batch-name: 'my_storage_name' # required - storage-key: '$AZURE_STORAGE_KEY' # required - storage-name: 'my_storage_name' # required - overwrite: True # optional + - type: 'github' # required + name: 'my_github_credentials' # required + workspace: 'my_organization/my_workspace' # required + username: 'my_username' # required + password: '$SEQPLATFORM_GITHUB_PASSWORD' # required + overwrite: True # optional + - type: 'container-reg' # required + name: 'my_dockerhub_credentials' # required + workspace: 'my_organization/my_workspace' # required + username: 'my_username' # required + password: '$DOCKERHUB_PASSWORD' # required + registry: 'docker.io' # required + overwrite: True # optional + - type: 'google' # required + name: 'my_google_credentials' # required + workspace: 'my_organization/my_workspace' # required + key: '$GOOGLE_KEY' # required + overwrite: True # optional + - type: 'aws' # required + name: 'my_aws_credentials' # required + workspace: 'my_organization/my_workspace' # required + access-key: '$AWS_ACCESS_KEY_ID' # required + secret-key: '$AWS_SECRET_ACCESS_KEY' # required + assume-role-arn: '$AWS_ASSUME_ROLE_ARN' # required + overwrite: True # optional + - type: 'azure' # required + name: 'my_azure_credentials' # required + workspace: 'my_organization/my_workspace' # required + batch-key: '$AZURE_BATCH_KEY' # required + batch-name: 'my_storage_name' # required + storage-key: '$AZURE_STORAGE_KEY' # required + storage-name: 'my_storage_name' # required + overwrite: True # optional diff --git a/templates/launch.yml b/templates/launch.yml index d9d449b..c281c39 100644 --- a/templates/launch.yml +++ b/templates/launch.yml @@ -5,18 +5,18 @@ ## Note: overwrite is not supported for "tw launch" launch: - - name: 'my_launchpad_launch' # required - workspace: 'my_organization/my_workspace' # required - pipeline: 'my_launchpad_pipeline' # required - params: # optional + - name: 'my_launchpad_launch' # required + workspace: 'my_organization/my_workspace' # required + pipeline: 'my_launchpad_pipeline' # required + params: # optional outdir: 's3://my_bucket/my_results' - - name: 'my_remote_launch' # required - workspace: 'my_organization/my_workspace' # required - compute-env: 'my_aws_compute_environment' # required - pipeline: 'https://github.com/my_username/my_repo' # required - work-dir: 's3://my_bucket' # optional - profile: 'test' # optional - revision: 'main' # optional - params-file: './pipelines/my_params.yml' # optional - config: './pipelines/my_nextflow.config' # optional - pre-run: './pipelines/my_pre_run.txt' # optional + - name: 'my_remote_launch' # required + workspace: 'my_organization/my_workspace' # required + compute-env: 'my_aws_compute_environment' # required + pipeline: 'https://github.com/my_username/my_repo' # required + work-dir: 's3://my_bucket' # optional + profile: 'test' # optional + revision: 'main' # optional + params-file: './pipelines/my_params.yml' # optional + config: './pipelines/my_nextflow.config' # optional + pre-run: './pipelines/my_pre_run.txt' # optional diff --git a/templates/pipelines.yml b/templates/pipelines.yml index 1f08dd9..3088226 100644 --- a/templates/pipelines.yml +++ b/templates/pipelines.yml @@ -3,22 +3,22 @@ ## 1. Explicitly in this file ## 2. Via a JSON file exported from SeqeraPlatform with the "tw pipelines export" command pipelines: - - name: 'my_first_pipeline' # required - workspace: 'my_organization/my_workspace' # required - description: 'My test pipeline' # optional - compute-env: 'my_aws_compute_environment' # required - work-dir: 's3://my_bucket' # optional - profile: 'test' # optional - revision: 'main' # required - params: # optional + - name: 'my_first_pipeline' # required + workspace: 'my_organization/my_workspace' # required + description: 'My test pipeline' # optional + compute-env: 'my_aws_compute_environment' # required + work-dir: 's3://my_bucket' # optional + profile: 'test' # optional + revision: 'main' # required + params: # optional outdir: 's3://my-bucket/my_results' - config: './pipelines/my_nextflow.config' # optional - pre-run: './pipelines/my_pre_run.txt' # optional - url: 'https://github.com/my_username/my_repo' # required - overwrite: True # optional - - name: 'my_second_pipeline' # required - workspace: 'my_organization/my_workspace' # required - description: 'My test pipeline' # optional - compute-env: 'my_aws_compute_environment' # required - file-path: './pipelines/my_pipeline.json' # required - overwrite: True # optional + config: './pipelines/my_nextflow.config' # optional + pre-run: './pipelines/my_pre_run.txt' # optional + url: 'https://github.com/my_username/my_repo' # required + overwrite: True # optional + - name: 'my_second_pipeline' # required + workspace: 'my_organization/my_workspace' # required + description: 'My test pipeline' # optional + compute-env: 'my_aws_compute_environment' # required + file-path: './pipelines/my_pipeline.json' # required + overwrite: True # optional From ceae0947169b8f6acaa966e307522ddadde8b737 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Sat, 25 Nov 2023 22:56:14 -0500 Subject: [PATCH 13/28] fix: fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e35d4a9..c22114c 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ seqerakit hello-world-config.yml --cli="-Djavax.net.ssl.trustStore=/absolute/pat ## YAML Configuration Options -There are several options that can be provided in your YAML configuration file, that are handled specially by seqerakit and/or are not expoed as `tw` CLI options. +There are several options that can be provided in your YAML configuration file, that are handled specially by seqerakit and/or are not exposed as `tw` CLI options. ### 1. Pipeline parameters using `params` and `params-file` From 63596005b0eee890858e1d695dee620d7e1dd53a Mon Sep 17 00:00:00 2001 From: ejseqera Date: Sat, 25 Nov 2023 23:02:22 -0500 Subject: [PATCH 14/28] build: update version in setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index d6acc5b..cc08754 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -VERSION = "0.4.2" +VERSION = "0.4.3" with open("README.md") as f: readme = f.read() From 2e4f8b0e0a7c913d5794bde50e2d2910eb71232a Mon Sep 17 00:00:00 2001 From: Esha Joshi <128735622+ejseqera@users.noreply.github.com> Date: Sat, 25 Nov 2023 23:08:38 -0500 Subject: [PATCH 15/28] Update tests/e2e/aws-e2e.yml --- tests/e2e/aws-e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/e2e/aws-e2e.yml b/tests/e2e/aws-e2e.yml index 817c2f8..dd503e3 100644 --- a/tests/e2e/aws-e2e.yml +++ b/tests/e2e/aws-e2e.yml @@ -71,7 +71,7 @@ datasets: description: 'Samplesheet to run the nf-core/rnaseq pipeline from end-to-end' header: true workspace: '$SEQERA_ORGANIZATION_NAME/$SEQERA_WORKSPACE_NAME' - file-path: '../examples/yaml/datasets/rnaseq_samples.csv' + file-path: './examples/yaml/datasets/rnaseq_samples.csv' overwrite: True pipelines: - name: '$PIPELINE_NAME_PREFIX-hello-world' From 393719a492b4e16c011fbf75ac8f6c634ed0b191 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Tue, 28 Nov 2023 23:58:16 -0500 Subject: [PATCH 16/28] fix: fix env var parsing in params block with params-file --- seqerakit/helper.py | 6 ++++++ seqerakit/utils.py | 41 +++++++++++++++++++++-------------------- 2 files changed, 27 insertions(+), 20 deletions(-) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 1e95e27..597c4d7 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -241,6 +241,9 @@ def parse_pipelines_block(item): ) params_args.extend(["--params-file", temp_file_name]) + if params_file_path and "params" not in item: + params_args.extend(["--params-file", params_file_path]) + combined_args = cmd_args + repo_args + params_args return combined_args @@ -267,6 +270,9 @@ def parse_launch_block(item): ) params_args.extend(["--params-file", temp_file_name]) + if params_file_path and "params" not in item: + params_args.extend(["--params-file", params_file_path]) + cmd_args = cmd_args + repo_args + params_args return cmd_args diff --git a/seqerakit/utils.py b/seqerakit/utils.py index dd20d8d..81e52d3 100644 --- a/seqerakit/utils.py +++ b/seqerakit/utils.py @@ -111,40 +111,41 @@ def is_url(s): return False +class quoted_str(str): + pass + + +def quoted_str_representer(dumper, data): + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"') + + +yaml.add_representer(quoted_str, quoted_str_representer) + + def create_temp_yaml(params_dict, params_file=None): """ - Create a generic temporary yaml file given a dictionary + Create a temporary YAML file given a dictionary. Optionally combine with contents from a JSON or YAML file if provided. """ - combined_params = {} def read_file(file_path): with open(file_path, "r") as file: - if file_path.endswith(".json"): - return json.load(file) - else: - return yaml.safe_load(file) + return ( + json.load(file) if file_path.endswith(".json") else yaml.safe_load(file) + ) + + combined_params = {} - # If a params_file is provided, update the dict if params_file: file_params = read_file(params_file) combined_params.update(file_params) combined_params.update(params_dict) - class quoted_str(str): - pass - - def quoted_str_representer(dumper, data): - return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"') - - yaml.add_representer(quoted_str, quoted_str_representer) - - # # Expand environment variables and quote strings - params_dict = { - k: quoted_str(os.path.expandvars(v)) if isinstance(v, str) else v - for k, v in combined_params.items() - } + for key, value in combined_params.items(): + if isinstance(value, str): + expanded_value = re.sub(r"\$\{?\w+\}?", replace_env_var, value) + combined_params[key] = quoted_str(expanded_value) with tempfile.NamedTemporaryFile( mode="w", delete=False, suffix=".yaml" From bf2f3615d3d69b15169012deb0110fccbfc781f1 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Wed, 29 Nov 2023 00:02:46 -0500 Subject: [PATCH 17/28] test: missing param in test for parse pipelines yaml --- tests/unit/test_helper.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 8800bf3..20c06eb 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -211,6 +211,8 @@ def test_create_mock_pipeline_add_yaml(mock_yaml_file): "--workspace", "my_organization/my_workspace", "https://github.com/nf-core/test_pipeline1", + "--params-file", + "./examples/yaml/pipelines/test_pipeline1/params.yaml", ], "overwrite": True, } From 403f49040b45c6fb2daeda6096fc5fe2a8889d82 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Wed, 29 Nov 2023 00:25:35 -0500 Subject: [PATCH 18/28] build: update version in setup for patch --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index cc08754..23091c1 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -VERSION = "0.4.3" +VERSION = "0.4.4" with open("README.md") as f: readme = f.read() From fac54fe3bdb9e52c994c7b59df21ba3a1f82f1d3 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Fri, 1 Dec 2023 13:01:36 -0500 Subject: [PATCH 19/28] fix: boolean value parsing for keys --- seqerakit/helper.py | 16 ++++++++++++---- tests/unit/test_helper.py | 5 +++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 597c4d7..690409d 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -136,6 +136,9 @@ def parse_credentials_block(item): for key, value in item.items(): if key == "type": cmd_args.append(str(value)) + elif isinstance(value, bool): + if value: + cmd_args.append(f"--{key}") else: cmd_args.extend([f"--{key}", str(value)]) return cmd_args @@ -146,8 +149,9 @@ def parse_compute_envs_block(item): for key, value in item.items(): if key == "file-path" or key == "type" or key == "config-mode": cmd_args.append(str(value)) - elif isinstance(value, bool) and value: - cmd_args.append(f"--{key}") + elif isinstance(value, bool): + if value: + cmd_args.append(f"--{key}") else: cmd_args.extend([f"--{key}", str(value)]) return cmd_args @@ -229,8 +233,9 @@ def parse_pipelines_block(item): params_file_path = str(value) elif key == "file-path": repo_args.extend([str(value)]) - elif isinstance(value, bool) and value: - cmd_args.append(f"--{key}") + elif isinstance(value, bool): + if value: + cmd_args.append(f"--{key}") else: cmd_args.extend([f"--{key}", str(value)]) @@ -261,6 +266,9 @@ def parse_launch_block(item): params_dict = value elif key == "params-file": params_file_path = str(value) + elif isinstance(value, bool): + if value: + cmd_args.append(f"--{key}") else: cmd_args.extend([f"--{key}", str(value)]) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 20c06eb..45da878 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -139,6 +139,8 @@ def test_create_mock_computeevs_yaml(mock_yaml_file): "credentials": "my_credentials", "file-path": "./examples/yaml/computeenvs/computeenvs.yaml", "wait": "AVAILABLE", + "fusion-v2": True, + "fargate": False, "overwrite": True, } ], @@ -150,6 +152,7 @@ def test_create_mock_computeevs_yaml(mock_yaml_file): "--credentials", "my_credentials", "./examples/yaml/computeenvs/computeenvs.yaml", + "--fusion-v2", "--name", "test_computeenv", "--wait", @@ -184,6 +187,7 @@ def test_create_mock_pipeline_add_yaml(mock_yaml_file): "pre-run": "./examples/yaml/pipelines/test_pipeline1/pre_run.sh", "revision": "master", "overwrite": True, + "stub-run": True, } ] } @@ -206,6 +210,7 @@ def test_create_mock_pipeline_add_yaml(mock_yaml_file): "test", "--revision", "master", + "--stub-run", "--work-dir", "s3://work", "--workspace", From f740702211d2b4a34e39af61ed03686e259d472b Mon Sep 17 00:00:00 2001 From: Adam Talbot <12817534+adamrtalbot@users.noreply.github.com> Date: Wed, 6 Dec 2023 14:43:11 +0000 Subject: [PATCH 20/28] SD-39 add end to end integration testing for seqerakit (#102) Add e2e tests for seqerakit triggered on PR to main branch (release). --- .github/workflows/e2e-testing.yml | 69 +++++++++++++++++++------ .github/workflows/overwrite-testing.yml | 49 ------------------ .github/workflows/teardown.yml | 54 +++++++++++++++++++ README.md | 4 +- examples/yaml/e2e/actions.yml | 12 +++++ examples/yaml/e2e/compute-envs.yml | 19 +++++++ examples/yaml/e2e/credentials.yml | 34 ++++++++++++ examples/yaml/e2e/datasets.yml | 31 +++++++++++ examples/yaml/e2e/organizations.yml | 7 +++ examples/yaml/e2e/participants.yml | 11 ++++ examples/yaml/e2e/pipelines.yml | 36 +++++++++++++ examples/yaml/e2e/secrets.yml | 5 ++ examples/yaml/e2e/teams.yml | 9 ++++ examples/yaml/e2e/workspaces.yml | 7 +++ examples/yaml/pipelines/pre_run.txt | 2 +- seqerakit/helper.py | 5 +- seqerakit/utils.py | 3 +- 17 files changed, 287 insertions(+), 70 deletions(-) delete mode 100644 .github/workflows/overwrite-testing.yml create mode 100644 .github/workflows/teardown.yml create mode 100644 examples/yaml/e2e/actions.yml create mode 100644 examples/yaml/e2e/compute-envs.yml create mode 100644 examples/yaml/e2e/credentials.yml create mode 100644 examples/yaml/e2e/datasets.yml create mode 100644 examples/yaml/e2e/organizations.yml create mode 100644 examples/yaml/e2e/participants.yml create mode 100644 examples/yaml/e2e/pipelines.yml create mode 100644 examples/yaml/e2e/secrets.yml create mode 100644 examples/yaml/e2e/teams.yml create mode 100644 examples/yaml/e2e/workspaces.yml diff --git a/.github/workflows/e2e-testing.yml b/.github/workflows/e2e-testing.yml index ef65d62..f569933 100644 --- a/.github/workflows/e2e-testing.yml +++ b/.github/workflows/e2e-testing.yml @@ -4,16 +4,27 @@ run-name: Create all entities in Seqera Platform from end-to-end with seqerakit # It will automate the end-to-end creation all of the following entities in Seqera Platform. on: + pull_request_target: + branches: [main] workflow_dispatch: + inputs: + pre_delete: + description: 'Delete all entities in the yaml before testing' + required: false + type: boolean + default: false + clearup: + description: 'Clearup all entities in yaml after testing' + required: false + type: boolean + default: true jobs: e2e-testing: name: if: github.repository == 'seqeralabs/seqera-kit' runs-on: ubuntu-latest - defaults: - run: - shell: bash -el {0} + env: TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }} TOWER_GITHUB_PASSWORD: ${{ secrets.TOWER_GITHUB_PASSWORD }} @@ -25,31 +36,57 @@ jobs: AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} GOOGLE_KEY: ${{ secrets.GOOGLE_KEY }} SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + steps: - name: Check out source-code repository uses: actions/checkout@v3 - - name: Setup conda - uses: conda-incubator/setup-miniconda@v2.2.0 + - name: Setup Python + uses: actions/setup-python@v2 with: - auto-update-conda: true - environment-file: environment.yml - miniforge-variant: Mambaforge - miniforge-version: latest - python-version: '3.10' - activate-environment: seqerakit - use-mamba: true + python-version: '3.12' - name: Install pip & seqerakit run: | - mamba install -y -c conda-forge pip pip install -e . - - name: Create all entities on Seqera Platform from end-to-end + - name: Install Tower CLI v0.9.1 + run: | + wget https://github.com/seqeralabs/tower-cli/releases/download/v0.9.1/tw-linux-x86_64 \ + && chmod +x tw-linux-x86_64 \ + && sudo mv tw-linux-x86_64 /usr/local/bin/tw + + - name: pre-clear + # Note this task always 'succeeds' + if: ( success() || failure() ) && ( github.event_name != 'workflow_dispatch' || inputs.pre_delete ) + run: | + temp_file=$(mktemp -q) + echo $GOOGLE_KEY | base64 -d > $temp_file + export GOOGLE_KEY=$temp_file + + seqerakit examples/yaml/e2e/*.yml --delete || true + + - name: dryrun + run: | + temp_file=$(mktemp -q) + echo $GOOGLE_KEY | base64 -d > $temp_file + export GOOGLE_KEY=$temp_file + + seqerakit examples/yaml/e2e/*.yml --dryrun + + - name: create + run: | + temp_file=$(mktemp -q) + echo $GOOGLE_KEY | base64 -d > $temp_file + export GOOGLE_KEY=$temp_file + + seqerakit examples/yaml/e2e/*.yml + + - name: teardown + if: ( success() || failure() ) && ( github.event_name != 'workflow_dispatch' || inputs.clearup ) run: | - mamba init temp_file=$(mktemp -q) echo $GOOGLE_KEY | base64 -d > $temp_file export GOOGLE_KEY=$temp_file - seqerakit examples/yaml/seqerakit-e2e.yml + seqerakit examples/yaml/e2e/*.yml --delete diff --git a/.github/workflows/overwrite-testing.yml b/.github/workflows/overwrite-testing.yml deleted file mode 100644 index 8db318c..0000000 --- a/.github/workflows/overwrite-testing.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: overwrite-testing -run-name: Create all entities in Seqera Platform from end-to-end with seqerakit, then overwrite -# This workflow can be triggered manually with GitHub actions workflow dispatch button. -# It will automate the end-to-end creation all of the following entities in Seqera Platform. - -on: - workflow_dispatch: - -jobs: - test-overwrite: - name: - if: github.repository == 'seqeralabs/seqera-kit' - runs-on: ubuntu-latest - container: - image: quay.io/biocontainers/seqerakit:latest - env: - SEQERA_ORGANIZATION_NAME: ${{ secrets.SEQERA_ORGANIZATION_NAME }} - SEQERA_TEAM_NAME: ${{ secrets.SEQERA_TEAM_NAME }} - SEQERA_WORKSPACE_NAME: ${{ secrets.SEQERA_WORKSPACE_NAME }} - TEAM_MEMBER_EMAIL1: ${{ secrets.TEAM_MEMBER_EMAIL1 }} - SEQERA_GITHUB_USERNAME: ${{ secrets.GITHUB_USERNAME }} - TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }} - SEQERA_GITHUB_PASSWORD: ${{ secrets.TOWER_GITHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ASSUME_ROLE_ARN: ${{ secrets.AWS_ASSUME_ROLE_ARN }} - AWS_COMPUTE_ENV_NAME: ${{ secrets.AWS_COMPUTE_ENV_NAME }} - AWS_REGION: ${{ secrets.AWS_REGION }} - SEQERA_WORK_DIR: ${{ secrets.SEQERA_WORK_DIR }} - PIPELINE_NAME_PREFIX: ${{ secrets.PIPELINE_NAME_PREFIX }} - SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} - DATASET_NAME_PREFIX: ${{ secrets.DATASET_NAME_PREFIX }} - steps: - - name: Check out source-code repository - uses: actions/checkout@v3 - - - name: Create all entities for AWS on Seqera Platform from end-to-end - run: | - seqerakit tests/e2e/aws-e2e.yml - - - name: Overwrite all entities for AWS on Seqera Platform from end-to-end - run: | - seqerakit tests/e2e/aws-e2e.yml - - - name: Delete all entities for AWS on Seqera Platform from end-to-end - run: | - seqerakit tests/e2e/aws-e2e.yml --delete diff --git a/.github/workflows/teardown.yml b/.github/workflows/teardown.yml new file mode 100644 index 0000000..a44b93c --- /dev/null +++ b/.github/workflows/teardown.yml @@ -0,0 +1,54 @@ +name: teardown +run-name: Destroy all entities in Seqera Platform from end-to-end with seqerakit +# This workflow can be triggered manually with GitHub actions workflow dispatch button. +# It will automate removing the e2e testing workspace. + +on: + workflow_dispatch: +jobs: + e2e-testing: + name: + if: github.repository == 'seqeralabs/seqera-kit' + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + env: + TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }} + TOWER_GITHUB_PASSWORD: ${{ secrets.TOWER_GITHUB_PASSWORD }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ASSUME_ROLE_ARN: ${{ secrets.AWS_ASSUME_ROLE_ARN }} + AZURE_BATCH_KEY: ${{ secrets.AZURE_BATCH_KEY }} + AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} + GOOGLE_KEY: ${{ secrets.GOOGLE_KEY }} + SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + + steps: + - name: Check out source-code repository + uses: actions/checkout@v3 + + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + environment-file: environment.yml + python-version: '3.12' + mamba-version: '*' + channels: conda-forge,bioconda,defaults + activate-environment: seqerakit + use-mamba: true + + - name: Install pip & seqerakit + run: | + pip install -e . + + - name: teardown + run: | + mamba init + temp_file=$(mktemp -q) + echo $GOOGLE_KEY | base64 -d > $temp_file + export GOOGLE_KEY=$temp_file + + seqerakit --delete examples/yaml/seqerakit-e2e.yml diff --git a/README.md b/README.md index c22114c..3f99daa 100644 --- a/README.md +++ b/README.md @@ -160,7 +160,7 @@ params: **Note**: If duplicate parameters are provided, the parameters provided as key-value pairs inside the `params` nested dictionary of the YAML file will take precedence **over** values in the provided `params-file`. ### 2. `overwrite` Functionality -For every entity defined in your YAML file, you can specify `overwrite: True` to overwrite any existing entities in Seqera Platform of the same name. +For every entity defined in your YAML file, you can specify `overwrite: True` to overwrite any existing entities in Seqera Platform of the same name. `seqerakit` will first check to see if the name of the entity exists, if so, it will invoke a `tw delete` command before attempting to create it based on the options defined in the YAML file. @@ -232,7 +232,7 @@ You can also launch the same pipeline via a Python script. This will essentially ## Real world example -Please see [`seqerakit-e2e.yml`](https://github.com/seqeralabs/seqera-kit/blob/main/examples/yaml/seqerakit-e2e.yml) for an end-to-end example that highlights how you can use `seqerakit` to create everything sequentially in Seqera Platform all the way from creating a new Organization to launching a pipeline. +Please see [`seqerakit-e2e.yml`](./examples/yaml/seqerakit-e2e.yml) for an end-to-end example that highlights how you can use `seqerakit` to create everything sequentially in Seqera Platform all the way from creating a new Organization to launching a pipeline. You can modify this YAML to similarly create Seqera Platform resources end-to-end for your setup. This YAML encodes environment variables to protect sensitive keys, usernames, and passwords that are required to create or add certain resources (i.e. credentials, compute environments). Prior to running it with `seqerakit examples/yaml/seqerakit-e2e.yml`, you will have to set the following environment variables: diff --git a/examples/yaml/e2e/actions.yml b/examples/yaml/e2e/actions.yml new file mode 100644 index 0000000..95b7533 --- /dev/null +++ b/examples/yaml/e2e/actions.yml @@ -0,0 +1,12 @@ +actions: + - type: 'tower' + name: 'nf-core-rnaseq-action' + pipeline: 'https://github.com/nf-core/rnaseq' + workspace: 'seqerakit-e2e/showcase' + compute-env: 'seqera_aws_virginia_fusionv2_nvme' + work-dir: 's3://scidev-testing' + revision: 'main' + profile: 'test' + params: + outdir: s3://seqeralabs-showcase/nf-core-rnaseq/action/' + overwrite: True diff --git a/examples/yaml/e2e/compute-envs.yml b/examples/yaml/e2e/compute-envs.yml new file mode 100644 index 0000000..a01ff45 --- /dev/null +++ b/examples/yaml/e2e/compute-envs.yml @@ -0,0 +1,19 @@ +compute-envs: + - name: 'seqera_aws_virginia_fusionv2_nvme' + workspace: 'seqerakit-e2e/showcase' + credentials: 'aws_credentials' + wait: 'AVAILABLE' + file-path: './examples/yaml/compute-envs/seqera_aws_virginia_fusionv2_nvme.json' + overwrite: True + - name: 'seqera_azure_virginia' + workspace: 'seqerakit-e2e/showcase' + credentials: 'azure_credentials' + wait: 'AVAILABLE' + file-path: './examples/yaml/compute-envs/seqera_azure_virginia.json' + overwrite: True + - name: 'seqera_gcp_finland' + workspace: 'seqerakit-e2e/showcase' + credentials: 'google_credentials' + wait: 'AVAILABLE' + file-path: './examples/yaml/compute-envs/seqera_gcp_finland.json' + overwrite: True diff --git a/examples/yaml/e2e/credentials.yml b/examples/yaml/e2e/credentials.yml new file mode 100644 index 0000000..a6f62a0 --- /dev/null +++ b/examples/yaml/e2e/credentials.yml @@ -0,0 +1,34 @@ +credentials: + - type: 'github' + name: 'github_credentials' + workspace: 'seqerakit-e2e/showcase' + username: 'ejseqera' + password: '$TOWER_GITHUB_PASSWORD' + overwrite: True + - type: 'container-reg' + name: 'dockerhub_credentials' + workspace: 'seqerakit-e2e/showcase' + username: 'eshajoshi' + password: '$DOCKERHUB_PASSWORD' + registry: 'docker.io' + overwrite: True + - type: 'aws' + name: 'aws_credentials' + workspace: 'seqerakit-e2e/showcase' + access-key: '$AWS_ACCESS_KEY_ID' + secret-key: '$AWS_SECRET_ACCESS_KEY' + assume-role-arn: '$AWS_ASSUME_ROLE_ARN' + overwrite: True + - type: 'azure' + name: 'azure_credentials' + workspace: 'seqerakit-e2e/showcase' + batch-key: '$AZURE_BATCH_KEY' + batch-name: 'seqeralabs' + storage-key: '$AZURE_STORAGE_KEY' + storage-name: 'seqeralabs' + overwrite: True + - type: 'google' + name: 'google_credentials' + workspace: 'seqerakit-e2e/showcase' + key: '$GOOGLE_KEY' + overwrite: True diff --git a/examples/yaml/e2e/datasets.yml b/examples/yaml/e2e/datasets.yml new file mode 100644 index 0000000..50aa036 --- /dev/null +++ b/examples/yaml/e2e/datasets.yml @@ -0,0 +1,31 @@ +datasets: + - name: 'rnaseq_samples' + description: 'Samplesheet to run the nf-core/rnaseq pipeline from end-to-end' + header: true + workspace: 'seqerakit-e2e/showcase' + file-path: './examples/yaml/datasets/rnaseq_samples.csv' + overwrite: True + - name: 'sarek_samples' + description: 'Samplesheet to run the nf-core/sarek pipeline from end-to-end' + header: true + workspace: 'seqerakit-e2e/showcase' + file-path: './examples/yaml/datasets/sarek_samples.csv' + overwrite: True + - name: 'viralrecon_illumina_samples' + description: 'Samplesheet to run the nf-core/viralrecon pipeline from end-to-end with Illumina data' + header: true + workspace: 'seqerakit-e2e/showcase' + file-path: './examples/yaml/datasets/viralrecon_illumina_samples.csv' + overwrite: True + - name: 'viralrecon_nanopore_samples' + description: 'Samplesheet to run the nf-core/viralrecon pipeline from end-to-end with Nanopore data' + header: true + workspace: 'seqerakit-e2e/showcase' + file-path: './examples/yaml/datasets/viralrecon_nanopore_samples.csv' + overwrite: True + - name: 'sentieon_samples' + description: 'Samplesheet to run the seqeralabs/nf-sentieon pipeline from end-to-end' + header: true + workspace: 'seqerakit-e2e/showcase' + file-path: './examples/yaml/datasets/sentieon_samples.csv' + overwrite: True diff --git a/examples/yaml/e2e/organizations.yml b/examples/yaml/e2e/organizations.yml new file mode 100644 index 0000000..a61cff0 --- /dev/null +++ b/examples/yaml/e2e/organizations.yml @@ -0,0 +1,7 @@ +organizations: + - name: 'seqerakit-e2e' + full-name: 'seqerakit-e2e' + description: 'Organization created E2E with seqerakit CLI scripting' + location: 'Global' + website: 'https://seqera.io/' + overwrite: True diff --git a/examples/yaml/e2e/participants.yml b/examples/yaml/e2e/participants.yml new file mode 100644 index 0000000..660e9f8 --- /dev/null +++ b/examples/yaml/e2e/participants.yml @@ -0,0 +1,11 @@ +participants: + - name: 'scidev_team' + type: 'TEAM' + workspace: 'seqerakit-e2e/showcase' + role: 'ADMIN' + overwrite: True + - name: 'adam.talbot@seqera.io' + type: 'MEMBER' + workspace: 'seqerakit-e2e/showcase' + role: 'LAUNCH' + overwrite: True diff --git a/examples/yaml/e2e/pipelines.yml b/examples/yaml/e2e/pipelines.yml new file mode 100644 index 0000000..2e92335 --- /dev/null +++ b/examples/yaml/e2e/pipelines.yml @@ -0,0 +1,36 @@ +pipelines: + - name: 'nf-core-rnaseq' + url: 'https://github.com/nf-core/rnaseq' + workspace: 'seqerakit-e2e/showcase' + description: 'RNA sequencing analysis pipeline using STAR, RSEM, HISAT2 or Salmon with gene/isoform counts and extensive quality control.' + compute-env: 'seqera_aws_virginia_fusionv2_nvme' + work-dir: 's3://seqeralabs-showcase' + profile: 'test' + revision: '3.12.0' + params: + outdir: 's3://seqeralabs-showcase/nf-core-rnaseq/results' + config: './examples/yaml/pipelines/nextflow.config' + pre-run: './examples/yaml/pipelines/pre_run.txt' + overwrite: True + - name: 'nf-core-sarek' + workspace: 'seqerakit-e2e/showcase' + compute-env: 'seqera_azure_virginia' + file-path: './examples/yaml/pipelines/nf-core-sarek_pipeline.json' + overwrite: True + - name: 'nf-core-viralrecon-illumina' + url: 'https://github.com/nf-core/viralrecon' + workspace: 'seqerakit-e2e/showcase' + description: 'Pipeline for assembly and intrahost/low-frequency variant calling for viral samples.' + compute-env: 'seqera_gcp_finland' + work-dir: 'gs://seqeralabs-showcase-eu-north-1' + profile: 'test' + revision: '2.6.0' + params-file: './examples/yaml/pipelines/nf_core_viralrecon_illumina_params.yml' + config: './examples/yaml/pipelines/nextflow.config' + pre-run: './examples/yaml/pipelines/pre_run.txt' + overwrite: True + - name: 'nf-sentieon' + workspace: 'seqerakit-e2e/showcase' + compute-env: 'seqera_aws_virginia_fusionv2_nvme' + file-path: './examples/yaml/pipelines/nf_sentieon_pipeline.json' + overwrite: True diff --git a/examples/yaml/e2e/secrets.yml b/examples/yaml/e2e/secrets.yml new file mode 100644 index 0000000..c0f852f --- /dev/null +++ b/examples/yaml/e2e/secrets.yml @@ -0,0 +1,5 @@ +secrets: + - name: 'SENTIEON_LICENSE_BASE64' + workspace: 'seqerakit-e2e/showcase' + value: '$SENTIEON_LICENSE_BASE64' + overwrite: True diff --git a/examples/yaml/e2e/teams.yml b/examples/yaml/e2e/teams.yml new file mode 100644 index 0000000..81ce294 --- /dev/null +++ b/examples/yaml/e2e/teams.yml @@ -0,0 +1,9 @@ +teams: + - name: 'scidev_team' + organization: 'seqerakit-e2e' + description: 'Scientific Development team @ Seqera Labs' + members: + - 'esha.joshi@seqera.io' + - 'adam.talbot@seqera.io' + - 'drpatelhh@gmail.com' + overwrite: True diff --git a/examples/yaml/e2e/workspaces.yml b/examples/yaml/e2e/workspaces.yml new file mode 100644 index 0000000..cba3687 --- /dev/null +++ b/examples/yaml/e2e/workspaces.yml @@ -0,0 +1,7 @@ +workspaces: + - name: 'showcase' + full-name: 'showcase' + organization: 'seqerakit-e2e' + description: 'Workspace created E2E with seqerakit CLI scripting' + visibility: 'PRIVATE' + overwrite: True diff --git a/examples/yaml/pipelines/pre_run.txt b/examples/yaml/pipelines/pre_run.txt index 3e565b1..f8a81d2 100644 --- a/examples/yaml/pipelines/pre_run.txt +++ b/examples/yaml/pipelines/pre_run.txt @@ -1 +1 @@ -export NXF_VER=23.06.0-edge \ No newline at end of file +export NXF_VER=23.10.0 diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 690409d..c27b135 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -61,7 +61,10 @@ def parse_all_yaml(file_paths, destroy=False): data = yaml.safe_load(f) for key, value in data.items(): if key in merged_data: - merged_data[key].extend(value) + try: + merged_data[key].extend(value) + except AttributeError: + merged_data[key] = [merged_data[key], value] else: merged_data[key] = value diff --git a/seqerakit/utils.py b/seqerakit/utils.py index 81e52d3..b58c3d1 100644 --- a/seqerakit/utils.py +++ b/seqerakit/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import json import tempfile import os @@ -65,7 +66,7 @@ def check_if_exists(json_data, namekey, namevalue): """ if not json_data: return False - + logging.info(f"Checking if {namekey} {namevalue} exists in Seqera Platform...") # Regex pattern to match environment variables in the string env_var_pattern = re.compile(r"\$\{?[\w]+\}?") From efd2e8c10f377d828490a190b29326f5f6d6dea0 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Dec 2023 22:02:19 -0500 Subject: [PATCH 21/28] docs: add local installation details and yaml definition guidance --- README.md | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/README.md b/README.md index 3f99daa..734fd55 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,27 @@ You can force overwrite the installation to use the latest changes with the comm pip install --upgrade --force-reinstall seqerakit ``` +### Local development installation +You can install the development branch of `seqerakit` on your local machine to test feature updates of the tool. Before proceeding, ensure that you have [Python](https://www.python.org/downloads/) and [Git](https://git-scm.com/downloads) installed on your system. + +1. To install directly from pip: +```bash +pip install git+https://github.com/seqeralabs/seqera-kit.git@dev +``` + +2. Alternatively, you may clone the repository locally and install manually: +```bash +git clone https://github.com/seqeralabs/seqera-kit.git +cd seqera-kit +git checkout dev +pip install . +``` + +You can verify your installation with: +```bash +pip show seqerakit +``` + ## Configuration Create a Seqera Platform access token using the [Seqera Platform](https://tower.nf/) web interface via the **Your Tokens** page in your profile. @@ -264,6 +285,25 @@ We have provided template YAML files for each of the entities that can be create - [pipelines.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/pipelines.yml) - [launch.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/launch.yml) + +All possible options to provide as definitions in your YAML file can be determined by running the Seqera Platform CLI help command for your desired entity. For example,to determine how to define options to add a Pipeline to the Launchpad, run: + +```console +$ tw pipelines add -h + +Usage: tw pipelines add [OPTIONS] PIPELINE_URL + +Add a workspace pipeline. + +Parameters: +* PIPELINE_URL Nextflow pipeline URL. + +Options: +* -n, --name= Pipeline name. + -w, --workspace= Workspace numeric identifier (TOWER_WORKSPACE_ID as default) or workspace reference as OrganizationName/WorkspaceName + ... +``` + ## Contributions and Support If you would like to contribute to `seqerakit`, please see the [contributing guidelines](https://github.com/seqeralabs/seqera-kit/blob/main/.github/CONTRIBUTING.md). From bbd469d580d30ad5d2f59bd743426fab8ce79505 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Dec 2023 22:10:09 -0500 Subject: [PATCH 22/28] docs: add better templates for CE YAML --- templates/compute-envs.yml | 35 ++++++++++++++++++++++++----------- 1 file changed, 24 insertions(+), 11 deletions(-) diff --git a/templates/compute-envs.yml b/templates/compute-envs.yml index f3e3d08..b6bf171 100644 --- a/templates/compute-envs.yml +++ b/templates/compute-envs.yml @@ -3,21 +3,34 @@ ## 1. Explicitly in this file ## 2. Via a JSON file exported from Seqera Platform with the "tw compute-envs export" command compute-envs: +# To create a compute environment from a JSON configuration file (AWS Example) - name: 'my_aws_compute_environment' # required workspace: 'my_organization/my_workspace' # required credentials: 'my_aws_credentials' # required wait: 'AVAILABLE' # optional file-path: './compute-envs/my_aws_compute_environment.json' # required overwrite: True # optional - - name: 'my_azure_compute_environment' # required + +# To create a compute environment with options specified through YAML (AWS Example) + - type: aws-batch # required + config-mode: forge # required for AWS and Azure + name: 'my_aws_compute_environment' # required workspace: 'my_organization/my_workspace' # required - credentials: 'my_azure_credentials' # required - wait: 'AVAILABLE' # optional - file-path: './compute-envs/my_azure_compute_environment.json' # required - overwrite: True # optional - - name: 'my_google_compute_environment' # required - workspace: 'my_organization/my_workspace' # required - credentials: 'my_google_credentials' # required - wait: 'AVAILABLE' # optional - file-path: './compute-envs/my_google_compute_environment.json' # required - overwrite: True # optional + credentials: 'my_aws_credentials' # required + region: 'eu-west-1' # required + work-dir: 's3://my-bucket' # required + provisioning-model: 'SPOT' # optional + fusion-v2: False # optional + wave: False # optional + fargate: False # optional + fast-storage: False # optional + instance-types: 'c6i,r6i,m6i' # optional, comma separated list + no-ebs-auto-scale: True # optional + max-cpus: 500 # required + labels: 'label1,label2' # optional, comma separated list + vpc-id: 'vpc-1234567890' # optional + subnets: 'subnet-1234567890,subnet-1234567891' # optional, comma separated list + security-groups: 'sg-1234567890,sg-1234567891' # optional, comma separated list + allow-buckets: 's3://my-bucket,s3://my-other-bucket' # optional, comma separated list + wait: 'AVAILABLE' # optional + overwrite: False # optional \ No newline at end of file From 2a0eb45aeb53448fe74b878fafd2b88b7761c572 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Dec 2023 22:27:38 -0500 Subject: [PATCH 23/28] docs: add detail to yaml definition section --- README.md | 93 +++++++++++++++++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 26 deletions(-) diff --git a/README.md b/README.md index 734fd55..6817c6e 100644 --- a/README.md +++ b/README.md @@ -251,24 +251,66 @@ You can also launch the same pipeline via a Python script. This will essentially 3. Login to your Seqera Platform instance and check the Runs page in the appropriate Workspace for the pipeline you just launched! -## Real world example +## Defining your YAML file using CLI options -Please see [`seqerakit-e2e.yml`](./examples/yaml/seqerakit-e2e.yml) for an end-to-end example that highlights how you can use `seqerakit` to create everything sequentially in Seqera Platform all the way from creating a new Organization to launching a pipeline. +All available options to provide as definitions in your YAML file can be determined by running the Seqera Platform CLI help command for your desired entity. -You can modify this YAML to similarly create Seqera Platform resources end-to-end for your setup. This YAML encodes environment variables to protect sensitive keys, usernames, and passwords that are required to create or add certain resources (i.e. credentials, compute environments). Prior to running it with `seqerakit examples/yaml/seqerakit-e2e.yml`, you will have to set the following environment variables: +1. Retrieve CLI Options + +To obtain a list of available CLI options for defining your YAML file, use the help command of the Seqera Platform CLI. For instance, if you want to configure a pipeline to be added to the Launchpad, you can view the options as follows: ```console -$TOWER_GITHUB_PASSWORD -$DOCKERHUB_PASSWORD -$AWS_ACCESS_KEY_ID -$AWS_SECRET_ACCESS_KEY -$AWS_ASSUME_ROLE_ARN -$AZURE_BATCH_KEY -$AZURE_STORAGE_KEY -$GOOGLE_KEY -$SENTIEON_LICENSE_BASE64 +$ tw pipelines add -h + +Usage: tw pipelines add [OPTIONS] PIPELINE_URL + +Add a workspace pipeline. + +Parameters: +* PIPELINE_URL Nextflow pipeline URL. + +Options: +* -n, --name= Pipeline name. + -w, --workspace= Workspace numeric identifier (TOWER_WORKSPACE_ID as default) or workspace reference as OrganizationName/WorkspaceName + -d, --description= Pipeline description. + --labels=[,...] List of labels seperated by coma. + -c, --compute-env= Compute environment name. + --work-dir= Path where the pipeline scratch data is stored. + -p, --profile=[,...] Comma-separated list of one or more configuration profile names you want to use for this pipeline execution. + --params-file= Pipeline parameters in either JSON or YML format. + --revision= A valid repository commit Id, tag or branch name. + ... +``` +2. Define Key-Value Pairs in YAML + +Translate each CLI option into a key-value pair in the YAML file. The structure of your YAML file should reflect the hierarchy and format of the CLI options. For instance: + +```yaml +pipelines: + - name: 'my_first_pipeline' + url: 'https://github.com/username/my_pipeline' + workspace: 'my_organization/my_workspace' + description: 'My test pipeline' + labels: 'yeast,test_data' + compute-env: 'my_compute_environment' + work-dir: 's3://my_bucket' + profile: 'test' + params-file: '/path/to/params.yaml' + revision: '1.0' ``` +In this example: + +- `name`, `url`, `workspace`, etc., are the keys derived from the CLI options. +- The corresponding values are user-defined + +### Best Practices: +- Ensure that the indentation and structure of the YAML file are correct - YAML is sensitive to formatting. +- Use quotes around strings that contain special characters or spaces. +- When listing multiple values (`labels`, `instance-types`, `allow-buckets`, etc), separate them with commas as shown above. +- For complex configurations, refer to the [Templates](/templates/) provided in this repository. + + ## Templates We have provided template YAML files for each of the entities that can be created on Seqera Platform. These can be found in the [`templates/`](https://github.com/seqeralabs/blob/main/seqera-kit/templates) directory and should form a good starting point for you to add your own customization: @@ -285,23 +327,22 @@ We have provided template YAML files for each of the entities that can be create - [pipelines.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/pipelines.yml) - [launch.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/launch.yml) +## Real world example -All possible options to provide as definitions in your YAML file can be determined by running the Seqera Platform CLI help command for your desired entity. For example,to determine how to define options to add a Pipeline to the Launchpad, run: - -```console -$ tw pipelines add -h - -Usage: tw pipelines add [OPTIONS] PIPELINE_URL - -Add a workspace pipeline. +Please see [`seqerakit-e2e.yml`](./examples/yaml/seqerakit-e2e.yml) for an end-to-end example that highlights how you can use `seqerakit` to create everything sequentially in Seqera Platform all the way from creating a new Organization to launching a pipeline. -Parameters: -* PIPELINE_URL Nextflow pipeline URL. +You can modify this YAML to similarly create Seqera Platform resources end-to-end for your setup. This YAML encodes environment variables to protect sensitive keys, usernames, and passwords that are required to create or add certain resources (i.e. credentials, compute environments). Prior to running it with `seqerakit examples/yaml/seqerakit-e2e.yml`, you will have to set the following environment variables: -Options: -* -n, --name= Pipeline name. - -w, --workspace= Workspace numeric identifier (TOWER_WORKSPACE_ID as default) or workspace reference as OrganizationName/WorkspaceName - ... +```console +$TOWER_GITHUB_PASSWORD +$DOCKERHUB_PASSWORD +$AWS_ACCESS_KEY_ID +$AWS_SECRET_ACCESS_KEY +$AWS_ASSUME_ROLE_ARN +$AZURE_BATCH_KEY +$AZURE_STORAGE_KEY +$GOOGLE_KEY +$SENTIEON_LICENSE_BASE64 ``` ## Contributions and Support From 3da2563a8d4ffe792f2fc56ad8c06eb49fd01d67 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Dec 2023 22:48:21 -0500 Subject: [PATCH 24/28] feat: add better error handling for empty yaml --- seqerakit/cli.py | 23 +++++++++++++---------- seqerakit/helper.py | 5 +++++ tests/unit/test_helper.py | 8 ++++++++ 3 files changed, 26 insertions(+), 10 deletions(-) diff --git a/seqerakit/cli.py b/seqerakit/cli.py index 7c87322..47bad66 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -163,16 +163,19 @@ def main(args=None): # Parse the YAML file(s) by blocks # and get a dictionary of command line arguments - cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete) - for block, args_list in cmd_args_dict.items(): - for args in args_list: - try: - # Run the 'tw' methods for each block - block_manager.handle_block(block, args, destroy=options.delete) - except (ResourceExistsError, ResourceCreationError) as e: - logging.error(e) - sys.exit(1) - + try: + cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete) + for block, args_list in cmd_args_dict.items(): + for args in args_list: + try: + # Run the 'tw' methods for each block + block_manager.handle_block(block, args, destroy=options.delete) + except (ResourceExistsError, ResourceCreationError) as e: + logging.error(e) + sys.exit(1) + except ValueError as e: + logging.error(e) + sys.exit(1) if __name__ == "__main__": main() diff --git a/seqerakit/helper.py b/seqerakit/helper.py index c27b135..10f66eb 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -59,6 +59,11 @@ def parse_all_yaml(file_paths, destroy=False): for file_path in file_paths: with open(file_path, "r") as f: data = yaml.safe_load(f) + + # Check if the YAML file is empty or contains no valid data + if data is None or not data: + raise ValueError(f" The YAML file '{file_path}' is empty or does not contain valid data.") + for key, value in data.items(): if key in merged_data: try: diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 45da878..af7b3d7 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -228,3 +228,11 @@ def test_create_mock_pipeline_add_yaml(mock_yaml_file): assert "pipelines" in result assert result["pipelines"] == expected_block_output + +def test_empty_yaml_file(mock_yaml_file): + test_data = {} + file_path = mock_yaml_file(test_data) + + with pytest.raises(ValueError) as e: + helper.parse_all_yaml([file_path]) + assert f"The YAML file '{file_path}' is empty or does not contain valid data." in str(e.value) \ No newline at end of file From a93e40952d62c0624778390930df32d434f83cf0 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Dec 2023 22:52:34 -0500 Subject: [PATCH 25/28] fix: linting errors --- seqerakit/cli.py | 1 + seqerakit/helper.py | 6 ++++-- tests/unit/test_helper.py | 6 +++++- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/seqerakit/cli.py b/seqerakit/cli.py index 47bad66..79605ed 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -177,5 +177,6 @@ def main(args=None): logging.error(e) sys.exit(1) + if __name__ == "__main__": main() diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 10f66eb..2b47599 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -62,8 +62,10 @@ def parse_all_yaml(file_paths, destroy=False): # Check if the YAML file is empty or contains no valid data if data is None or not data: - raise ValueError(f" The YAML file '{file_path}' is empty or does not contain valid data.") - + raise ValueError( + f" The file '{file_path}' is empty or does not contain valid data." + ) + for key, value in data.items(): if key in merged_data: try: diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index af7b3d7..ec426ec 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -229,10 +229,14 @@ def test_create_mock_pipeline_add_yaml(mock_yaml_file): assert "pipelines" in result assert result["pipelines"] == expected_block_output + def test_empty_yaml_file(mock_yaml_file): test_data = {} file_path = mock_yaml_file(test_data) with pytest.raises(ValueError) as e: helper.parse_all_yaml([file_path]) - assert f"The YAML file '{file_path}' is empty or does not contain valid data." in str(e.value) \ No newline at end of file + assert ( + f"The YAML file '{file_path}' is empty or does not contain valid data." + in str(e.value) + ) From 3569fce3455081a8dfe1ecc8af6b4e01ca499342 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Dec 2023 22:54:17 -0500 Subject: [PATCH 26/28] fix: fix assertion statement for empty yaml test --- tests/unit/test_helper.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index ec426ec..6094013 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -237,6 +237,6 @@ def test_empty_yaml_file(mock_yaml_file): with pytest.raises(ValueError) as e: helper.parse_all_yaml([file_path]) assert ( - f"The YAML file '{file_path}' is empty or does not contain valid data." + f"The file '{file_path}' is empty or does not contain valid data." in str(e.value) ) From 96e25be043de8ae41094e7df3b4ca54dbdfb9bf5 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 18 Dec 2023 22:57:18 -0500 Subject: [PATCH 27/28] style: linting fix --- tests/unit/test_helper.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 6094013..ab7158c 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -236,7 +236,6 @@ def test_empty_yaml_file(mock_yaml_file): with pytest.raises(ValueError) as e: helper.parse_all_yaml([file_path]) - assert ( - f"The file '{file_path}' is empty or does not contain valid data." - in str(e.value) + assert f"The file '{file_path}' is empty or does not contain valid data." in str( + e.value ) From 649304368d17a74e75b892c6346e02e5896bbccc Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 21 Dec 2023 18:36:58 -0500 Subject: [PATCH 28/28] build: update version in setup --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 23091c1..f5bf293 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -VERSION = "0.4.4" +VERSION = "0.4.5" with open("README.md") as f: readme = f.read()