From f8269d5470d6bd41e03bbec4e0b2107a462d9535 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Mon, 4 Mar 2024 00:36:13 -0500 Subject: [PATCH 1/4] feat: add support for stdin to cli --- seqerakit/cli.py | 14 +++++++------ seqerakit/helper.py | 50 ++++++++++++++++++++++++++++++--------------- 2 files changed, 42 insertions(+), 22 deletions(-) diff --git a/seqerakit/cli.py b/seqerakit/cli.py index f3e1b80..b35d6d1 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -21,7 +21,6 @@ import logging import sys -from pathlib import Path from seqerakit import seqeraplatform, helper, overwrite from seqerakit.seqeraplatform import ResourceExistsError, ResourceCreationError from seqerakit import __version__ @@ -66,7 +65,6 @@ def parse_args(args=None): yaml_processing = parser.add_argument_group("YAML Processing Options") yaml_processing.add_argument( "yaml", - type=Path, nargs="*", help="One or more YAML files with Seqera Platform resource definitions.", ) @@ -154,10 +152,14 @@ def main(args=None): return if not options.yaml: - logging.error( - " No YAML(s) provided. Please provide atleast one YAML configuration file." - ) - sys.exit(1) + if sys.stdin.isatty(): + logging.error( + " No YAML(s) provided and no input from stdin. Please provide " + "at least one YAML configuration file or pipe input from stdin." + ) + sys.exit(1) + else: + options.yaml = [sys.stdin] # Parse CLI arguments into a list cli_args_list = options.cli_args.split() if options.cli_args else [] diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 0b71311..eb32e0d 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -19,6 +19,7 @@ """ import yaml from seqerakit import utils +import sys def parse_yaml_block(yaml_data, block_name): @@ -56,24 +57,41 @@ def parse_all_yaml(file_paths, destroy=False): # If multiple yamls, merge them into one dictionary merged_data = {} - for file_path in file_paths: - with open(file_path, "r") as f: - data = yaml.safe_load(f) + # Special handling for stdin represented by "-" + if not file_paths or "-" in file_paths: + # Read YAML directly from stdin + data = yaml.safe_load(sys.stdin) + if data is None or not data: + raise ValueError( + " The input from stdin is empty or does not contain valid YAML data." + ) - # Check if the YAML file is empty or contains no valid data - if data is None or not data: - raise ValueError( - f" The file '{file_path}' is empty or does not contain valid data." - ) + merged_data.update(data) - for key, value in data.items(): - if key in merged_data: - try: - merged_data[key].extend(value) - except AttributeError: - merged_data[key] = [merged_data[key], value] - else: - merged_data[key] = value + for file_path in file_paths: + if file_path == "-": + continue + try: + with open(file_path, "r") as f: + data = yaml.safe_load(f) + if data is None or not data: + raise ValueError( + f" The file '{file_path}' is empty or " + "does not contain valid data." + ) + merged_data.update(data) + except FileNotFoundError: + print(f"Error: The file '{file_path}' was not found.") + sys.exit(1) + + for key, value in data.items(): + if key in merged_data: + try: + merged_data[key].extend(value) + except AttributeError: + merged_data[key] = [merged_data[key], value] + else: + merged_data[key] = value block_names = list(merged_data.keys()) From fb977cfb136b2b464ad413b57837297c3b467da9 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 25 Apr 2024 19:31:44 -0400 Subject: [PATCH 2/4] fix: logic for merging list of dicts, tests and add docs --- README.md | 12 +++++++++ seqerakit/helper.py | 36 ++++++++++++++++++++++---- tests/unit/test_helper.py | 53 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 93 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 3e6e3de..59f8737 100644 --- a/README.md +++ b/README.md @@ -109,6 +109,18 @@ Use `--version` or `-v` to retrieve the current version of your seqerakit instal ```bash seqerakit --version ``` +### Input +`seqerakit` supports input through either file paths to YAMLs or directly from standard input (stdin). + +#### Using File Path +```bash +seqerakit /path/to/file.yaml +``` +#### Using stdin +```console +$ cat file.yaml | seqerakit - +``` +See the [Defining your YAML file using CLI options](#defining-your-yaml-file-using-cli-options) section for guidance on formatting your input YAML file(s). ### Dryrun diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 5cc1064..3d02c85 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -62,11 +62,10 @@ def parse_all_yaml(file_paths, destroy=False): if not file_paths or "-" in file_paths: # Read YAML directly from stdin data = yaml.safe_load(sys.stdin) - if data is None or not data: + if not data: raise ValueError( " The input from stdin is empty or does not contain valid YAML data." ) - merged_data.update(data) for file_path in file_paths: @@ -75,12 +74,30 @@ def parse_all_yaml(file_paths, destroy=False): try: with open(file_path, "r") as f: data = yaml.safe_load(f) - if data is None or not data: + if not data: raise ValueError( f" The file '{file_path}' is empty or " "does not contain valid data." ) - merged_data.update(data) + + for key, new_value in data.items(): + if key in merged_data: + if isinstance(new_value, list) and all( + isinstance(i, dict) for i in new_value + ): + # Handle list of dictionaries & merge without duplication + existing_items = { + tuple(sorted(d.items())) for d in merged_data[key] + } + for item in new_value: + if tuple(sorted(item.items())) not in existing_items: + merged_data[key].append(item) + else: + # override if not list of dictionaries + merged_data[key] = new_value + else: + merged_data[key] = new_value + except FileNotFoundError: print(f"Error: The file '{file_path}' was not found.") sys.exit(1) @@ -94,6 +111,15 @@ def parse_all_yaml(file_paths, destroy=False): else: merged_data[key] = value + for key, value in data.items(): + if key in merged_data: + try: + merged_data[key].extend(value) + except AttributeError: + merged_data[key] = [merged_data[key], value] + else: + merged_data[key] = value + block_names = list(merged_data.keys()) # Define the order in which the resources should be created. @@ -113,7 +139,7 @@ def parse_all_yaml(file_paths, destroy=False): "launch", ] - # Reverse the order of resources if destroy is True + # Reverse the order of resources to delete if destroy is True if destroy: resource_order = resource_order[:-1][::-1] diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index fcdccfc..8038d1a 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -1,8 +1,8 @@ -from unittest.mock import mock_open +from unittest.mock import patch, mock_open from seqerakit import helper import yaml import pytest - +from io import StringIO # Fixture to mock a YAML file @pytest.fixture @@ -48,8 +48,8 @@ def test_create_mock_organization_yaml(mock_yaml_file): "overwrite": True, } ] - file_path = mock_yaml_file(test_data) + print(f"debug - file_path: {file_path}") result = helper.parse_all_yaml([file_path]) assert "organizations" in result @@ -343,6 +343,53 @@ def test_empty_yaml_file(mock_yaml_file): ) +def test_empty_stdin_file(): + # Prepare the mock to simulate empty stdin + with patch("sys.stdin", StringIO("")): + # Use '-' to indicate that stdin should be read + with pytest.raises(ValueError) as e: + helper.parse_all_yaml(["-"]) + assert ( + "The input from stdin is empty or does not contain valid YAML data." + in str(e.value) + ) + + +def test_stdin_yaml_file(): + # Prepare the mock to simulate stdin + yaml_data = """ +compute-envs: + - name: test_computeenv + config-mode: forge + workspace: my_organization/my_workspace + credentials: my_credentials + type: aws-batch + wait: AVAILABLE + """ + with patch("sys.stdin", StringIO(yaml_data)): + result = helper.parse_all_yaml(["-"]) + + expected_block_output = [ + { + "cmd_args": [ + "aws-batch", + "forge", + "--name", + "test_computeenv", + "--workspace", + "my_organization/my_workspace", + "--credentials", + "my_credentials", + "--wait", + "AVAILABLE", + ], + "overwrite": False, + } + ] + assert "compute-envs" in result + assert result["compute-envs"] == expected_block_output + + def test_error_type_yaml_file(mock_yaml_file): test_data = { "compute-envs": [ From b41fe2a9b727705497929e73b17aa448e7d514b4 Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 25 Apr 2024 19:35:31 -0400 Subject: [PATCH 3/4] fix: black linting --- tests/unit/test_helper.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index 8038d1a..76f5b55 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -4,6 +4,7 @@ import pytest from io import StringIO + # Fixture to mock a YAML file @pytest.fixture def mock_yaml_file(mocker): From 7b60445887fd1e68542195dae2bd081bf3ace58c Mon Sep 17 00:00:00 2001 From: ejseqera Date: Thu, 25 Apr 2024 19:42:20 -0400 Subject: [PATCH 4/4] fix: remove extra logic that remained for some reason --- seqerakit/helper.py | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 3d02c85..b041230 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -97,29 +97,10 @@ def parse_all_yaml(file_paths, destroy=False): merged_data[key] = new_value else: merged_data[key] = new_value - except FileNotFoundError: print(f"Error: The file '{file_path}' was not found.") sys.exit(1) - for key, value in data.items(): - if key in merged_data: - try: - merged_data[key].extend(value) - except AttributeError: - merged_data[key] = [merged_data[key], value] - else: - merged_data[key] = value - - for key, value in data.items(): - if key in merged_data: - try: - merged_data[key].extend(value) - except AttributeError: - merged_data[key] = [merged_data[key], value] - else: - merged_data[key] = value - block_names = list(merged_data.keys()) # Define the order in which the resources should be created.