diff --git a/.github/workflows/e2e-testing.yml b/.github/workflows/e2e-testing.yml index ef65d62..f569933 100644 --- a/.github/workflows/e2e-testing.yml +++ b/.github/workflows/e2e-testing.yml @@ -4,16 +4,27 @@ run-name: Create all entities in Seqera Platform from end-to-end with seqerakit # It will automate the end-to-end creation all of the following entities in Seqera Platform. on: + pull_request_target: + branches: [main] workflow_dispatch: + inputs: + pre_delete: + description: 'Delete all entities in the yaml before testing' + required: false + type: boolean + default: false + clearup: + description: 'Clearup all entities in yaml after testing' + required: false + type: boolean + default: true jobs: e2e-testing: name: if: github.repository == 'seqeralabs/seqera-kit' runs-on: ubuntu-latest - defaults: - run: - shell: bash -el {0} + env: TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }} TOWER_GITHUB_PASSWORD: ${{ secrets.TOWER_GITHUB_PASSWORD }} @@ -25,31 +36,57 @@ jobs: AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} GOOGLE_KEY: ${{ secrets.GOOGLE_KEY }} SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + steps: - name: Check out source-code repository uses: actions/checkout@v3 - - name: Setup conda - uses: conda-incubator/setup-miniconda@v2.2.0 + - name: Setup Python + uses: actions/setup-python@v2 with: - auto-update-conda: true - environment-file: environment.yml - miniforge-variant: Mambaforge - miniforge-version: latest - python-version: '3.10' - activate-environment: seqerakit - use-mamba: true + python-version: '3.12' - name: Install pip & seqerakit run: | - mamba install -y -c conda-forge pip pip install -e . - - name: Create all entities on Seqera Platform from end-to-end + - name: Install Tower CLI v0.9.1 + run: | + wget https://github.com/seqeralabs/tower-cli/releases/download/v0.9.1/tw-linux-x86_64 \ + && chmod +x tw-linux-x86_64 \ + && sudo mv tw-linux-x86_64 /usr/local/bin/tw + + - name: pre-clear + # Note this task always 'succeeds' + if: ( success() || failure() ) && ( github.event_name != 'workflow_dispatch' || inputs.pre_delete ) + run: | + temp_file=$(mktemp -q) + echo $GOOGLE_KEY | base64 -d > $temp_file + export GOOGLE_KEY=$temp_file + + seqerakit examples/yaml/e2e/*.yml --delete || true + + - name: dryrun + run: | + temp_file=$(mktemp -q) + echo $GOOGLE_KEY | base64 -d > $temp_file + export GOOGLE_KEY=$temp_file + + seqerakit examples/yaml/e2e/*.yml --dryrun + + - name: create + run: | + temp_file=$(mktemp -q) + echo $GOOGLE_KEY | base64 -d > $temp_file + export GOOGLE_KEY=$temp_file + + seqerakit examples/yaml/e2e/*.yml + + - name: teardown + if: ( success() || failure() ) && ( github.event_name != 'workflow_dispatch' || inputs.clearup ) run: | - mamba init temp_file=$(mktemp -q) echo $GOOGLE_KEY | base64 -d > $temp_file export GOOGLE_KEY=$temp_file - seqerakit examples/yaml/seqerakit-e2e.yml + seqerakit examples/yaml/e2e/*.yml --delete diff --git a/.github/workflows/overwrite-testing.yml b/.github/workflows/overwrite-testing.yml deleted file mode 100644 index 8db318c..0000000 --- a/.github/workflows/overwrite-testing.yml +++ /dev/null @@ -1,49 +0,0 @@ -name: overwrite-testing -run-name: Create all entities in Seqera Platform from end-to-end with seqerakit, then overwrite -# This workflow can be triggered manually with GitHub actions workflow dispatch button. -# It will automate the end-to-end creation all of the following entities in Seqera Platform. - -on: - workflow_dispatch: - -jobs: - test-overwrite: - name: - if: github.repository == 'seqeralabs/seqera-kit' - runs-on: ubuntu-latest - container: - image: quay.io/biocontainers/seqerakit:latest - env: - SEQERA_ORGANIZATION_NAME: ${{ secrets.SEQERA_ORGANIZATION_NAME }} - SEQERA_TEAM_NAME: ${{ secrets.SEQERA_TEAM_NAME }} - SEQERA_WORKSPACE_NAME: ${{ secrets.SEQERA_WORKSPACE_NAME }} - TEAM_MEMBER_EMAIL1: ${{ secrets.TEAM_MEMBER_EMAIL1 }} - SEQERA_GITHUB_USERNAME: ${{ secrets.GITHUB_USERNAME }} - TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }} - SEQERA_GITHUB_PASSWORD: ${{ secrets.TOWER_GITHUB_PASSWORD }} - DOCKERHUB_USERNAME: ${{ secrets.DOCKERHUB_USERNAME }} - DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} - AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} - AWS_ASSUME_ROLE_ARN: ${{ secrets.AWS_ASSUME_ROLE_ARN }} - AWS_COMPUTE_ENV_NAME: ${{ secrets.AWS_COMPUTE_ENV_NAME }} - AWS_REGION: ${{ secrets.AWS_REGION }} - SEQERA_WORK_DIR: ${{ secrets.SEQERA_WORK_DIR }} - PIPELINE_NAME_PREFIX: ${{ secrets.PIPELINE_NAME_PREFIX }} - SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} - DATASET_NAME_PREFIX: ${{ secrets.DATASET_NAME_PREFIX }} - steps: - - name: Check out source-code repository - uses: actions/checkout@v3 - - - name: Create all entities for AWS on Seqera Platform from end-to-end - run: | - seqerakit tests/e2e/aws-e2e.yml - - - name: Overwrite all entities for AWS on Seqera Platform from end-to-end - run: | - seqerakit tests/e2e/aws-e2e.yml - - - name: Delete all entities for AWS on Seqera Platform from end-to-end - run: | - seqerakit tests/e2e/aws-e2e.yml --delete diff --git a/.github/workflows/python-testing.yml b/.github/workflows/python-testing.yml index 98a2a59..2ce0e91 100644 --- a/.github/workflows/python-testing.yml +++ b/.github/workflows/python-testing.yml @@ -22,6 +22,7 @@ jobs: run: | python -m pip install --upgrade pip pip install pytest + pip install pytest-mock pip install -e . - name: Run tests run: | diff --git a/.github/workflows/teardown.yml b/.github/workflows/teardown.yml new file mode 100644 index 0000000..a44b93c --- /dev/null +++ b/.github/workflows/teardown.yml @@ -0,0 +1,54 @@ +name: teardown +run-name: Destroy all entities in Seqera Platform from end-to-end with seqerakit +# This workflow can be triggered manually with GitHub actions workflow dispatch button. +# It will automate removing the e2e testing workspace. + +on: + workflow_dispatch: +jobs: + e2e-testing: + name: + if: github.repository == 'seqeralabs/seqera-kit' + runs-on: ubuntu-latest + defaults: + run: + shell: bash -el {0} + env: + TOWER_ACCESS_TOKEN: ${{ secrets.TOWER_ACCESS_TOKEN }} + TOWER_GITHUB_PASSWORD: ${{ secrets.TOWER_GITHUB_PASSWORD }} + DOCKERHUB_PASSWORD: ${{ secrets.DOCKERHUB_PASSWORD }} + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + AWS_ASSUME_ROLE_ARN: ${{ secrets.AWS_ASSUME_ROLE_ARN }} + AZURE_BATCH_KEY: ${{ secrets.AZURE_BATCH_KEY }} + AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} + GOOGLE_KEY: ${{ secrets.GOOGLE_KEY }} + SENTIEON_LICENSE_BASE64: ${{ secrets.SENTIEON_LICENSE_BASE64 }} + + steps: + - name: Check out source-code repository + uses: actions/checkout@v3 + + - name: Setup conda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + environment-file: environment.yml + python-version: '3.12' + mamba-version: '*' + channels: conda-forge,bioconda,defaults + activate-environment: seqerakit + use-mamba: true + + - name: Install pip & seqerakit + run: | + pip install -e . + + - name: teardown + run: | + mamba init + temp_file=$(mktemp -q) + echo $GOOGLE_KEY | base64 -d > $temp_file + export GOOGLE_KEY=$temp_file + + seqerakit --delete examples/yaml/seqerakit-e2e.yml diff --git a/README.md b/README.md index 1521fe6..6817c6e 100644 --- a/README.md +++ b/README.md @@ -44,17 +44,37 @@ conda activate seqerakit If you already have [Seqera Platform CLI](https://github.com/seqeralabs/tower-cli#1-installation) and Python installed on your system, you can install `seqerakit` directly from [PyPI](https://pypi.org/project/seqerakit/): - -``` +```console pip install seqerakit ``` You can force overwrite the installation to use the latest changes with the command below: -``` +```console pip install --upgrade --force-reinstall seqerakit ``` +### Local development installation +You can install the development branch of `seqerakit` on your local machine to test feature updates of the tool. Before proceeding, ensure that you have [Python](https://www.python.org/downloads/) and [Git](https://git-scm.com/downloads) installed on your system. + +1. To install directly from pip: +```bash +pip install git+https://github.com/seqeralabs/seqera-kit.git@dev +``` + +2. Alternatively, you may clone the repository locally and install manually: +```bash +git clone https://github.com/seqeralabs/seqera-kit.git +cd seqera-kit +git checkout dev +pip install . +``` + +You can verify your installation with: +```bash +pip show seqerakit +``` + ## Configuration Create a Seqera Platform access token using the [Seqera Platform](https://tower.nf/) web interface via the **Your Tokens** page in your profile. @@ -67,17 +87,23 @@ export TOWER_ACCESS_TOKEN= ## Usage -Use the -h or --help parameter to list the available commands and their associated options: +To confirm the installation of `seqerakit`, configuration of the Seqera Platform CLI and connection is working as expected: +```bash +seqerakit --info ``` -seqerakit -h + +Use the `-h` or `--help `parameter to list the available commands and their associated options: + +```bash +seqerakit --help ``` ### Dryrun To print the commands that would executed with `tw` when using a YAML file, you can run `seqerakit` with the `--dryrun` flag: -``` +```bash seqerakit file.yaml --dryrun ``` @@ -85,7 +111,7 @@ seqerakit file.yaml --dryrun Instead of adding or creating resources, you can recursively delete resources in your YAML file by specifying the `--delete` flag: -``` +```bash seqerakit file.yaml --delete ``` @@ -95,32 +121,95 @@ For example, if you have a YAML file that defines an Organization -> Workspace - `tw` specific CLI options can be specified with the `--cli=` flag: -``` +```bash seqerakit file.yaml --cli="--arg1 --arg2" ``` You can find the full list of options by running `tw -h`. -The Seqera Platform CLI expects to connect to a Seqera Platform instance that is secured by a TLS certificate. If your Seqera Platform instance does not present a certificate, you will need to qualify and run your `tw` commands with the `--insecure` flag. +The Seqera Platform CLI expects to connect to a Seqera Platform instance that is secured by a TLS certificate. If your Seqera Platform Enterprise instance does not present a certificate, you will need to qualify and run your `tw` commands with the `--insecure` flag. To use `tw` specific CLI options such as `--insecure`, use the `--cli=` flag, followed by the options you would like to use enclosed in double quotes. For example: -``` +```bash seqerakit file.yaml --cli="--insecure" ``` -To use an SSL certificate that is not accepted by the default Java certificate authorities and specify a custom `cacerts` store as accepted by the `tw` CLI, you can specify the `-Djavax.net.ssl.trustStore=/absolute/path/to/cacerts` option enclosed in double quotes to `seqerakit` as you would to `tw`, preceded by `--cli=`. +For Seqera Platform Enterprise, to use an SSL certificate that is not accepted by the default Java certificate authorities and specify a custom `cacerts` store as accepted by the `tw` CLI, you can specify the `-Djavax.net.ssl.trustStore=/absolute/path/to/cacerts` option enclosed in double quotes to `seqerakit` as you would to `tw`, preceded by `--cli=`. For example: -``` +```bash seqerakit hello-world-config.yml --cli="-Djavax.net.ssl.trustStore=/absolute/path/to/cacerts" ``` Note: Use of `--verbose` option for the `tw` CLI is currently not supported by `seqerakit`. Supplying `--cli="--verbose"` will raise an error. +## YAML Configuration Options + +There are several options that can be provided in your YAML configuration file, that are handled specially by seqerakit and/or are not exposed as `tw` CLI options. + +### 1. Pipeline parameters using `params` and `params-file` + +To specify pipeline parameters, you may either use `params:` to specify a list of parameters, or use `params-file:` to point to a parameters file. + +For example, to specify pipeline parameters within your YAML: + +```yaml +params: + outdir: 's3://path/to/outdir' + fasta: 's3://path/to/reference.fasta' +``` + +Alternatively, to specify a file containing pipeline parameters: + +```yaml +params-file: '/path/to/my/parameters.yaml' +``` + +Optionally, you may provide both: + +```yaml +params-file: '/path/to/my/parameters.yaml' +params: + outdir: 's3://path/to/outdir' + fasta: 's3://path/to/reference.fasta' +``` + +**Note**: If duplicate parameters are provided, the parameters provided as key-value pairs inside the `params` nested dictionary of the YAML file will take precedence **over** values in the provided `params-file`. + +### 2. `overwrite` Functionality +For every entity defined in your YAML file, you can specify `overwrite: True` to overwrite any existing entities in Seqera Platform of the same name. + +`seqerakit` will first check to see if the name of the entity exists, if so, it will invoke a `tw delete` command before attempting to create it based on the options defined in the YAML file. + +```console +DEBUG:root: Overwrite is set to 'True' for organizations + +DEBUG:root: Running command: tw -o json organizations list +DEBUG:root: The attempted organizations resource already exists. Overwriting. + +DEBUG:root: Running command: tw organizations delete --name $SEQERA_ORGANIZATION_NAME +DEBUG:root: Running command: tw organizations add --name $SEQERA_ORGANIZATION_NAME --full-name $SEQERA_ORGANIZATION_NAME --description 'Example of an organization' +``` +### 3. Specifying JSON configuration files with `file-path` +The Seqera Platform CLI allows export and import of entities through JSON configuration files for pipelines and compute environments. To use these files to add a pipeline or compute environment to a workspace, use the `file-path` key to specify a path to a JSON configuration file. + +An example of the `file-path` option is provided in the [compute-envs.yml](templates/compute-envs.yml) template: + +```yaml +compute-envs: + - name: 'my_aws_compute_environment' # required + workspace: 'my_organization/my_workspace' # required + credentials: 'my_aws_credentials' # required + wait: 'AVAILABLE' # optional + file-path: './compute-envs/my_aws_compute_environment.json' # required + overwrite: True +``` + + ## Quick start You must provide a YAML file that defines the options for each of the entities you would like to create in Seqera Platform. @@ -131,18 +220,18 @@ You will need to have an account on Seqera Platform (see [Plans and pricing](htt 1. Create a YAML file called `hello-world-config.yml` with the contents below, and customise the `` and `` entries as required: - ``` + ```yaml # noqa launch: - - name: 'hello-world' # Workflow name - workspace: '' # Workspace name - compute-env: '' # Compute environment - revision: 'master' # Pipeline revision - pipeline: 'https://github.com/nextflow-io/hello' # Pipeline URL + - name: 'hello-world' # Workflow name + workspace: '' # Workspace name + compute-env: '' # Compute environment + revision: 'master' # Pipeline revision + pipeline: 'https://github.com/nextflow-io/hello' # Pipeline URL ``` 2. Launch the pipeline with `seqerakit`: - ``` + ```bash seqerakit hello-world-config.yml ``` @@ -156,30 +245,72 @@ You can also launch the same pipeline via a Python script. This will essentially 2. Launch the pipeline with `seqerakit`: - ``` +```bash python launch_hello_world.py - ``` +``` 3. Login to your Seqera Platform instance and check the Runs page in the appropriate Workspace for the pipeline you just launched! -## Real world example +## Defining your YAML file using CLI options -Please see [`seqerakit-e2e.yml`](https://github.com/seqeralabs/seqera-kit/blob/main/examples/yaml/seqerakit-e2e.yml) for an end-to-end example that highlights how you can use `seqerakit` to create everything sequentially in Seqera Platform all the way from creating a new Organization to launching a pipeline. +All available options to provide as definitions in your YAML file can be determined by running the Seqera Platform CLI help command for your desired entity. -You can modify this YAML to similarly create Seqera Platform resources end-to-end for your setup. This YAML encodes environment variables to protect sensitive keys, usernames, and passwords that are required to create or add certain resources (i.e. credentials, compute environments). Prior to running it with `seqerakit examples/yaml/seqerakit-e2e.yml`, you will have to set the following environment variables: +1. Retrieve CLI Options + +To obtain a list of available CLI options for defining your YAML file, use the help command of the Seqera Platform CLI. For instance, if you want to configure a pipeline to be added to the Launchpad, you can view the options as follows: +```console +$ tw pipelines add -h + +Usage: tw pipelines add [OPTIONS] PIPELINE_URL + +Add a workspace pipeline. + +Parameters: +* PIPELINE_URL Nextflow pipeline URL. + +Options: +* -n, --name= Pipeline name. + -w, --workspace= Workspace numeric identifier (TOWER_WORKSPACE_ID as default) or workspace reference as OrganizationName/WorkspaceName + -d, --description= Pipeline description. + --labels=[,...] List of labels seperated by coma. + -c, --compute-env= Compute environment name. + --work-dir= Path where the pipeline scratch data is stored. + -p, --profile=[,...] Comma-separated list of one or more configuration profile names you want to use for this pipeline execution. + --params-file= Pipeline parameters in either JSON or YML format. + --revision= A valid repository commit Id, tag or branch name. + ... ``` -$TOWER_GITHUB_PASSWORD -$DOCKERHUB_PASSWORD -$AWS_ACCESS_KEY_ID -$AWS_SECRET_ACCESS_KEY -$AWS_ASSUME_ROLE_ARN -$AZURE_BATCH_KEY -$AZURE_STORAGE_KEY -$GOOGLE_KEY -$SENTIEON_LICENSE_BASE64 +2. Define Key-Value Pairs in YAML + +Translate each CLI option into a key-value pair in the YAML file. The structure of your YAML file should reflect the hierarchy and format of the CLI options. For instance: + +```yaml +pipelines: + - name: 'my_first_pipeline' + url: 'https://github.com/username/my_pipeline' + workspace: 'my_organization/my_workspace' + description: 'My test pipeline' + labels: 'yeast,test_data' + compute-env: 'my_compute_environment' + work-dir: 's3://my_bucket' + profile: 'test' + params-file: '/path/to/params.yaml' + revision: '1.0' ``` +In this example: + +- `name`, `url`, `workspace`, etc., are the keys derived from the CLI options. +- The corresponding values are user-defined + +### Best Practices: +- Ensure that the indentation and structure of the YAML file are correct - YAML is sensitive to formatting. +- Use quotes around strings that contain special characters or spaces. +- When listing multiple values (`labels`, `instance-types`, `allow-buckets`, etc), separate them with commas as shown above. +- For complex configurations, refer to the [Templates](/templates/) provided in this repository. + + ## Templates We have provided template YAML files for each of the entities that can be created on Seqera Platform. These can be found in the [`templates/`](https://github.com/seqeralabs/blob/main/seqera-kit/templates) directory and should form a good starting point for you to add your own customization: @@ -196,6 +327,24 @@ We have provided template YAML files for each of the entities that can be create - [pipelines.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/pipelines.yml) - [launch.yml](https://github.com/seqeralabs/seqera-kit/blob/main/templates/launch.yml) +## Real world example + +Please see [`seqerakit-e2e.yml`](./examples/yaml/seqerakit-e2e.yml) for an end-to-end example that highlights how you can use `seqerakit` to create everything sequentially in Seqera Platform all the way from creating a new Organization to launching a pipeline. + +You can modify this YAML to similarly create Seqera Platform resources end-to-end for your setup. This YAML encodes environment variables to protect sensitive keys, usernames, and passwords that are required to create or add certain resources (i.e. credentials, compute environments). Prior to running it with `seqerakit examples/yaml/seqerakit-e2e.yml`, you will have to set the following environment variables: + +```console +$TOWER_GITHUB_PASSWORD +$DOCKERHUB_PASSWORD +$AWS_ACCESS_KEY_ID +$AWS_SECRET_ACCESS_KEY +$AWS_ASSUME_ROLE_ARN +$AZURE_BATCH_KEY +$AZURE_STORAGE_KEY +$GOOGLE_KEY +$SENTIEON_LICENSE_BASE64 +``` + ## Contributions and Support If you would like to contribute to `seqerakit`, please see the [contributing guidelines](https://github.com/seqeralabs/seqera-kit/blob/main/.github/CONTRIBUTING.md). diff --git a/examples/yaml/e2e/actions.yml b/examples/yaml/e2e/actions.yml new file mode 100644 index 0000000..95b7533 --- /dev/null +++ b/examples/yaml/e2e/actions.yml @@ -0,0 +1,12 @@ +actions: + - type: 'tower' + name: 'nf-core-rnaseq-action' + pipeline: 'https://github.com/nf-core/rnaseq' + workspace: 'seqerakit-e2e/showcase' + compute-env: 'seqera_aws_virginia_fusionv2_nvme' + work-dir: 's3://scidev-testing' + revision: 'main' + profile: 'test' + params: + outdir: s3://seqeralabs-showcase/nf-core-rnaseq/action/' + overwrite: True diff --git a/examples/yaml/e2e/compute-envs.yml b/examples/yaml/e2e/compute-envs.yml new file mode 100644 index 0000000..a01ff45 --- /dev/null +++ b/examples/yaml/e2e/compute-envs.yml @@ -0,0 +1,19 @@ +compute-envs: + - name: 'seqera_aws_virginia_fusionv2_nvme' + workspace: 'seqerakit-e2e/showcase' + credentials: 'aws_credentials' + wait: 'AVAILABLE' + file-path: './examples/yaml/compute-envs/seqera_aws_virginia_fusionv2_nvme.json' + overwrite: True + - name: 'seqera_azure_virginia' + workspace: 'seqerakit-e2e/showcase' + credentials: 'azure_credentials' + wait: 'AVAILABLE' + file-path: './examples/yaml/compute-envs/seqera_azure_virginia.json' + overwrite: True + - name: 'seqera_gcp_finland' + workspace: 'seqerakit-e2e/showcase' + credentials: 'google_credentials' + wait: 'AVAILABLE' + file-path: './examples/yaml/compute-envs/seqera_gcp_finland.json' + overwrite: True diff --git a/examples/yaml/e2e/credentials.yml b/examples/yaml/e2e/credentials.yml new file mode 100644 index 0000000..a6f62a0 --- /dev/null +++ b/examples/yaml/e2e/credentials.yml @@ -0,0 +1,34 @@ +credentials: + - type: 'github' + name: 'github_credentials' + workspace: 'seqerakit-e2e/showcase' + username: 'ejseqera' + password: '$TOWER_GITHUB_PASSWORD' + overwrite: True + - type: 'container-reg' + name: 'dockerhub_credentials' + workspace: 'seqerakit-e2e/showcase' + username: 'eshajoshi' + password: '$DOCKERHUB_PASSWORD' + registry: 'docker.io' + overwrite: True + - type: 'aws' + name: 'aws_credentials' + workspace: 'seqerakit-e2e/showcase' + access-key: '$AWS_ACCESS_KEY_ID' + secret-key: '$AWS_SECRET_ACCESS_KEY' + assume-role-arn: '$AWS_ASSUME_ROLE_ARN' + overwrite: True + - type: 'azure' + name: 'azure_credentials' + workspace: 'seqerakit-e2e/showcase' + batch-key: '$AZURE_BATCH_KEY' + batch-name: 'seqeralabs' + storage-key: '$AZURE_STORAGE_KEY' + storage-name: 'seqeralabs' + overwrite: True + - type: 'google' + name: 'google_credentials' + workspace: 'seqerakit-e2e/showcase' + key: '$GOOGLE_KEY' + overwrite: True diff --git a/examples/yaml/e2e/datasets.yml b/examples/yaml/e2e/datasets.yml new file mode 100644 index 0000000..50aa036 --- /dev/null +++ b/examples/yaml/e2e/datasets.yml @@ -0,0 +1,31 @@ +datasets: + - name: 'rnaseq_samples' + description: 'Samplesheet to run the nf-core/rnaseq pipeline from end-to-end' + header: true + workspace: 'seqerakit-e2e/showcase' + file-path: './examples/yaml/datasets/rnaseq_samples.csv' + overwrite: True + - name: 'sarek_samples' + description: 'Samplesheet to run the nf-core/sarek pipeline from end-to-end' + header: true + workspace: 'seqerakit-e2e/showcase' + file-path: './examples/yaml/datasets/sarek_samples.csv' + overwrite: True + - name: 'viralrecon_illumina_samples' + description: 'Samplesheet to run the nf-core/viralrecon pipeline from end-to-end with Illumina data' + header: true + workspace: 'seqerakit-e2e/showcase' + file-path: './examples/yaml/datasets/viralrecon_illumina_samples.csv' + overwrite: True + - name: 'viralrecon_nanopore_samples' + description: 'Samplesheet to run the nf-core/viralrecon pipeline from end-to-end with Nanopore data' + header: true + workspace: 'seqerakit-e2e/showcase' + file-path: './examples/yaml/datasets/viralrecon_nanopore_samples.csv' + overwrite: True + - name: 'sentieon_samples' + description: 'Samplesheet to run the seqeralabs/nf-sentieon pipeline from end-to-end' + header: true + workspace: 'seqerakit-e2e/showcase' + file-path: './examples/yaml/datasets/sentieon_samples.csv' + overwrite: True diff --git a/examples/yaml/e2e/organizations.yml b/examples/yaml/e2e/organizations.yml new file mode 100644 index 0000000..a61cff0 --- /dev/null +++ b/examples/yaml/e2e/organizations.yml @@ -0,0 +1,7 @@ +organizations: + - name: 'seqerakit-e2e' + full-name: 'seqerakit-e2e' + description: 'Organization created E2E with seqerakit CLI scripting' + location: 'Global' + website: 'https://seqera.io/' + overwrite: True diff --git a/examples/yaml/e2e/participants.yml b/examples/yaml/e2e/participants.yml new file mode 100644 index 0000000..660e9f8 --- /dev/null +++ b/examples/yaml/e2e/participants.yml @@ -0,0 +1,11 @@ +participants: + - name: 'scidev_team' + type: 'TEAM' + workspace: 'seqerakit-e2e/showcase' + role: 'ADMIN' + overwrite: True + - name: 'adam.talbot@seqera.io' + type: 'MEMBER' + workspace: 'seqerakit-e2e/showcase' + role: 'LAUNCH' + overwrite: True diff --git a/examples/yaml/e2e/pipelines.yml b/examples/yaml/e2e/pipelines.yml new file mode 100644 index 0000000..2e92335 --- /dev/null +++ b/examples/yaml/e2e/pipelines.yml @@ -0,0 +1,36 @@ +pipelines: + - name: 'nf-core-rnaseq' + url: 'https://github.com/nf-core/rnaseq' + workspace: 'seqerakit-e2e/showcase' + description: 'RNA sequencing analysis pipeline using STAR, RSEM, HISAT2 or Salmon with gene/isoform counts and extensive quality control.' + compute-env: 'seqera_aws_virginia_fusionv2_nvme' + work-dir: 's3://seqeralabs-showcase' + profile: 'test' + revision: '3.12.0' + params: + outdir: 's3://seqeralabs-showcase/nf-core-rnaseq/results' + config: './examples/yaml/pipelines/nextflow.config' + pre-run: './examples/yaml/pipelines/pre_run.txt' + overwrite: True + - name: 'nf-core-sarek' + workspace: 'seqerakit-e2e/showcase' + compute-env: 'seqera_azure_virginia' + file-path: './examples/yaml/pipelines/nf-core-sarek_pipeline.json' + overwrite: True + - name: 'nf-core-viralrecon-illumina' + url: 'https://github.com/nf-core/viralrecon' + workspace: 'seqerakit-e2e/showcase' + description: 'Pipeline for assembly and intrahost/low-frequency variant calling for viral samples.' + compute-env: 'seqera_gcp_finland' + work-dir: 'gs://seqeralabs-showcase-eu-north-1' + profile: 'test' + revision: '2.6.0' + params-file: './examples/yaml/pipelines/nf_core_viralrecon_illumina_params.yml' + config: './examples/yaml/pipelines/nextflow.config' + pre-run: './examples/yaml/pipelines/pre_run.txt' + overwrite: True + - name: 'nf-sentieon' + workspace: 'seqerakit-e2e/showcase' + compute-env: 'seqera_aws_virginia_fusionv2_nvme' + file-path: './examples/yaml/pipelines/nf_sentieon_pipeline.json' + overwrite: True diff --git a/examples/yaml/e2e/secrets.yml b/examples/yaml/e2e/secrets.yml new file mode 100644 index 0000000..c0f852f --- /dev/null +++ b/examples/yaml/e2e/secrets.yml @@ -0,0 +1,5 @@ +secrets: + - name: 'SENTIEON_LICENSE_BASE64' + workspace: 'seqerakit-e2e/showcase' + value: '$SENTIEON_LICENSE_BASE64' + overwrite: True diff --git a/examples/yaml/e2e/teams.yml b/examples/yaml/e2e/teams.yml new file mode 100644 index 0000000..81ce294 --- /dev/null +++ b/examples/yaml/e2e/teams.yml @@ -0,0 +1,9 @@ +teams: + - name: 'scidev_team' + organization: 'seqerakit-e2e' + description: 'Scientific Development team @ Seqera Labs' + members: + - 'esha.joshi@seqera.io' + - 'adam.talbot@seqera.io' + - 'drpatelhh@gmail.com' + overwrite: True diff --git a/examples/yaml/e2e/workspaces.yml b/examples/yaml/e2e/workspaces.yml new file mode 100644 index 0000000..cba3687 --- /dev/null +++ b/examples/yaml/e2e/workspaces.yml @@ -0,0 +1,7 @@ +workspaces: + - name: 'showcase' + full-name: 'showcase' + organization: 'seqerakit-e2e' + description: 'Workspace created E2E with seqerakit CLI scripting' + visibility: 'PRIVATE' + overwrite: True diff --git a/examples/yaml/pipelines/pre_run.txt b/examples/yaml/pipelines/pre_run.txt index 3e565b1..f8a81d2 100644 --- a/examples/yaml/pipelines/pre_run.txt +++ b/examples/yaml/pipelines/pre_run.txt @@ -1 +1 @@ -export NXF_VER=23.06.0-edge \ No newline at end of file +export NXF_VER=23.10.0 diff --git a/seqerakit/cli.py b/seqerakit/cli.py index e75aeec..79605ed 100644 --- a/seqerakit/cli.py +++ b/seqerakit/cli.py @@ -39,6 +39,11 @@ def parse_args(args=None): help="The desired log level (default: INFO).", type=str.upper, ) + parser.add_argument( + "--info", + action="store_true", + help="Display information about the Seqera Platform and exit", + ) parser.add_argument( "--dryrun", action="store_true", @@ -47,7 +52,7 @@ def parse_args(args=None): parser.add_argument( "yaml", type=Path, - nargs="+", # allow multiple YAML paths + nargs="*", # allow multiple YAML paths help="One or more YAML files with Seqera Platform resources to create", ) parser.add_argument( @@ -127,6 +132,18 @@ def main(args=None): options = parse_args(args if args is not None else sys.argv[1:]) logging.basicConfig(level=options.log_level) + # If the info flag is set, run 'tw info' + if options.info: + sp = seqeraplatform.SeqeraPlatform() + print(sp.info()) + return + + if not options.yaml: + logging.error( + " No YAML(s) provided. Please provide atleast one YAML configuration file." + ) + sys.exit(1) + # Parse CLI arguments into a list cli_args_list = options.cli_args.split() if options.cli_args else [] @@ -146,15 +163,19 @@ def main(args=None): # Parse the YAML file(s) by blocks # and get a dictionary of command line arguments - cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete) - for block, args_list in cmd_args_dict.items(): - for args in args_list: - try: - # Run the 'tw' methods for each block - block_manager.handle_block(block, args, destroy=options.delete) - except (ResourceExistsError, ResourceCreationError) as e: - logging.error(e) - sys.exit(1) + try: + cmd_args_dict = helper.parse_all_yaml(options.yaml, destroy=options.delete) + for block, args_list in cmd_args_dict.items(): + for args in args_list: + try: + # Run the 'tw' methods for each block + block_manager.handle_block(block, args, destroy=options.delete) + except (ResourceExistsError, ResourceCreationError) as e: + logging.error(e) + sys.exit(1) + except ValueError as e: + logging.error(e) + sys.exit(1) if __name__ == "__main__": diff --git a/seqerakit/helper.py b/seqerakit/helper.py index 9751cd7..2b47599 100644 --- a/seqerakit/helper.py +++ b/seqerakit/helper.py @@ -59,9 +59,19 @@ def parse_all_yaml(file_paths, destroy=False): for file_path in file_paths: with open(file_path, "r") as f: data = yaml.safe_load(f) + + # Check if the YAML file is empty or contains no valid data + if data is None or not data: + raise ValueError( + f" The file '{file_path}' is empty or does not contain valid data." + ) + for key, value in data.items(): if key in merged_data: - merged_data[key].extend(value) + try: + merged_data[key].extend(value) + except AttributeError: + merged_data[key] = [merged_data[key], value] else: merged_data[key] = value @@ -136,6 +146,9 @@ def parse_credentials_block(item): for key, value in item.items(): if key == "type": cmd_args.append(str(value)) + elif isinstance(value, bool): + if value: + cmd_args.append(f"--{key}") else: cmd_args.extend([f"--{key}", str(value)]) return cmd_args @@ -146,8 +159,9 @@ def parse_compute_envs_block(item): for key, value in item.items(): if key == "file-path" or key == "type" or key == "config-mode": cmd_args.append(str(value)) - elif isinstance(value, bool) and value: - cmd_args.append(f"--{key}") + elif isinstance(value, bool): + if value: + cmd_args.append(f"--{key}") else: cmd_args.extend([f"--{key}", str(value)]) return cmd_args @@ -218,38 +232,66 @@ def parse_pipelines_block(item): cmd_args = [] repo_args = [] params_args = [] + params_file_path = None for key, value in item.items(): if key == "url": repo_args.extend([str(value)]) elif key == "params": - temp_file_name = utils.create_temp_yaml(value) - params_args.extend(["--params-file", temp_file_name]) + params_dict = value + elif key == "params-file": + params_file_path = str(value) elif key == "file-path": repo_args.extend([str(value)]) - elif isinstance(value, bool) and value: - cmd_args.append(f"--{key}") + elif isinstance(value, bool): + if value: + cmd_args.append(f"--{key}") else: cmd_args.extend([f"--{key}", str(value)]) - # First append the url related arguments then append the remaining ones - cmd_args = repo_args + params_args + cmd_args - return cmd_args + # Create the temporary YAML file after processing all items + if "params" in item: + temp_file_name = utils.create_temp_yaml( + params_dict, params_file=params_file_path + ) + params_args.extend(["--params-file", temp_file_name]) + + if params_file_path and "params" not in item: + params_args.extend(["--params-file", params_file_path]) + + combined_args = cmd_args + repo_args + params_args + return combined_args def parse_launch_block(item): repo_args = [] cmd_args = [] + params_args = [] + params_file_path = None + for key, value in item.items(): if key == "pipeline" or key == "url": repo_args.extend([str(value)]) elif key == "params": - temp_file_name = utils.create_temp_yaml(value) - cmd_args.extend(["--params-file", temp_file_name]) + params_dict = value + elif key == "params-file": + params_file_path = str(value) + elif isinstance(value, bool): + if value: + cmd_args.append(f"--{key}") else: cmd_args.extend([f"--{key}", str(value)]) - cmd_args = repo_args + cmd_args + if "params" in item: + temp_file_name = utils.create_temp_yaml( + params_dict, params_file=params_file_path + ) + params_args.extend(["--params-file", temp_file_name]) + + if params_file_path and "params" not in item: + params_args.extend(["--params-file", params_file_path]) + + cmd_args = cmd_args + repo_args + params_args return cmd_args diff --git a/seqerakit/seqeraplatform.py b/seqerakit/seqeraplatform.py index be34dd1..6381e10 100644 --- a/seqerakit/seqeraplatform.py +++ b/seqerakit/seqeraplatform.py @@ -101,6 +101,11 @@ def _execute_command(self, full_cmd, to_json=False): return json.loads(stdout) if to_json else stdout + def _execute_info_command(self): + # Directly execute 'tw info' command + command = "tw info" + return self._execute_command(command) + def _handle_command_errors(self, stdout): logging.error(stdout) @@ -126,7 +131,10 @@ def _tw_run(self, cmd, *args, **kwargs): # Allow any 'tw' subcommand to be called as a method. def __getattr__(self, cmd): - return self.TwCommand(self, cmd.replace("_", "-")) + if cmd == "info": + return self._execute_info_command + else: + return self.TwCommand(self, cmd.replace("_", "-")) class ResourceExistsError(Exception): diff --git a/seqerakit/utils.py b/seqerakit/utils.py index 6048c1a..b58c3d1 100644 --- a/seqerakit/utils.py +++ b/seqerakit/utils.py @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import json import tempfile import os @@ -65,7 +66,7 @@ def check_if_exists(json_data, namekey, namevalue): """ if not json_data: return False - + logging.info(f"Checking if {namekey} {namevalue} exists in Seqera Platform...") # Regex pattern to match environment variables in the string env_var_pattern = re.compile(r"\$\{?[\w]+\}?") @@ -111,27 +112,46 @@ def is_url(s): return False -def create_temp_yaml(params_dict): +class quoted_str(str): + pass + + +def quoted_str_representer(dumper, data): + return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"') + + +yaml.add_representer(quoted_str, quoted_str_representer) + + +def create_temp_yaml(params_dict, params_file=None): """ - Create a generic temporary yaml file given a dictionary + Create a temporary YAML file given a dictionary. + Optionally combine with contents from a JSON or YAML file if provided. """ - class quoted_str(str): - pass + def read_file(file_path): + with open(file_path, "r") as file: + return ( + json.load(file) if file_path.endswith(".json") else yaml.safe_load(file) + ) + + combined_params = {} + + if params_file: + file_params = read_file(params_file) + combined_params.update(file_params) - def quoted_str_representer(dumper, data): - return dumper.represent_scalar("tag:yaml.org,2002:str", data, style='"') + combined_params.update(params_dict) - yaml.add_representer(quoted_str, quoted_str_representer) - params_dict = { - k: quoted_str(os.path.expandvars(v)) if isinstance(v, str) else v - for k, v in params_dict.items() - } + for key, value in combined_params.items(): + if isinstance(value, str): + expanded_value = re.sub(r"\$\{?\w+\}?", replace_env_var, value) + combined_params[key] = quoted_str(expanded_value) with tempfile.NamedTemporaryFile( mode="w", delete=False, suffix=".yaml" ) as temp_file: - yaml.dump(params_dict, temp_file) + yaml.dump(combined_params, temp_file, default_flow_style=False) return temp_file.name diff --git a/setup.py b/setup.py index d6acc5b..f5bf293 100644 --- a/setup.py +++ b/setup.py @@ -2,7 +2,7 @@ from setuptools import find_packages, setup -VERSION = "0.4.2" +VERSION = "0.4.5" with open("README.md") as f: readme = f.read() diff --git a/templates/actions.yml b/templates/actions.yml index 9d0bf16..d243b9a 100644 --- a/templates/actions.yml +++ b/templates/actions.yml @@ -1,24 +1,24 @@ ## To see the full list of options available run: "tw actions add" actions: - - type: 'github' # required - name: 'my_github_action' # required - pipeline: 'https://github.com/my_username/my_repo' # required - workspace: 'my_organization/my_workspace' # required - compute-env: 'my_aws_compute_environment' # required - work-dir: 's3://my_bucket' # required - revision: 'main' # required - profile: 'test' # optional - params: # optional + - type: 'github' # required + name: 'my_github_action' # required + pipeline: 'https://github.com/my_username/my_repo' # required + workspace: 'my_organization/my_workspace' # required + compute-env: 'my_aws_compute_environment' # required + work-dir: 's3://my_bucket' # required + revision: 'main' # required + profile: 'test' # optional + params: # optional outdir: 's3://my_bucket/my_results' - overwrite: True # optional - - type: 'tower' # required # TODO - name: 'my_tower_action' # required - pipeline: 'https://github.com/my_username/my_repo' # required - workspace: 'my_organization/my_workspace' # required - compute-env: 'my_aws_compute_environment' # required - work-dir: 's3://my_bucket' # required - revision: 'main' # required - profile: 'test' # optional - params: # optional + overwrite: True # optional + - type: 'tower' # required + name: 'my_tower_action' # required + pipeline: 'https://github.com/my_username/my_repo' # required + workspace: 'my_organization/my_workspace' # required + compute-env: 'my_aws_compute_environment' # required + work-dir: 's3://my_bucket' # required + revision: 'main' # required + profile: 'test' # optional + params: # optional outdir: 's3://my_bucket/my_results' - overwrite: True # optional + overwrite: True # optional diff --git a/templates/compute-envs.yml b/templates/compute-envs.yml index e8f3c33..b6bf171 100644 --- a/templates/compute-envs.yml +++ b/templates/compute-envs.yml @@ -3,21 +3,34 @@ ## 1. Explicitly in this file ## 2. Via a JSON file exported from Seqera Platform with the "tw compute-envs export" command compute-envs: - - name: 'my_aws_compute_environment' # required - workspace: 'my_organization/my_workspace' # required - credentials: 'my_aws_credentials' # required - wait: 'AVAILABLE' # optional - file-path: './compute-envs/my_aws_compute_environment.json' # required - overwrite: True # optional - - name: 'my_azure_compute_environment' # required - workspace: 'my_organization/my_workspace' # required - credentials: 'my_azure_credentials' # required - wait: 'AVAILABLE' # optional - file-path: './compute-envs/my_azure_compute_environment.json' # required - overwrite: True # optional - - name: 'my_google_compute_environment' # required - workspace: 'my_organization/my_workspace' # required - credentials: 'my_google_credentials' # required - wait: 'AVAILABLE' # optional - file-path: './compute-envs/my_google_compute_environment.json' # required - overwrite: True # optional +# To create a compute environment from a JSON configuration file (AWS Example) + - name: 'my_aws_compute_environment' # required + workspace: 'my_organization/my_workspace' # required + credentials: 'my_aws_credentials' # required + wait: 'AVAILABLE' # optional + file-path: './compute-envs/my_aws_compute_environment.json' # required + overwrite: True # optional + +# To create a compute environment with options specified through YAML (AWS Example) + - type: aws-batch # required + config-mode: forge # required for AWS and Azure + name: 'my_aws_compute_environment' # required + workspace: 'my_organization/my_workspace' # required + credentials: 'my_aws_credentials' # required + region: 'eu-west-1' # required + work-dir: 's3://my-bucket' # required + provisioning-model: 'SPOT' # optional + fusion-v2: False # optional + wave: False # optional + fargate: False # optional + fast-storage: False # optional + instance-types: 'c6i,r6i,m6i' # optional, comma separated list + no-ebs-auto-scale: True # optional + max-cpus: 500 # required + labels: 'label1,label2' # optional, comma separated list + vpc-id: 'vpc-1234567890' # optional + subnets: 'subnet-1234567890,subnet-1234567891' # optional, comma separated list + security-groups: 'sg-1234567890,sg-1234567891' # optional, comma separated list + allow-buckets: 's3://my-bucket,s3://my-other-bucket' # optional, comma separated list + wait: 'AVAILABLE' # optional + overwrite: False # optional \ No newline at end of file diff --git a/templates/credentials.yml b/templates/credentials.yml index 490d5d5..d60dbf5 100644 --- a/templates/credentials.yml +++ b/templates/credentials.yml @@ -2,36 +2,36 @@ ## To avoid exposing sensitive information about your credentials, ## use environment variables to supply passwords and secret keys credentials: - - type: 'github' # required - name: 'my_github_credentials' # required - workspace: 'my_organization/my_workspace' # required - username: 'my_username' # required - password: '$SEQPLATFORM_GITHUB_PASSWORD' # required - overwrite: True # optional - - type: 'container-reg' # required - name: 'my_dockerhub_credentials' # required - workspace: 'my_organization/my_workspace' # required - username: 'my_username' # required - password: '$DOCKERHUB_PASSWORD' # required - registry: 'docker.io' # required - overwrite: True # optional - - type: 'google' # required - name: 'my_google_credentials' # required - workspace: 'my_organization/my_workspace' # required - key: '$GOOGLE_KEY' # required - overwrite: True # optional - - type: 'aws' # required - name: 'my_aws_credentials' # required - workspace: 'my_organization/my_workspace' # required - access-key: '$AWS_ACCESS_KEY_ID' # required - secret-key: '$AWS_SECRET_ACCESS_KEY' # required - assume-role-arn: '$AWS_ASSUME_ROLE_ARN' # required - overwrite: True # optional - - type: 'azure' # required - name: 'my_azure_credentials' # required - workspace: 'my_organization/my_workspace' # required - batch-key: '$AZURE_BATCH_KEY' # required - batch-name: 'my_storage_name' # required - storage-key: '$AZURE_STORAGE_KEY' # required - storage-name: 'my_storage_name' # required - overwrite: True # optional + - type: 'github' # required + name: 'my_github_credentials' # required + workspace: 'my_organization/my_workspace' # required + username: 'my_username' # required + password: '$SEQPLATFORM_GITHUB_PASSWORD' # required + overwrite: True # optional + - type: 'container-reg' # required + name: 'my_dockerhub_credentials' # required + workspace: 'my_organization/my_workspace' # required + username: 'my_username' # required + password: '$DOCKERHUB_PASSWORD' # required + registry: 'docker.io' # required + overwrite: True # optional + - type: 'google' # required + name: 'my_google_credentials' # required + workspace: 'my_organization/my_workspace' # required + key: '$GOOGLE_KEY' # required + overwrite: True # optional + - type: 'aws' # required + name: 'my_aws_credentials' # required + workspace: 'my_organization/my_workspace' # required + access-key: '$AWS_ACCESS_KEY_ID' # required + secret-key: '$AWS_SECRET_ACCESS_KEY' # required + assume-role-arn: '$AWS_ASSUME_ROLE_ARN' # required + overwrite: True # optional + - type: 'azure' # required + name: 'my_azure_credentials' # required + workspace: 'my_organization/my_workspace' # required + batch-key: '$AZURE_BATCH_KEY' # required + batch-name: 'my_storage_name' # required + storage-key: '$AZURE_STORAGE_KEY' # required + storage-name: 'my_storage_name' # required + overwrite: True # optional diff --git a/templates/launch.yml b/templates/launch.yml index d9d449b..c281c39 100644 --- a/templates/launch.yml +++ b/templates/launch.yml @@ -5,18 +5,18 @@ ## Note: overwrite is not supported for "tw launch" launch: - - name: 'my_launchpad_launch' # required - workspace: 'my_organization/my_workspace' # required - pipeline: 'my_launchpad_pipeline' # required - params: # optional + - name: 'my_launchpad_launch' # required + workspace: 'my_organization/my_workspace' # required + pipeline: 'my_launchpad_pipeline' # required + params: # optional outdir: 's3://my_bucket/my_results' - - name: 'my_remote_launch' # required - workspace: 'my_organization/my_workspace' # required - compute-env: 'my_aws_compute_environment' # required - pipeline: 'https://github.com/my_username/my_repo' # required - work-dir: 's3://my_bucket' # optional - profile: 'test' # optional - revision: 'main' # optional - params-file: './pipelines/my_params.yml' # optional - config: './pipelines/my_nextflow.config' # optional - pre-run: './pipelines/my_pre_run.txt' # optional + - name: 'my_remote_launch' # required + workspace: 'my_organization/my_workspace' # required + compute-env: 'my_aws_compute_environment' # required + pipeline: 'https://github.com/my_username/my_repo' # required + work-dir: 's3://my_bucket' # optional + profile: 'test' # optional + revision: 'main' # optional + params-file: './pipelines/my_params.yml' # optional + config: './pipelines/my_nextflow.config' # optional + pre-run: './pipelines/my_pre_run.txt' # optional diff --git a/templates/pipelines.yml b/templates/pipelines.yml index 1f08dd9..3088226 100644 --- a/templates/pipelines.yml +++ b/templates/pipelines.yml @@ -3,22 +3,22 @@ ## 1. Explicitly in this file ## 2. Via a JSON file exported from SeqeraPlatform with the "tw pipelines export" command pipelines: - - name: 'my_first_pipeline' # required - workspace: 'my_organization/my_workspace' # required - description: 'My test pipeline' # optional - compute-env: 'my_aws_compute_environment' # required - work-dir: 's3://my_bucket' # optional - profile: 'test' # optional - revision: 'main' # required - params: # optional + - name: 'my_first_pipeline' # required + workspace: 'my_organization/my_workspace' # required + description: 'My test pipeline' # optional + compute-env: 'my_aws_compute_environment' # required + work-dir: 's3://my_bucket' # optional + profile: 'test' # optional + revision: 'main' # required + params: # optional outdir: 's3://my-bucket/my_results' - config: './pipelines/my_nextflow.config' # optional - pre-run: './pipelines/my_pre_run.txt' # optional - url: 'https://github.com/my_username/my_repo' # required - overwrite: True # optional - - name: 'my_second_pipeline' # required - workspace: 'my_organization/my_workspace' # required - description: 'My test pipeline' # optional - compute-env: 'my_aws_compute_environment' # required - file-path: './pipelines/my_pipeline.json' # required - overwrite: True # optional + config: './pipelines/my_nextflow.config' # optional + pre-run: './pipelines/my_pre_run.txt' # optional + url: 'https://github.com/my_username/my_repo' # required + overwrite: True # optional + - name: 'my_second_pipeline' # required + workspace: 'my_organization/my_workspace' # required + description: 'My test pipeline' # optional + compute-env: 'my_aws_compute_environment' # required + file-path: './pipelines/my_pipeline.json' # required + overwrite: True # optional diff --git a/tests/e2e/aws-e2e.yml b/tests/e2e/aws-e2e.yml index 817c2f8..dd503e3 100644 --- a/tests/e2e/aws-e2e.yml +++ b/tests/e2e/aws-e2e.yml @@ -71,7 +71,7 @@ datasets: description: 'Samplesheet to run the nf-core/rnaseq pipeline from end-to-end' header: true workspace: '$SEQERA_ORGANIZATION_NAME/$SEQERA_WORKSPACE_NAME' - file-path: '../examples/yaml/datasets/rnaseq_samples.csv' + file-path: './examples/yaml/datasets/rnaseq_samples.csv' overwrite: True pipelines: - name: '$PIPELINE_NAME_PREFIX-hello-world' diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py index ac9f459..ab7158c 100644 --- a/tests/unit/test_helper.py +++ b/tests/unit/test_helper.py @@ -1,42 +1,241 @@ -import unittest -from unittest.mock import mock_open, patch +from unittest.mock import mock_open from seqerakit import helper +import yaml +import pytest -mocked_yaml = """ -datasets: - - name: 'test_dataset1' - description: 'My test dataset 1' - header: true - workspace: 'my_organization/my_workspace' - file-path: './examples/yaml/datasets/samples.csv' - overwrite: True -""" -mocked_file = mock_open(read_data=mocked_yaml) +# Fixture to mock a YAML file +@pytest.fixture +def mock_yaml_file(mocker): + def _mock_yaml_file(test_data, file_name="mock_file.yaml"): + # Convert test data to YAML format + yaml_content = yaml.dump(test_data, default_flow_style=False) + mock_file = mock_open(read_data=yaml_content) + mocker.patch("builtins.open", mock_file) + return file_name -class TestYamlParserFunctions(unittest.TestCase): - @patch("builtins.open", mocked_file) - def test_parse_datasets_yaml(self): - result = helper.parse_all_yaml(["test_path.yaml"]) - expected_block_output = [ + return _mock_yaml_file + + +def test_create_mock_organization_yaml(mock_yaml_file): + test_data = { + "organizations": [ + { + "name": "test_organization1", + "full-name": "My test organization 1", + "description": "My test organization 1", + "location": "Global", + "url": "https://example.com", + "overwrite": True, + } + ] + } + expected_block_output = [ + { + "cmd_args": [ + "--description", + "My test organization 1", + "--full-name", + "My test organization 1", + "--location", + "Global", + "--name", + "test_organization1", + "--url", + "https://example.com", + ], + "overwrite": True, + } + ] + + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "organizations" in result + assert result["organizations"] == expected_block_output + + +def test_create_mock_workspace_yaml(mock_yaml_file): + test_data = { + "workspaces": [ { - "cmd_args": [ - "--header", - "./examples/yaml/datasets/samples.csv", - "--name", - "test_dataset1", - "--workspace", - "my_organization/my_workspace", - "--description", - "My test dataset 1", - ], + "name": "test_workspace1", + "full-name": "My test workspace 1", + "organization": "my_organization", + "description": "My test workspace 1", + "visibility": "PRIVATE", "overwrite": True, } ] + } + expected_block_output = [ + { + "cmd_args": [ + "--description", + "My test workspace 1", + "--full-name", + "My test workspace 1", + "--name", + "test_workspace1", + "--organization", + "my_organization", + "--visibility", + "PRIVATE", + ], + "overwrite": True, + } + ] + + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "workspaces" in result + assert result["workspaces"] == expected_block_output + + +def test_create_mock_dataset_yaml(mock_yaml_file): + test_data = { + "datasets": [ + { + "name": "test_dataset1", + "description": "My test dataset 1", + "workspace": "my_organization/my_workspace", + "header": True, + "file-path": "./examples/yaml/datasets/samples.csv", + "overwrite": True, + } + ] + } + expected_block_output = [ + { + "cmd_args": [ + "./examples/yaml/datasets/samples.csv", + "--name", + "test_dataset1", + "--workspace", + "my_organization/my_workspace", + "--description", + "My test dataset 1", + "--header", + ], + "overwrite": True, + } + ] + + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "datasets" in result + assert result["datasets"] == expected_block_output + + +def test_create_mock_computeevs_yaml(mock_yaml_file): + test_data = { + "compute-envs": [ + { + "name": "test_computeenv", + "workspace": "my_organization/my_workspace", + "credentials": "my_credentials", + "file-path": "./examples/yaml/computeenvs/computeenvs.yaml", + "wait": "AVAILABLE", + "fusion-v2": True, + "fargate": False, + "overwrite": True, + } + ], + } + + expected_block_output = [ + { + "cmd_args": [ + "--credentials", + "my_credentials", + "./examples/yaml/computeenvs/computeenvs.yaml", + "--fusion-v2", + "--name", + "test_computeenv", + "--wait", + "AVAILABLE", + "--workspace", + "my_organization/my_workspace", + ], + "overwrite": True, + } + ] + + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "compute-envs" in result + assert result["compute-envs"] == expected_block_output + + +def test_create_mock_pipeline_add_yaml(mock_yaml_file): + test_data = { + "pipelines": [ + { + "name": "test_pipeline1", + "url": "https://github.com/nf-core/test_pipeline1", + "workspace": "my_organization/my_workspace", + "description": "My test pipeline 1", + "compute-env": "my_computeenv", + "work-dir": "s3://work", + "profile": "test", + "params-file": "./examples/yaml/pipelines/test_pipeline1/params.yaml", + "config": "./examples/yaml/pipelines/test_pipeline1/config.txt", + "pre-run": "./examples/yaml/pipelines/test_pipeline1/pre_run.sh", + "revision": "master", + "overwrite": True, + "stub-run": True, + } + ] + } + + # params file cmds parsed separately + expected_block_output = [ + { + "cmd_args": [ + "--compute-env", + "my_computeenv", + "--config", + "./examples/yaml/pipelines/test_pipeline1/config.txt", + "--description", + "My test pipeline 1", + "--name", + "test_pipeline1", + "--pre-run", + "./examples/yaml/pipelines/test_pipeline1/pre_run.sh", + "--profile", + "test", + "--revision", + "master", + "--stub-run", + "--work-dir", + "s3://work", + "--workspace", + "my_organization/my_workspace", + "https://github.com/nf-core/test_pipeline1", + "--params-file", + "./examples/yaml/pipelines/test_pipeline1/params.yaml", + ], + "overwrite": True, + } + ] + + file_path = mock_yaml_file(test_data) + result = helper.parse_all_yaml([file_path]) + + assert "pipelines" in result + assert result["pipelines"] == expected_block_output - self.assertIn("datasets", result) - self.assertEqual(result["datasets"], expected_block_output) +def test_empty_yaml_file(mock_yaml_file): + test_data = {} + file_path = mock_yaml_file(test_data) -# TODO: add more tests for other functions in helper.py + with pytest.raises(ValueError) as e: + helper.parse_all_yaml([file_path]) + assert f"The file '{file_path}' is empty or does not contain valid data." in str( + e.value + )