From 5904f3ed95026ceeb45ed5de96c6c67d1007ce32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Wed, 30 Oct 2024 10:20:10 +0000 Subject: [PATCH 1/5] added gpu profile for cellpose --- .gitignore | 1 + conf/base.config | 3 +++ modules.json | 3 ++- modules/nf-core/cellpose/cellpose.diff | 23 +++++++++++++++++++++++ modules/nf-core/cellpose/main.nf | 4 +++- nextflow.config | 9 +++++++++ nextflow_schema.json | 9 +++++++-- 7 files changed, 48 insertions(+), 4 deletions(-) create mode 100644 modules/nf-core/cellpose/cellpose.diff diff --git a/.gitignore b/.gitignore index 05aa7e0..9f86a51 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ testing* *.pyc .nf-test.log .nf-test/ +null diff --git a/conf/base.config b/conf/base.config index 71978a4..2d1927b 100644 --- a/conf/base.config +++ b/conf/base.config @@ -58,4 +58,7 @@ process { withName:'ILASTIK_PIXELCLASSIFICATION|ILASTIK_MULTICUT' { label = "process_high" } + withLabel:process_gpu { + ext.use_gpu = { use_gpu } + } } diff --git a/modules.json b/modules.json index 17f6235..040e65b 100644 --- a/modules.json +++ b/modules.json @@ -8,7 +8,8 @@ "cellpose": { "branch": "master", "git_sha": "7c1543a730197e2f0b8ca5141f851c4a2b08bed6", - "installed_by": ["modules"] + "installed_by": ["modules"], + "patch": "modules/nf-core/cellpose/cellpose.diff" }, "deepcell/mesmer": { "branch": "master", diff --git a/modules/nf-core/cellpose/cellpose.diff b/modules/nf-core/cellpose/cellpose.diff new file mode 100644 index 0000000..086698d --- /dev/null +++ b/modules/nf-core/cellpose/cellpose.diff @@ -0,0 +1,23 @@ +Changes in module 'nf-core/cellpose' +Changes in 'cellpose/main.nf': +--- modules/nf-core/cellpose/main.nf ++++ modules/nf-core/cellpose/main.nf +@@ -1,8 +1,10 @@ + process CELLPOSE { + tag "$meta.id" + label 'process_medium' ++ label 'process_gpu' + +- container "docker.io/biocontainers/cellpose:3.0.1_cv1" ++ container "${ task.ext.use_gpu ? 'docker.io/kbestak/cellpose:3.0.1_cuda12.2.0' : ++ 'docker.io/biocontainers/cellpose:3.0.1_cv1'}" + + input: + tuple val(meta), path(image) + +'modules/nf-core/cellpose/meta.yml' is unchanged +'modules/nf-core/cellpose/tests/main.nf.test' is unchanged +'modules/nf-core/cellpose/tests/main.nf.test.snap' is unchanged +'modules/nf-core/cellpose/tests/nextflow_wflows.config' is unchanged +'modules/nf-core/cellpose/tests/tags.yml' is unchanged +************************************************************ diff --git a/modules/nf-core/cellpose/main.nf b/modules/nf-core/cellpose/main.nf index f100904..cb32012 100644 --- a/modules/nf-core/cellpose/main.nf +++ b/modules/nf-core/cellpose/main.nf @@ -1,8 +1,10 @@ process CELLPOSE { tag "$meta.id" label 'process_medium' + label 'process_gpu' - container "docker.io/biocontainers/cellpose:3.0.1_cv1" + container "${ task.ext.use_gpu ? 'docker.io/kbestak/cellpose:3.0.1_cuda12.2.0' : + 'docker.io/biocontainers/cellpose:3.0.1_cv1'}" input: tuple val(meta), path(image) diff --git a/nextflow.config b/nextflow.config index aff1a48..11040a6 100644 --- a/nextflow.config +++ b/nextflow.config @@ -56,6 +56,9 @@ params { max_multiqc_email_size = '25.MB' multiqc_methods_description = null + // GPU for segmentation + use_gpu = false + // Boilerplate options outdir = null publish_dir_mode = 'copy' @@ -187,6 +190,12 @@ profiles { } test { includeConfig 'conf/test.config' } test_full { includeConfig 'conf/test_full.config' } + gpu { + docker.runOptions = '-u $(id -u):$(id -g) --gpus device=0' + apptainer.runOptions = '--no-mount tmp --writable-tmpfs --nv' + singularity.runOptions = '--no-mount tmp --writable-tmpfs --nv' + use_gpu = true + } } // Load nf-core custom profiles from different Institutions diff --git a/nextflow_schema.json b/nextflow_schema.json index 870e933..82e6266 100644 --- a/nextflow_schema.json +++ b/nextflow_schema.json @@ -145,7 +145,7 @@ }, "clahe_kernel": { "type": "number", - "default": 25, + "default": 25.0, "description": "Kernel size to be used by CLAHE.", "fa_icon": "far fa-object-group" }, @@ -411,5 +411,10 @@ { "$ref": "#/$defs/generic_options" } - ] + ], + "properties": { + "use_gpu": { + "type": "boolean" + } + } } From a36ccc57126aba9d66e497b142955d1018d5aeef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Wed, 30 Oct 2024 11:11:40 +0000 Subject: [PATCH 2/5] check --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2bdcc90..b9295ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 - Updated pipeline tests to account for Cellpose3 changes - Introduced bug - new MultiQC does not read the custom files - fix ASAP - updated Zenodo doi to cite all versions +- GPU support for Cellpose ### `Removed` From 90498aba5b6b591d8bd75e4d919b1e05b2cce5ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Fri, 1 Nov 2024 18:27:25 +0000 Subject: [PATCH 3/5] added documentation for gpu --- docs/usage.md | 54 ++++++++++++++++++++++++++++++++++++++++++++ workflows/molkart.nf | 16 ++++++------- 2 files changed, 62 insertions(+), 8 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index eb2bebb..b2b9d33 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -97,6 +97,7 @@ mesmer_image_mpp: 0.138 mesmer_compartment: "whole-cell" ilastik_pixel_project: null ilastik_multicut_project: null +skip_mindagap: false mindagap_tilesize: 2144 mindagap_boxsize: 3 mindagap_loopnum: 40 @@ -128,6 +129,57 @@ After training a Cellpose 2.0 model, or creating ilastik Pixel Classification an nextflow run nf-core/molkart --input ./samplesheet.csv --outdir ./results -profile docker --segmentation_method cellpose,ilastik --cellpose_custom_model /path/to/model --ilastik_pixel_project /path/to/pixel_classifier.ilp --ilastik_multicut_project /path/to/multicut.ilp ``` +### Skipping preprocessing steps + +:::tip If gaps are not present in the input data, MINDAGAP functionality can be turned off. ::: + +Depending on the usecase, Mindagap (both MINDAGAP/MINDAGAP and MINDAGAP/DUPLICATEFINDER) can be skipped if input data does not contain gaps. It should be noted the default action is to perform Mindagap, if not needed, it can be skipped by specifying the `skip_mindagap` parameter. + +:::tip If signal intensity is even across the input image, CLAHE can be skipped. Usage of CLAHE is a recommendation for membrane channels with highly specific, but uneven signal. ::: + +Contrast-limited adaptive histogram equalization (CLAHE) is run by default. Care should be taken to provide correct parameters for local histogram adjustment. If signal intensity is even across the input image, CLAHE can be skipped by specifying the `skip_clahe` parameter. + +### GPU acceleration + +:::warning{title="Experimental feature"} This is an experimental feature and may produce errors. If you encounter any issues, please report them on the nf-core/molkart GitHub repository. ::: + +- GPU acceleration has only been tested with Docker +- CUDA 12.2 is required. + +Tools with GPU acceleration support within the pipeline: + +- Cellpose + +To utilize GPU acceleration, you need to specify the gpu profile. This will make the tool steps use cuda-enabled environments and it will tell the tools to use the GPU. All processes which support GPU acceleration are marked with the process_gpu label. + +You also need to make sure that the tasks are run on a machine with a GPU. If all tasks are run on a machine with a GPU, no further action is needed. If you are running the pipeline on a slurm cluster, where there is dedicated queue for GPU jobs, you need additional configuration that might look like this: + +``` +process { + withLabel:process_gpu { + queue = '' + clusterOptions = '--gpus 1' + } +} +``` + +:::tip More information on how to configure Slurm in Nextflow can be found here. Depending on your cluster configuration, you might need to adjust the clusterOptions to one of the following: + + --gpus 1 (as in the example above) + --gpus-per-node=1 + --gres=gpu:1 + +::: + +:::tip If your jobs get assigned to the correct nodes, but the GPU is not utilized, you might need to add the following configuration: +`singularity.runOptions = '--no-mount tmp --writable-tmpfs --nv --env CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES --env ROCR_VISIBLE_DEVICES=$ROCR_VISIBLE_DEVICES --env ZE_AFFINITY_MASK=$ZE_AFFINITY_MASK --env NVIDIA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES` + +The first part (--no-mount tmp --writable-tmpfs --nv) is set by default in the gpu profile. The rest of this configuration is needed in some cases to make the GPU visible to the container. ::: + +For different executors, the configuration might look different. Once a wider range of users have tested the GPU support, we will provide more detailed instructions for different executors. + +:::tip Many thanks to the nf-core/scdownstream pipeline contributors with providing detailed explanations on having gpu profiles within nf-core pipelines ::: + ### Updating the pipeline When you run the above command, Nextflow automatically pulls the pipeline code from GitHub and stores it as a cached version. When running the pipeline after this, it will always use the cached version if available - even if the pipeline has been updated since. To make sure that you're running the latest version of the pipeline, make sure that you regularly update the cached version of the pipeline: @@ -192,6 +244,8 @@ If `-profile` is not specified, the pipeline will run locally and expect all sof - A generic configuration profile to enable [Wave](https://seqera.io/wave/) containers. Use together with one of the above (requires Nextflow ` 24.03.0-edge` or later). - `conda` - A generic configuration profile to be used with [Conda](https://conda.io/docs/). Please only use Conda as a last resort i.e. when it's not possible to run the pipeline with Docker, Singularity, Podman, Shifter, Charliecloud, or Apptainer. Currently not supported. +- `gpu` + - Configuration profile for GPU acceleration. Note the limitations and additional configurations needed for GPU usage. Currently only supporting Cellpose. ### `-resume` diff --git a/workflows/molkart.nf b/workflows/molkart.nf index 14d89e8..8e7a60e 100644 --- a/workflows/molkart.nf +++ b/workflows/molkart.nf @@ -131,8 +131,8 @@ workflow MOLKART { // Combine images with crop_summary for making the same training tiff stacks as ilastik tiff_crop = stack_mix.join(CROPHDF5.out.crop_summary) CROPTIFF( - tiff_crop.map(it -> tuple(it[0],it[1])), - tiff_crop.map(it -> tuple(it[0],it[2])), + tiff_crop.map{ it -> tuple(it[0],it[1]) }, + tiff_crop.map{ it -> tuple(it[0],it[2]) }, ) ch_versions = ch_versions.mix(CROPTIFF.out.versions) MOLKARTQCPNG(CROPTIFF.out.overview.map{ @@ -221,8 +221,8 @@ workflow MOLKART { .combine(Channel.of('ilastik'))) } segmentation_masks.map{ - meta, mask, segmentation -> - new_meta = meta.clone() + def (meta, mask, segmentation) = it + def new_meta = meta.clone() new_meta.segmentation = segmentation [new_meta, mask] }.set { matched_segmasks } @@ -243,16 +243,16 @@ workflow MOLKART { qc_spots .combine(filtered_masks, by: 0) .map { - meta, spots_table, mask, segmethod -> - new_meta = meta.clone() + def (meta, spots_table, mask, segmethod) = it + def new_meta = meta.clone() new_meta.segmentation = segmethod [new_meta, spots_table, mask] } .set { dedup_spots } SPOT2CELL( - dedup_spots.map(it -> tuple(it[0],it[1])), - dedup_spots.map(it -> tuple(it[0],it[2])) + dedup_spots.map{ it -> tuple(it[0],it[1]) }, + dedup_spots.map{ it -> tuple(it[0],it[2]) } ) ch_versions = ch_versions.mix(SPOT2CELL.out.versions) From e72bfd367e19a46d2c1b670eae39b257e3bea78c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Sat, 2 Nov 2024 17:27:41 +0000 Subject: [PATCH 4/5] Updated cellpose ext.args --- conf/modules.config | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index c723eff..84d6912 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -199,7 +199,8 @@ process { params.cellpose_chan2 ? "--chan2 ${params.cellpose_chan2}" : "", params.cellpose_custom_model ? "" : params.cellpose_pretrained_model ? "--pretrained_model ${params.cellpose_pretrained_model}" : "", params.cellpose_flow_threshold ? "--flow_threshold ${params.cellpose_flow_threshold}" : "", - params.cellpose_edge_exclude ? "--exclude_on_edges" : "" + params.cellpose_edge_exclude ? "--exclude_on_edges" : "", + params.use_gpu ? "--use_gpu --gpu_device 1" : "" ].join(" ").trim() publishDir = [ path: "${params.outdir}/segmentation/cellpose", From 67a5eeea476e16acceb573d914408e287bad6038 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Kre=C5=A1imir=20Be=C5=A1tak?= Date: Sat, 2 Nov 2024 17:37:15 +0000 Subject: [PATCH 5/5] Updated cellpose ext.args --- conf/modules.config | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conf/modules.config b/conf/modules.config index 84d6912..696a327 100644 --- a/conf/modules.config +++ b/conf/modules.config @@ -200,7 +200,7 @@ process { params.cellpose_custom_model ? "" : params.cellpose_pretrained_model ? "--pretrained_model ${params.cellpose_pretrained_model}" : "", params.cellpose_flow_threshold ? "--flow_threshold ${params.cellpose_flow_threshold}" : "", params.cellpose_edge_exclude ? "--exclude_on_edges" : "", - params.use_gpu ? "--use_gpu --gpu_device 1" : "" + params.use_gpu ? "--use_gpu --gpu_device 0" : "" ].join(" ").trim() publishDir = [ path: "${params.outdir}/segmentation/cellpose",