From 3a5127f9e5c02cafb3dfd36bee0b05f79ee5ec5b Mon Sep 17 00:00:00 2001 From: Giulia Garcia <147185635+giuliaelgarcia@users.noreply.github.com> Date: Wed, 29 May 2024 14:39:36 +0100 Subject: [PATCH 1/5] Update pipeline_integration.py testing possible solution for threads --- panpipes/panpipes/pipeline_integration.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/panpipes/panpipes/pipeline_integration.py b/panpipes/panpipes/pipeline_integration.py index 4b2ad944..ba9e071e 100644 --- a/panpipes/panpipes/pipeline_integration.py +++ b/panpipes/panpipes/pipeline_integration.py @@ -853,6 +853,9 @@ def run_scib_metrics(infile, outfile): if PARAMS['scib_atac']: cmd += " --atac_cell_type %(scib_atac)s" + job_kwargs["job_threads"] = PARAMS['resources_threads_medium'] + log_msg = f"TASK: 'run_scib_metrics'" + f" IN CASE OF ERROR, PLEASE REFER TO : '{outfile}' FOR MORE INFORMATION." + get_logger().info(log_msg) P.run(cmd, **job_kwargs) From 3e8100b7206d4a660ea37b8683ba4d439e26fa0d Mon Sep 17 00:00:00 2001 From: bio-la Date: Tue, 11 Jun 2024 14:28:52 +0200 Subject: [PATCH 2/5] changes to integration documentation --- docs/yaml_docs/pipeline_integration_yml.md | 10 +++++++--- panpipes/panpipes/pipeline_integration/pipeline.yml | 10 +++++++--- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/docs/yaml_docs/pipeline_integration_yml.md b/docs/yaml_docs/pipeline_integration_yml.md index d28758ea..ab81905c 100644 --- a/docs/yaml_docs/pipeline_integration_yml.md +++ b/docs/yaml_docs/pipeline_integration_yml.md @@ -28,18 +28,22 @@ Specified by the following parameters: - threads_high `Integer`, Default: 1
Number of threads used for high intensity computing tasks. For each thread, there must be enough memory to load your MuData object which was created in the preprocessing step of - the workflow. + the workflow. In this workflow, all the integration, batch correction and dimensionality reduction tasks run with threads high - threads_medium `Integer`, Default: 1
Number of threads used for medium intensity computing tasks. - For each thread, there must be enough memory to load your mudata and do computationally light tasks. + For each thread, there must be enough memory to load your mudata and do computationally light tasks. In this workflow, collating results after integration and scib metrics calculation run with threads_medium. + - threads_low `Integer`, Default: 1
Number of threads used for low intensity computing tasks. - For each thread, there must be enough memory to load text files and do plotting, requires much less memory than the other two. + For each thread, there must be enough memory to load text files and do plotting, requires much less memory than the other two. + In this workflow, plotting and lisi calculation run with threads_low + - threads_gpu `Integer`, Default: 2
Number of cores per gpu used for computing tasks. For each thread, there must be enough memory to compute the tasks above. + In this workflow, if the gpu queues are defined below, `scvi` algorithms and `mofa` can run on gpu, otherwise `threads_high` argument is used condaenv `String`
Path to conda environment that should be used to run panpipes. diff --git a/panpipes/panpipes/pipeline_integration/pipeline.yml b/panpipes/panpipes/pipeline_integration/pipeline.yml index 794580be..d37a2504 100644 --- a/panpipes/panpipes/pipeline_integration/pipeline.yml +++ b/panpipes/panpipes/pipeline_integration/pipeline.yml @@ -9,18 +9,21 @@ # Compute resources options #-------------------------- resources: + # all the integration,batch correction and dimensionality reduction tasks run with threads high threads_high: 1 + # collating results and scib metrics run with threads_medium threads_medium: 1 + # plotting and lisi calculation run with threads_low threads_low: 1 - + # if the gpu queues are defined below, scvi algorithms and mofa can run on gpu, otherwise threads_high argument is used threads_gpu: 2 -condaenv: - queues: long: gpu: +condaenv: + # -------------------------------- # Loading and merging data options # -------------------------------- @@ -136,6 +139,7 @@ atac: # multimodal integration # ---------------------- # remember to specify knn graph params in the section "neighbors" +# These processes will run with computational resources "high" multimodal: run: True tools: From e6ad31b6e656281a8ead91d95432d8a2be1c2602 Mon Sep 17 00:00:00 2001 From: bio-la Date: Tue, 11 Jun 2024 14:29:00 +0200 Subject: [PATCH 3/5] release --- docs/release_notes.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/docs/release_notes.md b/docs/release_notes.md index 606b64d0..5abd8418 100644 --- a/docs/release_notes.md +++ b/docs/release_notes.md @@ -5,7 +5,21 @@ ### added + +### fixed + + + +### dependencies + + +## v0.5.0 + +### added + - added Tangram to `deconvolution_spatial` +- added scib metrics calculation to `integration`, using the [scib-metrics package](https://scib-metrics.readthedocs.io/en/latest/index.html) +- [extra documentation](https://panpipes-pipelines.readthedocs.io/en/latest/) ### fixed From e67893e5aa936e5be07c8f2ddd7b41a1e710fb0a Mon Sep 17 00:00:00 2001 From: bio-la Date: Wed, 12 Jun 2024 14:25:52 +0200 Subject: [PATCH 4/5] fixes to pin scanpy --- panpipes/panpipes/pipeline_integration/pipeline.yml | 8 ++++---- pyproject.toml | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/panpipes/panpipes/pipeline_integration/pipeline.yml b/panpipes/panpipes/pipeline_integration/pipeline.yml index d37a2504..6ab08960 100644 --- a/panpipes/panpipes/pipeline_integration/pipeline.yml +++ b/panpipes/panpipes/pipeline_integration/pipeline.yml @@ -9,13 +9,13 @@ # Compute resources options #-------------------------- resources: - # all the integration,batch correction and dimensionality reduction tasks run with threads high + # all the uni/multimodal integrations and dimensionality reduction tasks run with threads high (CPU) threads_high: 1 - # collating results and scib metrics run with threads_medium + # collating results and scib metrics run with threads_medium (CPU) threads_medium: 1 - # plotting and lisi calculation run with threads_low + # plotting and lisi calculation run with threads_low (CPU) threads_low: 1 - # if the gpu queues are defined below, scvi algorithms and mofa can run on gpu, otherwise threads_high argument is used + # if the gpu queues are defined below, specify the gpu threads, otherwise threads_high argument (CPU) is used threads_gpu: 2 queues: diff --git a/pyproject.toml b/pyproject.toml index 8e3da669..c302a170 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "pyyaml", "ruffus", "scanorama", - "scanpy", + "scanpy<=1.9.3", "leidenalg", "louvain", "scib", From 72b990549612dfba88bff82d7508756dc37eb33f Mon Sep 17 00:00:00 2001 From: bio-la Date: Wed, 12 Jun 2024 14:45:31 +0200 Subject: [PATCH 5/5] unpin scanpy --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index c302a170..8e3da669 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,7 @@ dependencies = [ "pyyaml", "ruffus", "scanorama", - "scanpy<=1.9.3", + "scanpy", "leidenalg", "louvain", "scib",