Merge pull request #279 from DendrouLab/threads_g

Fixed threads number in pipeline_integration.py

docs/yaml_docs/pipeline_integration_yml.md

@@ -28,18 +28,22 @@ Specified by the following parameters:
   - <span class="parameter">threads_high</span> `Integer`, Default: 1<br>
    Number of threads used for high intensity computing tasks. 
    For each thread, there must be enough memory to load your MuData object which was created in the preprocessing step of 
-   the workflow.
+   the workflow. In this workflow, all the integration, batch correction and dimensionality reduction tasks run with threads high
 
   - <span class="parameter">threads_medium</span> `Integer`, Default: 1<br>
    Number of threads used for medium intensity computing tasks.
-   For each thread, there must be enough memory to load your mudata and do computationally light tasks.
+   For each thread, there must be enough memory to load your mudata and do computationally light tasks. In this workflow, collating results after integration and scib metrics calculation run with threads_medium.
+
 
   - <span class="parameter">threads_low</span> `Integer`, Default: 1<br>
    Number of threads used for low intensity computing tasks.
-   For each thread, there must be enough memory to load text files and do plotting, requires much less memory than the other two.  
+   For each thread, there must be enough memory to load text files and do plotting, requires much less memory than the other two. 
+   In this workflow, plotting and lisi calculation run with threads_low 
+
   - <span class="parameter">threads_gpu</span> `Integer`, Default: 2<br>
    Number of cores per gpu used for computing tasks.
    For each thread, there must be enough memory to compute the tasks above. 
+   In this workflow, if the gpu queues are defined below, `scvi` algorithms and `mofa` can run on gpu, otherwise `threads_high` argument is used
 
 <span class="parameter">condaenv</span> `String`<br>
   Path to conda environment that should be used to run panpipes.

panpipes/panpipes/pipeline_integration.py

@@ -853,6 +853,9 @@ def run_scib_metrics(infile, outfile):
     if PARAMS['scib_atac']:
         cmd += " --atac_cell_type %(scib_atac)s"
 
+    job_kwargs["job_threads"] = PARAMS['resources_threads_medium']
+    log_msg = f"TASK: 'run_scib_metrics'" + f" IN CASE OF ERROR, PLEASE REFER TO : '{outfile}' FOR MORE INFORMATION."
+    get_logger().info(log_msg)
     P.run(cmd, **job_kwargs)
 
 

panpipes/panpipes/pipeline_integration/pipeline.yml

@@ -9,18 +9,21 @@
 # Compute resources options
 #--------------------------
 resources:
+  # all the uni/multimodal integrations and dimensionality reduction tasks run with threads high (CPU)
   threads_high: 1
+  # collating results and scib metrics run with threads_medium (CPU)
   threads_medium: 1
+  # plotting and lisi calculation run with threads_low (CPU)
   threads_low: 1
-
+  # if the gpu queues are defined below, specify the gpu threads, otherwise threads_high argument (CPU) is used
   threads_gpu: 2
 
-condaenv:
-
 queues:
   long:
   gpu:
 
+condaenv:
+
 # --------------------------------
 # Loading and merging data options
 # --------------------------------
@@ -136,6 +139,7 @@ atac:
 # multimodal integration
 # ----------------------
 # remember to specify knn graph params in the section "neighbors"
+# These processes will run with computational resources "high"
 multimodal:
   run: True
   tools: