Skip to content

Commit 6e58b0c

Browse files
slsevillaslsevilla
authored
Merge pull request #82 from CDCgov/update-resource-management
Update resource management: completes #80
2 parents f336793 + ae5578f commit 6e58b0c

9 files changed

+55
-128
lines changed

‎conf/base.config

+18-19
Original file line numberDiff line numberDiff line change
@@ -10,11 +10,10 @@
1010

1111
process {
1212

13-
// TODO nf-core: Check the defaults for all processes
14-
cpus = { check_max( 1 * task.attempt, 'cpus' ) }
15-
memory = { check_max( 6.GB * task.attempt, 'memory' ) }
16-
time = { check_max( 4.h * task.attempt, 'time' ) }
17-
13+
// Add defaults
14+
cpus = { 1 * task.attempt }
15+
memory = { 6.GB * task.attempt }
16+
time = { 4.h * task.attempt }
1817
errorStrategy = { task.exitStatus in ((130..145) + 104) ? 'retry' : 'finish' }
1918
maxRetries = 1
2019
maxErrors = '-1'
@@ -27,30 +26,30 @@ process {
2726
// TODO nf-core: Customise requirements for specific processes.
2827
// See https://www.nextflow.io/docs/latest/config.html#config-process-selectors
2928
withLabel:process_single {
30-
cpus = { check_max( 1 , 'cpus' ) }
31-
memory = { check_max( 6.GB * task.attempt, 'memory' ) }
32-
time = { check_max( 4.h * task.attempt, 'time' ) }
29+
cpus = { 1 }
30+
memory = { 6.GB * task.attempt }
31+
time = { 4.h * task.attempt }
3332
}
3433
withLabel:process_low {
35-
cpus = { check_max( 2 * task.attempt, 'cpus' ) }
36-
memory = { check_max( 12.GB * task.attempt, 'memory' ) }
37-
time = { check_max( 4.h * task.attempt, 'time' ) }
34+
cpus = { 2 * task.attempt }
35+
memory = { 12.GB * task.attempt }
36+
time = { 4.h * task.attempt }
3837
}
3938
withLabel:process_medium {
40-
cpus = { check_max( 6 * task.attempt, 'cpus' ) }
41-
memory = { check_max( 36.GB * task.attempt, 'memory' ) }
42-
time = { check_max( 8.h * task.attempt, 'time' ) }
39+
cpus = { 6 * task.attempt }
40+
memory = { 36.GB * task.attempt }
41+
time = { 8.h * task.attempt }
4342
}
4443
withLabel:process_high {
45-
cpus = { check_max( 12 * task.attempt, 'cpus' ) }
46-
memory = { check_max( 72.GB * task.attempt, 'memory' ) }
47-
time = { check_max( 16.h * task.attempt, 'time' ) }
44+
cpus = { 12 * task.attempt }
45+
memory = { 72.GB * task.attempt }
46+
time = { 16.h * task.attempt }
4847
}
4948
withLabel:process_long {
50-
time = { check_max( 20.h * task.attempt, 'time' ) }
49+
time = { 20.h * task.attempt }
5150
}
5251
withLabel:process_high_memory {
53-
memory = { check_max( 200.GB * task.attempt, 'memory' ) }
52+
memory = { 200.GB * task.attempt }
5453
}
5554
withLabel:error_ignore {
5655
errorStrategy = 'ignore'

‎conf/scicomp.config

+7-40
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,11 @@ params {
2424
custom_config_version = 'master'
2525

2626
// Default resource parameters. Expecting to be overwritten.
27-
max_memory = '128.GB'
28-
max_cpus = 16
29-
max_time = '240.h'
27+
resourceLimits = [
28+
cpus: 16,
29+
memory: '128.GB',
30+
time: '240.h'
31+
]
3032
}
3133

3234
/*
@@ -121,7 +123,7 @@ profiles {
121123
time = '72.h'
122124

123125
// Set h_vmem option for qsub submissions. +20 memory to h_vmem prevents memory allocation errors.
124-
clusterOptions = { "-l h_vmem=${(check_max((task.memory.toGiga())+20), 'memory').toString().replaceAll(/[\sB]/,'')}G" }
126+
clusterOptions = { "-l h_vmem=${(task.memory.toGiga() + 20).toString()}G" }
125127
}
126128
}
127129
training {
@@ -146,7 +148,7 @@ profiles {
146148
time = '8.h'
147149

148150
// Set h_vmem option for qsub submissions. +20 memory to h_vmem prevents memory allocation errors.
149-
clusterOptions = { "-l h_vmem=${(check_max((task.memory.toGiga())+20), 'memory').toString().replaceAll(/[\sB]/,'')}G" }
151+
clusterOptions = { "-l h_vmem=${((task.memory.toGiga())+20).toString().replaceAll(/[\sB]/,'')}G" }
150152
}
151153
}
152154
debug {
@@ -175,39 +177,4 @@ profiles {
175177
}
176178
}
177179

178-
/*
179-
==========================================================================================
180-
Function to check max resources
181-
==========================================================================================
182-
*/
183180

184-
def check_max(obj, type) {
185-
if (type == 'memory') {
186-
try {
187-
if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
188-
return params.max_memory as nextflow.util.MemoryUnit
189-
else
190-
return obj
191-
} catch (all) {
192-
println "Updating max_memory: '${params.max_memory}' to $obj based on available resources"
193-
return obj
194-
}
195-
} else if (type == 'time') {
196-
try {
197-
if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
198-
return params.max_time as nextflow.util.Duration
199-
else
200-
return obj
201-
} catch (all) {
202-
println "Updating max_time: '${params.max_time}' to $obj based on available resources"
203-
return obj
204-
}
205-
} else if (type == 'cpus') {
206-
try {
207-
return Math.min( obj, params.max_cpus as int )
208-
} catch (all) {
209-
println "Updating max_cpus: '${params.max_cpus}' to $obj based on available resources"
210-
return obj
211-
}
212-
}
213-
}

‎conf/test.config

+5-5
Original file line numberDiff line numberDiff line change
@@ -15,13 +15,13 @@ params {
1515
config_profile_description = 'Minimal test dataset to check pipeline function'
1616

1717
// Limit resources so that this can run on GitHub Actions
18-
max_cpus = 2
19-
max_memory = '6.GB'
20-
max_time = '6.h'
18+
resourceLimits = [
19+
cpus: 2,
20+
memory: '6.GB',
21+
time: '6.h'
22+
]
2123

2224
// Input data
23-
// TODO nf-core: Specify the paths to your test data on nf-core/test-datasets
24-
// TODO nf-core: Give any required params for the test so that command line flags are not needed
2525
input = params.pipelines_testdata_base_path + 'viralrecon/samplesheet/samplesheet_test_illumina_amplicon.csv'
2626

2727
// Genome references

‎conf/test_bam.config

+5-3
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@ params {
1515
config_profile_description = 'Minimal test dataset to check pipeline function'
1616

1717
// Limit resources so that this can run on GitHub Actions
18-
max_cpus = 2
19-
max_memory = 6.GB
20-
max_time = 6.h
18+
resourceLimits = [
19+
cpus: 2,
20+
memory: '6.GB',
21+
time: '6.h'
22+
]
2123

2224
// Input data
2325
input = "${projectDir}/assets/samplesheet_test_bam.csv"

‎conf/test_illumina.config

+5-3
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@ params {
1515
config_profile_description = 'Minimal test dataset to check pipeline function'
1616

1717
// Limit resources so that this can run on GitHub Actions
18-
max_cpus = 2
19-
max_memory = 6.GB
20-
max_time = 6.h
18+
resourceLimits = [
19+
cpus: 2,
20+
memory: '6.GB',
21+
time: '6.h'
22+
]
2123

2224
// Input data
2325
input = "${projectDir}/assets/samplesheet_test_illumina.csv"

‎conf/test_iontorrent.config

+6-4
Original file line numberDiff line numberDiff line change
@@ -15,10 +15,12 @@ params {
1515
config_profile_description = 'Minimal test dataset to check pipeline function'
1616

1717
// Limit resources so that this can run on GitHub Actions
18-
max_cpus = 2
19-
max_memory = 6.GB
20-
max_time = 6.h
21-
18+
resourceLimits = [
19+
cpus: 2,
20+
memory: 6.GB,
21+
time: 6.h
22+
]
23+
2224
// Input data
2325
input = "${projectDir}/assets/samplesheet_test_iontorrent.csv"
2426

‎conf/test_ont.config

+5-3
Original file line numberDiff line numberDiff line change
@@ -15,9 +15,11 @@ params {
1515
config_profile_description = 'Minimal test dataset to check pipeline function'
1616

1717
// Limit resources so that this can run on GitHub Actions
18-
max_cpus = 2
19-
max_memory = 6.GB
20-
max_time = 6.h
18+
resourceLimits = [
19+
cpus: 2,
20+
memory: '6.GB',
21+
time: '6.h'
22+
]
2123

2224
// Input data
2325
input = "${projectDir}/assets/samplesheet_test_ont.csv"

‎modules/nf-core/fastqc/main.nf

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ process FASTQC {
2626
def rename_to = old_new_pairs*.join(' ').join(' ')
2727
def renamed_files = old_new_pairs.collect{ old_name, new_name -> new_name }.join(' ')
2828

29-
def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB')
29+
def memory_in_mb = MemoryUnit.of("${task.memory}").toUnit('MB') / task.cpus
3030
// FastQC memory value allowed range (100 - 10000)
3131
def fastqc_memory = memory_in_mb > 10000 ? 10000 : (memory_in_mb < 100 ? 100 : memory_in_mb)
3232

‎nextflow.config

+3-50
Original file line numberDiff line numberDiff line change
@@ -8,15 +8,14 @@
88

99
// Global default params, used in configs
1010
params {
11-
1211
// Input options
1312
input = null
1413

1514
// References
1615
fasta = "${projectDir}/assets/references/SARS-CoV-2.reference.fasta"
1716
gff = "${projectDir}/assets/references/SARS-CoV-2.reference.gff"
1817
gff3 = "${projectDir}/assets/references/SARS-CoV-2.reference.gff3"
19-
bed =
18+
bed = ""
2019

2120
// Freyja barcodes and metadata
2221
//freyja_repeats = 1000 //default, change it to at minimum 250 to run it faster
@@ -49,6 +48,7 @@ params {
4948
hook_url = null
5049
help = false
5150
version = null
51+
5252
// Schema validation default options
5353
validationFailUnrecognisedParams = false
5454
validationLenientMode = true
@@ -67,22 +67,6 @@ params {
6767
config_profile_url = null
6868
config_profile_name = null
6969
skip_multiqc = false
70-
71-
// Max resource options
72-
// Defaults only, expecting to be overwritten
73-
max_memory = '128.GB'
74-
max_cpus = 16
75-
max_time = '240.h'
76-
77-
}
78-
79-
// Default publishing logic for pipeline
80-
process {
81-
publishDir = [
82-
path: { "${params.outdir}/${task.process.tokenize(':')[-1].tokenize('_')[0].toLowerCase()}" },
83-
mode: params.publish_dir_mode,
84-
saveAs: { filename -> filename.equals('versions.yml') ? null : filename }
85-
]
8670
}
8771

8872
// Load base.config by default for all pipelines
@@ -196,35 +180,4 @@ manifest {
196180
doi = ' '
197181
}
198182

199-
// Function to ensure that resource requirements don't go beyond
200-
// a maximum limit
201-
def check_max(obj, type) {
202-
if (type == 'memory') {
203-
try {
204-
if (obj.compareTo(params.max_memory as nextflow.util.MemoryUnit) == 1)
205-
return params.max_memory as nextflow.util.MemoryUnit
206-
else
207-
return obj
208-
} catch (all) {
209-
println " ### ERROR ### Max memory '${params.max_memory}' is not valid! Using default value: $obj"
210-
return obj
211-
}
212-
} else if (type == 'time') {
213-
try {
214-
if (obj.compareTo(params.max_time as nextflow.util.Duration) == 1)
215-
return params.max_time as nextflow.util.Duration
216-
else
217-
return obj
218-
} catch (all) {
219-
println " ### ERROR ### Max time '${params.max_time}' is not valid! Using default value: $obj"
220-
return obj
221-
}
222-
} else if (type == 'cpus') {
223-
try {
224-
return Math.min( obj, params.max_cpus as int )
225-
} catch (all) {
226-
println " ### ERROR ### Max cpus '${params.max_cpus}' is not valid! Using default value: $obj"
227-
return obj
228-
}
229-
}
230-
}
183+

0 commit comments

Comments
 (0)