3
3
# # given a file of commands (one command per line), runs them in
4
4
# # parallel on cheyenne using commandfile MPMD approach
5
5
6
+ set defaultq = yes
6
7
set help = no
7
8
set submit = yes
8
9
set QUEUE = premium
10
+ set serial = no
9
11
set WALLTIME = 00:10:00
10
12
11
13
# # Argument parsing from util-linux-ng getopt-parse.tcsh example.
12
14
13
15
# Use a temp variable b/c eval nukes getopt return value. ':q' copies
14
16
# argv list w/ no substitutions
15
17
16
- set temp=(` getopt -a -n vectorize -s tcsh -o d:hj:np:q:w : --long dir:,help,nosub,project:,queue:,walltime: -- $argv :q` )
18
+ set temp=(` getopt -a -n vectorize -s tcsh -o d:hj:np:q:sw : --long dir:,help,nosub,project:,queue:,serial ,walltime: -- $argv :q` )
17
19
if ($? ! = 0) then
18
20
echo " Terminating..." > /dev/stderr
19
21
exit 1
@@ -47,8 +49,13 @@ while (1)
47
49
breaksw;
48
50
case -q:
49
51
case --queue:
52
+ set defaultq = no
50
53
set QUEUE = $2 :q ; shift ; shift
51
54
breaksw;
55
+ case -s:
56
+ case --serial:
57
+ set serial = yes ; shift
58
+ breaksw
52
59
case -w:
53
60
case --walltime:
54
61
set WALLTIME = $2 :q ; shift ; shift
@@ -78,12 +85,13 @@ endif
78
85
if($help == yes || $PROJECT == none || $# argv ! = 1) then
79
86
cat << EOUSAGE
80
87
Usage: vectorize [-d dir] [-h] [-j jobname] [-n] [-p project] [-q queue] [-w walltime] cmdfile
81
- -d, --dir: directory for output; defaults to ` mktemp -d`
88
+ -d, --dir: directory for output; defaults to \ ` mktemp -d\ `
82
89
-h, --help: prints this help message
83
90
-j, --jobname: qsub jobname; defaults to ` basename cmdfile .txt`
84
91
-n, --nosub: don't submit job, just set everything up
85
92
-p, --project: project code; defaults to PROJECT envariable (if set)
86
93
-q, --queue: queue to submit to; defaults to premium
94
+ -s, --serial: runs in serial on share queue instead (ignores -q)
87
95
-w, --walltime: wallclock limit for job; defaults to 00:10:00
88
96
cmdfile: a file with one command per line, to be run in parallel
89
97
@@ -103,24 +111,32 @@ if (! $?JOBNAME ) then
103
111
set JOBNAME = ` basename $1 .txt`
104
112
endif
105
113
114
+ if ( $serial == yes ) then
115
+ if($defaultq == no) then
116
+ echo " vectorize: WARNING: running in serial on share queue; ignoring user-specified queue ($QUEUE )"
117
+ endif
118
+ set QUEUE = share
119
+ endif
120
+
106
121
107
122
set CMDFILE = $dir /cmd
108
123
set OUTFILE = $dir /out
109
124
set ERRFILE = $dir /err
110
125
set pbsfile = $dir /pbs
111
126
rm -f $dir /log
112
127
113
- # # commands need to be wrapped in a shell invocation
128
+ # # commands need to be wrapped in a shell invocation for MPMD
114
129
115
- # sed "s|\(.*\)|tcsh -c '\1 >>\& $dir/log'|g" $1 > $CMDFILE
116
- # sed "s|\(.*\)|bash -c '\1 2>>$dir/errerr 1>>$dir/outout'|g" $1 > $CMDFILE
117
- sed " s|\(.*\)|bash -c '\1'|g" $1 > $CMDFILE
130
+ if ($serial == yes) then
131
+ cp $1 $CMDFILE
132
+ else
133
+ # sed "s|\(.*\)|tcsh -c '\1 >>\& $dir/log'|g" $1 > $CMDFILE
134
+ # sed "s|\(.*\)|bash -c '\1 2>>$dir/errerr 1>>$dir/outout'|g" $1 > $CMDFILE
135
+ sed " s|\(.*\)|bash -c '\1'|g" $1 > $CMDFILE
136
+ endif
118
137
119
- set NTASK = ` cat $CMDFILE | wc -l`
120
- set NNODE = ` perl -w -e " use POSIX; print ceil($NTASK /36)" `
121
- set NCPUS = ` perl -w -e " use POSIX; print ceil($NTASK /$NNODE )" `
122
138
123
- cat << EOPBS > $pbsfile
139
+ cat << EOPBSHEAD > $pbsfile
124
140
#!/bin/tcsh
125
141
126
142
##### parameters for PBS scheduling via qsub
@@ -144,6 +160,22 @@ cat <<EOPBS > $pbsfile
144
160
# runtime limit
145
161
#PBS -l walltime=$WALLTIME
146
162
163
+ EOPBSHEAD
164
+
165
+ if ($serial == yes) then
166
+
167
+ cat << EOSERIAL >> $pbsfile
168
+ # CPU resource request for serial job in shared queue
169
+ #PBS -l select=1:ncpus=1
170
+ EOSERIAL
171
+
172
+ else
173
+
174
+ set NTASK = ` cat $CMDFILE | wc -l`
175
+ set NNODE = ` perl -w -e " use POSIX; print ceil($NTASK /36)" `
176
+ set NCPUS = ` perl -w -e " use POSIX; print ceil($NTASK /$NNODE )" `
177
+
178
+ cat << EOPARALLEL >> $pbsfile
147
179
# CPU resource request
148
180
# ntask = number of lines in file [$NTASK ]
149
181
# nnode = ceiling(ntask / 36) [$NNODE ]
@@ -153,6 +185,11 @@ cat <<EOPBS > $pbsfile
153
185
# mpiprocs: matches ncpus (1 process per CPU)
154
186
# ompthreads: always set to 1 (1 thread per process)
155
187
#PBS -l select=${NNODE} :ncpus=${NCPUS} :mpiprocs=${NCPUS} :ompthreads=1
188
+ EOPARALLEL
189
+
190
+ endif
191
+
192
+ cat << EOPBSMID >> $pbsfile
156
193
157
194
#source /glade/u/apps/opt/lmod/4.2.1/init/tcsh
158
195
@@ -163,11 +200,18 @@ module load nco
163
200
module load cdo
164
201
module load ncl
165
202
166
- setenv MPI_SHEPHERD true
203
+ EOPBSMID
167
204
205
+ if ($serial == yes) then
206
+ cat << EOSTAIL >> $pbsfile
207
+ bash $CMDFILE
208
+ EOSTAIL
209
+ else
210
+ cat << EOPTAIL >> $pbsfile
211
+ setenv MPI_SHEPHERD true
168
212
mpiexec_mpt -n $NTASK launch_cf.sh $CMDFILE
169
-
170
- EOPBS
213
+ EOPTAIL
214
+ endif
171
215
172
216
173
217
if($submit == yes) then
0 commit comments