33# # given a file of commands (one command per line), runs them in
44# # parallel on cheyenne using commandfile MPMD approach
55
6+ set defaultq = yes
67set help = no
78set submit = yes
89set QUEUE = premium
10+ set serial = no
911set WALLTIME = 00:10:00
1012
1113# # Argument parsing from util-linux-ng getopt-parse.tcsh example.
1214
1315# Use a temp variable b/c eval nukes getopt return value. ':q' copies
1416# argv list w/ no substitutions
1517
16- set temp=(` getopt -a -n vectorize -s tcsh -o d:hj:np:q:w : --long dir:,help,nosub,project:,queue:,walltime: -- $argv :q` )
18+ set temp=(` getopt -a -n vectorize -s tcsh -o d:hj:np:q:sw : --long dir:,help,nosub,project:,queue:,serial ,walltime: -- $argv :q` )
1719if ($? ! = 0) then
1820 echo " Terminating..." > /dev/stderr
1921 exit 1
@@ -47,8 +49,13 @@ while (1)
4749 breaksw;
4850 case -q:
4951 case --queue:
52+ set defaultq = no
5053 set QUEUE = $2 :q ; shift ; shift
5154 breaksw;
55+ case -s:
56+ case --serial:
57+ set serial = yes ; shift
58+ breaksw
5259 case -w:
5360 case --walltime:
5461 set WALLTIME = $2 :q ; shift ; shift
@@ -78,12 +85,13 @@ endif
7885if($help == yes || $PROJECT == none || $# argv ! = 1) then
7986 cat << EOUSAGE
8087Usage: vectorize [-d dir] [-h] [-j jobname] [-n] [-p project] [-q queue] [-w walltime] cmdfile
81- -d, --dir: directory for output; defaults to ` mktemp -d`
88+ -d, --dir: directory for output; defaults to \ ` mktemp -d\ `
8289 -h, --help: prints this help message
8390 -j, --jobname: qsub jobname; defaults to ` basename cmdfile .txt`
8491 -n, --nosub: don't submit job, just set everything up
8592 -p, --project: project code; defaults to PROJECT envariable (if set)
8693 -q, --queue: queue to submit to; defaults to premium
94+ -s, --serial: runs in serial on share queue instead (ignores -q)
8795 -w, --walltime: wallclock limit for job; defaults to 00:10:00
8896 cmdfile: a file with one command per line, to be run in parallel
8997
@@ -103,24 +111,32 @@ if (! $?JOBNAME ) then
103111 set JOBNAME = ` basename $1 .txt`
104112endif
105113
114+ if ( $serial == yes ) then
115+ if($defaultq == no) then
116+ echo " vectorize: WARNING: running in serial on share queue; ignoring user-specified queue ($QUEUE )"
117+ endif
118+ set QUEUE = share
119+ endif
120+
106121
107122set CMDFILE = $dir /cmd
108123set OUTFILE = $dir /out
109124set ERRFILE = $dir /err
110125set pbsfile = $dir /pbs
111126rm -f $dir /log
112127
113- # # commands need to be wrapped in a shell invocation
128+ # # commands need to be wrapped in a shell invocation for MPMD
114129
115- # sed "s|\(.*\)|tcsh -c '\1 >>\& $dir/log'|g" $1 > $CMDFILE
116- # sed "s|\(.*\)|bash -c '\1 2>>$dir/errerr 1>>$dir/outout'|g" $1 > $CMDFILE
117- sed " s|\(.*\)|bash -c '\1'|g" $1 > $CMDFILE
130+ if ($serial == yes) then
131+ cp $1 $CMDFILE
132+ else
133+ # sed "s|\(.*\)|tcsh -c '\1 >>\& $dir/log'|g" $1 > $CMDFILE
134+ # sed "s|\(.*\)|bash -c '\1 2>>$dir/errerr 1>>$dir/outout'|g" $1 > $CMDFILE
135+ sed " s|\(.*\)|bash -c '\1'|g" $1 > $CMDFILE
136+ endif
118137
119- set NTASK = ` cat $CMDFILE | wc -l`
120- set NNODE = ` perl -w -e " use POSIX; print ceil($NTASK /36)" `
121- set NCPUS = ` perl -w -e " use POSIX; print ceil($NTASK /$NNODE )" `
122138
123- cat << EOPBS > $pbsfile
139+ cat << EOPBSHEAD > $pbsfile
124140#!/bin/tcsh
125141
126142##### parameters for PBS scheduling via qsub
@@ -144,6 +160,22 @@ cat <<EOPBS > $pbsfile
144160# runtime limit
145161#PBS -l walltime=$WALLTIME
146162
163+ EOPBSHEAD
164+
165+ if ($serial == yes) then
166+
167+ cat << EOSERIAL >> $pbsfile
168+ # CPU resource request for serial job in shared queue
169+ #PBS -l select=1:ncpus=1
170+ EOSERIAL
171+
172+ else
173+
174+ set NTASK = ` cat $CMDFILE | wc -l`
175+ set NNODE = ` perl -w -e " use POSIX; print ceil($NTASK /36)" `
176+ set NCPUS = ` perl -w -e " use POSIX; print ceil($NTASK /$NNODE )" `
177+
178+ cat << EOPARALLEL >> $pbsfile
147179# CPU resource request
148180# ntask = number of lines in file [$NTASK ]
149181# nnode = ceiling(ntask / 36) [$NNODE ]
@@ -153,6 +185,11 @@ cat <<EOPBS > $pbsfile
153185# mpiprocs: matches ncpus (1 process per CPU)
154186# ompthreads: always set to 1 (1 thread per process)
155187#PBS -l select=${NNODE} :ncpus=${NCPUS} :mpiprocs=${NCPUS} :ompthreads=1
188+ EOPARALLEL
189+
190+ endif
191+
192+ cat << EOPBSMID >> $pbsfile
156193
157194#source /glade/u/apps/opt/lmod/4.2.1/init/tcsh
158195
@@ -163,11 +200,18 @@ module load nco
163200module load cdo
164201module load ncl
165202
166- setenv MPI_SHEPHERD true
203+ EOPBSMID
167204
205+ if ($serial == yes) then
206+ cat << EOSTAIL >> $pbsfile
207+ bash $CMDFILE
208+ EOSTAIL
209+ else
210+ cat << EOPTAIL >> $pbsfile
211+ setenv MPI_SHEPHERD true
168212mpiexec_mpt -n $NTASK launch_cf.sh $CMDFILE
169-
170- EOPBS
213+ EOPTAIL
214+ endif
171215
172216
173217if($submit == yes) then
0 commit comments