Skip to content

Commit 2ba3ba6

Browse files
SebastienRietteMTOSebastienRietteMTO
committed
New pyft version (parallel tool)
1 parent fef003b commit 2ba3ba6

File tree

4 files changed

+78
-67
lines changed

4 files changed

+78
-67
lines changed

‎build/with_fcm/arch/arch-pgi_CPU_preGPU_MFflavour.env

+31-29
Original file line numberDiff line numberDiff line change
@@ -22,25 +22,28 @@ stackMode=AROME
2222
#* do not modify turb_hor* or mode_rotate_wind.F90 (non column subroutines)
2323
#* remove call to condensation from lima, do not touch lima routines (due to implicit dimensions at declaration)
2424
#* remove problematic call to BL_DEPTH_DIAG_3D from the unused (in testprogs and AROME) BL_DEPTH_DIAG_1D
25+
#* do not touch .f files which will be included
2526
# this part is written differently for each model. For MesoNH, LES diag needs this call
26-
specificTransfos="^progs/:=:
27-
^rain_ice/:=:
28-
^rain_ice_old/:=:
29-
^ice_adjust/:=:
30-
^turb_mnh/:=:
31-
^shallow/:=:
32-
^support/:=:
33-
^ext/:=:
34-
^micro/minpack.F90:=:
35-
^aux/tools.F90:=:
36-
^turb/mode_turb_hor:=:--empty --simplify
37-
^turb/mode_rotate_wind.F90:=:--empty --simplify
38-
^turb/mode_tridiag_w.F90:=:--empty --simplify
39-
^micro/mode_lima:=:
40-
^micro/lima:=:
41-
^micro/lima_adjust_split.F90:=:--removeCall sub:LIMA_ADJUST_SPLIT CONDENSATION"
27+
specificTransfos="^(./)?progs/:=:
28+
^(./)?rain_ice/:=:
29+
^(./)?rain_ice_old/:=:
30+
^(./)?ice_adjust/:=:
31+
^(./)?turb_mnh/:=:
32+
^(./)?shallow/:=:
33+
^(./)?support/:=:
34+
^(./)?ext/:=:
35+
^(./)?micro/minpack\.F90:=:
36+
^(./)?aux/tools\.F90:=:
37+
^(./)?turb/mode_turb_hor:=:--empty --simplify
38+
^(./)?turb/mode_rotate_wind\.F90:=:--empty --simplify
39+
^(./)?turb/mode_tridiag_w\.F90:=:--empty --simplify
40+
^(./)?micro/mode_lima:=:
41+
^(./)?micro/lima:=:
42+
^(./)?micro/lima_adjust_split\.F90:=:--removeCall sub:LIMA_ADJUST_SPLIT CONDENSATION
43+
^(./)?micro/[_.a-z0-9]*\.h:=:
44+
^(./)?turb/[_.a-z0-9]*\.h:=:"
4245

43-
#Temporary file for the description tree
46+
#Temporary file for the description tree, only for MesoNH and IAL
4447
descTree=${TMPDIR:-/tmp}/descTree_$$
4548
trap "\rm -f $descTree" EXIT
4649

@@ -52,15 +55,15 @@ trap "\rm -f $descTree" EXIT
5255
PYFT_OPTS_MNH="--tree . --addIncludes --descTree $descTree --simplify --enableCache" #General options
5356
PYFT_OPTS_MNH="$PYFT_OPTS_MNH --inlineContainedSubroutinesPHYEX" #Inlining
5457
PYFT_OPTS_MNH="$PYFT_OPTS_MNH --deleteDrHook --deleteBudgetDDH" #Not available on GPU
55-
PYFT_OPTS_MNH="$PYFT_OPTS_MNH --addACC_routine_seq" #add 'acc routine' directive
58+
PYFT_OPTS_MNH="$PYFT_OPTS_MNH --addACCRoutineSeq" #add 'acc routine' directive
5659
PYFT_OPTS_MNH="$PYFT_OPTS_MNH --stopScopes sub:SHALLOW_MF#sub:ICE_ADJUST#sub:RAIN_ICE#sub:RAIN_ICE_OLD#sub:TURB#sub:LIMA_ADJUST_SPLIT#sub:LIMA"
5760
PYFT_OPTS_MNH="$PYFT_OPTS_MNH --setFalseIfStmt TURBN%LTURB_DIAG --setFalseIfStmt TURBN%LTURB_FLX" #remove diagnostic output
5861
PYFT_OPTS_MNH="$PYFT_OPTS_MNH --buildACCTypeHelpers"
5962
PYFT_OPTS_MNH="$PYFT_OPTS_MNH --addArrayParentheses --expandAllArraysPHYEX --attachArraySpecToEntity --applyCPPifdef %REPRO48 --replaceAutomaticWithAllocatable"
6063
PYFT_OPTS_MNH="$PYFT_OPTS_MNH
6164
$specificTransfos
62-
^turb/mode_bl_depth_diag.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS_MNH
63-
^micro/rain_ice.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS_MNH"
65+
^(./)?turb/mode_bl_depth_diag.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS_MNH
66+
^(./)?micro/rain_ice.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS_MNH"
6467

6568
#For IAL, stack transformation is replaced by allocatables
6669
#source this file, then: "export PYFT_OPTS_IAL; check_commit_ial.sh --prep_code-opts '--pyft_opts_env PYFT_OPTS_IAL' ...."
@@ -70,34 +73,33 @@ $specificTransfos
7073
PYFT_OPTS_IAL="--tree . --addIncludes --descTree $descTree --simplify --enableCache" #General options
7174
PYFT_OPTS_IAL="$PYFT_OPTS_IAL --inlineContainedSubroutinesPHYEX" #Inlining
7275
PYFT_OPTS_IAL="$PYFT_OPTS_IAL --deleteDrHook --deleteBudgetDDH" #Not available on GPU
73-
PYFT_OPTS_IAL="$PYFT_OPTS_IAL --addACC_routine_seq" #add 'acc routine' directive
76+
PYFT_OPTS_IAL="$PYFT_OPTS_IAL --addACCRoutineSeq" #add 'acc routine' directive
7477
PYFT_OPTS_IAL="$PYFT_OPTS_IAL --stopScopes sub:SHALLOW_MF#sub:ICE_ADJUST#sub:RAIN_ICE#sub:RAIN_ICE_OLD#sub:TURB#sub:LIMA_ADJUST_SPLIT#sub:LIMA"
7578
PYFT_OPTS_IAL="$PYFT_OPTS_IAL --setFalseIfStmt TURBN%LTURB_DIAG --setFalseIfStmt TURBN%LTURB_FLX" #remove diagnostic output
7679
PYFT_OPTS_IAL="$PYFT_OPTS_IAL --buildACCTypeHelpers"
7780
PYFT_OPTS_IAL="$PYFT_OPTS_IAL --addArrayParentheses --expandAllArraysPHYEX --attachArraySpecToEntity --applyCPPifdef REPRO48 --deleteNonColumnCallsPHYEX --replaceAutomaticWithAllocatable"
7881
PYFT_OPTS_IAL="$PYFT_OPTS_IAL
7982
$specificTransfos
80-
^turb/mode_bl_depth_diag.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS_IAL
81-
^micro/rain_ice.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS_IAL"
83+
^(./)?turb/mode_bl_depth_diag.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS_IAL
84+
^(./)?micro/rain_ice.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS_IAL"
8285

8386
#For testprogs
8487
#check_commit_testprogs.sh -a <this conf>
8588
#For --removeIJDim, the USE_COLCALL compilation key must be set in the *.fcm file
8689
#--removeIJDim internally applies --addArrayParentheses --expandAllArraysPHYEX and, if --simplify is set, attachArraySpecToEntity
8790
#The --wrapH is automatically set by prep_code
8891
#The --mnhExpand is controlled by the different check_commit_* scripts
89-
#--addIncludes must be before --descTree because --addIncludes removes a wrong "file" node
90-
PYFT_OPTS="--tree . --addIncludes --descTree $descTree --simplify --enableCache" #General options
92+
#make_fcm automatically adds the --tree and --descTree options
93+
PYFT_OPTS="--addIncludes --simplify --enableCache" #General options
9194
PYFT_OPTS="$PYFT_OPTS --inlineContainedSubroutinesPHYEX" #Inlining
9295
PYFT_OPTS="$PYFT_OPTS --deleteDrHook --deleteBudgetDDH" #Not available on GPU
93-
#PYFT_OPTS="$PYFT_OPTS --addACC_routine_seq" #add 'acc routine' directive
9496
PYFT_OPTS="$PYFT_OPTS --stopScopes sub:SHALLOW_MF#sub:ICE_ADJUST#sub:RAIN_ICE#sub:RAIN_ICE_OLD#sub:TURB#sub:LIMA_ADJUST_SPLIT#sub:LIMA"
9597
PYFT_OPTS="$PYFT_OPTS --setFalseIfStmt TURBN%LTURB_DIAG --setFalseIfStmt TURBN%LTURB_FLX" #remove diagnostic output
9698
PYFT_OPTS="$PYFT_OPTS --buildACCTypeHelpers"
9799
PYFT_OPTS="$PYFT_OPTS --applyCPPifdef REPRO48 --deleteNonColumnCallsPHYEX --removeIJDim --addStack $stackMode"
98-
PYFT_OPTS="$PYFT_OPTS --removeACC --addACC_routine_seq" #remove pre-existing ACC directives, then add 'acc routine seq' directive
100+
PYFT_OPTS="$PYFT_OPTS --removeACC --addACCRoutineSeq" #remove pre-existing ACC directives, then add 'acc routine seq' directive
99101
PYFT_OPTS="$PYFT_OPTS
100102
$specificTransfos
101-
^turb/mode_bl_depth_diag.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS
102-
^micro/rain_ice.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS"
103+
^(./)?turb/mode_bl_depth_diag\.F90:=:--removeCall module:MODE_BL_DEPTH_DIAG/sub:BL_DEPTH_DIAG_1D BL_DEPTH_DIAG_3D $PYFT_OPTS
104+
^(./)?micro/rain_ice\.F90:=:--removeCall sub:RAIN_ICE ELEC_TENDENCIES $PYFT_OPTS"
103105

‎build/with_fcm/make_fcm.sh

+14-5
Original file line numberDiff line numberDiff line change
@@ -392,6 +392,7 @@ if [ $packupdate -eq 1 -o $packcreation -eq 1 ]; then
392392
rm -rf src
393393
fi
394394
. arch.env
395+
[ -z ${PYFT_OPTS+x} ] && PYFT_OPTS=''
395396
export PYFT_OPTS
396397

397398
# Populate the source directory with (modified) PHYEX source code
@@ -411,20 +412,28 @@ if [ $packupdate -eq 1 -o $packcreation -eq 1 ]; then
411412
fi
412413
PHYEXTOOLSDIR="$DIR/../../../tools" #if run from within a PHYEX repository
413414
UPDATEDPATH=$PATH
415+
416+
#Temporary file for the description tree
417+
descTree=${TMPDIR:-/tmp}/descTree_$$
418+
trap "\rm -f $descTree" EXIT
419+
414420
which prep_code.sh > /dev/null || export UPDATEDPATH=$PHYEXTOOLSDIR:$PATH
415421
subs="$subs -s turb -s shallow -s turb_mnh -s micro -s aux -s ice_adjust -s rain_ice -s rain_ice_old -s support -s progs"
416422
if [ "$fromdir" == '' ]; then
417423
echo "Clone repository, and checkout commit $commit (using prep_code.sh)"
418424
if [[ $commit == testprogs${separator}* || $commit == offline${separator}* ]]; then
419-
PATH=$UPDATEDPATH prep_code.sh --pyft_opts_env PYFT_OPTS -c $commit src #This commit is ready for inclusion
425+
#This commit is ready for inclusion
426+
PATH=$UPDATEDPATH prep_code.sh -c $commit src
420427
else
421-
PATH=$UPDATEDPATH prep_code.sh --pyft_opts_env PYFT_OPTS -c $commit $expand_options $subs -m offline src
428+
PATH=$UPDATEDPATH prep_code.sh --pyft_opts_env PYFT_OPTS -c $commit $expand_options $subs \
429+
-m offline src --useParallelPyft -- --tree . --descTree $descTree
422430
fi
423431
else
424432
echo "Copy $fromdir"
425433
mkdir src
426434
scp -q -r $fromdir/src src/
427-
PATH=$UPDATEDPATH prep_code.sh --pyft_opts_env PYFT_OPTS $expand_options $subs -m offline src
435+
PATH=$UPDATEDPATH prep_code.sh --pyft_opts_env PYFT_OPTS $expand_options $subs \
436+
-m offline src --useParallelPyft -- --tree . --descTree $descTree
428437
fi
429438

430439
# Add some code
@@ -445,8 +454,8 @@ if [ $packupdate -eq 1 -o $packcreation -eq 1 ]; then
445454
pybinding.py micro/lima.F90 sub:LIMA pyphyex.F90 ../build/bin/pyphyex.py ./libphyex.so
446455
else
447456
cat <<....EOF > pyphyex.F90
448-
SUBROUTINE PYPHYEX
449-
END SUBROUTINE PYPHYEX
457+
SUBROUTINE PYPHYEXSUB
458+
END SUBROUTINE PYPHYEXSUB
450459
....EOF
451460
fi
452461
ln -s ../../fiat/src fiat

‎tools/INSTALL.sh

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
set -e
55
set -o pipefail #abort if left command on a pipe fails
66

7-
pyft_version=f63de396176e3e91a5cdedae2d098dbbe3497ae2
7+
pyft_version=76f61bcd9eae523ced16e8d854e0f065fa82078e
88

99
#This script installs PHYEX
1010
#Call the script with the -h option to get more information.

‎tools/prep_code.sh

+32-32
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@ function usage {
3333
echo "-v add verbosity (up to 3 -v)"
3434
echo "--pyft_opts_env VAR name of an environment variable containing options to use to call"
3535
echo " the pyft_tool.py script"
36+
echo "--useParallelPyft use the parallel version of the pyft tool (pyft_parallel_tool.py)"
3637
echo "-- PYFT_OPTIONS everything after '--' are used as options for pyft_tool.py"
3738
echo " These options are used for all the files."
3839
echo ""
@@ -81,6 +82,7 @@ subs=""
8182
renameFf=0
8283
verbose=0
8384
ilooprm=0
85+
useParallelPyft=0
8486
forpyft=0
8587
pyft_opts_env=""
8688

@@ -109,6 +111,7 @@ while [ -n "$1" ]; do
109111
'--ilooprm') ilooprm=1;;
110112
'--repo') repository=$2; shift;;
111113
'-v') verbose=$(($verbose+1));;
114+
'--useParallelPyft') useParallelPyft=1;;
112115
'--pyft_opts_env') pyft_opts_env=$2; shift;;
113116
'--') forpyft=1;;
114117
*) if [ $forpyft -eq 0 ]; then
@@ -120,11 +123,6 @@ while [ -n "$1" ]; do
120123
shift
121124
done
122125

123-
if [ "$pyft_opts_env" != "" ]; then
124-
#pyft_opts_env contains the name of the environment variable to use
125-
pyft_opts_env=${!pyft_opts_env}
126-
#now, pyft_opts_env contains the configuration to use
127-
fi
128126
if [ $verbose -ge 3 ]; then
129127
set -x
130128
fi
@@ -297,7 +295,11 @@ if [ "$pyft_opts_env" != "" -o -n "${pyft_options-}" ]; then
297295
[ $verbose -gt 0 ] && echo "Applying pyft_tool"
298296

299297
#Update PATH and PYTHONPATH if needed
300-
which pyft_tool.py > /dev/null || . $PHYEXTOOLSDIR/site/pyft/bin/env.sh
298+
if [ $useParallelPyft -eq 1 ]; then
299+
which pyft_parallel_tool.py > /dev/null || . $PHYEXTOOLSDIR/site/pyft/bin/env.sh
300+
else
301+
which pyft_tool.py > /dev/null || . $PHYEXTOOLSDIR/site/pyft/bin/env.sh
302+
fi
301303

302304
if [ -n "${model-}" ]; then
303305
reps=$subs
@@ -307,35 +309,33 @@ if [ "$pyft_opts_env" != "" -o -n "${pyft_options-}" ]; then
307309
reps="$reps src/*/$sub"
308310
done
309311
fi
310-
for rep in $reps; do
311-
if [ -d $rep ]; then
312-
find $rep -type f -not -name '.*.swp' -not -name '.*.swo' | while read file; do
313-
if [ "$(echo $file | grep '\.')" != '' -a $(echo $file | rev | cut -d. -f1 | rev) != 'fypp' ]; then
314-
#Files without extension are certainly not source code files
315-
#.fypp files cannot be read by pyft_tool.py
316-
extra_opts=""
317-
if [ "$pyft_opts_env" != "" ]; then
318-
while read line; do
319-
if echo $line | grep ':=:' > /dev/null; then
320-
#This line has the form FILE_DESCRIPTOR:OPTIONS
321-
fd="$(echo $line | awk -F :=: '{print $1}')"
322-
if echo $file | grep -e "$fd" > /dev/null; then
323-
extra_opts=$(echo $line | awk -F :=: '{$1=""; print $0}') #equivalent to cut -d:=: -f2- (multi characters separator are forbiden with cut)
324-
fi
325-
else
326-
extra_opts=$line
327-
fi
328-
done < <(echo "$pyft_opts_env")
329-
fi
330-
if [ "$extra_opts" != "" -o -n "${pyft_options-}" ]; then
331-
cmd="pyft_tool.py --wrapH $pyft_options $extra_opts" #--wrapH allows to deal with h files
332-
[ $verbose -gt 1 ] && echo $cmd "$file"
333-
$cmd "$file"
334-
fi
312+
313+
extra_opts=""
314+
if [ "$pyft_opts_env" != "" ]; then
315+
extra_opts="--optsByEnv $pyft_opts_env"
316+
fi
317+
318+
if [ "$extra_opts" != "" -o -n "${pyft_options-}" ]; then
319+
if [ $useParallelPyft -eq 1 ]; then
320+
cmd="pyft_parallel_tool.py --wrapH $pyft_options $extra_opts --nbPar 8" #--wrapH allows to deal with h files
321+
[ $verbose -gt 1 ] && echo $cmd
322+
$cmd
323+
else
324+
for rep in $reps; do
325+
if [ -d $rep ]; then
326+
find $rep -type f -not -name '.*.swp' -not -name '.*.swo' | while read file; do
327+
if [ "$(echo $file | grep '\.')" != '' -a $(echo $file | rev | cut -d. -f1 | rev) != 'fypp' ]; then
328+
#Files without extension are certainly not source code files
329+
#.fypp files cannot be read by pyft_tool.py
330+
cmd="pyft_tool.py --wrapH $pyft_options $extra_opts" #--wrapH allows to deal with h files
331+
[ $verbose -gt 1 ] && echo $cmd "$file"
332+
$cmd "$file"
333+
fi
334+
done
335335
fi
336336
done
337337
fi
338-
done
338+
fi
339339
fi
340340

341341
###### PUSH

0 commit comments

Comments
 (0)