Initial pass at new testing functionality

jmohdyusof · jmohdyusof · commit 914906d4b305 · 2021-10-08T12:47:27.000-06:00
diff --git a/Pilot1/Attn/test.sh b/Pilot1/Attn/test.sh
@@ -1,3 +1,4 @@
+set -x
 python attn_baseline_keras2.py
-# python attn_abstention_keras2.py --epochs 1
+python attn_abstention_keras2.py --epochs 1
 python attn_bin_working_jan7_h5.py --in ../../Data/Pilot1/top_21_1fold_001.h5 --ep 1 --save_dir "./save"
diff --git a/Pilot1/Combo/test.sh b/Pilot1/Combo/test.sh
@@ -1,7 +1,9 @@
 #!/bin/bash
 
+set -x
+
 python combo_baseline_keras2.py --use_landmark_genes True --warmup_lr True --reduce_lr True -z 256 --epochs 4
-python infer.py --sample_set NCIPDM --drug_set ALMANAC --use_landmark_genes -m ./save/combo.A=relu.B=256.E=10.O=adam.LR=None.CF=e.DF=d.wu_lr.re_lr.L1000.D1=1000.D2=1000.D3=1000.model.h5 -w ./save/combo.A=relu.B=256.E=10.O=adam.LR=None.CF=e.DF=d.wu_lr.re_lr.L1000.D1=1000.D2=1000.D3=1000.weights.h5 
+python infer.py --sample_set NCIPDM --drug_set ALMANAC --use_landmark_genes -m ./save/combo.A=relu.B=256.E=4.O=adam.LR=None.CF=e.DF=d.wu_lr.re_lr.L1000.D1=1000.D2=1000.D3=1000.model.h5 -w ./save/combo.A=relu.B=256.E=4.O=adam.LR=None.CF=e.DF=d.wu_lr.re_lr.L1000.D1=1000.D2=1000.D3=1000.weights.h5 
 
 # Need to revisit combo_dose.py and infer_dose.py
 # python combo_dose.py --use_landmark_genes True -z 256
diff --git a/Pilot1/NT3/nt3_candle_wrappers_baseline_keras2.py b/Pilot1/NT3/nt3_candle_wrappers_baseline_keras2.py
@@ -72,8 +72,6 @@ def load_data(train_path, test_path, gParameters):
 
 def run(gParameters):
 
-    print('Params:', gParameters)
-
     file_train = gParameters['train_data']
     file_test = gParameters['test_data']
     url = gParameters['data_url']
diff --git a/Pilot1/NT3/test.sh b/Pilot1/NT3/test.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -x
+
 python nt3_baseline_keras2.py -e 2
 # python nt3_baseline_keras2_tensorrt.py -e 2
 python nt3_abstention_keras2.py -e 2
diff --git a/Pilot1/P1B1/test.sh b/Pilot1/P1B1/test.sh
@@ -1 +1,2 @@
+set -x
 python p1b1_baseline_keras2.py --use_landmark_genes True --train_samples 100 --epochs 5
diff --git a/Pilot1/P1B2/test.sh b/Pilot1/P1B2/test.sh
@@ -1 +1,2 @@
+set -x
 python p1b2_baseline_keras2.py --epochs 3
diff --git a/Pilot1/P1B3/test.sh b/Pilot1/P1B3/test.sh
@@ -1,2 +1,3 @@
+set -x
 python p1b3_baseline_keras2.py --feature_subsample 500 -e 5 --train_steps 100 --val_steps 10 --test_steps 10
 python p1b3_baseline_keras2.py --conf p1b3_conv_model.txt -e 5 --train_steps 100 --val_steps 10 --test_steps 10
diff --git a/Pilot1/T29/test.sh b/Pilot1/T29/test.sh
@@ -1,3 +1,4 @@
 ./prep.sh
+set -x
 python t29res.py -e 5
 python infer.py --model t29res.model.json --weights t29res.model.h5 --n_pred 10
diff --git a/Pilot1/TC1/test.sh b/Pilot1/TC1/test.sh
@@ -1,2 +1,3 @@
+set -x
 python tc1_baseline_keras2.py -e 2
 python tc1_baseline_keras2.py --conf tc1_modac_model.txt -e 2
diff --git a/Pilot1/Uno/test.sh b/Pilot1/Uno/test.sh
@@ -1,5 +1,7 @@
 #!/bin/bash
 
+set -x
+
 # AUC prediction model
 if [ ! -f "top_21_auc_1fold.uno.h5" ]; then
   curl -o top_21_auc_1fold.uno.h5 http://ftp.mcs.anl.gov/pub/candle/public/benchmarks/Pilot1/uno/top_21_auc_1fold.uno.h5
diff --git a/Pilot1/UnoMT/test.sh b/Pilot1/UnoMT/test.sh
@@ -1 +1,2 @@
+set -x
 python unoMT_baseline_pytorch.py --epochs 3 --resp_val_start_epoch 2
diff --git a/Pilot1/Uno_UQ/test.sh b/Pilot1/Uno_UQ/test.sh
diff --git a/Pilot1/Uno_UQ_old/test.sh b/Pilot1/Uno_UQ_old/test.sh
diff --git a/Pilot2/P2B1/test.sh b/Pilot2/P2B1/test.sh
@@ -0,0 +1,2 @@
+set -x
+python p2b1_baseline_keras2.py -e 1
diff --git a/Pilot3/P3B1/test.sh b/Pilot3/P3B1/test.sh
@@ -0,0 +1,2 @@
+set -x
+python p3b1_baseline_keras2.py -e 1
diff --git a/Pilot3/P3B2/test.sh b/Pilot3/P3B2/test.sh
@@ -0,0 +1,2 @@
+set -x
+python p3b2_baseline_keras2.py -e 1
diff --git a/Pilot3/P3B3/test.sh b/Pilot3/P3B3/test.sh
@@ -0,0 +1,2 @@
+set -x
+python p3b3_baseline_keras2.py -e 1
diff --git a/Pilot3/P3B4/test.sh b/Pilot3/P3B4/test.sh
@@ -0,0 +1,2 @@
+set -x
+python p3b4_baseline_tf2.py -e 1
diff --git a/Pilot3/P3B5/test.sh b/Pilot3/P3B5/test.sh
@@ -0,0 +1,2 @@
+set -x
+python p3b5_baseline_pytorch.py -e 1
diff --git a/examples/ADRP/test.sh b/examples/ADRP/test.sh
@@ -0,0 +1,2 @@
+set -ex
+python adrp_baseline_keras2.py -e 1
diff --git a/examples/M16/test.sh b/examples/M16/test.sh
@@ -0,0 +1,2 @@
+set -ex
+python m16_baseline_keras2.py
diff --git a/examples/darts/test.sh b/examples/darts/test.sh
@@ -0,0 +1,3 @@
+cd uno
+set -x
+python uno_example.py -e 1
diff --git a/examples/darts/uno/uno_example.py b/examples/darts/uno/uno_example.py
@@ -1,4 +1,5 @@
 import logging
+import sys
 
 import torch
 import torch.nn as nn
@@ -9,8 +10,11 @@
 import darts
 import candle
 
-logging.basicConfig(level=logging.INFO)
+# logging.basicConfig(sys.stdout, level=logging.INFO)
+# Set up the logger to go to stdout instead of stderr
 logger = logging.getLogger("darts_uno")
+logger.setLevel(logging.INFO)
+logger.addHandler(logging.StreamHandler(sys.stdout))
 
 
 def initialize_parameters():
diff --git a/examples/histogen/extract_code_baseline_pytorch.py b/examples/histogen/extract_code_baseline_pytorch.py
@@ -82,7 +82,7 @@ def extract(lmdb_env, loader, model, device):
     index = 0
 
     with lmdb_env.begin(write=True) as txn:
-        pbar = tqdm(loader)
+        pbar = tqdm(loader, file=sys.stdout)
 
         for img, _, filename in pbar:
             img = img.to(device)
diff --git a/examples/histogen/test.sh b/examples/histogen/test.sh
@@ -0,0 +1,4 @@
+set -ex
+python train_vqvae_baseline_pytorch.py -e 1
+python extract_code_baseline_pytorch.py
+python train_pixelsnail_baseline_pytorch.py -e 1
diff --git a/examples/histogen/train_pixelsnail_baseline_pytorch.py b/examples/histogen/train_pixelsnail_baseline_pytorch.py
@@ -119,7 +119,7 @@ def initialize_parameters(default_model='train_pixelsnail_default_model.txt'):
 
 
 def train(args, epoch, loader, model, optimizer, scheduler, device):
-    loader = tqdm(loader)
+    loader = tqdm(loader, file=sys.stdout)
 
     criterion = nn.CrossEntropyLoss()
 
diff --git a/examples/histogen/train_vqvae_baseline_pytorch.py b/examples/histogen/train_vqvae_baseline_pytorch.py
@@ -92,7 +92,7 @@ def initialize_parameters(default_model='train_vqvae_default_model.txt'):
 
 def train(epoch, loader, model, optimizer, scheduler, device):
     if dist.is_primary():
-        loader = tqdm(loader)
+        loader = tqdm(loader, file=sys.stdout)
 
     criterion = nn.MSELoss()
 
diff --git a/examples/image-vae/test.sh b/examples/image-vae/test.sh
@@ -0,0 +1,3 @@
+set -ex
+python image_vae_baseline_pytorch.py -e 1
+python sample_baseline_pytorch.py --checkpoint epoch_1.pt
diff --git a/examples/mnist/test.sh b/examples/mnist/test.sh
@@ -0,0 +1,3 @@
+set -ex
+python mnist_cnn_candle.py -e 3
+python mnist_mlp_candle.py -e 3
diff --git a/examples/rnagen/test.sh b/examples/rnagen/test.sh
@@ -0,0 +1,3 @@
+set -ex
+python rnagen_baseline_keras2.py -e 1
+python rnagen_baseline_keras2.py --plot True
diff --git a/examples/rnngen/test.sh b/examples/rnngen/test.sh
@@ -0,0 +1,2 @@
+set -ex
+python infer_rnngen_baseline_pytorch.py
diff --git a/examples/unet/test.sh b/examples/unet/test.sh
diff --git a/test.sh b/test.sh
@@ -0,0 +1,42 @@
+#!/bin/bash
+
+export TF_CPP_MIN_LOG_LEVEL=3
+set +x
+if [ $# == 0 ]; then
+for pilot in "Pilot1" "Pilot2" "Pilot3" "examples"
+do
+    for dir in $pilot/*/
+    do
+        echo $dir
+        cd $dir
+# more test.sh
+        ./test.sh > test.log
+        rm test.log
+        cd ../../
+    done
+done
+elif [ $# == 1 ]; then
+    for dir in $1/*/
+    do
+        echo $dir
+        cd $dir
+#        more test.sh
+        ./test.sh > test.log
+        rm test.log
+        cd ../../
+    done
+elif [ $# == 2 ]; then
+    for dir in $1/$2/
+    do
+        echo $dir
+        cd $dir
+#        more test.sh
+        ./test.sh > test.log
+        rm test.log
+        cd ../../
+    done
+else
+    echo "Too many arguments"
+    echo "With no arguments the script will loop over Pilot1, Pilot2, Pilot3 and examples"
+    echo "Or you can enter a directory, or a directory and benchmark to test a smaller set"
+fi

Original file line number	Diff line number	Diff line change
`@@ -1 +1,2 @@`
	`1`	`+set -x`
`1`	`2`	`python p1b1_baseline_keras2.py --use_landmark_genes True --train_samples 100 --epochs 5`
Original file line number	Diff line number	Diff line change
`@@ -1 +1,2 @@`
	`1`	`+set -x`
`1`	`2`	`python p1b2_baseline_keras2.py --epochs 3`
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,3 @@`
	`1`	`+set -x`
`1`	`2`	`python p1b3_baseline_keras2.py --feature_subsample 500 -e 5 --train_steps 100 --val_steps 10 --test_steps 10`
`2`	`3`	`python p1b3_baseline_keras2.py --conf p1b3_conv_model.txt -e 5 --train_steps 100 --val_steps 10 --test_steps 10`
Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,3 @@`
	`1`	`+set -x`
`1`	`2`	`python tc1_baseline_keras2.py -e 2`
`2`	`3`	`python tc1_baseline_keras2.py --conf tc1_modac_model.txt -e 2`