t-play: update

added efficient testing notes. single-threaded testing much easier thanks to previous commit (no need to set OMP_NUM_THREAD by hand anymore to control number of dcnn threads) runpachi takes path instead of dir
pasky · Jul 5, 2023 · 5dda7b5 · 5dda7b5
1 parent 6123ca4
commit 5dda7b5
Show file tree

Hide file tree

Showing 3 changed files with 65 additions and 42 deletions.
diff --git a/t-play/TESTS → t-play/TESTS_OLD b/t-play/TESTS → t-play/TESTS_OLD
diff --git a/t-play/autotest/bin/runpachi b/t-play/autotest/bin/runpachi
@@ -2,11 +2,15 @@
 # Simple wrapper that runs pachi from its directory
 
 die()   {  echo "$@"; exit 1;  }
-usage() {  die "Usage: runpachi <pachi_dir> <pachi_opts>";  }
+usage() {  die "Usage: runpachi <pachi_path> <pachi_opts>";  }
 
-dir=`eval echo "$1"`		# make '~' work
+path=`eval echo "$1"`		# make '~' work
 shift
 
+dir=`dirname $path`
+pachi=`basename $path`
+
 [ -d "$dir" ] || usage
-cd $dir
-exec ./pachi $@
+cd $dir || die "cd $dir failed"
+
+exec ./$pachi $@
diff --git a/t-play/autotest/rc b/t-play/autotest/rc
@@ -6,53 +6,70 @@
 # to perform. autotest clients will repeatedly pick a random pairing
 # from the defined ones, play a game and record the result.
 
-### General parameters:
+########################################################################################
+# General parameters
 
 # twogtp path
 # We need the gogui twogtp variant (the one that takes -black, not --black as parameter).
-# If you want handicap support, you will need a custom patch from:
-#   http://pasky.or.cz/dev/pachi/gogui-handicap.diff
-# Or download patched release from https://github.com/lemonsqueeze/gogui/releases
-twogtp_path="/home/go/gogui-1.4.9/bin/gogui-twogtp"
+# If you want handicap support you need gogui-1.4.10:
+#   https://github.com/lemonsqueeze/gogui/releases
+# (or patch gogui: http://pasky.or.cz/dev/pachi/gogui-handicap.diff)
+twogtp_path="/path/to/gogui-1.4.10/bin/gogui-twogtp"
 
 
-### Players:
+########################################################################################
+# Players
+
 # The player specification is later eval'd during the pairing. You can use
 # $1, $2, ... placeholders within the specification to substitute various
 # parameters at the pairing time. The number parameter after player command
 # specification specifies how many parameters the specification takes.
 
+# Note: Can't use special characters like: '.' '-' ',' ':' in player names:
+# Trying to do so will give an error like:
+#     Loading rc file...
+#     bin/autotest-worker: line 19: <bad player name>_spec=<bad player command>: No such file or directory
+#     *** WORKER FAILED ***
+# Use '_' instead.
+
 # Warning! If you change player definition, you WILL NOT be able to determine
 # what definition your historical results used. Once you define a player of
 # some id, it's bad idea to tweak the definition in any way.
 
-# This is the reference player we use for our tests.
-# Hiratuka 10_36I (19x19, 13x13 or 9x9). Link no longer available, here's my copy:
-#   https://www.dropbox.com/s/aotkraae8sg08aw/Hira_10_36I_en.zip?dl=0
-#          10_37B is good too. Official link: (or search 平塚の囲碁 on www.vector.co.jp)
-#   https://www.vector.co.jp/soft/winnt/game/se513389.html
-# These are nice for testing (cpu only), later versions uses gpu so ymmv.
-player hira_3d        "hira -po 15000" 0
-player hira_2d        "hira -po 6000"  0
-player hira_1d        "hira -po 3000"  0
-player hira_1k        "hira -po 1500"  0
-
 
+########################################################################################
 # GnuGo player, level 10
-# Probably best to avoid if possible: it doesn't resign lost games so wastes lots
-# of resources. The games also tend to be very noisy so have to play thousands of
-# them to get a good estimate. Better play a few games with a strong opponent which
-# is good at punishing Pachi's mistakes.
+
+# Probably best avoided if possible: doesn't resign lost games (wastes lots
+# of resources). Also games tend to be very noisy, have to play thousands of
+# them to get a good estimate. Better play a few games with a strong opponent.
 #player gnugo10 "/usr/games/gnugo --mode gtp --chinese-rules --capture-all-dead --level 10" 0
 
 
-# Pachi player being tested. (5k sims, multithreaded, no dcnn)
-# Make native build for wanted boardsize to speed things up.
-# To test pachi build in ~/src/pachi use:
-player pachi5k_nodcnn        "runpachi ~/src/pachi --nodcnn -t =5000 max_tree_size=40,pondering=0" 0
+########################################################################################
+# Pachi players
+
+# To speed up testing:
+#   - Test on 15x15 instead of 19x19
+#   - Resign early (resign_threshold=0.29, below 30% is almost sure loss)
+#   - Make custom build for target boardsize
+#   - Run multiple single threaded games rather than one multi-threaded game
+#     on multi-core machines.
+
+# Our reference player:
+# Pachi 12.80 (dcnn, 10k playouts, multithreaded)
+player pachi1280		"runpachi ~/build/12.80/pachi -t =10000" 0
+
 
-# Pachi                      (90k sims, multithreaded, no dcnn)
-player pachi90k_nodcnn       "runpachi ~/src/pachi --nodcnn -t =90000 max_tree_size=200,pondering=0" 0
+# Version we want to test: Pachi build in ~/build/master.
+# Pachi (dcnn, 10k playouts, multithreaded)
+player pachi			"runpachi ~/build/master/pachi -t =10000" 0
+
+# Pachi (nodcnn, 10k playouts, multithreaded)
+player pachi_nodcnn		"runpachi ~/build/master/pachi --nodcnn -t =10000" 0
+
+# Pachi (nodcnn, 10k playouts, single-threaded)
+#player pachi_nodcnn		"runpachi ~/build/master/pachi --nodcnn -t =10000 threads=1" 0
 
 # Can also test different pachi versions from current directory:
 # We substitute particular revision and specific parameters in the pairings.
@@ -61,10 +78,12 @@ player pachi90k_nodcnn       "runpachi ~/src/pachi --nodcnn -t =90000 max_tree_s
 #player zamafmoggy5k './pachi-$1 -t =5000 pass_all_alive,policy=ucb1amaf,playout=moggy${2:+,$2}' 2
 
 
-### Pairings:
-# The pairing call looks somewhat elaborate:
-#
-# pairing BOARDSIZE KOMI BLACK PLAYER1 PLAYER1ARGS PLAYER2 PLAYER2ARGS
+########################################################################################
+# Pairings
+
+# Pairing syntax:
+#   pairing BOARDSIZE KOMI BLACK PLAYER1 PLAYER1ARGS PLAYER2 PLAYER2ARGS
+# 
 # BOARDSIZE: 9, 19, or whatever flavor you fancy...
 # KOMI: 7.5, 0.5, ... OR h2, h6, ... (handicap amount, komi is forced to 0.5)
 # BLACK: 'a' - black is chosen randomly, useful only if komi is 7.5
@@ -73,22 +92,22 @@ player pachi90k_nodcnn       "runpachi ~/src/pachi --nodcnn -t =90000 max_tree_s
 # PLAYERARGS: Parameters for the player, substituted for $1, $2, ...
 #             Parameters must not contain whitespace.
 #
-# Win rates are always stored from PLAYER2 perspective, no matter
+# Winrates are always stored from PLAYER2 perspective, no matter
 # what the player colors are - we assume that PLAYER1 is your reference
 # player while PLAYER2 varies based on the particular configuration you
 # want to test.
-
+#
 # You will be editing this all the time. :)
 
 
-# hira vs pachi,   13x13 H2  (pachi is black)
-#pairing 13  h2  2 hira_1k  pachi5k_nodcnn
+# self-play:       19x19 even game  (second player is black)
+#pairing 19  7.5 2 pachi1280  pachi
 
 
-# hira vs pachi,   19x19 even game
-#pairing 19  7.5 2 hira_1k  pachi90k_nodcnn
+# dcnn vs nodcnn:  15x15 H4
+#pairing 15  h4  2 pachi1280  pachi_nodcnn
 
 
 # gnugo vs pachi,  15x15 reverse komi (pachi is white)
-#pairing 15 -7.5 1 gnugo10  pachi5k_nodcnn
+#pairing 15 -7.5 1 gnugo10  pachi_nodcnn