From 5dda7b52b252d57d98c05e1e7d9af09f8f04142e Mon Sep 17 00:00:00 2001 From: lemonsqueeze Date: Tue, 4 Jul 2023 10:36:47 +0200 Subject: [PATCH] t-play: update added efficient testing notes. single-threaded testing much easier thanks to previous commit (no need to set OMP_NUM_THREAD by hand anymore to control number of dcnn threads) runpachi takes path instead of dir --- t-play/{TESTS => TESTS_OLD} | 0 t-play/autotest/bin/runpachi | 12 +++-- t-play/autotest/rc | 95 +++++++++++++++++++++--------------- 3 files changed, 65 insertions(+), 42 deletions(-) rename t-play/{TESTS => TESTS_OLD} (100%) diff --git a/t-play/TESTS b/t-play/TESTS_OLD similarity index 100% rename from t-play/TESTS rename to t-play/TESTS_OLD diff --git a/t-play/autotest/bin/runpachi b/t-play/autotest/bin/runpachi index 2b092c428..2af434158 100755 --- a/t-play/autotest/bin/runpachi +++ b/t-play/autotest/bin/runpachi @@ -2,11 +2,15 @@ # Simple wrapper that runs pachi from its directory die() { echo "$@"; exit 1; } -usage() { die "Usage: runpachi "; } +usage() { die "Usage: runpachi "; } -dir=`eval echo "$1"` # make '~' work +path=`eval echo "$1"` # make '~' work shift +dir=`dirname $path` +pachi=`basename $path` + [ -d "$dir" ] || usage -cd $dir -exec ./pachi $@ +cd $dir || die "cd $dir failed" + +exec ./$pachi $@ diff --git a/t-play/autotest/rc b/t-play/autotest/rc index 5b1ead1b0..6cc136f4f 100644 --- a/t-play/autotest/rc +++ b/t-play/autotest/rc @@ -6,53 +6,70 @@ # to perform. autotest clients will repeatedly pick a random pairing # from the defined ones, play a game and record the result. -### General parameters: +######################################################################################## +# General parameters # twogtp path # We need the gogui twogtp variant (the one that takes -black, not --black as parameter). -# If you want handicap support, you will need a custom patch from: -# http://pasky.or.cz/dev/pachi/gogui-handicap.diff -# Or download patched release from https://github.com/lemonsqueeze/gogui/releases -twogtp_path="/home/go/gogui-1.4.9/bin/gogui-twogtp" +# If you want handicap support you need gogui-1.4.10: +# https://github.com/lemonsqueeze/gogui/releases +# (or patch gogui: http://pasky.or.cz/dev/pachi/gogui-handicap.diff) +twogtp_path="/path/to/gogui-1.4.10/bin/gogui-twogtp" -### Players: +######################################################################################## +# Players + # The player specification is later eval'd during the pairing. You can use # $1, $2, ... placeholders within the specification to substitute various # parameters at the pairing time. The number parameter after player command # specification specifies how many parameters the specification takes. +# Note: Can't use special characters like: '.' '-' ',' ':' in player names: +# Trying to do so will give an error like: +# Loading rc file... +# bin/autotest-worker: line 19: _spec=: No such file or directory +# *** WORKER FAILED *** +# Use '_' instead. + # Warning! If you change player definition, you WILL NOT be able to determine # what definition your historical results used. Once you define a player of # some id, it's bad idea to tweak the definition in any way. -# This is the reference player we use for our tests. -# Hiratuka 10_36I (19x19, 13x13 or 9x9). Link no longer available, here's my copy: -# https://www.dropbox.com/s/aotkraae8sg08aw/Hira_10_36I_en.zip?dl=0 -# 10_37B is good too. Official link: (or search 平塚の囲碁 on www.vector.co.jp) -# https://www.vector.co.jp/soft/winnt/game/se513389.html -# These are nice for testing (cpu only), later versions uses gpu so ymmv. -player hira_3d "hira -po 15000" 0 -player hira_2d "hira -po 6000" 0 -player hira_1d "hira -po 3000" 0 -player hira_1k "hira -po 1500" 0 - +######################################################################################## # GnuGo player, level 10 -# Probably best to avoid if possible: it doesn't resign lost games so wastes lots -# of resources. The games also tend to be very noisy so have to play thousands of -# them to get a good estimate. Better play a few games with a strong opponent which -# is good at punishing Pachi's mistakes. + +# Probably best avoided if possible: doesn't resign lost games (wastes lots +# of resources). Also games tend to be very noisy, have to play thousands of +# them to get a good estimate. Better play a few games with a strong opponent. #player gnugo10 "/usr/games/gnugo --mode gtp --chinese-rules --capture-all-dead --level 10" 0 -# Pachi player being tested. (5k sims, multithreaded, no dcnn) -# Make native build for wanted boardsize to speed things up. -# To test pachi build in ~/src/pachi use: -player pachi5k_nodcnn "runpachi ~/src/pachi --nodcnn -t =5000 max_tree_size=40,pondering=0" 0 +######################################################################################## +# Pachi players + +# To speed up testing: +# - Test on 15x15 instead of 19x19 +# - Resign early (resign_threshold=0.29, below 30% is almost sure loss) +# - Make custom build for target boardsize +# - Run multiple single threaded games rather than one multi-threaded game +# on multi-core machines. + +# Our reference player: +# Pachi 12.80 (dcnn, 10k playouts, multithreaded) +player pachi1280 "runpachi ~/build/12.80/pachi -t =10000" 0 + -# Pachi (90k sims, multithreaded, no dcnn) -player pachi90k_nodcnn "runpachi ~/src/pachi --nodcnn -t =90000 max_tree_size=200,pondering=0" 0 +# Version we want to test: Pachi build in ~/build/master. +# Pachi (dcnn, 10k playouts, multithreaded) +player pachi "runpachi ~/build/master/pachi -t =10000" 0 + +# Pachi (nodcnn, 10k playouts, multithreaded) +player pachi_nodcnn "runpachi ~/build/master/pachi --nodcnn -t =10000" 0 + +# Pachi (nodcnn, 10k playouts, single-threaded) +#player pachi_nodcnn "runpachi ~/build/master/pachi --nodcnn -t =10000 threads=1" 0 # Can also test different pachi versions from current directory: # We substitute particular revision and specific parameters in the pairings. @@ -61,10 +78,12 @@ player pachi90k_nodcnn "runpachi ~/src/pachi --nodcnn -t =90000 max_tree_s #player zamafmoggy5k './pachi-$1 -t =5000 pass_all_alive,policy=ucb1amaf,playout=moggy${2:+,$2}' 2 -### Pairings: -# The pairing call looks somewhat elaborate: -# -# pairing BOARDSIZE KOMI BLACK PLAYER1 PLAYER1ARGS PLAYER2 PLAYER2ARGS +######################################################################################## +# Pairings + +# Pairing syntax: +# pairing BOARDSIZE KOMI BLACK PLAYER1 PLAYER1ARGS PLAYER2 PLAYER2ARGS +# # BOARDSIZE: 9, 19, or whatever flavor you fancy... # KOMI: 7.5, 0.5, ... OR h2, h6, ... (handicap amount, komi is forced to 0.5) # BLACK: 'a' - black is chosen randomly, useful only if komi is 7.5 @@ -73,22 +92,22 @@ player pachi90k_nodcnn "runpachi ~/src/pachi --nodcnn -t =90000 max_tree_s # PLAYERARGS: Parameters for the player, substituted for $1, $2, ... # Parameters must not contain whitespace. # -# Win rates are always stored from PLAYER2 perspective, no matter +# Winrates are always stored from PLAYER2 perspective, no matter # what the player colors are - we assume that PLAYER1 is your reference # player while PLAYER2 varies based on the particular configuration you # want to test. - +# # You will be editing this all the time. :) -# hira vs pachi, 13x13 H2 (pachi is black) -#pairing 13 h2 2 hira_1k pachi5k_nodcnn +# self-play: 19x19 even game (second player is black) +#pairing 19 7.5 2 pachi1280 pachi -# hira vs pachi, 19x19 even game -#pairing 19 7.5 2 hira_1k pachi90k_nodcnn +# dcnn vs nodcnn: 15x15 H4 +#pairing 15 h4 2 pachi1280 pachi_nodcnn # gnugo vs pachi, 15x15 reverse komi (pachi is white) -#pairing 15 -7.5 1 gnugo10 pachi5k_nodcnn +#pairing 15 -7.5 1 gnugo10 pachi_nodcnn