On Mon 26 Apr 2010 (21:32 +0000), Michael Depreli wrote:
>
> Hi Guys
> For my bot comparison pruning was not used for any ply with gnubg although
> it's kind of a default setting.
> I'm mildly curious to compare the error rates between them for 2-ply.
>
> Is there anyone on the team that could write some code that could compare the
> analysis between 2-ply and 2-ply pruned and output
> the instances where they disagree?
>
> I'm not curious enough to manually go through the 500 games again but would
> do it if someone could automate it for me.
> And if it turns out there aren't many differences I'd probably do 3 and 4 ply
> too.
I wrote some scripts to do just that when pruning was first added. I
had a colection of a lot of sgf's analysed two different ways. The
files for any given match should have the same names in two different
directories, one directory with one analysis, one with the same
matches with the other analysis.
IT does a move by move comparision of the two files, checking that
they both agree on the best move (if not, then it reports the equity
difference betwen the fist analysis's best move and the first
analysis' evaluation of what the second file considered the best move.
It accumulates mean and std err of the difference in evaluation of
wins, win-gammons, win-backgammons, lose-gammons, lose-backgammons and
euity (cubeful) for every move evaluated in both files.
It keeps separate mean and stderr for the same items for moves where
the two files disagree in what the best move is.
Usage is simple - say you have ~/eval1/ containing a.sgf, b.sgf and c.sgg
where a, b and c are three matches evaluated with one standard
~eval2 has the same 3 sgf files, but this time evaluated with a
different standard.
You run the program, specfying the two directories and supply the
names of the .sgf files on stdin for all the files to be analysed
(which is most easily done by cd'ing to the first directory and
using ls -1 *.sgf | comparegun , path-to-other-sgffiles. In this
case that would be:
cd eval1
ls -1 *.sgf | comparegnu . ~/eval2
All output is to stdout.
IT's all in Perl, commented, but assumes you've read the perlref man
page (there are some commented out
--
Jim Segrave [email protected]
#!/usr/bin/perl
use strict;
use Data::Dumper;
my( $matches, $games, $forced_moves, $total_moves, $analysed_pos, $diff_best );
my( $file1, $file2 );
my( $line );
my( $dir1, $dir2 );
my ($gno, $mno);
# array of sum of discrepencies
# win, wing, winbg, loseg, loseb, equity
my @diffs;
my @squares;
# as above, but only for moves where best move decision differs
my @best_diffs;
my @best_squares;
open( BEST, "> /tmp/bestdata" ) or die "Can't open /tmp/best: $!";
# called with one line from a file and a hash reference (either empty
# or the returned value from a call of this routine from another file
# parameter update simply toggles so that only one file is used to count
# moves and forced moves.
# returns "" if there is no move in the input line (double/take/drop
# otherwise the input hash will have a set of keys for each move analysed
# (the key is one to four letter pairs indicating the source and destination
# of one chequer moved
# the value for each key will be a hash with keys rank - the rank, 1=best
# of the move corresponding to the key and key 'ana' which is an array
# reference - the array has the elements win, wing, winbg, loseg, losebg,
# equity
sub doline {
my( $l, $moves, $update ) = @_;
my $key;
# exit early if not a move
return "" if( $l !~ /;[BW]\[([^\]]*)\]/);
# extrace dice roll
my $m = $1;
# not a dice roll, exit early
return "" if( $m =~ /^\d+$/ );
$total_moves++ if( $update );
# look for the analysis of the move - it will be form
# A[n][rank of move made 0 = best][move encoded as letter string
# src/dst (there will be between 1 and 4 such pairs of source dest
# letters, then the evaluation
# force moves aren't analysed, so we simply record it as forced
if( $l !~ /^.*A\[\d+\](.*)/ ) {
++$forced_moves if( $update );
return "";
}
$l = $1;
# discard the leading '['
$l =~ s/\[//g;
then break the move evaluations into separate strings
my @evals = split( /\]/, $l );
my $rank = 0;
# the list of evaluations are in order best to worst
while( my $e = shift @evals ) {
# rank of move 1 = best
++$rank;
# skip if we can't make sense of it
next if( not $e =~ /^([a-z]+) E (.*) 2C .*/ );
# key is encoding of move, $e is the series of values
# win, wing, winbg, loseg, losebg, equitey
($key, $e) = ($1, $2);
$$moves{ $key }->{ 'rank' } = $rank;
$$moves{ $key }->{ 'ana' } = [ split( /\s+/, $e ) ];
}
return $m;
}
# called with one move from each of the two files to be compared
sub domove {
my( $l1, $l2 ) = @_;
my( $m1, $m2, %move1, %move2 );
# we count all lines as moves unless they are takes (don't know
# why I did this
++$mno if( $l1 !~ /^;[BW]\[take\]/ );
# get a hash from each file's line
$m1 = doline( $l1, \%move1, 0 );
$m2 = doline( $l2, \%move2, 1 );
# comparision is shallow, actually only checks that they have the
# the same first level keys, meaning each file analysed the same set
# of moves
if( $m1 ne $m2 ) {
die "mismatched moves at line $line of $file1\n";
}
# exit early if it's not a move or it's a forced move
return if( $m1 eq "" );
# now compare the ranks and analyses of the moves
foreach my $k1 (sort ( keys %move1 ) ) {
my ($r1, $r2);
$r1 = $move1{ $k1 }->{ 'rank' };
$r2 = $move2{ $k1 }->{ 'rank' };
# processing if best mvoe from file1 isn't the best mvoe in file2
if( ( $r1 == 1 ) and ( $r1 != $r2 ) ) {
if ( $move2{ $k1 }->{ 'rank ' } != 1 ) {
# print "Move1:\n";
# print Dumper($move1{ $k1 });
# print "Move2:\n";
# print Dumper($move2{ $k1 });
++$diff_best;
# find the best move in file 2
foreach my $k2 ( sort ( keys %move2 ) ) {
next if( $move2{ $k2 }->{ 'rank' } != 1 );
# see what rank it had in file 1
$r1 = $move1{ $k2 }->{ 'rank' };
my $d;
# if the best move in file 2 was analysed in file 1...
if( defined( $move1{ $k2 } ) ) {
# update stats on difference between file1's analysis
# of file1's best move with file1's analysis of the
# best move according to file2
my $max = 0;
for( my $i = 0; $i < 6; ++$i ) {
$d = $move1{ $k1 }->{ 'ana' }[ $i ] -
$move1{ $k2 }->{ 'ana' }[ $i ];
$best_diffs[ $i ] += abs $d ;
$best_squares[ $i ] += $d * $d;
$max = abs $d if(((abs $d) > $max) and ($i != 5));
# printf BEST "%g ", abs $d;
}
# report difference in equities between best move
# in file1 vs. file1's equitie for best mvoe according
# to file2
printf BEST "%5.5f %s %d:%d\n", $d, $file1, $gno,$mno;
}
# report files disagree on best move
print STDERR "Wrong rank $k1 ($r2, $r1) at game $gno, ".
"move $mno - $file1:$line\n$l1\n$file2:$line\n$l2\n";
}
}
}
next if( !defined( $move2{ $k1 }->{ 'ana' } ) );
++$analysed_pos;
# accumulate differences in the various figures for every move
# which was analysed in both matches
for( my $i = 0; $i < 6; ++$i ) {
my $d = $move1{ $k1 }->{ 'ana' }[ $i ] -
$move2{ $k1 }->{ 'ana' }[ $i ];
$diffs[ $i ] += abs $d ;
$squares[ $i ] += $d * $d;
}
}
}
# called after the first line of a game has been read, $fh1, $fh2 are
# the input handles for the files being compared, the next read will
# get a move line, the first line of the next game, or eof returns
# TRUE if there are more games to read
sub dogame {
my( $fh1, $fh2 ) = @_;
my( $l1, $l2 );
++$games;
++$gno;
$mno = 0;
while( $l1 = <$fh1> ) {
++$line;
$l1 =~ s/\s+$//;
$l2 = <$fh2> or die
"Mismatched games - eof at $line of $file2";
if( $l1 =~ /^\(/ ) {
return 1 if( $l2 =~ /^\(/ );
die "End of game at $line of $file1 but not in $file2";
}
die "End of game at $line of $file2 but not in $file1"
if ( $l2 =~ /^\(/ );
domove( $l1, $l2 );
}
while ( $l2 = <$fh2> ) {
die "Eof at $line of $file1, not in $file2";
}
return 0;
}
# process an entire match out of an sgf file
sub domatch {
my( $file ) = @_;
++$matches;
$gno = -1;
$file1 = "$dir1/$file";
$file2 = "$dir2/$file";
$line = 0;
open( FILE1, "< $file1" ) or die "Can't open $file1: $!";
open( FILE2, "< $file2" ) or die "Can't open $file2: $!";
while( dogame( \*FILE1, \*FILE2 ) ) {
# no body
}
close( FILE1 );
close( FILE2 );
}
sub usage {
print <<EOM
Usage: $0 path0 path1
compare sgf files named on stdin in the two directories
EOM
;
}
sub main {
usage if $#ARGV != 1;
$dir1 = $ARGV[ 0 ];
$dir2 = $ARGV[ 1 ];
while( <STDIN> ) {
s/\s+$//;
next if( /^$/ );
domatch( $_ );
}
print "$matches matches, $games games, $total_moves moves\n";
print "$forced_moves forced moves, $analysed_pos positions\n";
print "The best move was different in $diff_best cases\n";
print "Average Diff/Std Err\n";
for( my $i = 0; $i < 6; ++$i ) {
printf "%1.5f ", $diffs[ $i ] / $analysed_pos;
}
print "\n";
for( my $i = 0; $i < 6; ++$i ) {
printf "%1.5f ", sqrt( $squares[ $i ] / $analysed_pos );
}
print "\n";
if( $diff_best > 0 ) {
print
"Where best moves differed, $dir1 match evaluations of the choices\n";
print "Average Diff/Std Err\n";
for( my $i = 0; $i < 6; ++$i ) {
printf "%1.5f ", $best_diffs[ $i ] / $diff_best;
}
print "\n";
for( my $i = 0; $i < 6; ++$i ) {
printf "%1.5f ", sqrt( $best_squares[ $i ] / $diff_best );
}
print "\n";
}
}
main;
_______________________________________________
Bug-gnubg mailing list
[email protected]
http://lists.gnu.org/mailman/listinfo/bug-gnubg
