Author: bernhard
Date: Wed Dec 28 04:49:33 2005
New Revision: 10722
Added:
trunk/t/benchmark/ (props changed)
trunk/t/benchmark/benchmarks.t
- copied, changed from r10718, trunk/t/benchmarks.t
Removed:
trunk/t/benchmarks.t
Modified:
trunk/MANIFEST
trunk/MANIFEST.SKIP
trunk/config/gen/makefiles/root.in
trunk/t/README
Log:
Move t/benchmarks.t into the new directory t/benchmark.
Enable the tests again, marking the failing ones as TODO.
'make testbench' is now 'make benchmark_tests'.
'make doc_tests' now runs all tests in 't/doc'.
Modified: trunk/MANIFEST
==============================================================================
--- trunk/MANIFEST (original)
+++ trunk/MANIFEST Wed Dec 28 04:49:33 2005
@@ -1781,7 +1781,7 @@ src/utils.c
src/vtables.c []
src/warnings.c []
t/README []
-t/benchmarks.t []
+t/benchmark/benchmarks.t []
t/compilers/imcc/imcpasm/cfg.t []
t/compilers/imcc/imcpasm/opt0.t []
t/compilers/imcc/imcpasm/opt1.t []
Modified: trunk/MANIFEST.SKIP
==============================================================================
--- trunk/MANIFEST.SKIP (original)
+++ trunk/MANIFEST.SKIP Wed Dec 28 04:49:33 2005
@@ -1,5 +1,5 @@
# $Id$
-# generated by tools/dev/gen_manifest_skip.pl Wed Dec 28 01:35:04 2005
+# generated by tools/dev/gen_manifest_skip.pl Wed Dec 28 03:17:58 2005
#
# This file should contain a transcript of the svn:ignore properties
# of the directories in the Parrot subversion repository. (Needed for
@@ -1036,6 +1036,11 @@
^t/benchmarks_.*\.pir/
^t/benchmarks_.*\.pasm$
^t/benchmarks_.*\.pasm/
+# generated from svn:ignore of 't/benchmark/'
+^t/benchmark/.*\.pasm$
+^t/benchmark/.*\.pasm/
+^t/benchmark/.*\.pir$
+^t/benchmark/.*\.pir/
# generated from svn:ignore of 't/compilers/'
# generated from svn:ignore of 't/compilers/imcc/'
# generated from svn:ignore of 't/compilers/imcc/imcpasm/'
Modified: trunk/config/gen/makefiles/root.in
==============================================================================
--- trunk/config/gen/makefiles/root.in (original)
+++ trunk/config/gen/makefiles/root.in Wed Dec 28 04:49:33 2005
@@ -614,7 +614,7 @@ help :
@echo " src_tests: Run test in C-file."
@echo " perl_tests: Test the Perl modules in the distribution."
@echo " testexex: Testing the exex runcore."
- @echo " testbench: Run all benchmarks."
+ @echo " benchmark_tests: Run all benchmarks."
@echo " manitest: Check for new and missing files."
@echo " doc_tests: Check POD syntax of all file containing
POD."
@echo ""
@@ -1257,12 +1257,16 @@ editor-clean :
# The --running-make-test argument is currently used by long-running tests
# to disable GC_DEBUG.
EXTRA_TEST_ARGS = --gc-debug --running-make-test
-RUN_TEST_FILES = \
- t/run/*.t
-SRC_TEST_FILES = \
- t/src/*.t
+DISTRO_TEST_FILES = \
+ t/distro/*.t
+BENCHMARK_TEST_FILES = \
+ t/benchmark/*.t
+DOC_TEST_FILES = \
+ t/doc/*.t
PERL_TEST_FILES = \
t/perl/*.t
+RUN_TEST_FILES = \
+ t/run/*.t
RUNCORE_TEST_FILES = \
t/compilers/imcc/*/*.t \
t/op/*.t \
@@ -1277,10 +1281,10 @@ RUNCORE_TEST_FILES = \
t/compilers/tge/*.t \
t/library/*.t \
t/examples/*.t
+SRC_TEST_FILES = \
+ t/src/*.t
TOOLS_TEST_FILES = \
t/tools/*.t
-DISTRO_TEST_FILES = \
- t/distro/*.t
# Common prep for all test targets.
# We propably need a complete build before running the tests.
@@ -1306,7 +1310,7 @@ fulltest : \
testS \
src_tests \
perl_tests \
- testbench \
+ benchmark_tests \
doc_tests \
distro_tests
@@ -1355,13 +1359,17 @@ run_tests :
perl_tests :
$(PERL) t/harness $(PERL_TEST_FILES)
-# all benchmarks
-testbench: test_prep
- $(PERL) t/harness t/benchmarks.t
+# obsolete
+testbench :
+ @echo "Please use the target benchmark_tests."
+
+# benchmark tests
+benchmark_tests : test_prep
+ $(PERL) t/harness $(EXTRA_TEST_ARGS) $(BENCHMARK_TEST_FILES)
# doc tests
doc_tests :
- $(PERL) t/harness t/doc/pod.t
+ $(PERL) t/harness $(EXTRA_TEST_ARGS) $(DOC_TEST_FILES)
# Running a benchmark
mopsbench : test_prep
Modified: trunk/t/README
==============================================================================
--- trunk/t/README (original)
+++ trunk/t/README Wed Dec 28 04:49:33 2005
@@ -4,6 +4,8 @@
This directory contains the Parrot test suite.
For details see the documentation in 'docs/tests.pod'.
+benchmark: Run all benchmarks
+
compilers: Mini languages distributed with Parrot
configure: Configuration subsystem
Copied: trunk/t/benchmark/benchmarks.t (from r10718, trunk/t/benchmarks.t)
==============================================================================
--- trunk/t/benchmarks.t (original)
+++ trunk/t/benchmark/benchmarks.t Wed Dec 28 04:49:33 2005
@@ -20,10 +20,10 @@ use Test::More;
# Set up expected output from files in 'examples/benchmarks'
my %outputs = (
- q(addit.imc) => qq(21001097.970000\n),
- q(addit.pasm) => qq(21001097.970000\n),
- q(addit2.imc) => qq(21001097.970000\n),
- q(array_access.imc) => qr/^1\s\*\s1000\s=\s1000\n
+ q{addit.imc} => qq(21001097.970000\n),
+ q{addit.pasm} => qq(21001097.970000\n),
+ q{addit2.imc} => qq(21001097.970000\n),
+ q{array_access.imc} => qr/^1\s\*\s1000\s=\s1000\n
1000\s\*\s1000\s=\s1000000\n
Array:\s\d+\.\d+s\n
\n
@@ -78,9 +78,9 @@ my %outputs = (
1\s\*\s1000\s=\s1000\n
1000\s\*\s1000\s=\s1000000\n
StringArray:\s\d+\.\d+s\n$/x,
- q(arriter.imc) => qq(100000\n100000\n100000\n111111\n),
- q(arriter_o1.imc) => qq(100000\n100000\n100000\n111111\n),
- q(bench_newp.pasm) => qr/^\d+\.\d+\sseconds.\s\d+\.\d+\sloops\/sec\n
+ q{arriter.imc} => qq(100000\n100000\n100000\n111111\n),
+ q{arriter_o1.imc} => qq(100000\n100000\n100000\n111111\n),
+ q{bench_newp.pasm} => qr/^\d+\.\d+\sseconds.\s\d+\.\d+\sloops\/sec\n
A\stotal\sof\s\d+\sbytes\swere\sallocated\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
A\stotal\sof\s\d+\scollection\sruns\swere\smade\n
@@ -89,69 +89,69 @@ my %outputs = (
There\sare\s\d+\stotal\sPMC\sstructs\n
There\sare\s\d+\sactive\sBuffer\sstructs\n
There\sare\s\d+\stotal\sBuffer\sstructs\n$/x,
- q(fib.imc) => qr/^fib\(28\)\s=\s317811\s\d+\.\d+s$/x,
- q(freeze.pasm) => qr/^constr.time\s\d+\.\d+\n
+ q{fib.imc} => qr/^fib\(28\)\s=\s317811\s\d+\.\d+s$/x,
+ q{freeze.pasm} => qr/^constr.time\s\d+\.\d+\n
freeze\stime\s\d+\.\d+\n
\s\sthaw\stime\s\d+\.\d+\n
Image\slen\s\d+\n
PerlArray\s100000\n$/x,
- q(gc_alloc_new.pasm) => qr/^\d+\.\d+\sseconds\.\n
+ q{gc_alloc_new.pasm} => qr/^\d+\.\d+\sseconds\.\n
A\stotal\sof\s\d+\sbytes\swere\sallocated\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
A\stotal\sof\s\d+\scollection\sruns\swere\smade\n
Copying\sa\stotal\sof\s\d+\sbytes\n
There\sare\s\d+\sactive\sBuffer\sstructs\n
There\sare\s\d+\stotal\sBuffer\sstructs\n$/x,
- q(gc_alloc_reuse.pasm) => qr/^\d+\.\d+\sseconds\.\n
+ q{gc_alloc_reuse.pasm} => qr/^\d+\.\d+\sseconds\.\n
A\stotal\sof\s\d+\sbytes\swere\sallocated\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
A\stotal\sof\s\d+\scollection\sruns\swere\smade\n
Copying\sa\stotal\sof\s\d+\sbytes\n
There\sare\s\d+\sactive\sBuffer\sstructs\n
There\sare\s\d+\stotal\sBuffer\sstructs\n$/x,
- q(gc_generations.pasm) => qr/^\d+\.\d+\sseconds\.\n
+ q{gc_generations.pasm} => qr/^\d+\.\d+\sseconds\.\n
A\stotal\sof\s\d+\sbytes\swere\sallocated\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
A\stotal\sof\s\d+\scollection\sruns\swere\smade\n
Copying\sa\stotal\sof\s\d+\sbytes\n
There\sare\s\d+\sactive\sBuffer\sstructs\n
There\sare\s\d+\stotal\sBuffer\sstructs\n$/x,
- q(gc_header_new.pasm) => qr/^\d+\.\d+\sseconds\.\n
+ q{gc_header_new.pasm} => qr/^\d+\.\d+\sseconds\.\n
A\stotal\sof\s\d+\sbytes\swere\sallocated\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
A\stotal\sof\s\d+\scollection\sruns\swere\smade\n
Copying\sa\stotal\sof\s\d+\sbytes\n
There\sare\s\d+\sactive\sBuffer\sstructs\n
There\sare\s\d+\stotal\sBuffer\sstructs\n$/x,
- q(gc_header_reuse.pasm) => qr/^\d+\.\d+\sseconds\.\n
+ q{gc_header_reuse.pasm} => qr/^\d+\.\d+\sseconds\.\n
A\stotal\sof\s\d+\sbytes\swere\sallocated\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
A\stotal\sof\s\d+\scollection\sruns\swere\smade\n
Copying\sa\stotal\sof\s\d+\sbytes\n
There\sare\s\d+\sactive\sBuffer\sstructs\n
There\sare\s\d+\stotal\sBuffer\sstructs\n$/x,
- q(gc_waves_headers.pasm) => qr/^\d+\.\d+\sseconds\.\n
+ q{gc_waves_headers.pasm} => qr/^\d+\.\d+\sseconds\.\n
A\stotal\sof\s\d+\sbytes\swere\sallocated\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
A\stotal\sof\s\d+\scollection\sruns\swere\smade\n
Copying\sa\stotal\sof\s\d+\sbytes\n
There\sare\s\d+\sactive\sBuffer\sstructs\n
There\sare\s\d+\stotal\sBuffer\sstructs\n$/x,
- q(gc_waves_sizeable_data.pasm) => qr/^\d+\.\d+\sseconds\.\n
+ q{gc_waves_sizeable_data.pasm} => qr/^\d+\.\d+\sseconds\.\n
A\stotal\sof\s\d+\sbytes\swere\sallocated\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
A\stotal\sof\s\d+\scollection\sruns\swere\smade\n
Copying\sa\stotal\sof\s\d+\sbytes\n
There\sare\s\d+\sactive\sBuffer\sstructs\n
There\sare\s\d+\stotal\sBuffer\sstructs\n$/x,
- q(gc_waves_sizeable_headers.pasm) => qr/^\d+\.\d+\sseconds\.\n
+ q{gc_waves_sizeable_headers.pasm} => qr/^\d+\.\d+\sseconds\.\n
A\stotal\sof\s\d+\sbytes\swere\sallocated\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
A\stotal\sof\s\d+\scollection\sruns\swere\smade\n
Copying\sa\stotal\sof\s\d+\sbytes\n
There\sare\s\d+\sactive\sBuffer\sstructs\n
There\sare\s\d+\stotal\sBuffer\sstructs\n$/x,
- q(mops.pasm) => qr/^Iterations:\s\s\s\s10000000\n
+ q{mops.pasm} => qr/^Iterations:\s\s\s\s10000000\n
Estimated\sops:\s20000000\n
done\n
Elapsed\stime:\s\s\d+\.\d+\n
@@ -160,27 +160,27 @@ my %outputs = (
Estimated\sops:\s200000000\n
Elapsed\stime:\s\s\d+\.\d+\n
M\sop\/s:\s\s\s\s\s\s\s\s\d+\.\d+\n$/x,
- q(oo1.pasm) => qq(10\n),
- q(oo2.pasm) => qq(10\n),
- q(oo3.pasm) => qq(10\n),
- q(oo4.pasm) => qq(500000\n),
- q(oo5.imc) => qq(10\n),
- q(oo6.imc) => qq(500000\n),
- q(oofib.imc) => qr/^fib\(28\)\s=\s317811\s\d+\.\d+s$/x,
- q(overload.imc) => qq(42\n),
- q(primes.pasm) => qr/^N\sprimes\sup\sto\s10000\sis:\s1229\n
+ q{oo1.pasm} => qq(10\n),
+ q{oo2.pasm} => qq(10\n),
+ q{oo3.pasm} => qq(10\n),
+ q{oo4.pasm} => qq(500000\n),
+ q{oo5.imc} => qq(10\n),
+ q{oo6.imc} => qq(500000\n),
+ q{oofib.imc} => qr/^fib\(28\)\s=\s317811\s\d+\.\d+s$/x,
+ q{overload.imc} => qq(42\n),
+ q{primes.pasm} => qr/^N\sprimes\sup\sto\s10000\sis:\s1229\n
last\sis:\s10001\n
Elapsed\stime:\s\d+\.\d+\n$/x,
- q(primes2.pasm) => qr/^N\sprimes\scalculated\sto\s5000\sis\s670\n
+ q{primes2.pasm} => qr/^N\sprimes\scalculated\sto\s5000\sis\s670\n
last\sis:\s4999\n$/x,
- q(primes2_i.pasm) => qr/^N\sprimes\scalculated\sto\s10000\s
+ q{primes2_i.pasm} => qr/^N\sprimes\scalculated\sto\s10000\s
is\s1230\nlast\sis:\s9973\n$/x,
- q(primes_i.pasm) => qr/^N\sprimes\sup\sto\s10000\sis:\s1229\n
+ q{primes_i.pasm} => qr/^N\sprimes\sup\sto\s10000\sis:\s1229\n
last\sis:\s9973\nElapsed\stime:\s\d+\.\d+\n$/x,
- q(shared_ref.pasm) => qq(),
- q(stress.pasm) => qr/^A\stotal\sof\s\d+\sDOD\sruns\s
+ q{shared_ref.pasm} => qq(),
+ q{stress.pasm} => qr/^A\stotal\sof\s\d+\sDOD\sruns\s
were\smade\n$/x,
- q(stress1.pasm) => qr/^\d+\.\d+\n
+ q{stress1.pasm} => qr/^\d+\.\d+\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
\d+\.\d+\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
@@ -200,13 +200,21 @@ my %outputs = (
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
\d+\.\d+\n
A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n$/x,
- q(stress2.pasm) => qq(),
- q(stress3.pasm) => qr/^A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
+ q{stress2.pasm} => qq(),
+ q{stress3.pasm} => qr/^A\stotal\sof\s\d+\sDOD\sruns\swere\smade\n
\d+\sactive\sPMCs\n
\d+\stotal\s\sPMCs\n$/x,
- q(vpm.imc) => qq(100000;\nl hackerjust another per\n)
+ q{vpm.imc} => qq(100000;\nl hackerjust another per\n)
);
+my %todo = ( q{arriter.imc} => 'syntax error',
+ q{arriter_o1.imc} => 'syntax error',
+ q{gc_header_new.pasm} => 'syntax error',
+ q{gc_waves_headers.pasm} => 'syntax error',
+ q{gc_waves_sizeable_headers.pasm} => 'syntax error',
+ q{stress3.pasm} => 'Null PMC access in
get_integer()',
+ q{vpm.imc} => 'delete_keyed() not
implemented',
+ );
# array_access.imc has weird output
my %filters = ( q(array_access.imc) => \&array_access_imc_filter );
@@ -214,8 +222,8 @@ sub array_access_imc_filter {
$_[ 0 ] =~ s/arr_size = S1/arr_size = 1000/;
}
-#plan tests => scalar keys %outputs;
-plan skip_all => 'currently not working';
+plan tests => scalar keys %outputs;
+# plan skip_all => 'currently not working';1
foreach ( sort keys %outputs ) {
SKIP: {
@@ -223,7 +231,7 @@ foreach ( sort keys %outputs ) {
eval {
my $file = q(examples/benchmarks/) . $_;
open( BENCH, qq(examples/benchmarks/$_) )
- or die qq(Couldn't open $_: $!.\n);
+ or die qq(Could not open $_: $!.\n);
while ( my $line = <BENCH> ) { $bench .= $line; }
close( BENCH );
@@ -233,12 +241,15 @@ foreach ( sort keys %outputs ) {
};
skip( $@, 1 ) if $@;
+ my @todo = $todo{$_} ? ( todo => $todo{$_} ) : ();
+
+ # XXX use example_output_is() and example_output_like()
if ( ref $outputs{ $_ } eq q(Regexp) ) {
if ( /\.pasm$/ ) {
- pasm_output_like( $bench, $outputs{ $_ }, $_ );
+ pasm_output_like( $bench, $outputs{ $_ }, $_, @todo );
}
elsif ( /\.imc$/ ) {
- pir_output_like( $bench, $outputs{ $_ }, $_ );
+ pir_output_like( $bench, $outputs{ $_ }, $_, @todo );
}
else {
skip( qq(Unknown file type: $_.), 1 );
@@ -246,10 +257,10 @@ foreach ( sort keys %outputs ) {
}
else {
if ( /\.pasm$/ ) {
- pasm_output_is( $bench, $outputs{ $_ }, $_ );
+ pasm_output_is( $bench, $outputs{ $_ }, $_, @todo );
}
elsif ( /\.imc$/ ) {
- pir_output_is( $bench, $outputs{ $_ }, $_ );
+ pir_output_is( $bench, $outputs{ $_ }, $_, @todo );
}
else {
skip( qq(Unknown file type: $_.), 1 );
