Sweet! I guess the only ''drawback'' is the meta-data is only accessable to perl, but who cares. I bet this fix will make #118 easier.
http://svn.open-mpi.org/trac/mtt/ticket/118 "Slim down result submissions to the bare essentials" On Thu, Nov/30/2006 07:55:14AM, jsquy...@osl.iu.edu wrote: > Author: jsquyres > Date: 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006) > New Revision: 461 > > Removed: > trunk/lib/XML/ > Modified: > trunk/lib/MTT/Files.pm > trunk/lib/MTT/MPI.pm > trunk/lib/MTT/Test.pm > trunk/lib/MTT/Test/RunEngine.pm > > Log: > Fixes #144 > > This commit is a bunch of things to improve the meta data handling and > storage in the MTT client: > > * Remove the use of XML::Simple and use Data::Dumper instead. This > allows us to remove the costly/slow (and error-prone when coding up > new data layers) data transformation for each of the datasets that > were saved out to disk. The output data is about the same size, > but we avoid the [potentially] huge memory transformation before > the write, and for large data sets (e.g., Test run data), can save > quite a bit of time (e.g., ~3 seconds with Data::Dumper vs. ~10 > seconds with XML::Simple). > * Split the Test run data up into multiple files instead of one huge > file. This allows us to write only the new data after tests have > run rather than having to write ''all'' the data ''every'' time. > It makes the reading and writing of the test run data a little more > complex, but the performance win is worth it. > * Write the Test Run data [slightly] less frequently -- i.e., write > out a [small] batch of new Test Run data periodically rather than > writing after ''every'' test. > > > Modified: trunk/lib/MTT/Files.pm > ============================================================================== > --- trunk/lib/MTT/Files.pm (original) > +++ trunk/lib/MTT/Files.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006) > @@ -40,8 +40,15 @@ > > #-------------------------------------------------------------------------- > > -sub mkdir { > +sub safe_mkdir { > my ($dir) = @_; > + MTT::Files::mkdir($dir, 1); > +} > + > +#-------------------------------------------------------------------------- > + > +sub mkdir { > + my ($dir, $safe) = @_; > > my $c = cwd(); > Debug("Making dir: $dir (cwd: $c)\n"); > @@ -58,6 +65,9 @@ > foreach my $p (@parts) { > next if (! $p); > > + $p = make_safe_filename($p) > + if ($safe); > + > $str .= "$p"; > if (! -d $str) { > Debug("$str does not exist -- creating\n"); > @@ -402,6 +412,8 @@ > return 1; > } > > +#-------------------------------------------------------------------------- > + > # Copy infile or stdin to a unique file in /tmp > sub copyfile { > > @@ -432,4 +444,64 @@ > return $outfile; > } > > +#-------------------------------------------------------------------------- > + > +sub load_dumpfile { > + my ($filename, $data) = @_; > + > + # Check that the file is there > + return > + if (! -r $filename); > + > + # Get the file size > + my ($dev, $ino, $mode, $nlink, $uid, $gid, $rdev, $size, > + $atime, $mtime, $ctime, $blksize, $blocks) = stat($filename); > + > + # Read it in > + open IN, $filename; > + my $tmp; > + read(IN, $tmp, $size); > + close IN; > + > + # It's in Dumper format. How many $VARx's are there? > + return > + if (! $tmp =~ m/\$VAR[0-9]+/g); > + my $count = 0; > + ++$count > + while ($tmp =~ m/\$VAR[0-9]+/g); > + > + # We know how many $VARx's there are. Build up a string to eval. > + my $str; > + my $var_num = 1; > + while ($var_num <= $count) { > + $str .= "my \$VAR$var_num;\n"; > + ++$var_num; > + } > + $str .= "eval \$tmp;\n"; > + my $var_num = 1; > + while ($var_num <= $count) { > + $str .= "\$\$data->{VAR$var_num} = \$VAR$var_num;\n"; > + ++$var_num; > + } > + eval $str; > +} > + > +#-------------------------------------------------------------------------- > + > +sub save_dumpfile { > + my ($filename) = @_; > + shift; > + > + # Serialize > + my $d = new Data::Dumper([@_]); > + $d->Purity(1)->Indent(1); > + > + open FILE, ">$filename.new"; > + print FILE $d->Dump; > + close FILE; > + > + # Atomically move it onto the old file > + rename("$filename.new", $filename); > +} > + > 1; > > Modified: trunk/lib/MTT/MPI.pm > ============================================================================== > --- trunk/lib/MTT/MPI.pm (original) > +++ trunk/lib/MTT/MPI.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006) > @@ -15,10 +15,9 @@ > package MTT::MPI; > > use strict; > -use XML::Simple qw(:strict); > use MTT::MPI::Get; > use MTT::MPI::Install; > -use Data::Dumper; > +use MTT::Files; > > #-------------------------------------------------------------------------- > > @@ -31,16 +30,10 @@ > #-------------------------------------------------------------------------- > > # Filename where list of MPI sources is kept > -my $sources_data_filename = "mpi_sources.xml"; > - > -# XML options for the MPI sources > -my $sources_xs; > +my $sources_data_filename = "mpi_sources.dump"; > > # Filename where list of MPI installs is kept > -my $installs_data_filename = "mpi_installs.xml"; > - > -# XML options for the MPI installs > -my $installs_xs; > +my $installs_data_filename = "mpi_installs.dump"; > > #-------------------------------------------------------------------------- > > @@ -60,65 +53,17 @@ > > #-------------------------------------------------------------------------- > > -sub _scrub { > - my $h = shift; > - > - foreach my $k (keys(%$h)) { > - if (ref($h->{$k}) eq "") { > - # Remove bad characters > - $h->{$k} =~ s/\000/<NULL>/g; > - } else { > - _scrub($h->{$k}); > - } > - } > -} > - > -#-------------------------------------------------------------------------- > - > -sub _setup_sources_xml { > - $sources_xs = new XML::Simple(KeyAttr => { mpi_get => > "simple_section_name", > - mpi_version => "version", > - }, > - ForceArray => [ "mpi_get", > - "mpi_version", > - ], > - AttrIndent => 1, > - RootName => "mpi_sources", > - ); > -} > - > -#-------------------------------------------------------------------------- > - > +use Data::Dumper; > sub LoadSources { > my ($dir) = @_; > > # Explicitly delete anything that was there > $MTT::MPI::sources = undef; > > - _setup_sources_xml() > - if (!$sources_xs); > - > # If the file exists, read it in > - if (-f "$dir/$sources_data_filename") { > - my $in = $sources_xs->XMLin("$dir/$sources_data_filename"); > - > - # Now transform this to the form suitable for > - # $MTT::MPI::sources (see comment in SaveSources) > - > - # For each MPI get section > - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) { > - my $mpi_get = $in->{mpi_get}->{$mpi_get_key}; > - > - # For each version > - foreach my $mpi_version_key (keys(%{$mpi_get->{mpi_version}})) { > - my $mpi_version = > $mpi_get->{mpi_version}->{$mpi_version_key}; > - > - $MTT::MPI::sources->{$mpi_get_key}->{$mpi_version_key} = > $mpi_version; > - > $MTT::MPI::sources->{$mpi_get_key}->{$mpi_version_key}->{simple_section_name} > = $mpi_get_key; > - > $MTT::MPI::sources->{$mpi_get_key}->{$mpi_version_key}->{version} = > $mpi_version_key; > - } > - } > - } > + my $data; > + MTT::Files::load_dumpfile("$dir/$sources_data_filename", \$data); > + $MTT::MPI::sources = $data->{VAR1}; > } > > #-------------------------------------------------------------------------- > @@ -126,54 +71,8 @@ > sub SaveSources { > my ($dir) = @_; > > - _setup_sources_xml() > - if (!$sources_xs); > - > - # Transform $MTT::MPI::sources to something XML::Simple can write > - # into valid XML (since our values can [and will] contain :'s, > - # which are the namespace identifiers in XML) > - my $transformed; > - > - # For each MPI get section > - foreach my $mpi_get_key (keys(%$MTT::MPI::sources)) { > - my $mpi_get = $MTT::MPI::sources->{$mpi_get_key}; > - > - # For each version > - foreach my $mpi_version_key (keys(%$mpi_get)) { > - my $mpi_version = $mpi_get->{$mpi_version_key}; > - > - # Deep copy and scrub > - my $h; > - %$h = %$mpi_version; > - _scrub($h); > - > - > $transformed->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key} = > $h; > - } > - } > - > - # Write out the file > - my $xml = $sources_xs->XMLout($transformed); > - my $file = "$dir/$sources_data_filename"; > - open(FILE, ">$file.new"); > - print FILE $xml; > - close(FILE); > - system("mv $file.new $file"); > -} > - > -#-------------------------------------------------------------------------- > - > -sub _setup_installs_xml { > - $installs_xs = new XML::Simple(KeyAttr => { mpi_get => > "simple_section_name", > - mpi_version => "version", > - mpi_install => > "simple_section_name", > - }, > - ForceArray => [ "mpi_get", > - "mpi_version", > - "mpi_install", > - ], > - AttrIndent => 1, > - RootName => "mpi_installs", > - ); > + MTT::Files::save_dumpfile("$dir/$sources_data_filename", > + $MTT::MPI::sources); > } > > #-------------------------------------------------------------------------- > @@ -184,33 +83,10 @@ > # Explicitly delete anything that was there > $MTT::MPI::installs = undef; > > - _setup_installs_xml() > - if (!$installs_xs); > - > # If the file exists, read it in > - if (-f "$dir/$installs_data_filename") { > - my $in = $installs_xs->XMLin("$dir/$installs_data_filename"); > - > - # Now transform this to the form suitable for > - # $MTT::MPI::installs (see comment in SaveSources). Wow. > - > - # For each MPI get section > - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) { > - my $mpi_get = $in->{mpi_get}->{$mpi_get_key}; > - > - # For each version of each MPI get > - foreach my $mpi_version_key (keys(%{$mpi_get->{mpi_version}})) { > - my $mpi_version = > $mpi_get->{mpi_version}->{$mpi_version_key}; > - > - # For each MPI install section > - foreach my $mpi_install_key > (keys(%{$mpi_version->{mpi_install}})) { > - > $MTT::MPI::installs->{$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key} = > - > $in->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key}; > - > $MTT::MPI::installs->{$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key}->{simple_section_name} > = $mpi_install_key; > - } > - } > - } > - } > + my $data; > + MTT::Files::load_dumpfile("$dir/$installs_data_filename", \$data); > + $MTT::MPI::installs = $data->{VAR1}; > } > > #-------------------------------------------------------------------------- > @@ -218,42 +94,8 @@ > sub SaveInstalls { > my ($dir) = @_; > > - _setup_installs_xml() > - if (!$installs_xs); > - > - # Transform $MTT::MPI::installs to something XML::Simple can write > - # into valid XML (see comment in SaveSources). Wow. > - my $transformed; > - > - # For each MPI get section > - foreach my $mpi_get_key (keys(%{$MTT::MPI::installs})) { > - my $mpi_get = $MTT::MPI::installs->{$mpi_get_key}; > - > - # For each version of that get > - foreach my $mpi_version_key (keys(%{$mpi_get})) { > - my $mpi_version = $mpi_get->{$mpi_version_key}; > - > - # For each MPI install action > - foreach my $mpi_install_key (keys(%{$mpi_version})) { > - my $mpi_install = $mpi_version->{$mpi_install_key}; > - > - # Deep copy and scrub > - my $h; > - %$h = %$mpi_install; > - _scrub($h); > - > - > $transformed->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key} > = $h; > - } > - } > - } > - > - # Write out the file > - my $xml = $installs_xs->XMLout($transformed); > - my $file = "$dir/$installs_data_filename"; > - open(FILE, ">$file.new"); > - print FILE $xml; > - close(FILE); > - system("mv $file.new $file"); > + MTT::Files::save_dumpfile("$dir/$installs_data_filename", > + $MTT::MPI::installs); > } > > 1; > > Modified: trunk/lib/MTT/Test.pm > ============================================================================== > --- trunk/lib/MTT/Test.pm (original) > +++ trunk/lib/MTT/Test.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov 2006) > @@ -13,11 +13,11 @@ > package MTT::Test; > > use strict; > +use File::Find; > use MTT::Test::Get; > use MTT::Test::Build; > use MTT::Test::Run; > -use XML::Simple; > -use Data::Dumper; > +use MTT::Files; > > #-------------------------------------------------------------------------- > > @@ -29,26 +29,24 @@ > > # Exported run tests handle > our $runs; > +our $runs_to_be_saved; > > #-------------------------------------------------------------------------- > > # Filename where list of test sources information is kept > -my $sources_data_filename = "test_sources.xml"; > - > -# XML options for the test builds > -my $sources_xs; > +my $sources_data_filename = "test_sources.dump"; > > # Filename where list of test build information is kept > -my $builds_data_filename = "test_builds.xml"; > +my $builds_data_filename = "test_builds.dump"; > > -# XML options for the test builds > -my $builds_xs; > +# Subdir where test runs are kept > +my $runs_subdir = "test_runs"; > > # Filename where list of test run information is kept > -my $runs_data_filename = "test_runs.xml"; > +my $runs_data_filename = "test_runs.dump"; > > -# XML options for the test runs > -my $runs_xs; > +# Helper variable for when we're loading test run data > +my $load_run_file_start_dir; > > #-------------------------------------------------------------------------- > > @@ -76,57 +74,16 @@ > > #-------------------------------------------------------------------------- > > -sub _scrub { > - my $h = shift; > - > - foreach my $k (keys(%$h)) { > - if (ref($h->{$k}) eq "") { > - # Remove bad characters > - $h->{$k} =~ s/\000/<NULL>/g; > - } else { > - _scrub($h->{$k}); > - } > - } > -} > - > -#-------------------------------------------------------------------------- > - > -sub _setup_sources_xml { > - $sources_xs = new XML::Simple(KeyAttr => { test_get => > "simple_section_name", > - }, > - ForceArray => [ "test_get", > - ], > - AttrIndent => 1, > - RootName => "test_sources", > - ); > -} > - > -#-------------------------------------------------------------------------- > - > sub LoadSources { > my ($dir) = @_; > > # Explicitly delete anything that was there > $MTT::Test::sources = undef; > > - _setup_sources_xml() > - if (!$sources_xs); > - > # If the file exists, read it in > - if (-f "$dir/$sources_data_filename") { > - my $in = $sources_xs->XMLin("$dir/$sources_data_filename"); > - > - # Now transform this to the form suitable for > - # $MTT::Test::sources (see comment in SaveSources) > - > - # For each Test get section > - foreach my $test_get_key (keys(%{$in->{test_get}})) { > - my $test_get = $in->{test_get}->{$test_get_key}; > - > - $MTT::Test::sources->{$test_get_key} = $test_get; > - $MTT::Test::sources->{$test_get_key}->{simple_section_name} = > $test_get_key; > - } > - } > + my $data; > + MTT::Files::load_dumpfile("$dir/$sources_data_filename", \$data); > + $MTT::Test::sources = $data->{VAR1}; > } > > #-------------------------------------------------------------------------- > @@ -134,51 +91,8 @@ > sub SaveSources { > my ($dir) = @_; > > - _setup_sources_xml() > - if (!$sources_xs); > - > - # Transform $MTT::Test::sources to something XML::Simple can write > - # into valid XML (since our values can [and will] contain :'s, > - # which are the namespace identifiers in XML) > - my $transformed; > - > - # For each Test get section > - foreach my $test_get_key (keys(%$MTT::Test::sources)) { > - my $test_get = $MTT::Test::sources->{$test_get_key}; > - > - # Deep copy and scrub > - my $h; > - %$h = %$test_get; > - _scrub($h); > - > - $transformed->{test_get}->{$test_get_key} = $h; > - } > - > - # Write out the file > - my $xml = $sources_xs->XMLout($transformed); > - my $file = "$dir/$sources_data_filename"; > - open(FILE, ">$file.new"); > - print FILE $xml; > - close(FILE); > - system("mv $file.new $file"); > -} > - > -#-------------------------------------------------------------------------- > - > -sub _setup_builds_xml { > - $builds_xs = new XML::Simple(KeyAttr => { mpi_get => > "simple_section_name", > - mpi_version => "version", > - mpi_install => > "simple_section_name", > - test_build => > "simple_section_name", > - }, > - ForceArray => [ "mpi_get", > - "mpi_version", > - "mpi_install", > - "test_build", > - ], > - AttrIndent => 1, > - RootName => "test_builds", > - ); > + MTT::Files::save_dumpfile("$dir/$sources_data_filename", > + $MTT::Test::sources); > } > > #-------------------------------------------------------------------------- > @@ -189,39 +103,10 @@ > # Explicitly delete anything that was there > $MTT::Test::builds = undef; > > - _setup_builds_xml() > - if (!$builds_xs); > - > # If the file exists, read it in > - if (-f "$dir/$builds_data_filename") { > - my $in = $builds_xs->XMLin("$dir/$builds_data_filename"); > - > - # Now transform this to the form suitable for > - # $MTT::Test::builds (see comment in SaveSources). Wow. > - # For each MPI get section > - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) { > - my $mpi_get = $in->{mpi_get}->{$mpi_get_key}; > - > - # For each version of that MPI > - foreach my $mpi_version_key (keys(%{$mpi_get->{mpi_version}})) { > - my $mpi_version = > $mpi_get->{mpi_version}->{$mpi_version_key}; > - > - # For each MPI install section > - foreach my $mpi_install_key > (keys(%{$mpi_version->{mpi_install}})) { > - my $mpi_install = > $mpi_version->{mpi_install}->{$mpi_install_key}; > - > - # For each test build section > - foreach my $test_build_key > (keys(%{$mpi_install->{test_build}})) { > - my $test_build = > $mpi_install->{test_build}->{$test_build_key}; > - > - > $MTT::Test::builds->{$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key}->{$test_build_key} > = > - > $in->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key}->{test_build}->{$test_build_key}; > - > $MTT::Test::builds->{$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key}->{$test_build_key}->{simple_section_name} > = $test_build_key; > - } > - } > - } > - } > - } > + my $data; > + MTT::Files::load_dumpfile("$dir/$builds_data_filename", \$data); > + $MTT::Test::builds = $data->{VAR1}; > } > > #-------------------------------------------------------------------------- > @@ -229,135 +114,48 @@ > sub SaveBuilds { > my ($dir) = @_; > > - _setup_builds_xml() > - if (!$builds_xs); > - > - # Transform $MTT::Test::builds to something XML::Simple can write > - # into valid XML (see comment in SaveSources). Wow. > - my $transformed; > - > - # For each MPI get section > - foreach my $mpi_get_key (keys(%{$MTT::Test::builds})) { > - my $mpi_get = $MTT::Test::builds->{$mpi_get_key}; > - > - # For each version of that MPI > - foreach my $mpi_version_key (keys(%{$mpi_get})) { > - my $mpi_version = $mpi_get->{$mpi_version_key}; > - > - # For each MPI install section > - foreach my $mpi_install_key (keys(%{$mpi_version})) { > - my $mpi_install = $mpi_version->{$mpi_install_key}; > - > - # For each test build section > - foreach my $test_build_key (keys(%{$mpi_install})) { > - my $test_build = $mpi_install->{$test_build_key}; > - > - # Deep copy and scrub > - my $h; > - %$h = %$test_build; > - _scrub($h); > - > - > $transformed->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key}->{test_build}->{$test_build_key} > = $h; > - } > - } > - } > - } > - > - # Write out the file > - my $xml = $builds_xs->XMLout($transformed); > - my $file = "$dir/$builds_data_filename"; > - open(FILE, ">$file.new"); > - print FILE $xml; > - close(FILE); > - system("mv $file.new $file"); > -} > - > -#-------------------------------------------------------------------------- > - > -sub _setup_runs_xml { > - $runs_xs = new XML::Simple(KeyAttr => { mpi_get => "simple_section_name", > - mpi_version => "version", > - mpi_install => > "simple_section_name", > - test_build => > "simple_section_name", > - test_run => > "simple_section_name", > - test_name => "name", > - test_np => "nprocs", > - test_cmd => "argv", > - }, > - ForceArray => [ "mpi_get", > - "mpi_version", > - "mpi_install", > - "test_build", > - "test_run", > - "test_name", > - "test_np", > - "test_cmd", > - ], > - AttrIndent => 1, > - RootName => "test_runs", > - ); > + MTT::Files::save_dumpfile("$dir/$builds_data_filename", > + $MTT::Test::builds); > } > > #-------------------------------------------------------------------------- > > +use Data::Dumper; > sub LoadRuns { > my ($dir) = @_; > > # Explicitly delete anything that was there > $MTT::Test::runs = undef; > > - _setup_runs_xml() > - if (!$runs_xs); > - > - # If the file exists, read it in > - if (-f "$dir/$runs_data_filename") { > - my $in = $runs_xs->XMLin("$dir/$runs_data_filename"); > - > - # Now transform this to the form suitable for > - # $MTT::Test::runs (see comment in SaveSources). Wow. > - # For each MPI get section > - foreach my $mpi_get_key (keys(%{$in->{mpi_get}})) { > - my $mpi_get = $in->{mpi_get}->{$mpi_get_key}; > - > - # For each MPI source > - foreach my $mpi_version_key (keys(%{$mpi_get->{mpi_version}})) { > - my $mpi_version = > $mpi_get->{mpi_version}->{$mpi_version_key}; > - > - # For each MPI install section > - foreach my $mpi_install_key > (keys(%{$mpi_version->{mpi_install}})) { > - my $mpi_install = > $mpi_version->{mpi_install}->{$mpi_install_key}; > - > - # For each test build section > - foreach my $test_build_key > (keys(%{$mpi_install->{test_build}})) { > - my $test_build = > $mpi_install->{test_build}->{$test_build_key}; > - > - # For each test run section > - foreach my $test_run_key > (keys(%{$test_build->{test_run}})) { > - my $test_run = > $test_build->{test_run}->{$test_run_key}; > - > - # For each test name > - foreach my $test_name_key > (keys(%{$test_run->{test_name}})) { > - my $test_name = > $test_run->{test_name}->{$test_name_key}; > - > - # For each np > - foreach my $test_np_key > (keys(%{$test_name->{test_np}})) { > - my $test_np = > $test_name->{test_np}->{$test_np_key}; > - > - # For each test command > - foreach my $test_cmd_key > (keys(%{$test_np->{test_cmd}})) { > - my $test_cmd = > $test_np->{test_cmd}->{$test_cmd_key}; > - > - > $MTT::Test::runs->{$mpi_get_key}->{$mpi_version_key}->{$mpi_install_key}->{$test_build_key}->{$test_run_key}->{$test_name_key}->{$test_np_key}->{$test_cmd_key} > = > - > $in->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key}->{test_build}->{$test_build_key}->{test_run}->{$test_run_key}->{test_name}->{$test_name_key}->{test_np}->{$test_np_key}->{test_cmd}->{$test_cmd_key}; > - } > - } > - } > - } > - } > - } > - } > - } > - } > + # See SaveRuns, below, for an explanation. We traverse > + # directories looking for dump files and read them into the > + # appropriate section of the $MTT::Test::runs hash. > + > + $load_run_file_start_dir = "$dir/$runs_subdir"; > + find(\&load_run_file, $load_run_file_start_dir) > + if (-d $load_run_file_start_dir); > +} > + > +sub load_run_file { > + # We only want files named $runs_data_filename > + return 0 > + if (! -f $_ || $runs_data_filename ne $_); > + > + # Read in the file > + my $data; > + MTT::Files::load_dumpfile($File::Find::name, \$data); > + > + # Put the loaded data in the hash in the right place. Per > + # SaveRuns(), below, we look at the key values in $data->{VAR1} to > + # know where to put $data->{VAR2} in the $MTT::Test::runs hash. > + my $str = "\$MTT::Test::runs"; > + my $k = 1; > + do { > + $str .= "->{\"" . $data->{VAR1}->{$k} . "\"}"; > + ++$k; > + } while (exists($data->{VAR1}->{$k})); > + $str .= " = \$data->{VAR2}"; > + eval $str; > } > > #-------------------------------------------------------------------------- > @@ -365,15 +163,25 @@ > sub SaveRuns { > my ($dir) = @_; > > - _setup_runs_xml() > - if (!$runs_xs); > + # Because test run data can get very, very large, we break it up > + # and store it in lots of smaller files so that we can write out > + # to disk in small portions. > + > + # Test runs are stored in the hash with this order of keys: > + > + # mpi_get_simple_section_name > + # mpi_version > + # mpi_install_simple_section_name > + # test_build_simple_section_name > + # test_run_simple_section_name > + # test_name > + # np > + # command > > - # Transform $MTT::Test::runs to something XML::Simple can write > - # into valid XML (see comment in SaveSources). Wow. > - my $transformed; > + # We save from test_name and down in a single file. > > # For each MPI get section > - foreach my $mpi_get_key (keys(%{$MTT::Test::runs})) { > + foreach my $mpi_get_key (keys(%{$MTT::Test::runs_to_be_saved})) { > my $mpi_get = $MTT::Test::runs->{$mpi_get_key}; > > # For each source of that MPI > @@ -395,23 +203,22 @@ > # For each test name > foreach my $test_name_key (keys(%{$test_run})) { > my $test_name = $test_run->{$test_name_key}; > - > - # For each np > - foreach my $test_np_key (keys(%{$test_name})) { > - my $test_np = $test_name->{$test_np_key}; > - > - # For each cmd > - foreach my $test_cmd_key (keys(%{$test_np})) > { > - my $test_cmd = $test_np->{$test_cmd_key}; > - > - # Deep copy and scrub > - my $h; > - %$h = %$test_cmd; > - _scrub($h); > - > - > $transformed->{mpi_get}->{$mpi_get_key}->{mpi_version}->{$mpi_version_key}->{mpi_install}->{$mpi_install_key}->{test_build}->{$test_build_key}->{test_run}->{$test_run_key}->{test_name}->{$test_name_key}->{test_np}->{$test_np_key}->{test_cmd}->{$test_cmd_key} > = $h; > - } > - } > + > + my $file = > MTT::Files::safe_mkdir("$dir/$runs_subdir/$mpi_get_key/$mpi_version_key/$mpi_install_key/$test_build_key/$test_run_key/$test_name_key") > . "/$runs_data_filename"; > + > + # We need to save two items in test run > + # data files -- the actual data and where > + # it belongs in the $MTT::Test::runs hash. > + my $hashname = { > + 1 => $mpi_get_key, > + 2 => $mpi_version_key, > + 3 => $mpi_install_key, > + 4 => $test_build_key, > + 5 => $test_run_key, > + 6 => $test_name_key, > + }; > + MTT::Files::save_dumpfile($file, $hashname, > + $test_name); > } > } > } > @@ -419,13 +226,8 @@ > } > } > > - # Write out the file > - my $xml = $runs_xs->XMLout($transformed); > - my $file = "$dir/$runs_data_filename"; > - open(FILE, ">$file.new"); > - print FILE $xml; > - close(FILE); > - system("mv $file.new $file"); > + # Explicitly reset the test runs to be saved > + $MTT::Test::runs_to_be_saved = undef; > } > > 1; > > Modified: trunk/lib/MTT/Test/RunEngine.pm > ============================================================================== > --- trunk/lib/MTT/Test/RunEngine.pm (original) > +++ trunk/lib/MTT/Test/RunEngine.pm 2006-11-30 07:55:07 EST (Thu, 30 Nov > 2006) > @@ -67,6 +67,9 @@ > } > } > ++$count; > + > + # Write out the "to be saved" test run results > + MTT::Test::SaveRuns($top_dir); > > # Output a progress bar > if ($verbose_out > 50) { > @@ -189,8 +192,21 @@ > # Assume that the Analyze module will output one line > ++$verbose_out; > > - > $MTT::Test::runs->{$mpi_details->{mpi_get_simple_section_name}}->{$mpi_details->{version}}->{$mpi_details->{mpi_install_simple_section_name}}->{$run->{test_build_simple_section_name}}->{$run->{simple_section_name}}->{$name}->{$MTT::Test::Run::test_np}->{$cmd} > = $report; > - MTT::Test::SaveRuns($top_dir); > + # For Test Runs data, we have two datasets: the "to be saved" set > + # and the "all results" set. The "to be saved" set is a > + # relatively small set of data that is written out to disk > + # periodically (i.e., augmenting what has already been written > + # out). The "all results" set is everything that has occurred so > + # far. We do this because the "all results" set can get *very* > + # large, so we don't want to write out the whole thing every time > + # we save the results to disk. > + > + # So save this new result in both the "to be saved" and "all > + # results" sets. We'll write out the "to be saved" results > + # shortly. > + > + > $MTT::Test::runs_to_be_saved->{$mpi_details->{mpi_get_simple_section_name}}->{$mpi_details->{version}}->{$mpi_details->{mpi_install_simple_section_name}}->{$run->{test_build_simple_section_name}}->{$run->{simple_section_name}}->{$name}->{$MTT::Test::Run::test_np}->{$cmd} > = > + > $MTT::Test::runs->{$mpi_details->{mpi_get_simple_section_name}}->{$mpi_details->{version}}->{$mpi_details->{mpi_install_simple_section_name}}->{$run->{test_build_simple_section_name}}->{$run->{simple_section_name}}->{$name}->{$MTT::Test::Run::test_np}->{$cmd} > = $report; > MTT::Reporter::QueueAdd("Test Run", $run->{simple_section_name}, > $report); > > # If there is an after_each step, run it > _______________________________________________ > mtt-svn mailing list > mtt-...@open-mpi.org > http://www.open-mpi.org/mailman/listinfo.cgi/mtt-svn -- -Ethan
