Sweet! Can you document this on the wiki in the listing of all the INI fields?
On Mar 2, 2012, at 10:27 AM, mi...@osl.iu.edu wrote: > Author: miked > Date: 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012) > New Revision: 1440 > URL: https://svn.open-mpi.org/trac/mtt/changeset/1440 > > Log: > Now it is possible to shuffle tests in mtt. The parameter that enables this > feature is: > shuffle_tests= > it can be assigned a comma separated list of allowed values: sections, tests, > execs, nps, args, all. > > Notations: > sections - shuffle test_run sections found in mtt > tests - shuffle test executables if any (we have single executable per > test_run) > execs - shuffle exec command line present in main [MTT] section > nps - shuffle runs with different nps > args - shuffle executable arguments > all - enable all the above. > > > Text files modified: > trunk/lib/MTT/Globals.pm | 65 > +++++++++++---------------------------- > trunk/lib/MTT/Reporter/Email.pm | 6 ++- > > trunk/lib/MTT/Reporter/TextFile.pm | 2 > > trunk/lib/MTT/Test/Run.pm | 7 +++ > > trunk/lib/MTT/Test/RunEngine.pm | 61 > +++++++++++++++++++++++++++++-------- > trunk/lib/MTT/Util.pm | 14 ++++++- > > 6 files changed, 89 insertions(+), 66 deletions(-) > > Modified: trunk/lib/MTT/Globals.pm > ============================================================================== > --- trunk/lib/MTT/Globals.pm (original) > +++ trunk/lib/MTT/Globals.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012) > @@ -80,8 +80,7 @@ > > save_intermediate_report => undef, > save_intermediate_report_enable => undef, > - ini_value_run_for => undef, > - start_time => localtime, > + shuffle_tests => undef, > description => undef, > }; > > @@ -181,50 +180,24 @@ > $Values->{save_intermediate_report_enable} = $val; > } > > -# $val = MTT::Values::Value($ini, "MTT", "finish_at"); > -# if (defined($val)) { > -# #finish_at format: hh:mm[dd/MM] > -# $val =~ m/(\d\d:\d\d)(\[(\d\d)\/(\d\d)\])*/; > -# my $stop_time = $1.":00"; > -# my $stop_day = $3; > -# my $stop_month =$4; > -# my @timeData = localtime(time); > -# #@timedata : [0]secs [1]minutes [2]hours [3]days [4]month-1 > -# my $finish_time = MTT::Util::parse_time_to_seconds($stop_time); > -# > -# > -# my $local_time=$timeData[2].":".$timeData[1].":".$timeData[0]; > -# my $current_time = > MTT::Util::parse_time_to_seconds($local_time); > -# > -# > -# > -# my $secs_left; > -# if ($current_time < $finish_time){ > -# $secs_left = $finish_time-$current_time; > -# } else { > -# $secs_left = $finish_time+24*3600-$current_time; > -# } > -# $Values->{ini_value_run_for} = $secs_left; > -# > -# if (!$stop_month){ > -# if ($stop_day){ > -# if ($stop_day < $timeData[3]){ > -# MTT::Messages::Warning("Stop date is less than > start date: disabling stop_at feature"); > -# $Values->{ini_value_run_for} = undef; > -# }else if ($stop_day == $timeData[3]){ > -# if ($finish_time < $current_time){ > -# MTT::Messages::Warning("The stop is > less than start time: disabling stop_at feature"); > -# } > -# }else{ > -# if ($finish_time > $current_time){ > -# $secs_left += 24*3600; > -# } > -# } > -# } > -# } > -# printf "secs_left = ",$secs_left, "hours_left = > ",$secs_left/3600,"\n"; > -# > -# } > + > + $val = MTT::Values::Value($ini, "MTT", "shuffle_tests"); > + if (defined($val)) { > + my @shuffles = grep length, split(/\s*,\s*/,$val); > + my @allowed_shuffles = ('sections', 'tests', 'execs', 'nps', > 'args', 'all'); > + my %allowed_map = map { $_ => 1} @allowed_shuffles; > + foreach my $sh (@shuffles){ > + if (!exists($allowed_map{$sh})){ > + MTT::Messages::Error("<$sh> is not allowed > shuffle_tests value. Allowed are:", join(',',@allowed_shuffles),"."); > + } > + $Values->{shuffle_tests}->{$sh} = 1; > + } > + if ($Values->{shuffle_tests}->{all}){ > + foreach my $allow (@allowed_shuffles){ > + $Values->{shuffle_tests}->{$allow} = 1; > + } > + } > + } > } > > #-------------------------------------------------------------------------- > > Modified: trunk/lib/MTT/Reporter/Email.pm > ============================================================================== > --- trunk/lib/MTT/Reporter/Email.pm (original) > +++ trunk/lib/MTT/Reporter/Email.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar > 2012) > @@ -148,8 +148,10 @@ > $to = Value( $Ini, $Section, "email_to" ); > } > } > - my $body = $subject."\n".$footer."\n"; > - MTT::Mail::Send($subject, $to, $from, $body ); > + my $body = $footer."\n"; > + if ($to){ > + MTT::Mail::Send($subject, $to, $from, $body ); > + } > } > > 1; > > Modified: trunk/lib/MTT/Reporter/TextFile.pm > ============================================================================== > --- trunk/lib/MTT/Reporter/TextFile.pm (original) > +++ trunk/lib/MTT/Reporter/TextFile.pm 2012-03-02 10:27:34 EST (Fri, > 02 Mar 2012) > @@ -142,7 +142,7 @@ > sub Flush{ > my ($info, $entries) = @_; > my @results_to_flush = @results; > - push(@results_to_flush, $entries); > + push(@results_to_flush, $entries) if $entries; > _summary_report(\@results_to_flush, "yes") > if (@results_to_flush); > > > Modified: trunk/lib/MTT/Test/Run.pm > ============================================================================== > --- trunk/lib/MTT/Test/Run.pm (original) > +++ trunk/lib/MTT/Test/Run.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012) > @@ -77,7 +77,12 @@ > > # Go through all the sections in the ini file looking for section > # names that begin with "Test run:" > - foreach my $section ($ini->Sections()) { > + my @sects = $ini->Sections(); > + if ($MTT::Globals::Values->{shuffle_tests}->{sections}){ > + MTT::Util::shuffle(\@sects); > + } > + > + foreach my $section (@sects) { > # See if we're supposed to terminate. Only check in the > # outtermost and innermost loops (even though we *could* check > # at every loop level); that's good enough. > > Modified: trunk/lib/MTT/Test/RunEngine.pm > ============================================================================== > --- trunk/lib/MTT/Test/RunEngine.pm (original) > +++ trunk/lib/MTT/Test/RunEngine.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar > 2012) > @@ -39,7 +39,7 @@ > my $mpi_details_name; > my $test_run_full_name; > my $break_threshold; > - > +my $shuffle; > # Keep track of how many tests have passed, failed, skipped, and timed out > my $test_results_count; > > @@ -52,7 +52,6 @@ > my $report_after_n_results; > my $prev_section_name = undef; > my $group_reports = 0; > -my $tests_performed = 0; > > #-------------------------------------------------------------------------- > > @@ -96,6 +95,10 @@ > $break_threshold->{MTT::Values::TIMED_OUT} = Value($ini, $section, > "break_threshold_timeout"); > $break_threshold->{MTT::Values::SKIPPED} = Value($ini, $section, > "break_threshold_skipped"); > $break_threshold->{MTT::Values::TIMED_OUT_OR_FAIL} = Value($ini, "mtt", > "break_threshold_timeout_and_fail"); > + > + $shuffle = undef; > + _register_shuffles($ini,$section); > + > my $reports_per_job = Value($ini, "mtt", "reports_per_job"); > # This boolean value defaults to 0, and allows the user to submit results > # after each test to ensure at least *some* results are submitted (in case > @@ -167,6 +170,10 @@ > $verbose_out = 0; > my $test_count = 0; > my $printed = 0; > + if ($shuffle->{tests}){ > + MTT::Util::shuffle(\@{$ret->{tests}}); > + } > + > foreach my $run (@{$ret->{tests}}) { > > # See if we're supposed to terminate. > @@ -221,6 +228,9 @@ > if (ref($all_np) eq "") { > $test_results->{$all_np} = _run_one_np($install_dir, $run, > $mpi_details, $all_np, $force); > } else { > + if ($shuffle->{nps}){ > + MTT::Util::shuffle($all_np); > + } > foreach my $this_np (@$all_np) { > # See if we're supposed to terminate. > last > @@ -288,13 +298,19 @@ > if (ref($all_argv) eq "") { > $all_argv = [$all_argv]; > } > - > + if ($shuffle->{args}){ > + MTT::Util::shuffle($all_argv); > + } > foreach my $this_argv (@$all_argv) { > $MTT::Test::Run::test_argv = $this_argv; > > # Get all the exec's for this one np > my $execs = MTT::Values::EvaluateString($mpi_details->{exec}, > $ini, $mpi_details_name); > - > + if (ref($execs) ne "" && $shuffle->{execs}){ > + MTT::Util::shuffle($execs); > + } > + > + > # If we just got one, run it. Otherwise, loop over running them. > if (ref($execs) eq "") { > _run_one_test($install_dir, $run, $mpi_details, $execs, > $name, 1, > @@ -497,14 +513,7 @@ > if (exists($report->{test_result}) && > (MTT::Values::FAIL == $report->{test_result} || > MTT::Values::TIMED_OUT == $report->{test_result})); > > - $tests_performed++; > > - #print "\t\t--> Test #",$tests_performed," done\n"; > - > - > - #MTT::Reporter::Flush(); > - #MTT::Reporter::TextFile::deSubmit(); > - > # If there is an after_each step, run it > $ENV{MTT_TEST_RUN_RESULT_MESSAGE} = > (MTT::Values::PASS == $report->{test_result} ? "passed" : > @@ -558,6 +567,32 @@ > } > } > } > - > - > +sub _register_shuffles{ > + my ($ini,$section) = @_; > + > + my $val = MTT::Values::Value($ini, $section, "shuffle_tests"); > + if (defined($val)) { > + my @shuffles = grep length, split(/\s*,\s*/,$val); > + my @allowed_shuffles = ('tests', 'execs', 'nps', 'args', 'all', > 'none'); > + my %allowed_map = map { $_ => 1} @allowed_shuffles; > + foreach my $sh (@shuffles){ > + if (!exists($allowed_map{$sh})){ > + MTT::Messages::Error("<$sh> is not allowed section > specific shuffle_tests value. Allowed are:", join(',',@allowed_shuffles),"."); > + } > + $shuffle->{$sh} = 1; > + } > + if ($shuffle->{none}){ > + $shuffle = undef; > + } > + if ($shuffle->{all}){ > + foreach my $allow (@allowed_shuffles){ > + $shuffle->{$allow} = 1; > + } > + } > + > + } > + else{ > + $shuffle = $MTT::Globals::Values->{shuffle_tests}; > + } > +} > 1; > > Modified: trunk/lib/MTT/Util.pm > ============================================================================== > --- trunk/lib/MTT/Util.pm (original) > +++ trunk/lib/MTT/Util.pm 2012-03-02 10:27:34 EST (Fri, 02 Mar 2012) > @@ -208,9 +208,6 @@ > $MTT::Globals::Internals->{is_stopped_on_break_threshold} = > "true"; > $MTT::Globals::Internals->{stopped_on_break_threshold_message} = > "--> Threshold ($per) exceeded for \"$result_label\": $count->{$result} out > of $total.\n"; > print STDOUT "--> Threshold ($per) exceeded for > \"$result_label\": $count->{$result} out of $total.\n"; > - if ($MTT::Globals::Internals->{is_stopped_on_break_threshold}){ > - print STDOUT "0xdeadbeef: it works"; > - } > return 1; > } > } > @@ -395,4 +392,15 @@ > } > } > > +#-------------------------------------------------------------------------- > +sub shuffle{ > + # Shuffle an array given via reference randomly > + my $array = shift; > + my $i; > + for ($i = @$array; --$i; ) { > + my $j = int rand ($i+1); > + next if $i == $j; > + @$array[$i,$j] = @$array[$j,$i]; > + } > +} > 1; > _______________________________________________ > mtt-svn mailing list > mtt-...@open-mpi.org > http://www.open-mpi.org/mailman/listinfo.cgi/mtt-svn -- Jeff Squyres jsquy...@cisco.com For corporate legal information go to: http://www.cisco.com/web/about/doing_business/legal/cri/
