Author: rohini
Date: Tue Jan 6 15:21:38 2015
New Revision: 1649829
URL: http://svn.apache.org/r1649829
Log:
PIG-4339: e2e test framework assumes default exectype as mapred (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/harness/build.xml
pig/trunk/test/e2e/harness/xmlReport.pl
pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1649829&r1=1649828&r2=1649829&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Jan 6 15:21:38 2015
@@ -30,6 +30,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4339: e2e test framework assumes default exectype as mapred (rohini)
+
PIG-4360: HBaseStorage should support setting the timestamp field (bridiver
via daijy)
PIG-2949: JsonLoader only reads arrays of objects (eyal via daijy)
Modified: pig/trunk/test/e2e/harness/build.xml
URL:
http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/build.xml?rev=1649829&r1=1649828&r2=1649829&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/build.xml (original)
+++ pig/trunk/test/e2e/harness/build.xml Tue Jan 6 15:21:38 2015
@@ -41,6 +41,7 @@
<include name="Log.pm"/>
<include name="Properties.pm"/>
<include name="test_harness.pl"/>
+ <include name="xmlReport.pl"/>
</fileset>
</copy>
Modified: pig/trunk/test/e2e/harness/xmlReport.pl
URL:
http://svn.apache.org/viewvc/pig/trunk/test/e2e/harness/xmlReport.pl?rev=1649829&r1=1649828&r2=1649829&view=diff
==============================================================================
--- pig/trunk/test/e2e/harness/xmlReport.pl (original)
+++ pig/trunk/test/e2e/harness/xmlReport.pl Tue Jan 6 15:21:38 2015
@@ -15,7 +15,7 @@ unshift( @INC, ".");
########################################################################
# Sub: testcase
# Prints the result for a test case.
-#
+#
# Parameters:
# $class - the test file
# $test - the test name
@@ -24,115 +24,122 @@ unshift( @INC, ".");
# Returns:
# None
#
-sub getTestcase($$$$$$){
-
- my ( $class_name, $group_name, $test_name, $time, $status, $msg )=@_;
- my $result = "";
-
- if ( $status =~ "passed" ){
- $result = "\n<testcase classname=\"$class_name\" name=\"$test_name\"
time=\"$time\"/>";
+sub getTestcase($$$$$){
- } elsif ( $status =~ "skipped" ){
- $result = "\n<testcase classname=\"$class_name\" name=\"$test_name\"
time=\"0\">"
- . "\n<skipped type=\"$status\">$msg</skipped>"
- . "\n</testcase>"
- ;
- } else {
- $result = "\n<testcase classname=\"$class_name\" name=\"$test_name\"
time=\"$time\">"
- . "\n<error type=\"$status\">$msg</error>"
- . "\n</testcase>"
- ;
- } return $result;
+ my ( $exec_type, $test_name, $time, $status, $msg )=@_;
+ my $result = "";
+ my $group_name = substr ($test_name, 0, rindex ($test_name, "_"));
+
+ if ( $status =~ "passed" ){
+ $result = "\n<testcase classname=\"$exec_type.$group_name\"
name=\"$test_name\" time=\"$time\"/>";
+ } elsif ( $status =~ "skipped" ){
+ $result = "\n<testcase classname=\"$exec_type.$group_name\"
name=\"$test_name\" time=\"0\">"
+ . "\n<skipped type=\"$status\">$msg</skipped>"
+ . "\n</testcase>"
+ ;
+ } else {
+ $result = "\n<testcase classname=\"$exec_type.$group_name\"
name=\"$test_name\" time=\"$time\">"
+ . "\n<error type=\"$status\">$msg</error>"
+ . "\n</testcase>"
+ ;
+ }
+ return $result;
}
-sub printXmlReport($) {
+sub printXmlReport($$) {
- my $host = Sys::Hostname::hostname();
- my $tmpFileName = shift;
- my $reportFileName = shift;
-
- my $testcases = "";
- my $total_time= 0;
- my $testnameSuffix = "";
-
- if ($tmpFileName =~ m/-local/) {
- $testnameSuffix = "_local";
- }
-
- my $passedCount = 0;
- my $failureCount = 0;
- my $errorCount = 0;
- my $skippedCount = 0;
- my $totalCount = 0;
-
- my %test2starttime = { '', 0 };
-
- open( IN, "$tmpFileName")
- || die "Could not open $tmpFileName\n";
-
- while(<IN>){
-
- my $line = $_;
- # e.g.: "Beginning test Checkin_1 at 1346855793"
- if ( $line =~ m/Beginning test (.*) at (.*)/ ) {
- # put "test - start time" pair to the hash:
- $test2starttime{ $1 } = $2;
- } else {
- if ( $line =~ "TestDriver::run" ) {
- my $duration = 0;
- if ( $line =~ m/TestDriver::run.*Failed to run test (.*) <(.*)/ ) {
- # ERROR TestDriver::run at : 470 Failed to run test
ClassResolution_1 <Failed running
./out/pigtest/hadoopqa/hadoopqa.1327755958/ClassResolution_1_benchmark.pig
- #print "test aborted: $line";
- $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix, 0,
"aborted", $2 );
- $errorCount++;
- } elsif ( $line =~ m/TestDriver::run.*Test (.*) SUCCEEDED at (.*)/ ) {
- # INFO: TestDriver::run() at 444:Test Unicode_cmdline_1 SUCCEEDED
at 1327751873
- #print "test passed: $1 $2 line: $line";
- $passedCount++;
- $duration = $2 - $test2starttime{ $1 };
- $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix,
$duration, "passed", "" );
- } elsif ( $line =~ m/TestDriver::run.*Test (.*) FAILED at (.*)/ ) {
- $failureCount++;
- $duration = $2 - $test2starttime{ $1 };
- $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix,
$duration, "failed", "" );
- #print "test failed: $1 $2 line: $line";
- } elsif ($line =~ "Running TEST GROUP") {
- next;
- } elsif ($line =~ m/TestDriver::run.*Test (.*) SKIPPED at (.*)/) {
- # INFO: TestDriver::run() at 444:Test StreamingLocalErrors_1
SKIPPED at 1327923441
- $skippedCount++;
- $duration = $2 - $test2starttime{ $1 };
- $testcases .= getTestcase ( $1, "group", $1 . $testnameSuffix,
$duration, "skipped", "" );
- } else {
- print STDERR "Ignored line: $line";
- next;
- }
- #$total_time= $total_time + $time;
- $totalCount++;
- }
- }
-
- }
- close(IN);
- #Report
- my $host = Sys::Hostname::hostname();
- my $run_name = "e2e tests";
- my $report=
- '<?xml version="1.0" encoding="UTF-8" ?>'
- . "\n<testsuite errors=\"$errorCount\" failures=\"$failureCount\"
skips=\"$skippedCount\" hostname=\"$host\" name=\"$run_name\"
tests=\"$totalCount\" time=\"$total_time\">"
- . "\n$testcases"
- . "\n</testsuite>"
- . "\n";
- print $report;
+ my $host = Sys::Hostname::hostname();
+ my $reportFileName = shift;
+ my $execType = shift;
+
+ my $testcases = "";
+ my $testRunStartTime = 0;
+ my $testRunEndTime = 0;
+
+ my $passedCount = 0;
+ my $failureCount = 0;
+ my $errorCount = 0;
+ my $skippedCount = 0;
+ my $totalCount = 0;
+
+ my %test2starttime = { '', 0 };
+
+ open( IN, "$reportFileName")
+ || die "Could not open $reportFileName\n";
+
+ while(<IN>) {
+
+ my $line = $_;
+ if ($line =~ m/Beginning test run at (.*)/ ) {
+ # e.g: Beginning test run at 1416422503
+ $testRunStartTime = $1;
+ } elsif ($line =~ m/Finished test run at (.*)/ ) {
+ # e.g: Finished test run at 1416428021
+ $testRunEndTime = $1;
+ } elsif ( $line =~ m/Beginning test (.*) at (.*)/ ) {
+ # e.g.: "Beginning test Checkin_1 at 1346855793"
+ # put "test - start time" pair to the hash:
+ $test2starttime{ $1 } = $2;
+ } elsif ( $line =~ "TestDriver::run" ) {
+ my $duration = 0;
+ if ( $line =~ m/TestDriver::run.*Failed to run test (.*) <(.*)/ ) {
+ # ERROR TestDriver::run at : 470 Failed to run test
ClassResolution_1 <Failed running
./out/pigtest/hadoopqa/hadoopqa.1327755958/ClassResolution_1_benchmark.pig
+ #print "test aborted: $line";
+ $testcases .= getTestcase ( $execType, $1 , 0, "aborted", $2 );
+ $errorCount++;
+ } elsif ( $line =~ m/TestDriver::run.*Test (.*) SUCCEEDED at (.*)/
) {
+ # INFO: TestDriver::run() at 444:Test Unicode_cmdline_1
SUCCEEDED at 1327751873
+ #print "test passed: $1 $2 line: $line";
+ $passedCount++;
+ $duration = $2 - $test2starttime{ $1 };
+ $testcases .= getTestcase ( $execType, $1 , $duration,
"passed", "" );
+ } elsif ( $line =~ m/TestDriver::run.*Test (.*) FAILED at (.*)/ ) {
+ $failureCount++;
+ $duration = $2 - $test2starttime{ $1 };
+ $testcases .= getTestcase ( $execType, $1 , $duration,
"failed", "" );
+ #print "test failed: $1 $2 line: $line";
+ } elsif ($line =~ "Running TEST GROUP") {
+ next;
+ } elsif ($line =~ m/TestDriver::run.*Test (.*) SKIPPED at (.*)/) {
+ # INFO: TestDriver::run() at 444:Test StreamingLocalErrors_1
SKIPPED at 1327923441
+ $skippedCount++;
+ $duration = $2 - $test2starttime{ $1 };
+ $testcases .= getTestcase ( $execType, $1 , $duration,
"skipped", "" );
+ } else {
+ print STDERR "Ignored line: $line";
+ next;
+ }
+ $totalCount++;
+ }
+
+ }
+ close(IN);
+ #Report
+ my $host = Sys::Hostname::hostname();
+ my $run_name = $execType . " e2e tests";
+ my $total_time = $testRunEndTime - $testRunStartTime;
+ my $report= '<?xml version="1.0" encoding="UTF-8" ?>'
+ . "\n<testsuite errors=\"$errorCount\"
failures=\"$failureCount\" skips=\"$skippedCount\" hostname=\"$host\"
name=\"$run_name\" tests=\"$totalCount\" time=\"$total_time\">"
+ . "\n$testcases"
+ . "\n</testsuite>"
+ . "\n";
+ print $report;
}
if (!defined( $ARGV[0] )) {
- die "No input log file specified\n";
+ die "No input log file specified\n";
+}
+
+my $execType = $ARGV[1];
+if (!defined( $ARGV[1] )) {
+ print stderr "No execution type specified. Using mapred as default\n";
+ $execType = "mapred";
}
-printXmlReport($ARGV[0]);
+# Harness log file name, exec type which will be used as test group name
+printXmlReport($ARGV[0], $execType);
Modified: pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm
URL:
http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm?rev=1649829&r1=1649828&r2=1649829&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm (original)
+++ pig/trunk/test/e2e/pig/drivers/TestDriverPig.pm Tue Jan 6 15:21:38 2015
@@ -277,9 +277,7 @@ sub runPigCmdLine
# Build the command
my @baseCmd = $self->getPigCmd($testCmd, $log);
- if ($testCmd->{'exectype'} eq "tez") {
- push(@baseCmd, ("-x", "tez"));
- }
+ push(@baseCmd, ("-x", $testCmd->{'exectype'}));
my @cmd = @baseCmd;
# Add option -l giving location for secondary logs
@@ -437,9 +435,9 @@ sub getPigCmd($$$)
$additionalJavaParams .= " -Dmapred.local.dir=$hadoopTmpDir
-Dmapreduce.cluster.local.dir=$hadoopTmpDir";
}
TestDriver::dbg("Additional java parameters:
[$additionalJavaParams].\n");
-
- push(@pigCmd, ("-x", "local"));
}
+
+ push(@pigCmd, ("-x", $testCmd->{'exectype'}));
if (defined($testCmd->{'java_params'}) || defined($additionalJavaParams)) {
if (defined($testCmd->{'java_params'})) {
@@ -503,9 +501,7 @@ sub runPig
# Build the command
my @baseCmd = $self->getPigCmd($testCmd, $log);
- if ($testCmd->{'exectype'} eq "tez") {
- push(@baseCmd, ("-x", "tez"));
- }
+ push(@baseCmd, ("-x", $testCmd->{'exectype'}));
my @cmd = @baseCmd;
# Add option -l giving location for secondary logs
