Author: rohini
Date: Wed Jan 28 17:32:52 2015
New Revision: 1655399
URL: http://svn.apache.org/r1655399
Log:
PIG-4391: Fix TestPigStats test failure (rohini)
Added:
pig/trunk/test/excluded-tests-mr
pig/trunk/test/excluded-tests-tez
Modified:
pig/trunk/CHANGES.txt
pig/trunk/build.xml
pig/trunk/test/org/apache/pig/test/TestPigStats.java
pig/trunk/test/org/apache/pig/test/TestPigStatsMR.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1655399&r1=1655398&r2=1655399&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jan 28 17:32:52 2015
@@ -44,6 +44,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4391: Fix TestPigStats test failure (rohini)
+
PIG-4387: Honor yarn settings in tez-site.xml and optimize dag status fetch
(rohini)
PIG-4352: Port local mode tests to Tez - TestUnionOnSchema (daijy)
Modified: pig/trunk/build.xml
URL:
http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1655399&r1=1655398&r2=1655399&view=diff
==============================================================================
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Wed Jan 28 17:32:52 2015
@@ -108,6 +108,8 @@
<property name="test.exclude.file" value="${test.src.dir}/excluded-tests"/>
<property name="test.exclude.file.20"
value="${test.src.dir}/excluded-tests-20"/>
<property name="test.exclude.file.23"
value="${test.src.dir}/excluded-tests-23"/>
+ <property name="test.exclude.file.mr"
value="${test.src.dir}/excluded-tests-mr"/>
+ <property name="test.exclude.file.tez"
value="${test.src.dir}/excluded-tests-tez"/>
<property name="pigunit.jarfile" value="pigunit.jar" />
<property name="piggybank.jarfile"
value="${basedir}/contrib/piggybank/java/piggybank.jar" />
<property name="smoke.tests.jarfile"
value="${build.dir}/${final.name}-smoketests.jar" />
@@ -780,6 +782,14 @@
project.setProperty(attributes.get("name"),
attributes.get("value"));
</scriptdef>
+ <macrodef name="propertycopy">
+ <attribute name="name"/>
+ <attribute name="from"/>
+ <sequential>
+ <propertyreset name="@{name}" value="${@{from}}"/>
+ </sequential>
+ </macrodef>
+
<target name="mvn-jar" description="Create pig.jar for maven deploy for
Hadoop2/23 and Hadoop1/20">
<propertyreset name="hadoopversion" value="23" />
<propertyreset name="isHadoop" value="true" />
@@ -910,6 +920,8 @@
<mkdir dir="${test.log.dir}"/>
<tempfile property="junit.tmp.dir" prefix="pig_junit_tmp"
destDir="${java.io.tmpdir}" />
<mkdir dir="${junit.tmp.dir}/"/>
+ <propertycopy name="test.exclude.file.for.exectype"
from="test.exclude.file.${test.exec.type}"/>
+ <echo>Tests in ${test.exclude.file.for.exectype} will be excluded</echo>
<junit showoutput="${test.output}" printsummary="yes"
haltonfailure="no" fork="yes" maxmemory="2048m" dir="${basedir}"
timeout="${test.timeout}" errorProperty="@{tests.failed}"
failureProperty="@{tests.failed}">
<sysproperty key="hadoopversion" value="${hadoopversion}" />
<sysproperty key="test.exec.type" value="${test.exec.type}" />
@@ -942,6 +954,7 @@
<excludesfile name="${test.exclude.file}"
if="test.exclude.file"/>
<excludesfile name="${test.exclude.file.20}"
unless="isHadoop23"/>
<excludesfile name="${test.exclude.file.23}"
if="isHadoop23"/>
+ <excludesfile name="${test.exclude.file.for.exectype}"/>
</patternset>
<exclude name="**/${exclude.testcase}.java"
if="exclude.testcase" />
<exclude name="**/TestRegisteredJarVisibility.java"
if="offline"/>
Added: pig/trunk/test/excluded-tests-mr
URL:
http://svn.apache.org/viewvc/pig/trunk/test/excluded-tests-mr?rev=1655399&view=auto
==============================================================================
--- pig/trunk/test/excluded-tests-mr (added)
+++ pig/trunk/test/excluded-tests-mr Wed Jan 28 17:32:52 2015
@@ -0,0 +1 @@
+**/tez/*.java
\ No newline at end of file
Added: pig/trunk/test/excluded-tests-tez
URL:
http://svn.apache.org/viewvc/pig/trunk/test/excluded-tests-tez?rev=1655399&view=auto
==============================================================================
--- pig/trunk/test/excluded-tests-tez (added)
+++ pig/trunk/test/excluded-tests-tez Wed Jan 28 17:32:52 2015
@@ -0,0 +1 @@
+**/Test*MR.java
\ No newline at end of file
Modified: pig/trunk/test/org/apache/pig/test/TestPigStats.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStats.java?rev=1655399&r1=1655398&r2=1655399&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStats.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStats.java Wed Jan 28 17:32:52
2015
@@ -26,8 +26,6 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
-import junit.framework.Assert;
-
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
@@ -46,10 +44,10 @@ import org.junit.Test;
@Ignore
abstract public class TestPigStats {
- private static final Log LOG = LogFactory.getLog(TestPigStats.class);
+ protected static final Log LOG = LogFactory.getLog(TestPigStats.class);
abstract public void addSettingsToConf(Configuration conf, String
scriptFileName);
-
+
@Test
public void testPigScriptInConf() throws Exception {
PrintWriter w = new PrintWriter(new FileWriter("test.pig"));
@@ -58,22 +56,22 @@ abstract public class TestPigStats {
w.println("register /mydir/lib/jackson-core-asl-1.4.2.jar");
w.println("register /mydir/lib/jackson-mapper-asl-1.4.2.jar");
w.close();
-
+
Configuration conf = new Configuration();
addSettingsToConf(conf, "test.pig");
-
+
String s = conf.get("pig.script");
String script = new String(Base64.decodeBase64(s.getBytes()));
-
- String expected =
+
+ String expected =
"register /mydir/sath.jar\n" +
"register /mydir/lib/hadoop-tools-0.20.201.0-SNAPSHOT.jar\n" +
"register /mydir/lib/jackson-core-asl-1.4.2.jar\n" +
"register /mydir/lib/jackson-mapper-asl-1.4.2.jar\n";
-
- Assert.assertEquals(expected, script);
+
+ assertEquals(expected, script);
}
-
+
@Test
public void testJythonScriptInConf() throws Exception {
String[] script = {
@@ -90,16 +88,16 @@ abstract public class TestPigStats {
"else:",
"\traise 'failed'"
};
-
+
Util.createLocalInputFile( "testScript.py", script);
-
+
Configuration conf = new Configuration();
addSettingsToConf(conf, "testScript.py");
-
+
String s = conf.get("pig.script");
String actual = new String(Base64.decodeBase64(s.getBytes()));
-
- String expected =
+
+ String expected =
"#!/usr/bin/python\n" +
"from org.apache.pig.scripting import *\n" +
"Pig.fs(\"rmr simple_out\")\n" +
@@ -112,10 +110,10 @@ abstract public class TestPigStats {
"\tprint 'success!'\n" +
"else:\n" +
"\traise 'failed'\n";
-
- Assert.assertEquals(expected, actual);
+
+ assertEquals(expected, actual);
}
-
+
@Test
public void testBytesWritten_JIRA_1027() throws Exception {
@@ -157,7 +155,7 @@ abstract public class TestPigStats {
pig.registerQuery("D = order C by $1;");
pig.registerQuery("E = limit D 10;");
pig.registerQuery("store E into 'alias_output';");
-
+
LogicalPlan lp = getLogicalPlan(pig);
lp.optimize(pig.getPigContext());
PhysicalPlan pp =
((HExecutionEngine)pig.getPigContext().getExecutionEngine()).compile(lp,
@@ -201,7 +199,7 @@ abstract public class TestPigStats {
}
}
}
-
+
private void deleteDirectory(File dir) {
try {
FileUtils.deleteDirectory(dir);
@@ -215,5 +213,5 @@ abstract public class TestPigStats {
buildLp.setAccessible(true);
return (LogicalPlan ) buildLp.invoke( pig );
}
-
+
}
Modified: pig/trunk/test/org/apache/pig/test/TestPigStatsMR.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStatsMR.java?rev=1655399&r1=1655398&r2=1655399&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPigStatsMR.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStatsMR.java Wed Jan 28 17:32:52
2015
@@ -18,22 +18,60 @@
package org.apache.pig.test;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import java.io.File;
+import java.io.IOException;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecJob;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import org.apache.pig.impl.PigContext;
+import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.impl.plan.OperatorKey;
-import org.apache.pig.tools.pigstats.ScriptState;
+import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStats.JobGraph;
+import org.apache.pig.tools.pigstats.ScriptState;
import org.apache.pig.tools.pigstats.mapreduce.MRScriptState;
+import org.junit.Test;
public class TestPigStatsMR extends TestPigStats {
+
+ @Override
+ @Test
+ public void testBytesWritten_JIRA_1027() throws Exception {
+
+ FileLocalizer.setInitialized(false);
+ // This test cannot be run in MR local mode due to lack of counters
+ MiniGenericCluster cluster = MiniGenericCluster.buildCluster();
+ try {
+ String filePath = "/tmp/" + this.getClass().getName() + "_"
+ + "testBytesWritten_JIRA_1027";
+
+ PigServer pig = new PigServer(cluster.getExecType(),
cluster.getProperties());
+ String inputFile = "test/org/apache/pig/test/data/passwd";
+ Util.copyFromLocalToCluster(cluster, inputFile, inputFile);
+ pig.registerQuery("A = load '" + inputFile + "';");
+ ExecJob job = pig.store("A", filePath);
+ PigStats stats = job.getStatistics();
+ Path dataFile = Util.getFirstPartFile(new Path(filePath));
+ FileStatus fs = cluster.getFileSystem().getFileStatus(dataFile);
+ assertEquals(fs.getLen(), stats.getBytesWritten());
+ } catch (IOException e) {
+ LOG.error("Error while generating file", e);
+ fail("Encountered IOException");
+ } finally {
+ FileLocalizer.setInitialized(false);
+ cluster.shutDown();
+ }
+ }
+
@Override
public void addSettingsToConf(Configuration conf, String scriptFileName) {
MRScriptState ss = MRScriptState.get();
@@ -71,7 +109,7 @@ public class TestPigStatsMR extends Test
compile.setAccessible(true);
return (MROperPlan) compile.invoke(launcher, new Object[] { pp, ctx });
}
-
+
private static String getAlias(MapReduceOper mro) throws Exception {
ScriptState ss = ScriptState.get();
java.lang.reflect.Method getAlias = ss.getClass()
