Author: knoguchi
Date: Wed Mar 9 18:30:02 2016
New Revision: 1734296
URL: http://svn.apache.org/viewvc?rev=1734296&view=rev
Log:
PIG-4833 TestBuiltin.testURIWithCurlyBrace in TEZ failing after PIG-4819
(knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
pig/trunk/test/org/apache/pig/test/TestBuiltin.java
pig/trunk/test/org/apache/pig/test/Util.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1734296&r1=1734295&r2=1734296&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Mar 9 18:30:02 2016
@@ -97,6 +97,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4833 TestBuiltin.testURIWithCurlyBrace in TEZ failing after PIG-4819
(knoguchi)
+
PIG-4819: RANDOM() udf can lead to missing or redundant records (knoguchi)
PIG-4816: Read a null scalar causing a Tez failure (daijy)
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java?rev=1734296&r1=1734295&r2=1734296&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/tez/TezResourceManager.java
Wed Mar 9 18:30:02 2016
@@ -55,6 +55,14 @@ public class TezResourceManager {
return instance;
}
+
+ /**
+ * This method is only used by test code to reset state.
+ */
+ public static void dropInstance() {
+ instance = null;
+ }
+
public void init(PigContext pigContext, Configuration conf) throws
IOException {
if (!inited) {
this.resourcesDir =
FileLocalizer.getTemporaryResourcePath(pigContext);
Modified: pig/trunk/test/org/apache/pig/test/TestBuiltin.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBuiltin.java?rev=1734296&r1=1734295&r2=1734296&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestBuiltin.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBuiltin.java Wed Mar 9 18:30:02 2016
@@ -3209,7 +3209,9 @@ public class TestBuiltin {
String inputFileName = "testUniqueID.txt";
Util.createInputFile(cluster, inputFileName, new String[]
{"1\n2\n3\n4\n5\n1\n2\n3\n4\n5\n"});
- PigServer pigServer = new PigServer(cluster.getExecType(),
cluster.getProperties());
+ Properties copyproperties = new Properties();
+ copyproperties.putAll(cluster.getProperties());
+ PigServer pigServer = new PigServer(cluster.getExecType(),
copyproperties);
pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size",
"10");
pigServer.getPigContext().getProperties().setProperty("pig.noSplitCombination",
"true");
pigServer.registerQuery("A = load '" + inputFileName + "' as (name);");
@@ -3225,6 +3227,7 @@ public class TestBuiltin {
assertEquals(iter.next().get(1),"1-2");
assertEquals(iter.next().get(1),"1-3");
assertEquals(iter.next().get(1),"1-4");
+ Util.deleteFile(cluster, inputFileName);
}
@Test
@@ -3233,7 +3236,10 @@ public class TestBuiltin {
String inputFileName = "testRANDOM.txt";
Util.createInputFile(cluster, inputFileName, new String[]
{"1\n2\n3\n4\n5\n1\n2\n3\n4\n5\n"});
- PigServer pigServer = new PigServer(cluster.getExecType(),
cluster.getProperties());
+
+ Properties copyproperties = new Properties();
+ copyproperties.putAll(cluster.getProperties());
+ PigServer pigServer = new PigServer(cluster.getExecType(),
copyproperties);
// running with two mappers
pigServer.getPigContext().getProperties().setProperty("mapred.max.split.size",
"10");
pigServer.getPigContext().getProperties().setProperty("pig.noSplitCombination",
"true");
@@ -3260,6 +3266,7 @@ public class TestBuiltin {
for( int i = 0; i < 5; i++ ){
assertNotEquals(mapper1[i], mapper2[i], 0.0001);
}
+ Util.deleteFile(cluster, inputFileName);
}
Modified: pig/trunk/test/org/apache/pig/test/Util.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/Util.java?rev=1734296&r1=1734295&r2=1734296&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ pig/trunk/test/org/apache/pig/test/Util.java Wed Mar 9 18:30:02 2016
@@ -75,6 +75,7 @@ import org.apache.pig.backend.hadoop.exe
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import org.apache.pig.backend.hadoop.executionengine.tez.TezResourceManager;
import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
import org.apache.pig.builtin.Utf8StorageConverter;
import org.apache.pig.data.BagFactory;
@@ -1340,6 +1341,11 @@ public class Util {
*/
public static void resetStateForExecModeSwitch() {
FileLocalizer.setInitialized(false);
+
+ // For tez testing, we want to avoid TezResourceManager/LocalResource
reuse
+ // (when switching between local and mapreduce/tez)
+ TezResourceManager.dropInstance();
+
// TODO: once we have Tez local mode, we can get rid of this. For now,
// if we run this test suite in Tez mode and there are some tests
// in LOCAL mode, we need to set ScriptState to
