Author: daijy
Date: Thu Sep 24 05:46:25 2015
New Revision: 1704997
URL: http://svn.apache.org/viewvc?rev=1704997&view=rev
Log:
PIG-4683: Nested order is broken after PIG-3591 in some cases
Modified:
pig/branches/branch-0.15/CHANGES.txt
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
pig/branches/branch-0.15/test/org/apache/pig/test/TestSecondarySort.java
Modified: pig/branches/branch-0.15/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.15/CHANGES.txt?rev=1704997&r1=1704996&r2=1704997&view=diff
==============================================================================
--- pig/branches/branch-0.15/CHANGES.txt (original)
+++ pig/branches/branch-0.15/CHANGES.txt Thu Sep 24 05:46:25 2015
@@ -28,6 +28,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4683: Nested order is broken after PIG-3591 in some cases (daijy)
+
PIG-4628: Pig 0.14 job with order by fails in mapreduce mode with Oozie
(knoguchi)
PIG-4627: [Pig on Tez] Self join does not handle null values correctly (rohini)
Modified:
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java?rev=1704997&r1=1704996&r2=1704997&view=diff
==============================================================================
---
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
(original)
+++
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPackage.java
Thu Sep 24 05:46:25 2015
@@ -257,8 +257,15 @@ public class POPackage extends PhysicalO
NullableTuple ntup = tupIter.next();
int index = ntup.getIndex();
if (index == numInputs - 1) {
- dbs[index] = new PeekedBag(pkgr, ntup, tupIter,
keyWritable);
- break;
+ if (pkgr.getUseSecondaryKey()) {
+ if (dbs[index] == null) {
+ dbs[index] = useDefaultBag ?
BagFactory.getInstance()
+ .newDefaultBag() : new
InternalCachedBag(numInputs);
+ }
+ } else {
+ dbs[index] = new PeekedBag(pkgr, ntup, tupIter,
keyWritable);
+ break;
+ }
}
Tuple copy = pkgr.getValueTuple(keyWritable, ntup, index);
Modified:
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java?rev=1704997&r1=1704996&r2=1704997&view=diff
==============================================================================
---
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
(original)
+++
pig/branches/branch-0.15/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/Packager.java
Thu Sep 24 05:46:25 2015
@@ -353,6 +353,9 @@ public class Packager implements Illustr
public void setUseSecondaryKey(boolean useSecondaryKey) {
this.useSecondaryKey = useSecondaryKey;
}
+ public boolean getUseSecondaryKey() {
+ return useSecondaryKey;
+ }
public void setPackageType(PackageType type) {
this.pkgType = type;
Modified:
pig/branches/branch-0.15/test/org/apache/pig/test/TestSecondarySort.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.15/test/org/apache/pig/test/TestSecondarySort.java?rev=1704997&r1=1704996&r2=1704997&view=diff
==============================================================================
--- pig/branches/branch-0.15/test/org/apache/pig/test/TestSecondarySort.java
(original)
+++ pig/branches/branch-0.15/test/org/apache/pig/test/TestSecondarySort.java
Thu Sep 24 05:46:25 2015
@@ -448,6 +448,38 @@ public abstract class TestSecondarySort
}
+ @Test
+ public void testNestedSortMultiQueryEndToEnd3() throws Exception {
+ File input1 = Util.createTempFileDelOnExit("test", "txt");
+ PrintStream ps1 = new PrintStream(new FileOutputStream(input1));
+ ps1.println("a\t0");
+ ps1.println("a\t2");
+ ps1.println("a\t1");
+ ps1.close();
+ Util.copyFromLocalToCluster(cluster, input1.getCanonicalPath(),
"testNestedSortMultiQueryEndToEnd3-input-1.txt");
+
+ File input2 = Util.createTempFileDelOnExit("test", "txt");
+ PrintStream ps2 = new PrintStream(new FileOutputStream(input2));
+ ps2.println("a");
+ ps2.close();
+ Util.copyFromLocalToCluster(cluster, input2.getCanonicalPath(),
"testNestedSortMultiQueryEndToEnd3-input-2.txt");
+
+ try {
+ pigServer.setBatchOn();
+ pigServer.registerQuery("a = load
'testNestedSortMultiQueryEndToEnd3-input-1.txt' as (a0:chararray,
a1:chararray);");
+ pigServer.registerQuery("b = load
'testNestedSortMultiQueryEndToEnd3-input-2.txt' as (b0);");
+ pigServer.registerQuery("c = cogroup b by b0, a by a0;");
+ pigServer.registerQuery("d = foreach c {a_sorted = order a by a1
desc;generate group, a_sorted, b;}");
+ Iterator<Tuple> iter = pigServer.openIterator("d");
+
+ assertEquals(iter.next().toString(),
"(a,{(a,2),(a,1),(a,0)},{(a)})");
+ } finally {
+ Util.deleteFile(cluster,
"testNestedSortMultiQueryEndToEnd3-input-1.txt");
+ Util.deleteFile(cluster,
"testNestedSortMultiQueryEndToEnd3-input-2.txt");
+ }
+
+ }
+
// See PIG-1978
@Test
public void testForEachTwoInput() throws Exception {
