Author: daijy
Date: Wed Apr 20 06:06:45 2011
New Revision: 1095283
URL: http://svn.apache.org/viewvc?rev=1095283&view=rev
Log:
PIG-1978: Secondary sort fail when dereferencing two fields inside foreach
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestSecondarySort.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1095283&r1=1095282&r2=1095283&view=diff
==============================================================================
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Wed Apr 20 06:06:45 2011
@@ -158,6 +158,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1978: Secondary sort fail when dereferencing two fields inside foreach
(daijy)
+
PIG-1962: Wrong alias assinged to store operator (daijy)
PIG-1975: Need to provide backward compatibility for legacy LoadCaster
(without bytesToMap(bytes, fieldSchema)) (daijy)
Modified:
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java?rev=1095283&r1=1095282&r2=1095283&view=diff
==============================================================================
---
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
(original)
+++
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/SecondaryKeyOptimizer.java
Wed Apr 20 06:06:45 2011
@@ -546,10 +546,8 @@ public class SecondaryKeyOptimizer exten
// Accumulate column info from nested project
public boolean processForEach(POForEach fe) throws FrontendException {
if (fe.getInputPlans().size() > 1) {
- // I may be wrong, but for now let's assume foreach plan before
- // sort/distinct only have one foreach plan
- throw new FrontendException(
- "POForEach has more than 1 input plans");
+ // We don't optimize the case when POForEach has more than 1
input plan
+ return true;
}
boolean r = false;
try {
Modified:
pig/branches/branch-0.9/test/org/apache/pig/test/TestSecondarySort.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestSecondarySort.java?rev=1095283&r1=1095282&r2=1095283&view=diff
==============================================================================
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestSecondarySort.java
(original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestSecondarySort.java Wed
Apr 20 06:06:45 2011
@@ -583,4 +583,31 @@ public class TestSecondarySort extends T
FileLocalizer.delete("/tmp/output2", pigServer.getPigContext());
Util.deleteFile(cluster,
"testNestedSortMultiQueryEndToEnd1-input.txt");
}
+
+ // See PIG-1978
+ @Test
+ public void testForEachTwoInput() throws Exception {
+ File tmpFile1 = Util.createTempFileDelOnExit("test", "txt");
+ PrintStream ps1 = new PrintStream(new FileOutputStream(tmpFile1));
+ ps1.println("1\t2\t3");
+ ps1.println("1\t3\t4");
+ ps1.println("1\t2\t4");
+ ps1.println("1\t2\t4");
+ ps1.println("1\t2\t4");
+ ps1.println("2\t3\t4");
+ ps1.close();
+ Util.copyFromLocalToCluster(cluster, tmpFile1.getCanonicalPath(),
tmpFile1.getCanonicalPath());
+ pigServer.registerQuery("A = LOAD '" + tmpFile1.getCanonicalPath() +
"' AS (a0, a1, a2);");
+ pigServer.registerQuery("B = group A by (a0, a1);");
+ pigServer.registerQuery("C = foreach B { C1 = A.(a1,a2); generate
group, C1;};");
+ Iterator<Tuple> iter = pigServer.openIterator("C");
+ assertTrue(iter.hasNext());
+ assertEquals("((1,2),{(2,3),(2,4),(2,4),(2,4)})",
iter.next().toString());
+ assertTrue(iter.hasNext());
+ assertEquals("((1,3),{(3,4)})", iter.next().toString());
+ assertTrue(iter.hasNext());
+ assertEquals("((2,3),{(3,4)})", iter.next().toString());
+ assertFalse(iter.hasNext());
+ Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
+ }
}
\ No newline at end of file
