Author: knoguchi
Date: Thu Jul 13 18:28:29 2017
New Revision: 1801860
URL: http://svn.apache.org/viewvc?rev=1801860&view=rev
Log:
PIG-4767: Partition filter not pushed down when filter clause references
variable from another load path (knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
pig/trunk/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java
pig/trunk/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1801860&r1=1801859&r2=1801860&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Jul 13 18:28:29 2017
@@ -36,6 +36,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-4767: Partition filter not pushed down when filter clause references
variable from another load path (knoguchi)
+
PIG-5270: Typo in Pig Logging (FromAlaska49 via daijy)
PIG-5263: Using wildcard doesn't work with OrcStorage (satishsaley via rohini)
Modified:
pig/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java?rev=1801860&r1=1801859&r2=1801860&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
(original)
+++
pig/trunk/src/org/apache/pig/newplan/logical/rules/PartitionFilterOptimizer.java
Thu Jul 13 18:28:29 2017
@@ -116,10 +116,6 @@ public class PartitionFilterOptimizer ex
return false;
loFilter = (LOFilter)succeds.get(0);
- // Filter has dependency other than load, skip optimization
- if (currentPlan.getSoftLinkPredecessors(loFilter)!=null)
- return false;
-
// we have to check more only if LoadFunc implements LoadMetada
loadFunc = loLoad.getLoadFunc();
if(!( loadFunc instanceof LoadMetadata ) ) {
Modified:
pig/trunk/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java?rev=1801860&r1=1801859&r2=1801860&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java
(original)
+++
pig/trunk/src/org/apache/pig/newplan/logical/rules/PredicatePushdownOptimizer.java
Thu Jul 13 18:28:29 2017
@@ -107,10 +107,6 @@ public class PredicatePushdownOptimizer
return false;
loFilter = (LOFilter) succeds.get(0);
- // Filter has dependency other than load, skip optimization
- if (currentPlan.getSoftLinkPredecessors(loFilter) != null)
- return false;
-
// we have to check more only if LoadFunc implements
LoadPredicatePushdown
loadFunc = loLoad.getLoadFunc();
if (!(loadFunc instanceof LoadPredicatePushdown)) {
Modified: pig/trunk/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java?rev=1801860&r1=1801859&r2=1801860&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java
(original)
+++ pig/trunk/test/org/apache/pig/test/TestNewPartitionFilterPushDown.java Thu
Jul 13 18:28:29 2017
@@ -61,10 +61,12 @@ import org.apache.pig.newplan.logical.ex
import org.apache.pig.newplan.logical.expression.NotExpression;
import org.apache.pig.newplan.logical.expression.OrExpression;
import org.apache.pig.newplan.logical.expression.ProjectExpression;
+import org.apache.pig.newplan.logical.expression.ScalarExpression;
import org.apache.pig.newplan.logical.optimizer.LogicalPlanOptimizer;
import org.apache.pig.newplan.logical.relational.LOFilter;
import org.apache.pig.newplan.logical.relational.LogToPhyTranslationVisitor;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.newplan.logical.relational.LogicalRelationalOperator;
import org.apache.pig.newplan.logical.rules.LoadTypeCastInserter;
import org.apache.pig.newplan.logical.rules.PartitionFilterOptimizer;
import org.apache.pig.newplan.optimizer.PlanOptimizer;
@@ -467,10 +469,14 @@ public class TestNewPartitionFilterPushD
Operator op = newLogicalPlan.getSinks().get(0);
LOFilter filter =
(LOFilter)newLogicalPlan.getPredecessors(op).get(0);
- String actual = new PartitionFilterExtractor(null, new
ArrayList<String>())
- .getExpression((LogicalExpression)
filter.getFilterPlan().getSources().get(0)).toString();
- Assert.assertEquals("checking trimmed filter expression:",
- filterExpr, actual);
+ if (unsupportedExpr) {
+ String actual = getTestExpression((LogicalExpression)
filter.getFilterPlan().getSources().get(0));
+ Assert.assertEquals("checking trimmed filter expression:",
filterExpr, actual);
+ } else {
+ String actual = new PartitionFilterExtractor(null, new
ArrayList<String>())
+ .getExpression((LogicalExpression)
filter.getFilterPlan().getSources().get(0)).toString();
+ Assert.assertEquals("checking trimmed filter expression:",
filterExpr, actual);
+ }
} else {
Iterator<Operator> it = newLogicalPlan.getOperators();
while( it.hasNext() ) {
@@ -707,6 +713,14 @@ public class TestNewPartitionFilterPushD
"(not (browser#'type' is null))", true);
}
+ @Test
+ public void testScalarExpressions() throws Exception {
+ String q = "z = load '1line' as (z1:int); " +
+ query3 + "b = filter a by srcid != 10 and srcid == z.z1;" +
+ "store b into 'out';";
+ testFull(q, "(srcid != 10)", "(srcid == z.z1)", true);
+ }
+
//// helper methods ///////
private PartitionFilterExtractor test(String query, List<String>
partitionCols,
String expPartFilterString, String expFilterString)
@@ -917,6 +931,10 @@ public class TestNewPartitionFilterPushD
} else if (op instanceof NotExpression) {
String expr = getTestExpression(((NotExpression)
op).getExpression());
return braketize("not " + expr);
+ } else if (op instanceof ScalarExpression) {
+ ScalarExpression scalar = (ScalarExpression) op;
+ return
((LogicalRelationalOperator)scalar.getImplicitReferencedOperator()).getAlias() +
+ "." + scalar.getFieldSchema().alias;
} else {
throw new FrontendException("Unsupported conversion of
LogicalExpression to Expression: " + op.getName());
}
