Author: praveen
Date: Tue Apr 7 09:25:18 2015
New Revision: 1671772
URL: http://svn.apache.org/r1671772
Log:
PIG-4438: Limit after sort situation does not work in spark mode (Liyun via
Praveen)
Modified:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkCompiler.java
Modified:
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkCompiler.java
URL:
http://svn.apache.org/viewvc/pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkCompiler.java?rev=1671772&r1=1671771&r2=1671772&view=diff
==============================================================================
---
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkCompiler.java
(original)
+++
pig/branches/spark/src/org/apache/pig/backend/hadoop/executionengine/spark/plan/SparkCompiler.java
Tue Apr 7 09:25:18 2015
@@ -435,8 +435,15 @@ public class SparkCompiler extends PhyPl
@Override
public void visitSort(POSort op) throws VisitorException {
try {
- nonBlocking(op);
- phyToSparkOpMap.put(op, curSparkOp);
+ nonBlocking(op);
+ POSort sort = op;
+ long limit = sort.getLimit();
+ if (limit!=-1) {
+ POLimit pLimit2 = new POLimit(new
OperatorKey(scope,nig.getNextNodeId(scope)));
+ pLimit2.setLimit(limit);
+ curSparkOp.physicalPlan.addAsLeaf(pLimit2);
+ }
+ phyToSparkOpMap.put(op, curSparkOp);
} catch (Exception e) {
int errCode = 2034;
String msg = "Error compiling operator "
