Author: gates
Date: Mon Aug 11 21:35:02 2008
New Revision: 685032
URL: http://svn.apache.org/viewvc?rev=685032&view=rev
Log:
PIG-343 Shravan's patch to fix split.
Modified:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java
incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld
incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld
Modified:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java?rev=685032&r1=685031&r2=685032&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/LogToPhyTranslationVisitor.java
Mon Aug 11 21:35:02 2008
@@ -17,6 +17,7 @@
*/
package org.apache.pig.backend.hadoop.executionengine.physicalLayer;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
@@ -28,6 +29,7 @@
import org.apache.commons.logging.LogFactory;
import org.apache.pig.ComparisonFunc;
import org.apache.pig.EvalFunc;
+import org.apache.pig.FuncSpec;
import org.apache.pig.LoadFunc;
import org.apache.pig.data.DataType;
import org.apache.pig.impl.PigContext;
@@ -38,6 +40,9 @@
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.ExpressionOperator;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.BinaryExpressionOperator;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.UnaryExpressionOperator;
+import org.apache.pig.builtin.BinStorage;
+import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.impl.io.FileSpec;
import org.apache.pig.impl.logicalLayer.*;
import org.apache.pig.impl.plan.DependencyOrderWalker;
import org.apache.pig.impl.plan.DependencyOrderWalkerWOSeenChk;
@@ -759,6 +764,14 @@
String scope = split.getOperatorKey().scope;
PhysicalOperator physOp = new POSplit(new OperatorKey(scope, nodeGen
.getNextNodeId(scope)), split.getRequestedParallelism());
+ FileSpec splStrFile;
+ try {
+ splStrFile = new FileSpec(FileLocalizer.getTemporaryPath(null,
pc).toString(),new FuncSpec(BinStorage.class.getName()));
+ } catch (IOException e1) {
+ log.error("Unable to obtain a temporary path because " +
e1.getMessage());
+ throw new VisitorException(e1);
+ }
+ ((POSplit)physOp).setSplitStore(splStrFile);
LogToPhyMap.put(split, physOp);
currentPlan.add(physOp);
Modified:
incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java?rev=685032&r1=685031&r2=685032&view=diff
==============================================================================
---
incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java
(original)
+++
incubator/pig/branches/types/test/org/apache/pig/test/TestLogToPhyCompiler.java
Mon Aug 11 21:35:02 2008
@@ -68,10 +68,16 @@
Random r = new Random();
PigContext pc = new PigContext(ExecType.LOCAL, new Properties());
-
+
private boolean generate = false;
+
+ @Override
+ protected void setUp() throws Exception {
+ pc.connect();
+ }
+
private void writeData(File input, int noTuples, int arityOfTuples, char
separator) throws IOException {
FileOutputStream dat = new FileOutputStream(input);
Modified:
incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld?rev=685032&r1=685031&r2=685032&view=diff
==============================================================================
---
incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld
(original)
+++
incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/IsNull1.gld
Mon Aug 11 21:35:02 2008
@@ -1,20 +1,20 @@
Filter[tuple] - Test-Plan-Builder-214
| |
-| Not[boolean] - Test-Plan-Builder-217
+| POIsNull[tuple] - Test-Plan-Builder-216
| |
-| |---POIsNull[tuple] - Test-Plan-Builder-216
-| |
-| |---Project[bytearray][0] - Test-Plan-Builder-215
+| |---Project[bytearray][0] - Test-Plan-Builder-215
|
|---Split - Test-Plan-Builder-213
|
|---Load(a:org.apache.pig.builtin.PigStorage) - Test-Plan-Builder-212
-Filter[tuple] - Test-Plan-Builder-218
+Filter[tuple] - Test-Plan-Builder-217
| |
-| POIsNull[tuple] - Test-Plan-Builder-220
+| Not[boolean] - Test-Plan-Builder-220
| |
-| |---Project[bytearray][0] - Test-Plan-Builder-219
+| |---POIsNull[tuple] - Test-Plan-Builder-219
+| |
+| |---Project[bytearray][0] - Test-Plan-Builder-218
|
|---Split - Test-Plan-Builder-213
|
Modified:
incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld?rev=685032&r1=685031&r2=685032&view=diff
==============================================================================
---
incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld
(original)
+++
incubator/pig/branches/types/test/org/apache/pig/test/data/GoldenFiles/Split1.gld
Mon Aug 11 21:35:02 2008
@@ -1,6 +1,6 @@
Filter[tuple] - Test-Plan-Builder-196
| |
-| Greater Than[boolean] - Test-Plan-Builder-199
+| Less Than[boolean] - Test-Plan-Builder-199
| |
| |---Project[bytearray][0] - Test-Plan-Builder-197
| |
@@ -12,7 +12,7 @@
Filter[tuple] - Test-Plan-Builder-200
| |
-| Less Than[boolean] - Test-Plan-Builder-203
+| Greater Than[boolean] - Test-Plan-Builder-203
| |
| |---Project[bytearray][0] - Test-Plan-Builder-201
| |
