svn commit: r654629 [2/4] - in /incubator/pig/branches/types: ./ src/org/apache/pig/impl/eval/ src/org/apache/pig/impl/logicalLayer/ src/org/apache/pig/impl/logicalLayer/parser/ src/org/apache/pig/imp
Added:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java?rev=654629view=auto
==
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java
(added)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LORegexp.java
Thu May 8 14:25:22 2008
@@ -0,0 +1,87 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.impl.logicalLayer;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.pig.data.DataType;
+import org.apache.pig.impl.plan.VisitorException;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class LORegexp extends ExpressionOperator {
+private static final long serialVersionUID = 2L;
+
+/**
+ * The expression and the column to be projected.
+ */
+private ExpressionOperator mOperand;
+private String mRegexp;
+private static Log log = LogFactory.getLog(LORegexp.class);
+
+/**
+ *
+ * @param plan
+ *Logical plan this operator is a part of.
+ * @param k
+ *Operator key to assign to this node.
+ * @param exp
+ *the expression which might contain the column to project
+ * @param projection
+ *the list of columns to project
+ */
+public LORegexp(LogicalPlan plan, OperatorKey key,
+ExpressionOperator operand, String regexp) {
+super(plan, key);
+mOperand = operand;
+mRegexp = regexp;
+}
+
+public ExpressionOperator getOperand() {
+return mOperand;
+}
+
+public String getRegexp() {
+return mRegexp;
+}
+
+@Override
+public String name() {
+return Project + mKey.scope + - + mKey.id;
+}
+
+@Override
+public boolean supportsMultipleInputs() {
+return false;
+}
+
+@Override
+public Schema getSchema() {
+return mSchema;
+}
+
+@Override
+public void visit(LOVisitor v) throws VisitorException {
+v.visit(this);
+}
+
+}
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java?rev=654629r1=654628r2=654629view=diff
==
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/LOSort.java
Thu May 8 14:25:22 2008
@@ -26,19 +26,26 @@
import org.apache.pig.impl.logicalLayer.schema.Schema;
import org.apache.pig.impl.plan.VisitorException;
import org.apache.pig.impl.plan.PlanVisitor;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
public class LOSort extends LogicalOperator {
private static final long serialVersionUID = 2L;
-private ListInteger mSortCols;
+private LogicalOperator mInput;
private ListBoolean mAscCols;
-private LOUserFunc mSortFunc;
+private String mSortFunc;
+private boolean mIsStar = false;
+private ListLogicalPlan mSortColPlans;
+ private static Log log = LogFactory.getLog(LOSort.class);
/**
* @param plan
*LogicalPlan this operator is a part of.
* @param key
*OperatorKey for this operator
+ * @param input
+ *Input to sort
* @param sortCols
*Array of column numbers that will be used for sorting data.
* @param ascCols
@@ -47,33 +54,40 @@
*this array is null, then all columns will be sorted
ascending.
* @param sorFunc
*the user defined sorting function
- * @param rp
- *Requested level of parallelism to be used in the sort.
svn commit: r654629 [4/4] - in /incubator/pig/branches/types: ./ src/org/apache/pig/impl/eval/ src/org/apache/pig/impl/logicalLayer/ src/org/apache/pig/impl/logicalLayer/parser/ src/org/apache/pig/imp
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java?rev=654629r1=654628r2=654629view=diff
==
---
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/validators/TypeCheckingVisitor.java
Thu May 8 14:25:22 2008
@@ -349,7 +349,7 @@
private void insertLeftCastForBinaryOp(BinaryExpressionOperator binOp,
byte toType ) {
OperatorKey newKey = genNewOperatorKey(binOp) ;
-LOCast cast = new LOCast(mPlan, newKey, 1, binOp.getLhsOperand(),
toType) ;
+LOCast cast = new LOCast(mPlan, newKey, binOp.getLhsOperand(), toType)
;
mPlan.add(cast) ;
mPlan.disconnect(binOp.getLhsOperand(), binOp) ;
try {
@@ -367,7 +367,7 @@
private void insertRightCastForBinaryOp(BinaryExpressionOperator binOp,
byte toType ) {
OperatorKey newKey = genNewOperatorKey(binOp) ;
-LOCast cast = new LOCast(mPlan, newKey, 1, binOp.getRhsOperand(),
toType) ;
+LOCast cast = new LOCast(mPlan, newKey, binOp.getRhsOperand(), toType)
;
mPlan.add(cast) ;
mPlan.disconnect(binOp.getRhsOperand(), binOp) ;
try {
@@ -425,7 +425,7 @@
// All uniOps at the moment only work with Expression input
ExpressionOperator input = (ExpressionOperator) list.get(0) ;
OperatorKey newKey = genNewOperatorKey(uniOp) ;
-LOCast cast = new LOCast(mPlan, newKey, 1, input, toType) ;
+LOCast cast = new LOCast(mPlan, newKey, input, toType) ;
mPlan.disconnect(input, uniOp) ;
try {
@@ -518,7 +518,7 @@
private void insertLeftCastForBinCond(LOBinCond binCond, byte toType) {
OperatorKey newKey = genNewOperatorKey(binCond) ;
-LOCast cast = new LOCast(mPlan, newKey, 1, binCond.getLhsOp(), toType)
;
+LOCast cast = new LOCast(mPlan, newKey, binCond.getLhsOp(), toType) ;
mPlan.add(cast) ;
mPlan.disconnect(binCond.getLhsOp(), binCond) ;
try {
@@ -535,7 +535,7 @@
private void insertRightCastForBinCond(LOBinCond binCond, byte toType) {
OperatorKey newKey = genNewOperatorKey(binCond) ;
-LOCast cast = new LOCast(mPlan, newKey, 1, binCond.getRhsOp(), toType)
;
+LOCast cast = new LOCast(mPlan, newKey, binCond.getRhsOp(), toType) ;
mPlan.add(cast) ;
mPlan.disconnect(binCond.getRhsOp(), binCond) ;
try {
@@ -831,6 +831,8 @@
*/
// TODO: NOT DONE YET
+ //COmmenting out this method as its not using the new APIs
+ /*
protected void visit(LOGenerate g) throws VisitorException {
// Visit each of generates projection elements.
IteratorExpressionOperator i = g.getProjections().iterator();
@@ -838,6 +840,7 @@
i.next().visit(this);
}
}
+ */
/***
* This does:-
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java?rev=654629r1=654628r2=654629view=diff
==
--- incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
(original)
+++ incubator/pig/branches/types/src/org/apache/pig/impl/plan/OperatorPlan.java
Thu May 8 14:25:22 2008
@@ -26,6 +26,9 @@
import java.util.Map;
import org.apache.pig.impl.logicalLayer.OperatorKey;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
//import org.apache.commons.collections.map.MultiValueMap;
@@ -40,7 +43,7 @@
private ListE mRoots;
private ListE mLeaves;
-
+private Log log = LogFactory.getLog(OperatorPlan.class);
public OperatorPlan() {
mRoots = new ArrayListE();
@@ -100,6 +103,25 @@
}
/**
+ * Get the map of operator key and associated operators
+ * @return map of operator key and operators.
+ */
+/*
+public MapOperatorKey, E getKeys() {
+return mKeys;
+}
+*/
+/**
+ * Get the map of operators and associated operator keys
+ * @return map of operator and operator keys.
+ */
+/*
+public MapE, OperatorKey getOps() {
+return mOps;
+}
+*/
+
+/**
* Insert an operator into the plan. This only inserts it as a node in
* the graph, it does not connect it to any other operators. That should
* be done as a separate step using connect.
@@ -232,6 +254,8 @@
private
svn commit: r654669 - in /incubator/pig/trunk: CHANGES.txt src/org/apache/pig/impl/io/FileLocalizer.java src/org/apache/pig/impl/streaming/StreamingCommand.java test/org/apache/pig/test/TestStreaming.
Author: olga
Date: Thu May 8 19:40:10 2008
New Revision: 654669
URL: http://svn.apache.org/viewvc?rev=654669view=rev
Log:
PIG-232: let valid cache statements in
Modified:
incubator/pig/trunk/CHANGES.txt
incubator/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
incubator/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java
incubator/pig/trunk/test/org/apache/pig/test/TestStreaming.java
Modified: incubator/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=654669r1=654668r2=654669view=diff
==
--- incubator/pig/trunk/CHANGES.txt (original)
+++ incubator/pig/trunk/CHANGES.txt Thu May 8 19:40:10 2008
@@ -289,3 +289,5 @@
PIG-232: fix for number of input records when BinaryStirage is used
(acmurthy via olgan)
+
+PIG-232: let valid cache specifications through (acmurthy via olgan)
Modified: incubator/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java
URL:
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java?rev=654669r1=654668r2=654669view=diff
==
--- incubator/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java (original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java Thu May
8 19:40:10 2008
@@ -344,6 +344,27 @@
return elem.exists() || globMatchesFiles(elem, store);
}
+public static boolean isFile(String filename, PigContext context)
+throws IOException {
+return !isDirectory(filename, context.getDfs());
+}
+
+public static boolean isFile(String filename, DataStorage store)
+throws IOException {
+return !isDirectory(filename, store);
+}
+
+public static boolean isDirectory(String filename, PigContext context)
+throws IOException {
+return isDirectory(filename, context.getDfs());
+}
+
+public static boolean isDirectory(String filename, DataStorage store)
+throws IOException {
+ElementDescriptor elem = store.asElement(filename);
+return (elem instanceof ContainerDescriptor);
+}
+
private static boolean globMatchesFiles(ElementDescriptor elem,
DataStorage fs)
throws IOException
Modified:
incubator/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java
URL:
http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java?rev=654669r1=654668r2=654669view=diff
==
--- incubator/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java
(original)
+++ incubator/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java
Thu May 8 19:40:10 2008
@@ -3,6 +3,8 @@
import java.io.File;
import java.io.IOException;
import java.io.Serializable;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -142,7 +144,11 @@
// Validate
File file = new File(path);
if (!file.exists()) {
-throw new IOException(Invalid ship specification: + path);
+throw new IOException(Invalid ship specification: ' + path +
+ ' does not exist!);
+} else if (file.isDirectory()) {
+throw new IOException(Invalid ship specification: ' + path +
+ ' is a directory and can't be shipped!);
}
shipSpec.add(path);
@@ -156,10 +162,35 @@
*/
public void addPathToCache(String path) throws IOException {
// Validate
-if (!FileLocalizer.fileExists(path, pigContext)) {
+URI pathUri = null;
+URI dfsPath = null;
+try {
+pathUri = new URI(path);
+
+// Strip away the URI's _fragment_ and _query_
+dfsPath = new URI(pathUri.getScheme(), pathUri.getAuthority(),
+ pathUri.getPath(), null, null);
+} catch (URISyntaxException urise) {
throw new IOException(Invalid cache specification: + path);
}
+boolean exists = false;
+try {
+exists = FileLocalizer.fileExists(dfsPath.toString(), pigContext);
+} catch (IOException ioe) {
+// Throw a better error message...
+throw new IOException(Invalid cache specification: ' + dfsPath +
+ ' does not exist!);
+}
+
+if (!exists) {
+throw new IOException(Invalid cache specification: ' + dfsPath +
+ ' does not exist!);
+} else if (FileLocalizer.isDirectory(dfsPath.toString(), pigContext)) {
+throw new IOException(Invalid
