Author: daijy
Date: Fri Sep 4 02:26:59 2009
New Revision: 811203
URL: http://svn.apache.org/viewvc?rev=811203&view=rev
Log:
making dump and PigDump independent from Tuple.toString
Added:
hadoop/pig/trunk/src/org/apache/pig/impl/util/BagFormat.java
hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java
hadoop/pig/trunk/test/org/apache/pig/test/TestBagFormat.java
hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/builtin/PigDump.java
hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java
hadoop/pig/trunk/src/org/apache/pig/data/DefaultTuple.java
hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=811203&r1=811202&r2=811203&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Sep 4 02:26:59 2009
@@ -28,6 +28,8 @@
IMPROVEMENTS
+PIG-936: making dump and PigDump independent from Tuple.toString (daijy)
+
PIG-890: Create a sampler interface and improve the skewed join sampler
(sriranjan via daijy)
PIG-922: Logical optimizer: push up project part 1 (daijy)
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/PigDump.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/PigDump.java?rev=811203&r1=811202&r2=811203&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/PigDump.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/PigDump.java Fri Sep 4
02:26:59 2009
@@ -15,32 +15,33 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-package org.apache.pig.builtin;
-
-import java.io.IOException;
-import java.io.OutputStream;
+package org.apache.pig.builtin;
+
+import java.io.IOException;
+import java.io.OutputStream;
import org.apache.pig.StoreFunc;
import org.apache.pig.data.Tuple;
-
-
-public class PigDump implements StoreFunc {
-
- public static String recordDelimiter = "\n";
-
-
- OutputStream os;
-
- public void bindTo(OutputStream os) throws IOException {
- this.os = os;
- }
-
- public void finish() throws IOException {
-
- }
-
- public void putNext(Tuple f) throws IOException {
- os.write((f.toString() + recordDelimiter).getBytes());
+import org.apache.pig.impl.util.TupleFormat;
+
+
+public class PigDump implements StoreFunc {
+
+ public static String recordDelimiter = "\n";
+
+
+ OutputStream os;
+
+ public void bindTo(OutputStream os) throws IOException {
+ this.os = os;
+ }
+
+ public void finish() throws IOException {
+
+ }
+
+ public void putNext(Tuple f) throws IOException {
+ os.write((TupleFormat.format(f) + recordDelimiter).getBytes());
}
/* (non-Javadoc)
@@ -50,6 +51,6 @@
public Class getStorePreparationClass() throws IOException {
// TODO Auto-generated method stub
return null;
- }
-
-}
+ }
+
+}
Modified: hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java?rev=811203&r1=811202&r2=811203&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java Fri Sep 4
02:26:59 2009
@@ -32,6 +32,7 @@
import org.apache.pig.backend.executionengine.ExecException;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PigLogger;
+import org.apache.pig.impl.util.BagFormat;
import org.apache.pig.impl.util.Spillable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -264,17 +265,7 @@
* Write the bag into a string. */
@Override
public String toString() {
- StringBuffer sb = new StringBuffer();
- sb.append('{');
- Iterator<Tuple> it = iterator();
- while ( it.hasNext() ) {
- Tuple t = it.next();
- String s = t.toString();
- sb.append(s);
- if (it.hasNext()) sb.append(",");
- }
- sb.append('}');
- return sb.toString();
+ return BagFormat.format(this);
}
@Override
Modified: hadoop/pig/trunk/src/org/apache/pig/data/DefaultTuple.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/DefaultTuple.java?rev=811203&r1=811202&r2=811203&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/data/DefaultTuple.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/data/DefaultTuple.java Fri Sep 4
02:26:59 2009
@@ -33,6 +33,7 @@
import org.apache.pig.PigException;
import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.impl.util.TupleFormat;
/**
* A default implementation of Tuple. This class will be created by the
@@ -214,29 +215,7 @@
@Override
public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append('(');
- for (Iterator<Object> it = mFields.iterator(); it.hasNext();) {
- Object d = it.next();
- if(d != null) {
- if(d instanceof Map) {
- sb.append(DataType.mapToString((Map<String, Object>)d));
- } else {
- sb.append(d.toString());
- if(d instanceof Long) {
- sb.append("L");
- } else if(d instanceof Float) {
- sb.append("F");
- }
- }
- } else {
- sb.append("");
- }
- if (it.hasNext())
- sb.append(",");
- }
- sb.append(')');
- return sb.toString();
+ return TupleFormat.format(this);
}
public int compareTo(Object other) {
Added: hadoop/pig/trunk/src/org/apache/pig/impl/util/BagFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/util/BagFormat.java?rev=811203&view=auto
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/util/BagFormat.java (added)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/util/BagFormat.java Fri Sep 4
02:26:59 2009
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+package org.apache.pig.impl.util;
+
+import java.util.Iterator;
+
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+
+public class BagFormat {
+
+ public static String format(DataBag bag) {
+ StringBuffer sb = new StringBuffer();
+ sb.append('{');
+
+ Iterator<Tuple> it = bag.iterator();
+ while (it.hasNext()) {
+ Tuple t = it.next();
+ String s = TupleFormat.format(t);
+ sb.append(s);
+ if (it.hasNext())
+ sb.append(",");
+ }
+ sb.append('}');
+ return sb.toString();
+ }
+}
Added: hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java?rev=811203&view=auto
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java (added)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java Fri Sep 4
02:26:59 2009
@@ -0,0 +1,84 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.impl.util;
+
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+
+/**
+ * Default implementation of format of Tuple. Dump and PigDump use this default
+ * implementation
+ *
+ */
+public class TupleFormat {
+
+ protected static final Log mLog = LogFactory.getLog(TupleFormat.class);
+
+ /**
+ * Default implementation of format of tuple (each filed is delimited by
+ * tab)
+ *
+ * @param tuple
+ * @return Default format of Tuple
+ */
+ public static String format(Tuple tuple) {
+ StringBuilder sb = new StringBuilder();
+ sb.append('(');
+ for (int i = 0; i < tuple.size(); ++i) {
+ try {
+ Object d = tuple.get(i);
+ if (d != null) {
+ if (d instanceof Map) {
+ sb.append(DataType.mapToString((Map<String, Object>)
d));
+ } else if (d instanceof Tuple) {
+ Tuple t = (Tuple) d;
+ sb.append(TupleFormat.format(t));
+ } else if (d instanceof DataBag){
+ DataBag bag=(DataBag)d;
+ sb.append(BagFormat.format(bag));
+ }
+ else {
+ sb.append(d.toString());
+ if (d instanceof Long) {
+ sb.append("L");
+ } else if (d instanceof Float) {
+ sb.append("F");
+ }
+ }
+ } else {
+ sb.append("");
+ }
+ if (i != tuple.size() - 1)
+ sb.append(",");
+ } catch (ExecException e) {
+ e.printStackTrace();
+ mLog.warn("Exception when format tuple", e);
+ }
+
+ }
+ sb.append(')');
+ return sb.toString();
+ }
+}
Modified: hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=811203&r1=811202&r2=811203&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Fri Sep 4
02:26:59 2009
@@ -61,6 +61,7 @@
import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS;
import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.util.TupleFormat;
import org.apache.pig.impl.util.WrappedIOException;
import org.apache.pig.tools.pigscript.parser.ParseException;
import org.apache.pig.tools.pigscript.parser.PigScriptParser;
@@ -532,7 +533,7 @@
while (result.hasNext())
{
Tuple t = result.next();
- System.out.println(t);
+ System.out.println(TupleFormat.format(t));
}
}
Added: hadoop/pig/trunk/test/org/apache/pig/test/TestBagFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestBagFormat.java?rev=811203&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestBagFormat.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestBagFormat.java Fri Sep 4
02:26:59 2009
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test;
+
+import junit.framework.TestCase;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.util.BagFormat;
+
+public class TestBagFormat extends TestCase {
+
+ public static void testBagFormat() {
+ try {
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+
+ Tuple tuple_1 = TupleFactory.getInstance().newTuple(1);
+ tuple_1.set(0, 12);
+ bag.add(tuple_1);
+
+ Tuple tuple_2 = TupleFactory.getInstance().newTuple(1);
+ DataBag innerBag = BagFactory.getInstance().newDefaultBag();
+ innerBag.add(tuple_1);
+ tuple_2.set(0, (innerBag));
+ bag.add(tuple_2);
+
+ System.out.println(BagFormat.format(bag));
+ assertEquals("{(12),({(12)})}", BagFormat.format(bag));
+ } catch (ExecException e) {
+ e.printStackTrace();
+ fail();
+ }
+ }
+}
Added: hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java?rev=811203&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java Fri Sep 4
02:26:59 2009
@@ -0,0 +1,63 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.util.HashMap;
+import java.util.Map;
+
+import junit.framework.TestCase;
+
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.BagFactory;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.data.TupleFactory;
+import org.apache.pig.impl.util.TupleFormat;
+
+public class TestTupleFormat extends TestCase {
+
+ public void testTupleFormat() {
+
+ try {
+ Tuple tuple = TupleFactory.getInstance().newTuple(7);
+ tuple.set(0, 12);
+ Map<String, String> map = new HashMap<String, String>();
+ map.put("pig", "scalability");
+ tuple.set(1, map);
+ tuple.set(2, null);
+ tuple.set(3, 12L);
+ tuple.set(4, 1.2F);
+
+ Tuple innerTuple = TupleFactory.getInstance().newTuple(1);
+ innerTuple.set(0, "innerTuple");
+ tuple.set(5, innerTuple);
+
+ DataBag bag = BagFactory.getInstance().newDefaultBag();
+ bag.add(innerTuple);
+ tuple.set(6, bag);
+
+ assertEquals(
+
"(12,[pig#scalability],,12L,1.2F,(innerTuple),{(innerTuple)})",
+ TupleFormat.format(tuple));
+ } catch (ExecException e) {
+ e.printStackTrace();
+ fail();
+ }
+
+ }
+}
