Author: sms Date: Fri Jul 31 21:13:03 2009 New Revision: 799749 URL: http://svn.apache.org/viewvc?rev=799749&view=rev Log: PIG-880: Order by is borken with complex fields
Added: hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/data/parser/TextDataParser.jjt hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java hadoop/pig/trunk/test/org/apache/pig/test/utils/TestHelper.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=799749&r1=799748&r2=799749&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Fri Jul 31 21:13:03 2009 @@ -44,6 +44,8 @@ BUG FIXES + PIG-880: Order by is borken with complex fields (sms) + PIG-773: Empty complex constants (empty bag, empty tuple and empty map) should be supported (ashutoshc via sms) Modified: hadoop/pig/trunk/src/org/apache/pig/data/parser/TextDataParser.jjt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/parser/TextDataParser.jjt?rev=799749&r1=799748&r2=799749&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/parser/TextDataParser.jjt (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/parser/TextDataParser.jjt Fri Jul 31 21:13:03 2009 @@ -44,6 +44,7 @@ import org.apache.pig.data.Tuple; import org.apache.pig.data.BagFactory; import org.apache.pig.data.DataBag; +import org.apache.pig.data.DataByteArray; public class TextDataParser { @@ -141,13 +142,13 @@ void KeyValuePair(Map<String, Object> keyValues) : { String key = null; - Object value = null; + String value = null; log.trace("Entering KeyValuePair"); } { - (key = StringDatum() "#" value = Datum()) + (key = StringDatum() "#" value = StringDatum()) { - keyValues.put(key, value); + keyValues.put(key, new DataByteArray(value.getBytes("UTF-8"))); log.trace("Exiting KeyValuePair"); } @@ -161,11 +162,20 @@ } { ( - t = <STRING> { s = t.image; } -| t = <NULL> {} + ( + t = <SIGNEDINTEGER> +| t = <LONGINTEGER> +| t = <FLOATNUMBER> +| t = <DOUBLENUMBER> +| t = <STRING> + ) + { + s = t.image; + } + ) { - log.trace("Exiting String with " + t.image); + log.trace("Exiting String with " + s); return s; } } @@ -195,7 +205,6 @@ { obj = new Double(Double.parseDouble(t.image)); } -| t = <NULL> {} | t = <STRING> { obj = t.image; Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java?rev=799749&r1=799748&r2=799749&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java Fri Jul 31 21:13:03 2009 @@ -17,12 +17,14 @@ */ package org.apache.pig.test; +import java.util.Iterator; import java.util.Random; import java.util.Map; import java.io.IOException; import org.apache.pig.builtin.PigStorage; import org.apache.pig.test.utils.GenRandomData; +import org.apache.pig.test.utils.TestHelper; import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.apache.pig.data.DataBag; @@ -44,6 +46,7 @@ PigStorage ps = new PigStorage(); Random r = new Random(); final int MAX = 10; + @Test public void testBytesToInteger() throws IOException @@ -153,7 +156,7 @@ for (int i = 0; i < MAX; i++) { Tuple t = GenRandomData.genRandSmallBagTextTuple(r, 1, 100); Tuple convertedTuple = ps.bytesToTuple(t.toString().getBytes()); - assertTrue(t.equals(convertedTuple)); + assertTrue(TestHelper.tupleEquals(t, convertedTuple)); } } @@ -164,7 +167,7 @@ for (int i = 0; i < MAX; i++) { DataBag b = GenRandomData.genRandFullTupTextDataBag(r,5,100); DataBag convertedBag = ps.bytesToBag(b.toString().getBytes()); - assertTrue(b.equals(convertedBag)); + assertTrue(TestHelper.bagEquals(b, convertedBag)); } } @@ -177,7 +180,7 @@ Map<String, Object> m = GenRandomData.genRandMap(r,5); String expectedMapString = DataType.mapToString(m); Map<String, Object> convertedMap = ps.bytesToMap(expectedMapString.getBytes()); - assertTrue(m.equals(convertedMap)); + assertTrue(TestHelper.mapEquals(m, convertedMap)); } } Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java?rev=799749&r1=799748&r2=799749&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestStore.java Fri Jul 31 21:13:03 2009 @@ -207,7 +207,7 @@ t.append(flds[8].compareTo("")!=0 ? ps.bytesToTuple(flds[8].getBytes()) : null); t.append(flds[9].compareTo("")!=0 ? ps.bytesToCharArray(flds[9].getBytes()) : null); - assertTrue(inputTuple.equals(t)); + assertTrue(TestHelper.tupleEquals(inputTuple, t)); ++size; } FileLocalizer.delete(fSpec.getFileName(), pc); Added: hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java?rev=799749&view=auto ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java (added) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java Fri Jul 31 21:13:03 2009 @@ -0,0 +1,192 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.pig.test; + +import java.io.ByteArrayInputStream; +import java.util.Map; + +import org.junit.Test; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.pig.data.BagFactory; +import org.apache.pig.data.DataBag; +import org.apache.pig.data.DataByteArray; +import org.apache.pig.data.DefaultBagFactory; +import org.apache.pig.data.DefaultTupleFactory; +import org.apache.pig.data.Tuple; +import org.apache.pig.data.TupleFactory; +import org.apache.pig.data.parser.TextDataParser; +import org.apache.pig.data.parser.ParseException ; + + +public class TestTextDataParser extends junit.framework.TestCase { + + private final Log log = LogFactory.getLog(getClass()); + private TupleFactory tupleFactory = DefaultTupleFactory.getInstance(); + private BagFactory bagFactory = DefaultBagFactory.getInstance(); + + + @Test + public void testInteger() throws Exception{ + String myInteger = "1"; + Integer i = (Integer)parseTextData(myInteger); + assertTrue(i.equals(1)); + } + + @Test + public void testLong() throws Exception{ + String myLong = "1l"; + Long l = (Long)parseTextData(myLong); + assertTrue(l.equals(1l)); + } + + @Test + public void testFloat() throws Exception{ + String myFloat = "0.1f"; + Float f = (Float)parseTextData(myFloat); + assertTrue(f.equals(0.1f)); + } + + @Test + public void testDouble() throws Exception{ + String myDouble = "0.1"; + Double d = (Double)parseTextData(myDouble); + assertTrue(d.equals(0.1)); + } + + @Test + public void testString() throws Exception{ + String myString = "1a"; + String s = (String)parseTextData(myString); + assertTrue(s.equals(myString)); + } + + + //the value types of a map should always be a byte array + //irrespective of the actual type + @SuppressWarnings("unchecked") + @Test + public void testMapStringValueType() throws Exception{ + String myMap = "[key1#value1]"; + Map<String, Object> map = (Map<String, Object>)parseTextData(myMap); + String key = map.keySet().iterator().next(); + Object v = map.get("key1"); + assertTrue(key.equals("key1")); + assertTrue(v instanceof DataByteArray); + String value = new String(((DataByteArray)v).get()); + assertTrue(value.equals("value1")); + } + + //the value types of a map should always be a byte array + //irrespective of the actual type + @SuppressWarnings("unchecked") + @Test + public void testMapIntegerValueType() throws Exception{ + String myMap = "[key1#1]"; + Map<String, Object> map = (Map<String, Object>)parseTextData(myMap); + String key = map.keySet().iterator().next(); + Object v = map.get("key1"); + assertTrue(key.equals("key1")); + assertTrue(v instanceof DataByteArray); + String value = new String(((DataByteArray)v).get()); + assertTrue(value.equals("1")); + } + + //the value types of a map should always be a byte array + //irrespective of the actual type + @SuppressWarnings("unchecked") + @Test + public void testMapLongValueType() throws Exception{ + String myMap = "[key1#1l]"; + Map<String, Object> map = (Map<String, Object>)parseTextData(myMap); + String key = map.keySet().iterator().next(); + Object v = map.get("key1"); + assertTrue(key.equals("key1")); + assertTrue(v instanceof DataByteArray); + String value = new String(((DataByteArray)v).get()); + assertTrue(value.equals("1l")); + } + + //the value types of a map should always be a byte array + //irrespective of the actual type + @SuppressWarnings("unchecked") + @Test + public void testMapFloatValueType() throws Exception{ + String myMap = "[key1#0.1f]"; + Map<String, Object> map = (Map<String, Object>)parseTextData(myMap); + String key = map.keySet().iterator().next(); + Object v = map.get("key1"); + assertTrue(key.equals("key1")); + assertTrue(v instanceof DataByteArray); + String value = new String(((DataByteArray)v).get()); + assertTrue(value.equals("0.1f")); + } + + //the value types of a map should always be a byte array + //irrespective of the actual type + @SuppressWarnings("unchecked") + @Test + public void testMapDoubleValueType() throws Exception{ + String myMap = "[key1#0.1]"; + Map<String, Object> map = (Map<String, Object>)parseTextData(myMap); + String key = map.keySet().iterator().next(); + Object v = map.get("key1"); + assertTrue(key.equals("key1")); + assertTrue(v instanceof DataByteArray); + String value = new String(((DataByteArray)v).get()); + assertTrue(value.equals("0.1")); + } + + @Test + public void testTuple() throws Exception{ + String myTuple = "(1,a)"; + Object o = parseTextData(myTuple); + assertTrue(o instanceof Tuple); + Tuple t = (Tuple)o; + Tuple expectedTuple = tupleFactory.newTuple(2); + expectedTuple.set(0, 1); + expectedTuple.set(1, "a"); + assertTrue(t.equals(expectedTuple)); + } + + @Test + public void testBag() throws Exception{ + String myBag = "{(1,a),(2,b)}"; + Object o = parseTextData(myBag); + assertTrue(o instanceof DataBag); + DataBag b = (DataBag)o; + DataBag expectedBag = bagFactory.newDefaultBag(); + Tuple expectedTuple = tupleFactory.newTuple(2); + expectedTuple.set(0, 1); + expectedTuple.set(1, "a"); + expectedBag.add(expectedTuple); + expectedTuple = tupleFactory.newTuple(2); + expectedTuple.set(0, 2); + expectedTuple.set(1, "b"); + expectedBag.add(expectedTuple); + assertTrue(b.equals(expectedBag)); + } + + + private Object parseTextData(String pigConstantAsString) throws ParseException { + ByteArrayInputStream stream = new ByteArrayInputStream(pigConstantAsString.getBytes()) ; + TextDataParser textDataParser = new TextDataParser(stream) ; + return textDataParser.Datum(); + } +} Modified: hadoop/pig/trunk/test/org/apache/pig/test/utils/TestHelper.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/utils/TestHelper.java?rev=799749&r1=799748&r2=799749&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/utils/TestHelper.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/utils/TestHelper.java Fri Jul 31 21:13:03 2009 @@ -19,6 +19,7 @@ import java.io.*; import java.util.Iterator; +import java.util.Map; import java.util.Random; import org.apache.pig.backend.executionengine.ExecException; @@ -45,7 +46,7 @@ public static boolean bagContains(DataBag db, Tuple t) { Iterator<Tuple> iter = db.iterator(); for (Tuple tuple : db) { - if (tuple.compareTo(t) == 0) + if (tuple.compareTo(t) == 0 || tupleEquals(tuple, t)) return true; } return false; @@ -240,4 +241,134 @@ ps.close(); return fp1 ; } + + //a quick way to check for map equality as the map value returned by PigStorage has byte array + public static boolean mapEquals(Map<String, Object> expectedMap, Map<String, Object> convertedMap) { + if(expectedMap == null) { + if(convertedMap != null) { + return false; + } + } else { + if (convertedMap == null) { + return false; + } + } + + if(expectedMap.size() != convertedMap.size()) { + return false; + } + + for(String key: expectedMap.keySet()) { + Object v = convertedMap.get(key); + String convertedValue = new String(((DataByteArray)v).get()); + if(!expectedMap.get(key).equals(convertedValue)) { + return false; + } + } + return true; + } + + @SuppressWarnings("unchecked") + public static boolean tupleEquals(Tuple expectedTuple, Tuple convertedTuple) { + if(expectedTuple == null) { + if(convertedTuple != null) { + return false; + } + } else { + if(convertedTuple == null) { + return false; + } + } + + if(expectedTuple.size() != convertedTuple.size()) { + return false; + } + + for(int i = 0; i < expectedTuple.size(); ++i) { + Object e ; + Object c ; + + try { + e = expectedTuple.get(i); + c = convertedTuple.get(i); + } catch (Exception e1) { + return false; + } + + if(e instanceof Map) { + Map<String, Object> eMap = (Map<String, Object>)e; + if(c instanceof Map) { + Map<String, Object> cMap = (Map<String, Object>)c; + if(!mapEquals(eMap, cMap)) { + return false; + } + } else { + return false; + } + } else if (e instanceof Tuple) { + if(c instanceof Tuple) { + if(!tupleEquals((Tuple)e, (Tuple)c)) { + return false; + } + } else { + return false; + } + } else if (e instanceof DataBag){ + if(c instanceof DataBag) { + if(!bagEquals((DataBag)e, (DataBag)c)) { + return false; + } + } else { + return false; + } + } else { + if(e == null) { + if(c != null) { + return false; + } + } else { + if(c == null) { + return false; + } else { + if(!e.equals(c)) { + return false; + } + } + } + } + } + + return true; + } + + public static boolean bagEquals(DataBag expectedBag, DataBag convertedBag) { + if(expectedBag == null) { + if(convertedBag != null) { + return false; + } + } else { + if(convertedBag == null) { + return false; + } + } + + if(expectedBag.size() != convertedBag.size()) { + return false; + } + + Iterator<Tuple> expectedBagIterator = expectedBag.iterator(); + Iterator<Tuple> convertedBagIterator = convertedBag.iterator(); + + while(expectedBagIterator.hasNext()) { + Tuple expectedBagTuple = expectedBagIterator.next(); + Tuple convertedBagTuple = convertedBagIterator.next(); + if(!tupleEquals(expectedBagTuple, convertedBagTuple)) { + return false; + } + } + + return true; + + } + }