Author: daijy
Date: Mon May 17 17:24:40 2010
New Revision: 945254
URL: http://svn.apache.org/viewvc?rev=945254&view=rev
Log:
PIG-566: Dump and store outputs do not match for PigStorage (azaroth via daijy)
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java
hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java
hadoop/pig/trunk/test/org/apache/pig/test/TestCombiner.java
hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java
hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java
hadoop/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java
hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon May 17 17:24:40 2010
@@ -64,6 +64,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-566: Dump and store outputs do not match for PigStorage (azaroth via daijy)
+
PIG-1414: Problem with parameter substitution (rding)
PIG-1407: Logging starts before being configured (azaroth via daijy)
Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java
(original)
+++ hadoop/pig/trunk/src/org/apache/pig/builtin/Utf8StorageConverter.java Mon
May 17 17:24:40 2010
@@ -52,8 +52,10 @@ public class Utf8StorageConverter implem
protected TupleFactory mTupleFactory = TupleFactory.getInstance();
protected final Log mLog = LogFactory.getLog(getClass());
- private Integer mMaxInt = Integer.valueOf(Integer.MAX_VALUE);
- private Long mMaxLong = Long.valueOf(Long.MAX_VALUE);
+ private static final Integer mMaxInt = Integer.valueOf(Integer.MAX_VALUE);
+ private static final Integer mMinInt = Integer.valueOf(Integer.MIN_VALUE);
+ private static final Long mMaxLong = Long.valueOf(Long.MAX_VALUE);
+ private static final Long mMinLong = Long.valueOf(Long.MIN_VALUE);
private static final int BUFFER_SIZE = 1024;
public Utf8StorageConverter() {
@@ -328,14 +330,12 @@ public class Utf8StorageConverter implem
if(b == null)
return null;
String s;
- if(b.length > 0 &&
- (b[b.length - 1] == 'F' || b[b.length - 1] == 'f') ){
+ if (b.length > 0 && (b[b.length - 1] == 'F' || b[b.length - 1] ==
'f')) {
s = new String(b, 0, b.length - 1);
- }
- else {
+ } else {
s = new String(b);
}
-
+
try {
return Float.valueOf(s);
} catch (NumberFormatException nfe) {
@@ -368,9 +368,10 @@ public class Utf8StorageConverter implem
try {
Double d = Double.valueOf(s);
// Need to check for an overflow error
- if (d.doubleValue() > mMaxInt.doubleValue() + 1.0) {
+ if (Double.compare(d.doubleValue(), mMaxInt.doubleValue() + 1)
>= 0 ||
+ Double.compare(d.doubleValue(), mMinInt.doubleValue()
- 1) <= 0) {
LogUtils.warn(this, "Value " + d + " too large for
integer",
- PigWarning.TOO_LARGE_FOR_INT, mLog);
+ PigWarning.TOO_LARGE_FOR_INT, mLog);
return null;
}
return Integer.valueOf(d.intValue());
@@ -385,18 +386,15 @@ public class Utf8StorageConverter implem
}
public Long bytesToLong(byte[] b) throws IOException {
- if(b == null)
+ if (b == null)
return null;
-
String s;
- if(b.length > 0 &&
- (b[b.length - 1] == 'L' || b[b.length - 1] == 'l') ){
+ if (b.length > 0 && (b[b.length - 1] == 'L' || b[b.length - 1] ==
'l')) {
s = new String(b, 0, b.length - 1);
- }
- else {
+ } else {
s = new String(b);
}
-
+
try {
return Long.valueOf(s);
} catch (NumberFormatException nfe) {
@@ -407,8 +405,9 @@ public class Utf8StorageConverter implem
try {
Double d = Double.valueOf(s);
// Need to check for an overflow error
- if (d.doubleValue() > mMaxLong.doubleValue() + 1.0) {
- LogUtils.warn(this, "Value " + d + " too large for
integer",
+ if (Double.compare(d.doubleValue(), mMaxLong.doubleValue() +
1) > 0 ||
+ Double.compare(d.doubleValue(), mMinLong.doubleValue()
- 1) < 0) {
+ LogUtils.warn(this, "Value " + d + " too large for
long",
PigWarning.TOO_LARGE_FOR_INT, mLog);
return null;
}
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/util/TupleFormat.java Mon May 17
17:24:40 2010
@@ -62,11 +62,6 @@ public class TupleFormat {
}
else {
sb.append(d.toString());
- if (d instanceof Long) {
- sb.append("L");
- } else if (d instanceof Float) {
- sb.append("F");
- }
}
} else {
sb.append("");
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestCombiner.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestCombiner.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestCombiner.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestCombiner.java Mon May 17
17:24:40 2010
@@ -127,9 +127,9 @@ public class TestCombiner extends TestCa
pig.registerQuery("C = foreach B generate flatten(group), COUNT($1);");
Iterator<Tuple> resultIterator = pig.openIterator("C");
Tuple tuple = resultIterator.next();
- assertEquals("(a,b,2L)", tuple.toString());
+ assertEquals("(a,b,2)", tuple.toString());
tuple = resultIterator.next();
- assertEquals("(a,c,1L)", tuple.toString());
+ assertEquals("(a,c,1)", tuple.toString());
return inputFileName;
}
@@ -364,9 +364,9 @@ public class TestCombiner extends TestCa
"pig1\t20\t3.1" };
String expected[] = {
-
"(pig1,75L,{(pig1,18,2.1),(pig1,18,2.1),(pig1,19,2.1),(pig1,20,3.1)})",
- "(pig2,48L,{(pig2,24,3.3),(pig2,24,4.5)})",
- "(pig5,45L,{(pig5,45,2.4)})"
+
"(pig1,75,{(pig1,18,2.1),(pig1,18,2.1),(pig1,19,2.1),(pig1,20,3.1)})",
+ "(pig2,48,{(pig2,24,3.3),(pig2,24,4.5)})",
+ "(pig5,45,{(pig5,45,2.4)})"
};
try {
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestConversions.java Mon May 17
17:24:40 2010
@@ -40,7 +40,6 @@ import org.apache.pig.impl.util.Utils;
import org.junit.Test;
import junit.framework.TestCase;
-import junit.framework.AssertionFailedError;
/**
* Test class to test conversions from bytes to types
@@ -80,7 +79,7 @@ public class TestConversions extends Tes
{
// valid floats
String[] a = {"1", "-2.345", "12.12334567", "1.02e-2",".23344",
- "23.1234567897", "12312.33f", "002312.33F", "1.02e-2f",
""};
+ "23.1234567897", "12312.33", "002312.33", "1.02e-2", ""};
Float[] f = {1f, -2.345f, 12.12334567f, 1.02e-2f,.23344f,
23.1234567f, // 23.1234567f is a truncation case
12312.33f, 2312.33f, 1.02e-2f };
@@ -126,7 +125,7 @@ public class TestConversions extends Tes
{
// valid Longs
String[] a = {"1", "-2345", "123456789012345678", "1.1", "-23.45",
- "21345345l", "3422342L", ""};
+ "21345345", "3422342", ""};
Long[] la = {1L, -2345L, 123456789012345678L, 1L, -23L,
21345345L, 3422342L};
@@ -304,6 +303,7 @@ public class TestConversions extends Tes
}
}
+ @Test
public void testBytesToComplexTypeMisc() throws IOException,
ParseException {
String s = "(a,b";
Schema schema = Utils.getSchemaFromString("t:tuple(a:chararray,
b:chararray)");
@@ -370,4 +370,48 @@ public class TestConversions extends Tes
t = ps.getLoadCaster().bytesToTuple(s.getBytes(), rfs);
assertTrue(t==null);
}
+
+ @Test
+ public void testOverflow() throws IOException, ParseException {
+ Schema schema;
+ ResourceFieldSchema rfs;
+ Tuple tuple, convertedTuple;
+ tuple = TupleFactory.getInstance().newTuple(1);
+
+ schema = Utils.getSchemaFromString("t:tuple(a:int)");
+ rfs = new ResourceSchema(schema).getFields()[0];
+
+ // long bigger than Integer.MAX_VALUE
+ tuple.set(0, Integer.valueOf(Integer.MAX_VALUE).longValue() + 1);
+ convertedTuple =
ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+ assertNull("Invalid cast to int: " + tuple.get(0) + " -> " +
convertedTuple.get(0), convertedTuple.get(0));
+
+ // long smaller than Integer.MIN_VALUE
+ tuple.set(0, Integer.valueOf(Integer.MIN_VALUE).longValue() - 1);
+ convertedTuple =
ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+ assertNull("Invalid cast to int: " + tuple.get(0) + " -> " +
convertedTuple.get(0), convertedTuple.get(0));
+
+ // double bigger than Integer.MAX_VALUE
+ tuple.set(0, Integer.valueOf(Integer.MAX_VALUE).doubleValue() + 1);
+ convertedTuple =
ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+ assertNull("Invalid cast to int: " + tuple.get(0) + " -> " +
convertedTuple.get(0), convertedTuple.get(0));
+
+ // double smaller than Integer.MIN_VALUE
+ tuple.set(0, Integer.valueOf(Integer.MIN_VALUE).doubleValue() - 1);
+ convertedTuple =
ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+ assertNull("Invalid cast to int: " + tuple.get(0) + " -> " +
convertedTuple.get(0), convertedTuple.get(0));
+
+ schema = Utils.getSchemaFromString("t:tuple(a:long)");
+ rfs = new ResourceSchema(schema).getFields()[0];
+
+ // double bigger than Long.MAX_VALUE
+ tuple.set(0, Long.valueOf(Long.MAX_VALUE).doubleValue() + 10000);
+ convertedTuple =
ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+ assertNull("Invalid cast to long: " + tuple.get(0) + " -> " +
convertedTuple.get(0), convertedTuple.get(0));
+
+ // double smaller than Long.MIN_VALUE
+ tuple.set(0, Long.valueOf(Long.MIN_VALUE).doubleValue() - 10000);
+ convertedTuple =
ps.getLoadCaster().bytesToTuple(tuple.toString().getBytes(), rfs);
+ assertNull("Invalid cast to long: " + tuple.get(0) + " -> " +
convertedTuple.get(0), convertedTuple.get(0));
+ }
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestDataModel.java Mon May 17
17:24:40 2010
@@ -266,7 +266,7 @@ public class TestDataModel extends junit
public void testTupleToString() throws Exception {
Tuple t = giveMeOneOfEach();
- assertEquals("toString", "((3,3.0F),{(4),(mary had a little
lamb)},[hello#world,goodbye#all],42,5000000000L,3.1415927F,2.99792458E8,true,hello,goodbye,)",
t.toString());
+ assertEquals("toString", "((3,3.0),{(4),(mary had a little
lamb)},[hello#world,goodbye#all],42,5000000000,3.1415927,2.99792458E8,true,hello,goodbye,)",
t.toString());
}
@Test
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestSecondarySort.java Mon May 17
17:24:40 2010
@@ -441,9 +441,9 @@ public class TestSecondarySort extends T
pigServer.registerQuery("D = foreach C { E = limit A 10; F = E.a1; G =
DISTINCT F; generate group, COUNT(G);};");
Iterator<Tuple> iter = pigServer.openIterator("D");
assertTrue(iter.hasNext());
- assertTrue(iter.next().toString().equals("(1,2L)"));
+ assertTrue(iter.next().toString().equals("(1,2)"));
assertTrue(iter.hasNext());
- assertTrue(iter.next().toString().equals("(2,1L)"));
+ assertTrue(iter.next().toString().equals("(2,1)"));
assertFalse(iter.hasNext());
Util.deleteFile(cluster, tmpFile1.getCanonicalPath());
Util.deleteFile(cluster, tmpFile2.getCanonicalPath());
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java Mon May
17 17:24:40 2010
@@ -86,14 +86,14 @@ public class TestTextDataParser extends
@Test
public void testLong() throws Exception{
- String myLong = "1l";
+ String myLong = "1";
Long l = ps.getLoadCaster().bytesToLong(myLong.getBytes());
assertTrue(l.equals(1l));
}
@Test
public void testFloat() throws Exception{
- String myFloat = "0.1f";
+ String myFloat = "0.1";
Float f = ps.getLoadCaster().bytesToFloat(myFloat.getBytes());
assertTrue(f.equals(0.1f));
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java?rev=945254&r1=945253&r2=945254&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestTupleFormat.java Mon May 17
17:24:40 2010
@@ -52,7 +52,7 @@ public class TestTupleFormat extends Tes
tuple.set(6, bag);
assertEquals(
-
"(12,[pig#scalability],,12L,1.2F,(innerTuple),{(innerTuple)})",
+
"(12,[pig#scalability],,12,1.2,(innerTuple),{(innerTuple)})",
TupleFormat.format(tuple));
} catch (ExecException e) {
e.printStackTrace();
