Author: namit
Date: Thu Nov 3 17:17:08 2011
New Revision: 1197227
URL: http://svn.apache.org/viewvc?rev=1197227&view=rev
Log:
HIVE-2465 Primitive Data Types returning null if the data is out of range of
the data type. (Ashutosh Chauhan via namit)
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBinary.java
Thu Nov 3 17:17:08 2011
@@ -18,12 +18,19 @@
package org.apache.hadoop.hive.serde2.lazy;
+import java.nio.charset.CharacterCodingException;
+
import org.apache.commons.codec.binary.Base64;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import
org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyBinaryObjectInspector;
import org.apache.hadoop.io.BytesWritable;
+import org.apache.hadoop.io.Text;
public class LazyBinary extends LazyPrimitive<LazyBinaryObjectInspector,
BytesWritable> {
+ private static final Log LOG = LogFactory.getLog(LazyBinary.class);
+
LazyBinary(LazyBinaryObjectInspector oi) {
super(oi);
data = new BytesWritable();
@@ -42,7 +49,12 @@ public class LazyBinary extends LazyPrim
byte[] recv = new byte[length];
System.arraycopy(bytes.getData(), start, recv, 0, length);
- byte[] decoded = Base64.isArrayByteBase64(recv) ?
Base64.decodeBase64(recv) : recv;
+ boolean arrayByteBase64 = Base64.isArrayByteBase64(recv);
+ if (arrayByteBase64) {
+ LOG.debug("Data not contains valid characters within the Base64 alphabet
so " +
+ "decoded the data.");
+ }
+ byte[] decoded = arrayByteBase64 ? Base64.decodeBase64(recv) : recv;
data.set(decoded, 0, decoded.length);
}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyBoolean.java
Thu Nov 3 17:17:08 2011
@@ -64,6 +64,7 @@ public class LazyBoolean extends
isNull = false;
} else {
isNull = true;
+ logExceptionMessage(bytes, start, length, "BOOLEAN");
}
}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyByte.java
Thu Nov 3 17:17:08 2011
@@ -53,6 +53,7 @@ public class LazyByte extends
isNull = false;
} catch (NumberFormatException e) {
isNull = true;
+ logExceptionMessage(bytes, start, length, "TINYINT");
}
}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyDouble.java
Thu Nov 3 17:17:08 2011
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.serde2.la
import java.nio.charset.CharacterCodingException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
import
org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyDoubleObjectInspector;
import org.apache.hadoop.io.Text;
@@ -30,6 +32,7 @@ import org.apache.hadoop.io.Text;
public class LazyDouble extends
LazyPrimitive<LazyDoubleObjectInspector, DoubleWritable> {
+ private static final Log LOG = LogFactory.getLog(LazyDouble.class);
public LazyDouble(LazyDoubleObjectInspector oi) {
super(oi);
data = new DoubleWritable();
@@ -42,13 +45,18 @@ public class LazyDouble extends
@Override
public void init(ByteArrayRef bytes, int start, int length) {
+ String byteData = null;
try {
- data.set(Double.parseDouble(Text.decode(bytes.getData(), start,
length)));
+ byteData = Text.decode(bytes.getData(), start, length);
+ data.set(Double.parseDouble(byteData));
isNull = false;
} catch (NumberFormatException e) {
isNull = true;
+ LOG.debug("Data not in the Double data type range so converted to null.
Given data is :"
+ + byteData, e);
} catch (CharacterCodingException e) {
isNull = true;
+ LOG.debug("Data not in the Double data type range so converted to
null.", e);
}
}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyFloat.java
Thu Nov 3 17:17:08 2011
@@ -19,6 +19,8 @@ package org.apache.hadoop.hive.serde2.la
import java.nio.charset.CharacterCodingException;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import
org.apache.hadoop.hive.serde2.lazy.objectinspector.primitive.LazyFloatObjectInspector;
import org.apache.hadoop.io.FloatWritable;
import org.apache.hadoop.io.Text;
@@ -30,6 +32,7 @@ import org.apache.hadoop.io.Text;
public class LazyFloat extends
LazyPrimitive<LazyFloatObjectInspector, FloatWritable> {
+ private static final Log LOG = LogFactory.getLog(LazyFloat.class);
public LazyFloat(LazyFloatObjectInspector oi) {
super(oi);
data = new FloatWritable();
@@ -42,13 +45,18 @@ public class LazyFloat extends
@Override
public void init(ByteArrayRef bytes, int start, int length) {
+ String byteData = null;
try {
- data.set(Float.parseFloat(Text.decode(bytes.getData(), start, length)));
+ byteData = Text.decode(bytes.getData(), start, length);
+ data.set(Float.parseFloat(byteData));
isNull = false;
} catch (NumberFormatException e) {
isNull = true;
+ LOG.debug("Data not in the Float data type range so converted to null.
Given data is :"
+ + byteData, e);
} catch (CharacterCodingException e) {
isNull = true;
+ LOG.debug("Data not in the Float data type range so converted to null.",
e);
}
}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyInteger.java
Thu Nov 3 17:17:08 2011
@@ -56,6 +56,7 @@ public class LazyInteger extends
isNull = false;
} catch (NumberFormatException e) {
isNull = true;
+ logExceptionMessage(bytes, start, length, "INT");
}
}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyLong.java
Thu Nov 3 17:17:08 2011
@@ -56,6 +56,7 @@ public class LazyLong extends
isNull = false;
} catch (NumberFormatException e) {
isNull = true;
+ logExceptionMessage(bytes, start, length, "BIGINT");
}
}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyPrimitive.java
Thu Nov 3 17:17:08 2011
@@ -17,7 +17,12 @@
*/
package org.apache.hadoop.hive.serde2.lazy;
+import java.nio.charset.CharacterCodingException;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
/**
@@ -26,6 +31,7 @@ import org.apache.hadoop.io.Writable;
public abstract class LazyPrimitive<OI extends ObjectInspector, T extends
Writable>
extends LazyObject<OI> {
+ private static final Log LOG = LogFactory.getLog(LazyPrimitive.class);
LazyPrimitive(OI oi) {
super(oi);
}
@@ -61,4 +67,15 @@ public abstract class LazyPrimitive<OI e
return isNull ? 0 : data.hashCode();
}
+ public void logExceptionMessage(ByteArrayRef bytes, int start, int length,
String dataType) {
+ try {
+ String byteData = Text.decode(bytes.getData(), start, length);
+ LOG.debug("Data not in the " + dataType
+ + " data type range so converted to null. Given data is :" +
+ byteData);
+ } catch (CharacterCodingException e1) {
+ LOG.debug("Data not in the " + dataType + " data type range so converted
to null.", e1);
+ }
+ }
+
}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
--- hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
(original)
+++ hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyShort.java
Thu Nov 3 17:17:08 2011
@@ -53,6 +53,7 @@ public class LazyShort extends
isNull = false;
} catch (NumberFormatException e) {
isNull = true;
+ logExceptionMessage(bytes, start, length, "SMALLINT");
}
}
Modified:
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
---
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java
(original)
+++
hive/trunk/serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyTimestamp.java
Thu Nov 3 17:17:08 2011
@@ -69,6 +69,7 @@ public class LazyTimestamp extends LazyP
Timestamp t;
if (s.compareTo("NULL") == 0) {
t = null;
+ logExceptionMessage(bytes, start, length, "TIMESTAMP");
} else {
t = Timestamp.valueOf(s);
}
Modified:
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java
URL:
http://svn.apache.org/viewvc/hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java?rev=1197227&r1=1197226&r2=1197227&view=diff
==============================================================================
---
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java
(original)
+++
hive/trunk/serde/src/test/org/apache/hadoop/hive/serde2/lazy/TestLazyPrimitive.java
Thu Nov 3 17:17:08 2011
@@ -375,16 +375,17 @@ public class TestLazyPrimitive extends T
}
public void testLazyBinary() {
-
LazyBinary ba = new
LazyBinary(LazyPrimitiveObjectInspectorFactory.LAZY_BINARY_OBJECT_INSPECTOR);
- initLazyObject(ba, new byte[]{}, 0, 0);
+ initLazyObject(ba, new byte[] {}, 0, 0);
assertEquals(new BytesWritable(), ba.getWritableObject());
- initLazyObject(ba, new byte[]{'%'}, 0, 1);
- assertEquals(new BytesWritable(new byte[]{'%'}), ba.getWritableObject());
- initLazyObject(ba, new byte[]{'2','>','3'}, 1, 1);
- assertEquals(new BytesWritable(new byte[]{'>'}), ba.getWritableObject());
- initLazyObject(ba, new byte[]{'2','?','3'}, 0, 3);
- assertEquals(new BytesWritable(new byte[]{'2','?','3'}),
ba.getWritableObject());
+ initLazyObject(ba, new byte[] {'%'}, 0, 1);
+ assertEquals(new BytesWritable(new byte[] {'%'}), ba.getWritableObject());
+ initLazyObject(ba, new byte[] {'2', '>', '3'}, 1, 1);
+ assertEquals(new BytesWritable(new byte[] {'>'}), ba.getWritableObject());
+ initLazyObject(ba, new byte[] {'2', '?', '3'}, 0, 3);
+ assertEquals(new BytesWritable(new byte[] {'2', '?', '3'}),
ba.getWritableObject());
+ initLazyObject(ba, new byte[] {'\n'}, 0, 1);
+ assertEquals(new BytesWritable(new byte[] {}), ba.getWritableObject());
}
public void testLazyIntegerWrite() throws Throwable {
