spark git commit: [SPARK-11908][SQL] Add NullType support to RowEncoder

2015-11-22 Thread marmbrus 
Repository: spark
Updated Branches:
  refs/heads/master ff442bbcf -> 426004a9c


[SPARK-11908][SQL] Add NullType support to RowEncoder

JIRA: https://issues.apache.org/jira/browse/SPARK-11908

We should add NullType support to RowEncoder.

Author: Liang-Chi Hsieh 

Closes #9891 from viirya/rowencoder-nulltype.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/426004a9
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/426004a9
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/426004a9

Branch: refs/heads/master
Commit: 426004a9c9a864f90494d08601e6974709091a56
Parents: ff442bb
Author: Liang-Chi Hsieh 
Authored: Sun Nov 22 10:36:47 2015 -0800
Committer: Michael Armbrust 
Committed: Sun Nov 22 10:36:47 2015 -0800

--
 .../org/apache/spark/sql/catalyst/encoders/RowEncoder.scala | 5 +++--
 .../org/apache/spark/sql/catalyst/expressions/objects.scala | 3 +++
 .../apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala| 3 +++
 3 files changed, 9 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/426004a9/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
index 4cda482..fa553e7 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
@@ -48,7 +48,7 @@ object RowEncoder {
   private def extractorsFor(
   inputObject: Expression,
   inputType: DataType): Expression = inputType match {
-case BooleanType | ByteType | ShortType | IntegerType | LongType |
+case NullType | BooleanType | ByteType | ShortType | IntegerType | 
LongType |
  FloatType | DoubleType | BinaryType => inputObject
 
 case udt: UserDefinedType[_] =>
@@ -143,6 +143,7 @@ object RowEncoder {
 case _: MapType => ObjectType(classOf[scala.collection.Map[_, _]])
 case _: StructType => ObjectType(classOf[Row])
 case udt: UserDefinedType[_] => ObjectType(udt.userClass)
+case _: NullType => ObjectType(classOf[java.lang.Object])
   }
 
   private def constructorFor(schema: StructType): Expression = {
@@ -158,7 +159,7 @@ object RowEncoder {
   }
 
   private def constructorFor(input: Expression): Expression = input.dataType 
match {
-case BooleanType | ByteType | ShortType | IntegerType | LongType |
+case NullType | BooleanType | ByteType | ShortType | IntegerType | 
LongType |
  FloatType | DoubleType | BinaryType => input
 
 case udt: UserDefinedType[_] =>

http://git-wip-us.apache.org/repos/asf/spark/blob/426004a9/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
index ef7399e..82317d3 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
@@ -369,6 +369,9 @@ case class MapObjects(
   private lazy val completeFunction = function(loopAttribute)
 
   private def itemAccessorMethod(dataType: DataType): String => String = 
dataType match {
+case NullType =>
+  val nullTypeClassName = NullType.getClass.getName + ".MODULE$"
+  (i: String) => s".get($i, $nullTypeClassName)"
 case IntegerType => (i: String) => s".getInt($i)"
 case LongType => (i: String) => s".getLong($i)"
 case FloatType => (i: String) => s".getFloat($i)"

http://git-wip-us.apache.org/repos/asf/spark/blob/426004a9/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
index 46c6e0d..0ea51ec 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
@@ -80,11 +80,13 @@ class RowEncoderSuite extends SparkFunSuite {
   private val structOfString = new StructType().add("str", StringType)
   private val structOfUDT = new StructType().add("udt", new ExamplePointUDT, 
false)
   private val arrayOf

spark git commit: [SPARK-11908][SQL] Add NullType support to RowEncoder

2015-11-22 Thread marmbrus 
Repository: spark
Updated Branches:
  refs/heads/branch-1.6 8c718a577 -> b004a104f


[SPARK-11908][SQL] Add NullType support to RowEncoder

JIRA: https://issues.apache.org/jira/browse/SPARK-11908

We should add NullType support to RowEncoder.

Author: Liang-Chi Hsieh 

Closes #9891 from viirya/rowencoder-nulltype.

(cherry picked from commit 426004a9c9a864f90494d08601e6974709091a56)
Signed-off-by: Michael Armbrust 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/b004a104
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/b004a104
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/b004a104

Branch: refs/heads/branch-1.6
Commit: b004a104f62849b393047aa8ea45542c871198e7
Parents: 8c718a5
Author: Liang-Chi Hsieh 
Authored: Sun Nov 22 10:36:47 2015 -0800
Committer: Michael Armbrust 
Committed: Sun Nov 22 10:38:20 2015 -0800

--
 .../org/apache/spark/sql/catalyst/encoders/RowEncoder.scala | 5 +++--
 .../org/apache/spark/sql/catalyst/expressions/objects.scala | 3 +++
 .../apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala| 3 +++
 3 files changed, 9 insertions(+), 2 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/b004a104/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
index 4cda482..fa553e7 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/encoders/RowEncoder.scala
@@ -48,7 +48,7 @@ object RowEncoder {
   private def extractorsFor(
   inputObject: Expression,
   inputType: DataType): Expression = inputType match {
-case BooleanType | ByteType | ShortType | IntegerType | LongType |
+case NullType | BooleanType | ByteType | ShortType | IntegerType | 
LongType |
  FloatType | DoubleType | BinaryType => inputObject
 
 case udt: UserDefinedType[_] =>
@@ -143,6 +143,7 @@ object RowEncoder {
 case _: MapType => ObjectType(classOf[scala.collection.Map[_, _]])
 case _: StructType => ObjectType(classOf[Row])
 case udt: UserDefinedType[_] => ObjectType(udt.userClass)
+case _: NullType => ObjectType(classOf[java.lang.Object])
   }
 
   private def constructorFor(schema: StructType): Expression = {
@@ -158,7 +159,7 @@ object RowEncoder {
   }
 
   private def constructorFor(input: Expression): Expression = input.dataType 
match {
-case BooleanType | ByteType | ShortType | IntegerType | LongType |
+case NullType | BooleanType | ByteType | ShortType | IntegerType | 
LongType |
  FloatType | DoubleType | BinaryType => input
 
 case udt: UserDefinedType[_] =>

http://git-wip-us.apache.org/repos/asf/spark/blob/b004a104/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
index ef7399e..82317d3 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/objects.scala
@@ -369,6 +369,9 @@ case class MapObjects(
   private lazy val completeFunction = function(loopAttribute)
 
   private def itemAccessorMethod(dataType: DataType): String => String = 
dataType match {
+case NullType =>
+  val nullTypeClassName = NullType.getClass.getName + ".MODULE$"
+  (i: String) => s".get($i, $nullTypeClassName)"
 case IntegerType => (i: String) => s".getInt($i)"
 case LongType => (i: String) => s".getLong($i)"
 case FloatType => (i: String) => s".getFloat($i)"

http://git-wip-us.apache.org/repos/asf/spark/blob/b004a104/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
--
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
index 46c6e0d..0ea51ec 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/encoders/RowEncoderSuite.scala
@@ -80,11 +80,13 @@ class RowEncoderSuite extends SparkFunSuite {
   private val structOfString = new StructType().add("str", StringTyp