spark git commit: [SPARK-11819][SQL] nice error message for missing encoder

2015-11-20 Thread marmbrus 
Repository: spark
Updated Branches:
  refs/heads/master 60bfb1133 -> 3b9d2a347


[SPARK-11819][SQL] nice error message for missing encoder

before this PR, when users try to get an encoder for an un-supported class, 
they will only get a very simple error message like `Encoder for type xxx is 
not supported`.

After this PR, the error message become more friendly, for example:
```
No Encoder found for abc.xyz.NonEncodable
- array element class: "abc.xyz.NonEncodable"
- field (class: "scala.Array", name: "arrayField")
- root class: "abc.xyz.AnotherClass"
```

Author: Wenchen Fan 

Closes #9810 from cloud-fan/error-message.


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/3b9d2a34
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/3b9d2a34
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/3b9d2a34

Branch: refs/heads/master
Commit: 3b9d2a347f9c796b90852173d84189834e499e25
Parents: 60bfb11
Author: Wenchen Fan 
Authored: Fri Nov 20 12:04:42 2015 -0800
Committer: Michael Armbrust 
Committed: Fri Nov 20 12:04:42 2015 -0800

--
 .../spark/sql/catalyst/ScalaReflection.scala| 90 +++-
 .../encoders/EncoderErrorMessageSuite.scala | 62 ++
 2 files changed, 129 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/3b9d2a34/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 33ae700..918050b 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -63,7 +63,7 @@ object ScalaReflection extends ScalaReflection {
   case t if t <:< definitions.BooleanTpe => BooleanType
   case t if t <:< localTypeOf[Array[Byte]] => BinaryType
   case _ =>
-val className: String = tpe.erasure.typeSymbol.asClass.fullName
+val className = getClassNameFromType(tpe)
 className match {
   case "scala.Array" =>
 val TypeRef(_, _, Seq(elementType)) = tpe
@@ -320,9 +320,23 @@ object ScalaReflection extends ScalaReflection {
 }
   }
 
-  /** Returns expressions for extracting all the fields from the given type. */
+  /**
+   * Returns expressions for extracting all the fields from the given type.
+   *
+   * If the given type is not supported, i.e. there is no encoder can be built 
for this type,
+   * an [[UnsupportedOperationException]] will be thrown with detailed error 
message to explain
+   * the type path walked so far and which class we are not supporting.
+   * There are 4 kinds of type path:
+   *  * the root type: `root class: "abc.xyz.MyClass"`
+   *  * the value type of [[Option]]: `option value class: "abc.xyz.MyClass"`
+   *  * the element type of [[Array]] or [[Seq]]: `array element class: 
"abc.xyz.MyClass"`
+   *  * the field of [[Product]]: `field (class: "abc.xyz.MyClass", name: 
"myField")`
+   */
   def extractorsFor[T : TypeTag](inputObject: Expression): CreateNamedStruct = 
{
-extractorFor(inputObject, localTypeOf[T]) match {
+val tpe = localTypeOf[T]
+val clsName = getClassNameFromType(tpe)
+val walkedTypePath = s"""- root class: "${clsName} :: Nil
+extractorFor(inputObject, tpe, walkedTypePath) match {
   case s: CreateNamedStruct => s
   case other => CreateNamedStruct(expressions.Literal("value") :: other :: 
Nil)
 }
@@ -331,7 +345,28 @@ object ScalaReflection extends ScalaReflection {
   /** Helper for extracting internal fields from a case class. */
   private def extractorFor(
   inputObject: Expression,
-  tpe: `Type`): Expression = ScalaReflectionLock.synchronized {
+  tpe: `Type`,
+  walkedTypePath: Seq[String]): Expression = 
ScalaReflectionLock.synchronized {
+
+def toCatalystArray(input: Expression, elementType: `Type`): Expression = {
+  val externalDataType = dataTypeFor(elementType)
+  val Schema(catalystType, nullable) = silentSchemaFor(elementType)
+  if (isNativeType(catalystType)) {
+NewInstance(
+  classOf[GenericArrayData],
+  input :: Nil,
+  dataType = ArrayType(catalystType, nullable))
+  } else {
+val clsName = getClassNameFromType(elementType)
+val newPath = s"""- array element class: "$clsName +: 
walkedTypePath
+// `MapObjects` will run `extractorFor` lazily, we need to eagerly 
call `extractorFor` here
+// to trigger the type check.
+

spark git commit: [SPARK-11819][SQL] nice error message for missing encoder

2015-11-20 Thread marmbrus 
Repository: spark
Updated Branches:
  refs/heads/branch-1.6 119f92b4e -> ff156a3a6


[SPARK-11819][SQL] nice error message for missing encoder

before this PR, when users try to get an encoder for an un-supported class, 
they will only get a very simple error message like `Encoder for type xxx is 
not supported`.

After this PR, the error message become more friendly, for example:
```
No Encoder found for abc.xyz.NonEncodable
- array element class: "abc.xyz.NonEncodable"
- field (class: "scala.Array", name: "arrayField")
- root class: "abc.xyz.AnotherClass"
```

Author: Wenchen Fan 

Closes #9810 from cloud-fan/error-message.

(cherry picked from commit 3b9d2a347f9c796b90852173d84189834e499e25)
Signed-off-by: Michael Armbrust 


Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ff156a3a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ff156a3a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ff156a3a

Branch: refs/heads/branch-1.6
Commit: ff156a3a660e1730de220b404a61e1bda8b7682e
Parents: 119f92b
Author: Wenchen Fan 
Authored: Fri Nov 20 12:04:42 2015 -0800
Committer: Michael Armbrust 
Committed: Fri Nov 20 12:04:53 2015 -0800

--
 .../spark/sql/catalyst/ScalaReflection.scala| 90 +++-
 .../encoders/EncoderErrorMessageSuite.scala | 62 ++
 2 files changed, 129 insertions(+), 23 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/spark/blob/ff156a3a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
--
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index 33ae700..918050b 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -63,7 +63,7 @@ object ScalaReflection extends ScalaReflection {
   case t if t <:< definitions.BooleanTpe => BooleanType
   case t if t <:< localTypeOf[Array[Byte]] => BinaryType
   case _ =>
-val className: String = tpe.erasure.typeSymbol.asClass.fullName
+val className = getClassNameFromType(tpe)
 className match {
   case "scala.Array" =>
 val TypeRef(_, _, Seq(elementType)) = tpe
@@ -320,9 +320,23 @@ object ScalaReflection extends ScalaReflection {
 }
   }
 
-  /** Returns expressions for extracting all the fields from the given type. */
+  /**
+   * Returns expressions for extracting all the fields from the given type.
+   *
+   * If the given type is not supported, i.e. there is no encoder can be built 
for this type,
+   * an [[UnsupportedOperationException]] will be thrown with detailed error 
message to explain
+   * the type path walked so far and which class we are not supporting.
+   * There are 4 kinds of type path:
+   *  * the root type: `root class: "abc.xyz.MyClass"`
+   *  * the value type of [[Option]]: `option value class: "abc.xyz.MyClass"`
+   *  * the element type of [[Array]] or [[Seq]]: `array element class: 
"abc.xyz.MyClass"`
+   *  * the field of [[Product]]: `field (class: "abc.xyz.MyClass", name: 
"myField")`
+   */
   def extractorsFor[T : TypeTag](inputObject: Expression): CreateNamedStruct = 
{
-extractorFor(inputObject, localTypeOf[T]) match {
+val tpe = localTypeOf[T]
+val clsName = getClassNameFromType(tpe)
+val walkedTypePath = s"""- root class: "${clsName} :: Nil
+extractorFor(inputObject, tpe, walkedTypePath) match {
   case s: CreateNamedStruct => s
   case other => CreateNamedStruct(expressions.Literal("value") :: other :: 
Nil)
 }
@@ -331,7 +345,28 @@ object ScalaReflection extends ScalaReflection {
   /** Helper for extracting internal fields from a case class. */
   private def extractorFor(
   inputObject: Expression,
-  tpe: `Type`): Expression = ScalaReflectionLock.synchronized {
+  tpe: `Type`,
+  walkedTypePath: Seq[String]): Expression = 
ScalaReflectionLock.synchronized {
+
+def toCatalystArray(input: Expression, elementType: `Type`): Expression = {
+  val externalDataType = dataTypeFor(elementType)
+  val Schema(catalystType, nullable) = silentSchemaFor(elementType)
+  if (isNativeType(catalystType)) {
+NewInstance(
+  classOf[GenericArrayData],
+  input :: Nil,
+  dataType = ArrayType(catalystType, nullable))
+  } else {
+val clsName = getClassNameFromType(elementType)
+val newPath = s"""- array element class: "$clsName +: 
walkedTypePath
+//