incubator-livy git commit: [LIVY-455][REPL] Fix json4s doesn't support java.math.BigDecimal issue

2018-04-10 Thread jshao 
Repository: incubator-livy
Updated Branches:
  refs/heads/branch-0.5 4783fbc25 -> 88d66a95f


[LIVY-455][REPL] Fix json4s doesn't support java.math.BigDecimal issue

## What changes were proposed in this pull request?

Livy's SQLInterpreter will throw exception when rows contain 
java.math.BigDecimal data. This is because current version of json4s doesn't 
treat java.math.BigDecimal type as primitive type. On the contrary, json4s 
supports Scala BigDecimal as primitive type. So the fix is to convert java 
BigDecimal to Scala BigDecimal.

## How was this patch tested?

Unit test is added.

Author: jerryshao 

Closes #85 from jerryshao/LIVY-455.

(cherry picked from commit 7e4bb3bd6888742e43aed7396ed490d343a063a2)
Signed-off-by: jerryshao 


Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/88d66a95
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/88d66a95
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/88d66a95

Branch: refs/heads/branch-0.5
Commit: 88d66a95fb6737a1e3bdeed34138e16f64fc23af
Parents: 4783fbc
Author: jerryshao 
Authored: Wed Apr 11 10:24:20 2018 +0800
Committer: jerryshao 
Committed: Wed Apr 11 10:24:31 2018 +0800

--
 .../org/apache/livy/repl/SQLInterpreter.scala   | 10 +-
 .../apache/livy/repl/SQLInterpreterSpec.scala   | 36 
 2 files changed, 45 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/88d66a95/repl/src/main/scala/org/apache/livy/repl/SQLInterpreter.scala
--
diff --git a/repl/src/main/scala/org/apache/livy/repl/SQLInterpreter.scala 
b/repl/src/main/scala/org/apache/livy/repl/SQLInterpreter.scala
index a98223e..afef09b 100644
--- a/repl/src/main/scala/org/apache/livy/repl/SQLInterpreter.scala
+++ b/repl/src/main/scala/org/apache/livy/repl/SQLInterpreter.scala
@@ -97,7 +97,15 @@ class SQLInterpreter(
   val rows = result.getClass.getMethod("take", classOf[Int])
 .invoke(result, maxResult: java.lang.Integer)
 .asInstanceOf[Array[Row]]
-  .map(_.toSeq)
+.map {
+  _.toSeq.map {
+// Convert java BigDecimal type to Scala BigDecimal, because 
current version of
+// Json4s doesn't support java BigDecimal as a primitive type 
(LIVY-455).
+case i: java.math.BigDecimal => BigDecimal(i)
+case e => e
+  }
+}
+
   val jRows = Extraction.decompose(rows)
 
   Interpreter.ExecuteSuccess(

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/88d66a95/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
--
diff --git a/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala 
b/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
index 58e89ee..d2c12eb 100644
--- a/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
+++ b/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
@@ -95,6 +95,42 @@ class SQLInterpreterSpec extends BaseInterpreterSpec {
 ))
   }
 
+  it should "handle java BigDecimal" in withInterpreter { interpreter =>
+val rdd = sparkEntries.sc().parallelize(Seq(
+  ("1", new java.math.BigDecimal(1.0)),
+  ("2", new java.math.BigDecimal(2.0
+val df = sparkEntries.sqlctx().createDataFrame(rdd).selectExpr("_1 as 
col1", "_2 as col2")
+df.registerTempTable("test")
+
+val resp1 = interpreter.execute(
+  """
+|SELECT * FROM test
+  """.stripMargin)
+
+val expectedResult = (nullable: Boolean) => {
+  Interpreter.ExecuteSuccess(
+APPLICATION_JSON -> (("schema" ->
+  (("type" -> "struct") ~
+("fields" -> List(
+  ("name" -> "col1") ~ ("type" -> "string") ~ ("nullable" -> true) 
~
+("metadata" -> List()),
+  ("name" -> "col2") ~ ("type" -> "decimal(38,18)") ~ ("nullable" 
-> nullable) ~
+("metadata" -> List())
+ ~
+  ("data" -> List(
+List[JValue]("1", 1.0d),
+List[JValue]("2", 2.0d)
+  )))
+  )
+}
+
+val result = Try { resp1 should equal(expectedResult(false))}
+  .orElse(Try { resp1 should equal(expectedResult(true)) })
+if (result.isFailure) {
+  fail(s"$resp1 doesn't equal to expected result")
+}
+  }
+
   it should "throw exception for illegal query" in withInterpreter { 
interpreter =>
 val resp = interpreter.execute(
   """

incubator-livy git commit: [LIVY-455][REPL] Fix json4s doesn't support java.math.BigDecimal issue

2018-04-10 Thread jshao 
Repository: incubator-livy
Updated Branches:
  refs/heads/master 06cfa7bce -> 7e4bb3bd6


[LIVY-455][REPL] Fix json4s doesn't support java.math.BigDecimal issue

## What changes were proposed in this pull request?

Livy's SQLInterpreter will throw exception when rows contain 
java.math.BigDecimal data. This is because current version of json4s doesn't 
treat java.math.BigDecimal type as primitive type. On the contrary, json4s 
supports Scala BigDecimal as primitive type. So the fix is to convert java 
BigDecimal to Scala BigDecimal.

## How was this patch tested?

Unit test is added.

Author: jerryshao 

Closes #85 from jerryshao/LIVY-455.


Project: http://git-wip-us.apache.org/repos/asf/incubator-livy/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-livy/commit/7e4bb3bd
Tree: http://git-wip-us.apache.org/repos/asf/incubator-livy/tree/7e4bb3bd
Diff: http://git-wip-us.apache.org/repos/asf/incubator-livy/diff/7e4bb3bd

Branch: refs/heads/master
Commit: 7e4bb3bd6888742e43aed7396ed490d343a063a2
Parents: 06cfa7b
Author: jerryshao 
Authored: Wed Apr 11 10:24:20 2018 +0800
Committer: jerryshao 
Committed: Wed Apr 11 10:24:20 2018 +0800

--
 .../org/apache/livy/repl/SQLInterpreter.scala   | 10 +-
 .../apache/livy/repl/SQLInterpreterSpec.scala   | 36 
 2 files changed, 45 insertions(+), 1 deletion(-)
--


http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/7e4bb3bd/repl/src/main/scala/org/apache/livy/repl/SQLInterpreter.scala
--
diff --git a/repl/src/main/scala/org/apache/livy/repl/SQLInterpreter.scala 
b/repl/src/main/scala/org/apache/livy/repl/SQLInterpreter.scala
index a98223e..afef09b 100644
--- a/repl/src/main/scala/org/apache/livy/repl/SQLInterpreter.scala
+++ b/repl/src/main/scala/org/apache/livy/repl/SQLInterpreter.scala
@@ -97,7 +97,15 @@ class SQLInterpreter(
   val rows = result.getClass.getMethod("take", classOf[Int])
 .invoke(result, maxResult: java.lang.Integer)
 .asInstanceOf[Array[Row]]
-  .map(_.toSeq)
+.map {
+  _.toSeq.map {
+// Convert java BigDecimal type to Scala BigDecimal, because 
current version of
+// Json4s doesn't support java BigDecimal as a primitive type 
(LIVY-455).
+case i: java.math.BigDecimal => BigDecimal(i)
+case e => e
+  }
+}
+
   val jRows = Extraction.decompose(rows)
 
   Interpreter.ExecuteSuccess(

http://git-wip-us.apache.org/repos/asf/incubator-livy/blob/7e4bb3bd/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
--
diff --git a/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala 
b/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
index 58e89ee..d2c12eb 100644
--- a/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
+++ b/repl/src/test/scala/org/apache/livy/repl/SQLInterpreterSpec.scala
@@ -95,6 +95,42 @@ class SQLInterpreterSpec extends BaseInterpreterSpec {
 ))
   }
 
+  it should "handle java BigDecimal" in withInterpreter { interpreter =>
+val rdd = sparkEntries.sc().parallelize(Seq(
+  ("1", new java.math.BigDecimal(1.0)),
+  ("2", new java.math.BigDecimal(2.0
+val df = sparkEntries.sqlctx().createDataFrame(rdd).selectExpr("_1 as 
col1", "_2 as col2")
+df.registerTempTable("test")
+
+val resp1 = interpreter.execute(
+  """
+|SELECT * FROM test
+  """.stripMargin)
+
+val expectedResult = (nullable: Boolean) => {
+  Interpreter.ExecuteSuccess(
+APPLICATION_JSON -> (("schema" ->
+  (("type" -> "struct") ~
+("fields" -> List(
+  ("name" -> "col1") ~ ("type" -> "string") ~ ("nullable" -> true) 
~
+("metadata" -> List()),
+  ("name" -> "col2") ~ ("type" -> "decimal(38,18)") ~ ("nullable" 
-> nullable) ~
+("metadata" -> List())
+ ~
+  ("data" -> List(
+List[JValue]("1", 1.0d),
+List[JValue]("2", 2.0d)
+  )))
+  )
+}
+
+val result = Try { resp1 should equal(expectedResult(false))}
+  .orElse(Try { resp1 should equal(expectedResult(true)) })
+if (result.isFailure) {
+  fail(s"$resp1 doesn't equal to expected result")
+}
+  }
+
   it should "throw exception for illegal query" in withInterpreter { 
interpreter =>
 val resp = interpreter.execute(
   """