Repository: spark Updated Branches: refs/heads/master 594b14f1e -> ae5b2d3e4
[SPARK-18746][SQL] Add implicit encoder for BigDecimal, timestamp and date ## What changes were proposed in this pull request? Add implicit encoders for BigDecimal, timestamp and date. ## How was this patch tested? Add an unit test. Pass build, unit tests, and some tests below . Before: ``` scala> spark.createDataset(Seq(new java.math.BigDecimal(10))) <console>:24: error: Unable to find encoder for type stored in a Dataset. Primitive types (Int, String, etc) and Product types (case classes) are supported by importing spark.implicits._ Support for serializing other types will be added in future releases. spark.createDataset(Seq(new java.math.BigDecimal(10))) ^ scala> ``` After: ``` scala> spark.createDataset(Seq(new java.math.BigDecimal(10))) res0: org.apache.spark.sql.Dataset[java.math.BigDecimal] = [value: decimal(38,18)] ``` Author: Weiqing Yang <yangweiqing...@gmail.com> Closes #16176 from weiqingy/SPARK-18746. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ae5b2d3e Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ae5b2d3e Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ae5b2d3e Branch: refs/heads/master Commit: ae5b2d3e46cc4c460f539c4db1688309d1cdc66a Parents: 594b14f Author: Weiqing Yang <yangweiqing...@gmail.com> Authored: Wed Dec 14 09:48:38 2016 +0800 Committer: Wenchen Fan <wenc...@databricks.com> Committed: Wed Dec 14 09:48:38 2016 +0800 ---------------------------------------------------------------------- .../scala/org/apache/spark/sql/SQLImplicits.scala | 15 ++++++++++++++- .../scala/org/apache/spark/sql/DatasetSuite.scala | 16 +++++++++++++++- 2 files changed, 29 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/ae5b2d3e/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala index 73d16d8..872a78b 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala @@ -74,6 +74,19 @@ abstract class SQLImplicits { /** @since 1.6.0 */ implicit def newStringEncoder: Encoder[String] = Encoders.STRING + /** @since 2.2.0 */ + implicit def newJavaDecimalEncoder: Encoder[java.math.BigDecimal] = Encoders.DECIMAL + + /** @since 2.2.0 */ + implicit def newScalaDecimalEncoder: Encoder[scala.math.BigDecimal] = ExpressionEncoder() + + /** @since 2.2.0 */ + implicit def newDateEncoder: Encoder[java.sql.Date] = Encoders.DATE + + /** @since 2.2.0 */ + implicit def newTimeStampEncoder: Encoder[java.sql.Timestamp] = Encoders.TIMESTAMP + + // Boxed primitives /** @since 2.0.0 */ @@ -141,7 +154,7 @@ abstract class SQLImplicits { implicit def newFloatArrayEncoder: Encoder[Array[Float]] = ExpressionEncoder() /** @since 1.6.1 */ - implicit def newByteArrayEncoder: Encoder[Array[Byte]] = ExpressionEncoder() + implicit def newByteArrayEncoder: Encoder[Array[Byte]] = Encoders.BINARY /** @since 1.6.1 */ implicit def newShortArrayEncoder: Encoder[Array[Short]] = ExpressionEncoder() http://git-wip-us.apache.org/repos/asf/spark/blob/ae5b2d3e/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala index 3742115..c27b815 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala @@ -26,7 +26,6 @@ import org.apache.spark.sql.execution.{LogicalRDD, RDDScanExec, SortExec} import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec, ShuffleExchange} import org.apache.spark.sql.execution.streaming.MemoryStream import org.apache.spark.sql.functions._ -import org.apache.spark.sql.internal.SQLConf import org.apache.spark.sql.test.SharedSQLContext import org.apache.spark.sql.types._ @@ -1129,6 +1128,21 @@ class DatasetSuite extends QueryTest with SharedSQLContext { val ds2 = Seq(WithMap("hi", Map(42L -> "foo"))).toDS checkDataset(ds2.map(t => t), WithMap("hi", Map(42L -> "foo"))) } + + test("SPARK-18746: add implicit encoder for BigDecimal, date, timestamp") { + // For this implicit encoder, 18 is the default scale + assert(spark.range(1).map { x => new java.math.BigDecimal(1) }.head == + new java.math.BigDecimal(1).setScale(18)) + + assert(spark.range(1).map { x => scala.math.BigDecimal(1, 18) }.head == + scala.math.BigDecimal(1, 18)) + + assert(spark.range(1).map { x => new java.sql.Date(2016, 12, 12) }.head == + new java.sql.Date(2016, 12, 12)) + + assert(spark.range(1).map { x => new java.sql.Timestamp(100000) }.head == + new java.sql.Timestamp(100000)) + } } case class WithImmutableMap(id: String, map_test: scala.collection.immutable.Map[Long, String]) --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org