Repository: spark
Updated Branches:
refs/heads/master 594b14f1e -> ae5b2d3e4
[SPARK-18746][SQL] Add implicit encoder for BigDecimal, timestamp and date
## What changes were proposed in this pull request?
Add implicit encoders for BigDecimal, timestamp and date.
## How was this patch tested?
Add an unit test. Pass build, unit tests, and some tests below .
Before:
```
scala> spark.createDataset(Seq(new java.math.BigDecimal(10)))
<console>:24: error: Unable to find encoder for type stored in a Dataset.
Primitive types (Int, String, etc) and Product types (case classes) are
supported by importing spark.implicits._ Support for serializing other types
will be added in future releases.
spark.createDataset(Seq(new java.math.BigDecimal(10)))
^
scala>
```
After:
```
scala> spark.createDataset(Seq(new java.math.BigDecimal(10)))
res0: org.apache.spark.sql.Dataset[java.math.BigDecimal] = [value:
decimal(38,18)]
```
Author: Weiqing Yang <[email protected]>
Closes #16176 from weiqingy/SPARK-18746.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ae5b2d3e
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ae5b2d3e
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ae5b2d3e
Branch: refs/heads/master
Commit: ae5b2d3e46cc4c460f539c4db1688309d1cdc66a
Parents: 594b14f
Author: Weiqing Yang <[email protected]>
Authored: Wed Dec 14 09:48:38 2016 +0800
Committer: Wenchen Fan <[email protected]>
Committed: Wed Dec 14 09:48:38 2016 +0800
----------------------------------------------------------------------
.../scala/org/apache/spark/sql/SQLImplicits.scala | 15 ++++++++++++++-
.../scala/org/apache/spark/sql/DatasetSuite.scala | 16 +++++++++++++++-
2 files changed, 29 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ae5b2d3e/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
index 73d16d8..872a78b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/SQLImplicits.scala
@@ -74,6 +74,19 @@ abstract class SQLImplicits {
/** @since 1.6.0 */
implicit def newStringEncoder: Encoder[String] = Encoders.STRING
+ /** @since 2.2.0 */
+ implicit def newJavaDecimalEncoder: Encoder[java.math.BigDecimal] =
Encoders.DECIMAL
+
+ /** @since 2.2.0 */
+ implicit def newScalaDecimalEncoder: Encoder[scala.math.BigDecimal] =
ExpressionEncoder()
+
+ /** @since 2.2.0 */
+ implicit def newDateEncoder: Encoder[java.sql.Date] = Encoders.DATE
+
+ /** @since 2.2.0 */
+ implicit def newTimeStampEncoder: Encoder[java.sql.Timestamp] =
Encoders.TIMESTAMP
+
+
// Boxed primitives
/** @since 2.0.0 */
@@ -141,7 +154,7 @@ abstract class SQLImplicits {
implicit def newFloatArrayEncoder: Encoder[Array[Float]] =
ExpressionEncoder()
/** @since 1.6.1 */
- implicit def newByteArrayEncoder: Encoder[Array[Byte]] = ExpressionEncoder()
+ implicit def newByteArrayEncoder: Encoder[Array[Byte]] = Encoders.BINARY
/** @since 1.6.1 */
implicit def newShortArrayEncoder: Encoder[Array[Short]] =
ExpressionEncoder()
http://git-wip-us.apache.org/repos/asf/spark/blob/ae5b2d3e/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
index 3742115..c27b815 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DatasetSuite.scala
@@ -26,7 +26,6 @@ import org.apache.spark.sql.execution.{LogicalRDD,
RDDScanExec, SortExec}
import org.apache.spark.sql.execution.exchange.{BroadcastExchangeExec,
ShuffleExchange}
import org.apache.spark.sql.execution.streaming.MemoryStream
import org.apache.spark.sql.functions._
-import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSQLContext
import org.apache.spark.sql.types._
@@ -1129,6 +1128,21 @@ class DatasetSuite extends QueryTest with
SharedSQLContext {
val ds2 = Seq(WithMap("hi", Map(42L -> "foo"))).toDS
checkDataset(ds2.map(t => t), WithMap("hi", Map(42L -> "foo")))
}
+
+ test("SPARK-18746: add implicit encoder for BigDecimal, date, timestamp") {
+ // For this implicit encoder, 18 is the default scale
+ assert(spark.range(1).map { x => new java.math.BigDecimal(1) }.head ==
+ new java.math.BigDecimal(1).setScale(18))
+
+ assert(spark.range(1).map { x => scala.math.BigDecimal(1, 18) }.head ==
+ scala.math.BigDecimal(1, 18))
+
+ assert(spark.range(1).map { x => new java.sql.Date(2016, 12, 12) }.head ==
+ new java.sql.Date(2016, 12, 12))
+
+ assert(spark.range(1).map { x => new java.sql.Timestamp(100000) }.head ==
+ new java.sql.Timestamp(100000))
+ }
}
case class WithImmutableMap(id: String, map_test:
scala.collection.immutable.Map[Long, String])
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
