This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 6f394a2 [SPARK-24601][SPARK-27051][BACKPORT][CORE] Update to Jackson
2.9.8
6f394a2 is described below
commit 6f394a20bf49f67b4d6329a1c25171c8024a2fae
Author: Sean Owen <[email protected]>
AuthorDate: Sun Apr 21 09:56:48 2019 -0700
[SPARK-24601][SPARK-27051][BACKPORT][CORE] Update to Jackson 2.9.8
## What changes were proposed in this pull request?
This backports:
https://github.com/apache/spark/commit/ab1650d2938db4901b8c28df945d6a0691a19d31
https://github.com/apache/spark/commit/7857c6d633f3df426a6ac4618316eb83b1cefe2b
which collectively updates Jackson to 2.9.8.
## How was this patch tested?
Existing tests.
Closes #24418 from srowen/SPARK-24601.2.
Authored-by: Sean Owen <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
core/pom.xml | 4 +++
.../deploy/rest/SubmitRestProtocolMessage.scala | 2 +-
.../org/apache/spark/rdd/RDDOperationScope.scala | 2 +-
.../scala/org/apache/spark/status/KVUtils.scala | 2 +-
.../spark/status/api/v1/JacksonMessageWriter.scala | 2 +-
dev/deps/spark-deps-hadoop-2.6 | 16 +++++------
dev/deps/spark-deps-hadoop-2.7 | 16 +++++------
dev/deps/spark-deps-hadoop-3.1 | 16 +++++------
pom.xml | 7 ++---
.../expressions/JsonExpressionsSuite.scala | 7 +++++
.../datasources/json/JsonBenchmarks.scala | 33 ++++++++++++----------
11 files changed, 60 insertions(+), 47 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 59caf2e..521b952 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -225,6 +225,10 @@
<artifactId>scala-library</artifactId>
</dependency>
<dependency>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-reflect</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-jackson_${scala.binary.version}</artifactId>
</dependency>
diff --git
a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala
b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala
index ef5a7e3..97b689c 100644
---
a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala
@@ -36,7 +36,7 @@ import org.apache.spark.util.Utils
* (2) the Spark version of the client / server
* (3) an optional message
*/
-@JsonInclude(Include.NON_NULL)
+@JsonInclude(Include.NON_ABSENT)
@JsonAutoDetect(getterVisibility = Visibility.ANY, setterVisibility =
Visibility.ANY)
@JsonPropertyOrder(alphabetic = true)
private[rest] abstract class SubmitRestProtocolMessage {
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
index 53d69ba..3abb2d8 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
@@ -41,7 +41,7 @@ import org.apache.spark.internal.Logging
* There is no particular relationship between an operation scope and a stage
or a job.
* A scope may live inside one stage (e.g. map) or span across multiple jobs
(e.g. take).
*/
-@JsonInclude(Include.NON_NULL)
+@JsonInclude(Include.NON_ABSENT)
@JsonPropertyOrder(Array("id", "name", "parent"))
private[spark] class RDDOperationScope(
val name: String,
diff --git a/core/src/main/scala/org/apache/spark/status/KVUtils.scala
b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
index 99b1843..45348be 100644
--- a/core/src/main/scala/org/apache/spark/status/KVUtils.scala
+++ b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
@@ -42,7 +42,7 @@ private[spark] object KVUtils extends Logging {
private[spark] class KVStoreScalaSerializer extends KVStoreSerializer {
mapper.registerModule(DefaultScalaModule)
- mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
+ mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
}
diff --git
a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala
b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala
index 4560d30..50a286d 100644
---
a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala
+++
b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala
@@ -49,7 +49,7 @@ private[v1] class JacksonMessageWriter extends
MessageBodyWriter[Object]{
}
mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
mapper.enable(SerializationFeature.INDENT_OUTPUT)
- mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
+ mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)
override def isWriteable(
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
index b2bc8e2..c60c7125 100644
--- a/dev/deps/spark-deps-hadoop-2.6
+++ b/dev/deps/spark-deps-hadoop-2.6
@@ -87,16 +87,16 @@ htrace-core-3.0.4.jar
httpclient-4.5.6.jar
httpcore-4.4.10.jar
ivy-2.4.0.jar
-jackson-annotations-2.6.7.jar
-jackson-core-2.6.7.jar
+jackson-annotations-2.9.8.jar
+jackson-core-2.9.8.jar
jackson-core-asl-1.9.13.jar
-jackson-databind-2.6.7.1.jar
-jackson-dataformat-yaml-2.6.7.jar
+jackson-databind-2.9.8.jar
+jackson-dataformat-yaml-2.9.8.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations-2.6.7.jar
-jackson-module-paranamer-2.7.9.jar
-jackson-module-scala_2.11-2.6.7.1.jar
+jackson-module-jaxb-annotations-2.9.8.jar
+jackson-module-paranamer-2.9.8.jar
+jackson-module-scala_2.11-2.9.8.jar
jackson-xc-1.9.13.jar
janino-3.0.9.jar
javassist-3.18.1-GA.jar
@@ -179,7 +179,7 @@ shapeless_2.11-2.3.2.jar
shims-0.7.45.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
-snakeyaml-1.15.jar
+snakeyaml-1.23.jar
snappy-0.2.jar
snappy-java-1.1.7.3.jar
spire-macros_2.11-0.13.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index b2767d3..d261b38 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -87,16 +87,16 @@ htrace-core-3.1.0-incubating.jar
httpclient-4.5.6.jar
httpcore-4.4.10.jar
ivy-2.4.0.jar
-jackson-annotations-2.6.7.jar
-jackson-core-2.6.7.jar
+jackson-annotations-2.9.8.jar
+jackson-core-2.9.8.jar
jackson-core-asl-1.9.13.jar
-jackson-databind-2.6.7.1.jar
-jackson-dataformat-yaml-2.6.7.jar
+jackson-databind-2.9.8.jar
+jackson-dataformat-yaml-2.9.8.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations-2.6.7.jar
-jackson-module-paranamer-2.7.9.jar
-jackson-module-scala_2.11-2.6.7.1.jar
+jackson-module-jaxb-annotations-2.9.8.jar
+jackson-module-paranamer-2.9.8.jar
+jackson-module-scala_2.11-2.9.8.jar
jackson-xc-1.9.13.jar
janino-3.0.9.jar
javassist-3.18.1-GA.jar
@@ -180,7 +180,7 @@ shapeless_2.11-2.3.2.jar
shims-0.7.45.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
-snakeyaml-1.15.jar
+snakeyaml-1.23.jar
snappy-0.2.jar
snappy-java-1.1.7.3.jar
spire-macros_2.11-0.13.0.jar
diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1
index 9b2cb8d..3c01372 100644
--- a/dev/deps/spark-deps-hadoop-3.1
+++ b/dev/deps/spark-deps-hadoop-3.1
@@ -86,17 +86,17 @@ htrace-core4-4.1.0-incubating.jar
httpclient-4.5.6.jar
httpcore-4.4.10.jar
ivy-2.4.0.jar
-jackson-annotations-2.6.7.jar
-jackson-core-2.6.7.jar
+jackson-annotations-2.9.8.jar
+jackson-core-2.9.8.jar
jackson-core-asl-1.9.13.jar
-jackson-databind-2.6.7.1.jar
-jackson-dataformat-yaml-2.6.7.jar
+jackson-databind-2.9.8.jar
+jackson-dataformat-yaml-2.9.8.jar
jackson-jaxrs-base-2.7.8.jar
jackson-jaxrs-json-provider-2.7.8.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations-2.6.7.jar
-jackson-module-paranamer-2.7.9.jar
-jackson-module-scala_2.11-2.6.7.1.jar
+jackson-module-jaxb-annotations-2.9.8.jar
+jackson-module-paranamer-2.9.8.jar
+jackson-module-scala_2.11-2.9.8.jar
janino-3.0.9.jar
javassist-3.18.1-GA.jar
javax.annotation-api-1.2.jar
@@ -199,7 +199,7 @@ shapeless_2.11-2.3.2.jar
shims-0.7.45.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
-snakeyaml-1.15.jar
+snakeyaml-1.23.jar
snappy-0.2.jar
snappy-java-1.1.7.3.jar
spire-macros_2.11-0.13.0.jar
diff --git a/pom.xml b/pom.xml
index 0b395f5..c8811e8 100644
--- a/pom.xml
+++ b/pom.xml
@@ -158,8 +158,7 @@
<scala.version>2.11.12</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
- <fasterxml.jackson.version>2.6.7</fasterxml.jackson.version>
-
<fasterxml.jackson.databind.version>2.6.7.1</fasterxml.jackson.databind.version>
+ <fasterxml.jackson.version>2.9.8</fasterxml.jackson.version>
<snappy.version>1.1.7.3</snappy.version>
<netlib.java.version>1.1.2</netlib.java.version>
<calcite.version>1.2.0-incubating</calcite.version>
@@ -629,7 +628,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>${fasterxml.jackson.databind.version}</version>
+ <version>${fasterxml.jackson.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -641,7 +640,7 @@
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
- <version>${fasterxml.jackson.databind.version}</version>
+ <version>${fasterxml.jackson.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
index 0e9c8ab..c3ea57a 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
@@ -244,6 +244,13 @@ class JsonExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper with
"1234")
}
+ test("some big value") {
+ val value = "x" * 3000
+ checkEvaluation(
+ GetJsonObject(NonFoldableLiteral((s"""{"big": "$value"}""")),
+ NonFoldableLiteral("$.big")), value)
+ }
+
val jsonTupleQuery = Literal("f1") ::
Literal("f2") ::
Literal("f3") ::
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala
index 5592aa6..50fe7a3 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala
@@ -72,12 +72,13 @@ object JSONBenchmarks {
}
/*
- Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+ Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5
+ Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
- JSON schema inferring: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
-
--------------------------------------------------------------------------------------------
- No encoding 38902 / 39282 2.6
389.0 1.0X
- UTF-8 is set 56959 / 57261 1.8
569.6 0.7X
+ JSON schema inferring: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
+
---------------------------------------------------------------------------------------------
+ No encoding 45908 / 46480 2.2
459.1 1.0X
+ UTF-8 is set 68469 / 69762 1.5
684.7 0.7X
*/
benchmark.run()
}
@@ -113,12 +114,13 @@ object JSONBenchmarks {
}
/*
- Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+ Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5
+ Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
- JSON per-line parsing: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
-
--------------------------------------------------------------------------------------------
- No encoding 25947 / 26188 3.9
259.5 1.0X
- UTF-8 is set 46319 / 46417 2.2
463.2 0.6X
+ JSON per-line parsing: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
+
---------------------------------------------------------------------------------------------
+ No encoding 9982 / 10237 10.0
99.8 1.0X
+ UTF-8 is set 16373 / 16806 6.1
163.7 0.6X
*/
benchmark.run()
}
@@ -161,12 +163,13 @@ object JSONBenchmarks {
}
/*
- Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
+ Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5
+ Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
- JSON parsing of wide lines: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
-
--------------------------------------------------------------------------------------------
- No encoding 45543 / 45660 0.2
4554.3 1.0X
- UTF-8 is set 65737 / 65957 0.2
6573.7 0.7X
+ JSON parsing of wide lines: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
+
---------------------------------------------------------------------------------------------
+ No encoding 26038 / 26386 0.4
2603.8 1.0X
+ UTF-8 is set 28343 / 28557 0.4
2834.3 0.9X
*/
benchmark.run()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
