This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new 1323ddc Revert "[SPARK-24601][SPARK-27051][BACKPORT][CORE] Update to
Jackson 2.9.8
1323ddc is described below
commit 1323ddcc17c27f8f3b5e4b2eaf5a41ec10212dfd
Author: gatorsmile <[email protected]>
AuthorDate: Tue Apr 30 08:55:41 2019 -0700
Revert "[SPARK-24601][SPARK-27051][BACKPORT][CORE] Update to Jackson 2.9.8
## What changes were proposed in this pull request?
This reverts commit 6f394a20bf49f67b4d6329a1c25171c8024a2fae.
In general, we need to be very cautious about the Jackson upgrade in the
patch releases, especially when this upgrade could break the existing behaviors
of the external packages or data sources, and generate different results after
the upgrade. The external packages and data sources need to change their source
code to keep the original behaviors. The upgrade requires more discussions
before releasing it, I think.
In the previous PR https://github.com/apache/spark/pull/22071, we turned
off `spark.master.rest.enabled` by default and added the following claim in our
security doc:
> The Rest Submission Server and the MesosClusterDispatcher do not support
authentication. You should ensure that all network access to the REST API &
MesosClusterDispatcher (port 6066 and 7077 respectively by default) are
restricted to hosts that are trusted to submit jobs.
We need to understand whether this Jackson CVE applies to Spark. Before
officially releasing it, we need more inputs from all of you. Currently, I
would suggest to revert this upgrade from the upcoming 2.4.3 release, which is
trying to fix the accidental default Scala version changes in pre-built
artifacts.
## How was this patch tested?
N/A
Closes #24493 from gatorsmile/revert24418.
Authored-by: gatorsmile <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
core/pom.xml | 4 ---
.../deploy/rest/SubmitRestProtocolMessage.scala | 2 +-
.../org/apache/spark/rdd/RDDOperationScope.scala | 2 +-
.../scala/org/apache/spark/status/KVUtils.scala | 2 +-
.../spark/status/api/v1/JacksonMessageWriter.scala | 2 +-
dev/deps/spark-deps-hadoop-2.6 | 16 +++++------
dev/deps/spark-deps-hadoop-2.7 | 16 +++++------
dev/deps/spark-deps-hadoop-3.1 | 16 +++++------
pom.xml | 7 +++--
.../expressions/JsonExpressionsSuite.scala | 7 -----
.../datasources/json/JsonBenchmarks.scala | 33 ++++++++++------------
11 files changed, 47 insertions(+), 60 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index 521b952..59caf2e 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -225,10 +225,6 @@
<artifactId>scala-library</artifactId>
</dependency>
<dependency>
- <groupId>org.scala-lang</groupId>
- <artifactId>scala-reflect</artifactId>
- </dependency>
- <dependency>
<groupId>org.json4s</groupId>
<artifactId>json4s-jackson_${scala.binary.version}</artifactId>
</dependency>
diff --git
a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala
b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala
index 97b689c..ef5a7e3 100644
---
a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala
+++
b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala
@@ -36,7 +36,7 @@ import org.apache.spark.util.Utils
* (2) the Spark version of the client / server
* (3) an optional message
*/
-@JsonInclude(Include.NON_ABSENT)
+@JsonInclude(Include.NON_NULL)
@JsonAutoDetect(getterVisibility = Visibility.ANY, setterVisibility =
Visibility.ANY)
@JsonPropertyOrder(alphabetic = true)
private[rest] abstract class SubmitRestProtocolMessage {
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
index 3abb2d8..53d69ba 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala
@@ -41,7 +41,7 @@ import org.apache.spark.internal.Logging
* There is no particular relationship between an operation scope and a stage
or a job.
* A scope may live inside one stage (e.g. map) or span across multiple jobs
(e.g. take).
*/
-@JsonInclude(Include.NON_ABSENT)
+@JsonInclude(Include.NON_NULL)
@JsonPropertyOrder(Array("id", "name", "parent"))
private[spark] class RDDOperationScope(
val name: String,
diff --git a/core/src/main/scala/org/apache/spark/status/KVUtils.scala
b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
index 45348be..99b1843 100644
--- a/core/src/main/scala/org/apache/spark/status/KVUtils.scala
+++ b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
@@ -42,7 +42,7 @@ private[spark] object KVUtils extends Logging {
private[spark] class KVStoreScalaSerializer extends KVStoreSerializer {
mapper.registerModule(DefaultScalaModule)
- mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
+ mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
}
diff --git
a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala
b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala
index 50a286d..4560d30 100644
---
a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala
+++
b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala
@@ -49,7 +49,7 @@ private[v1] class JacksonMessageWriter extends
MessageBodyWriter[Object]{
}
mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule)
mapper.enable(SerializationFeature.INDENT_OUTPUT)
- mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT)
+ mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL)
mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat)
override def isWriteable(
diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6
index e7fdbf9..ca13c96 100644
--- a/dev/deps/spark-deps-hadoop-2.6
+++ b/dev/deps/spark-deps-hadoop-2.6
@@ -86,16 +86,16 @@ htrace-core-3.0.4.jar
httpclient-4.5.6.jar
httpcore-4.4.10.jar
ivy-2.4.0.jar
-jackson-annotations-2.9.8.jar
-jackson-core-2.9.8.jar
+jackson-annotations-2.6.7.jar
+jackson-core-2.6.7.jar
jackson-core-asl-1.9.13.jar
-jackson-databind-2.9.8.jar
-jackson-dataformat-yaml-2.9.8.jar
+jackson-databind-2.6.7.1.jar
+jackson-dataformat-yaml-2.6.7.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations-2.9.8.jar
-jackson-module-paranamer-2.9.8.jar
-jackson-module-scala_2.11-2.9.8.jar
+jackson-module-jaxb-annotations-2.6.7.jar
+jackson-module-paranamer-2.7.9.jar
+jackson-module-scala_2.11-2.6.7.1.jar
jackson-xc-1.9.13.jar
janino-3.0.9.jar
javassist-3.18.1-GA.jar
@@ -178,7 +178,7 @@ shapeless_2.11-2.3.2.jar
shims-0.7.45.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
-snakeyaml-1.23.jar
+snakeyaml-1.15.jar
snappy-0.2.jar
snappy-java-1.1.7.3.jar
spire-macros_2.11-0.13.0.jar
diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7
index 91fe6e8..95b778f 100644
--- a/dev/deps/spark-deps-hadoop-2.7
+++ b/dev/deps/spark-deps-hadoop-2.7
@@ -86,16 +86,16 @@ htrace-core-3.1.0-incubating.jar
httpclient-4.5.6.jar
httpcore-4.4.10.jar
ivy-2.4.0.jar
-jackson-annotations-2.9.8.jar
-jackson-core-2.9.8.jar
+jackson-annotations-2.6.7.jar
+jackson-core-2.6.7.jar
jackson-core-asl-1.9.13.jar
-jackson-databind-2.9.8.jar
-jackson-dataformat-yaml-2.9.8.jar
+jackson-databind-2.6.7.1.jar
+jackson-dataformat-yaml-2.6.7.jar
jackson-jaxrs-1.9.13.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations-2.9.8.jar
-jackson-module-paranamer-2.9.8.jar
-jackson-module-scala_2.11-2.9.8.jar
+jackson-module-jaxb-annotations-2.6.7.jar
+jackson-module-paranamer-2.7.9.jar
+jackson-module-scala_2.11-2.6.7.1.jar
jackson-xc-1.9.13.jar
janino-3.0.9.jar
javassist-3.18.1-GA.jar
@@ -179,7 +179,7 @@ shapeless_2.11-2.3.2.jar
shims-0.7.45.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
-snakeyaml-1.23.jar
+snakeyaml-1.15.jar
snappy-0.2.jar
snappy-java-1.1.7.3.jar
spire-macros_2.11-0.13.0.jar
diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1
index 3c01372..9b2cb8d 100644
--- a/dev/deps/spark-deps-hadoop-3.1
+++ b/dev/deps/spark-deps-hadoop-3.1
@@ -86,17 +86,17 @@ htrace-core4-4.1.0-incubating.jar
httpclient-4.5.6.jar
httpcore-4.4.10.jar
ivy-2.4.0.jar
-jackson-annotations-2.9.8.jar
-jackson-core-2.9.8.jar
+jackson-annotations-2.6.7.jar
+jackson-core-2.6.7.jar
jackson-core-asl-1.9.13.jar
-jackson-databind-2.9.8.jar
-jackson-dataformat-yaml-2.9.8.jar
+jackson-databind-2.6.7.1.jar
+jackson-dataformat-yaml-2.6.7.jar
jackson-jaxrs-base-2.7.8.jar
jackson-jaxrs-json-provider-2.7.8.jar
jackson-mapper-asl-1.9.13.jar
-jackson-module-jaxb-annotations-2.9.8.jar
-jackson-module-paranamer-2.9.8.jar
-jackson-module-scala_2.11-2.9.8.jar
+jackson-module-jaxb-annotations-2.6.7.jar
+jackson-module-paranamer-2.7.9.jar
+jackson-module-scala_2.11-2.6.7.1.jar
janino-3.0.9.jar
javassist-3.18.1-GA.jar
javax.annotation-api-1.2.jar
@@ -199,7 +199,7 @@ shapeless_2.11-2.3.2.jar
shims-0.7.45.jar
slf4j-api-1.7.16.jar
slf4j-log4j12-1.7.16.jar
-snakeyaml-1.23.jar
+snakeyaml-1.15.jar
snappy-0.2.jar
snappy-java-1.1.7.3.jar
spire-macros_2.11-0.13.0.jar
diff --git a/pom.xml b/pom.xml
index 0c2666f..704e258 100644
--- a/pom.xml
+++ b/pom.xml
@@ -158,7 +158,8 @@
<scala.version>2.11.12</scala.version>
<scala.binary.version>2.11</scala.binary.version>
<codehaus.jackson.version>1.9.13</codehaus.jackson.version>
- <fasterxml.jackson.version>2.9.8</fasterxml.jackson.version>
+ <fasterxml.jackson.version>2.6.7</fasterxml.jackson.version>
+
<fasterxml.jackson.databind.version>2.6.7.1</fasterxml.jackson.databind.version>
<snappy.version>1.1.7.3</snappy.version>
<netlib.java.version>1.1.2</netlib.java.version>
<calcite.version>1.2.0-incubating</calcite.version>
@@ -633,7 +634,7 @@
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
- <version>${fasterxml.jackson.version}</version>
+ <version>${fasterxml.jackson.databind.version}</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
@@ -645,7 +646,7 @@
<dependency>
<groupId>com.fasterxml.jackson.module</groupId>
<artifactId>jackson-module-scala_${scala.binary.version}</artifactId>
- <version>${fasterxml.jackson.version}</version>
+ <version>${fasterxml.jackson.databind.version}</version>
<exclusions>
<exclusion>
<groupId>com.google.guava</groupId>
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
index c3ea57a..0e9c8ab 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala
@@ -244,13 +244,6 @@ class JsonExpressionsSuite extends SparkFunSuite with
ExpressionEvalHelper with
"1234")
}
- test("some big value") {
- val value = "x" * 3000
- checkEvaluation(
- GetJsonObject(NonFoldableLiteral((s"""{"big": "$value"}""")),
- NonFoldableLiteral("$.big")), value)
- }
-
val jsonTupleQuery = Literal("f1") ::
Literal("f2") ::
Literal("f3") ::
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala
index 50fe7a3..5592aa6 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala
@@ -72,13 +72,12 @@ object JSONBenchmarks {
}
/*
- Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5
- Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+ Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
- JSON schema inferring: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
-
---------------------------------------------------------------------------------------------
- No encoding 45908 / 46480 2.2
459.1 1.0X
- UTF-8 is set 68469 / 69762 1.5
684.7 0.7X
+ JSON schema inferring: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
+
--------------------------------------------------------------------------------------------
+ No encoding 38902 / 39282 2.6
389.0 1.0X
+ UTF-8 is set 56959 / 57261 1.8
569.6 0.7X
*/
benchmark.run()
}
@@ -114,13 +113,12 @@ object JSONBenchmarks {
}
/*
- Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5
- Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+ Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
- JSON per-line parsing: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
-
---------------------------------------------------------------------------------------------
- No encoding 9982 / 10237 10.0
99.8 1.0X
- UTF-8 is set 16373 / 16806 6.1
163.7 0.6X
+ JSON per-line parsing: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
+
--------------------------------------------------------------------------------------------
+ No encoding 25947 / 26188 3.9
259.5 1.0X
+ UTF-8 is set 46319 / 46417 2.2
463.2 0.6X
*/
benchmark.run()
}
@@ -163,13 +161,12 @@ object JSONBenchmarks {
}
/*
- Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5
- Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz
+ Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz
- JSON parsing of wide lines: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
-
---------------------------------------------------------------------------------------------
- No encoding 26038 / 26386 0.4
2603.8 1.0X
- UTF-8 is set 28343 / 28557 0.4
2834.3 0.9X
+ JSON parsing of wide lines: Best/Avg Time(ms) Rate(M/s)
Per Row(ns) Relative
+
--------------------------------------------------------------------------------------------
+ No encoding 45543 / 45660 0.2
4554.3 1.0X
+ UTF-8 is set 65737 / 65957 0.2
6573.7 0.7X
*/
benchmark.run()
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
