Repository: spark Updated Branches: refs/heads/master 459700727 -> ab1650d29
[SPARK-24601] Update Jackson to 2.9.6 Hi all, Jackson is incompatible with upstream versions, therefore bump the Jackson version to a more recent one. I bumped into some issues with Azure CosmosDB that is using a more recent version of Jackson. This can be fixed by adding exclusions and then it works without any issues. So no breaking changes in the API's. I would also consider bumping the version of Jackson in Spark. I would suggest to keep up to date with the dependencies, since in the future this issue will pop up more frequently. ## What changes were proposed in this pull request? Bump Jackson to 2.9.6 ## How was this patch tested? Compiled and tested it locally to see if anything broke. Please review http://spark.apache.org/contributing.html before opening a pull request. Closes #21596 from Fokko/fd-bump-jackson. Authored-by: Fokko Driesprong <fokkodriespr...@godatadriven.com> Signed-off-by: hyukjinkwon <gurwls...@apache.org> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ab1650d2 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ab1650d2 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ab1650d2 Branch: refs/heads/master Commit: ab1650d2938db4901b8c28df945d6a0691a19d31 Parents: 4597007 Author: Fokko Driesprong <fokkodriespr...@godatadriven.com> Authored: Fri Oct 5 16:40:08 2018 +0800 Committer: hyukjinkwon <gurwls...@apache.org> Committed: Fri Oct 5 16:40:08 2018 +0800 ---------------------------------------------------------------------- .../deploy/rest/SubmitRestProtocolMessage.scala | 2 +- .../apache/spark/rdd/RDDOperationScope.scala | 2 +- .../scala/org/apache/spark/status/KVUtils.scala | 2 +- .../status/api/v1/JacksonMessageWriter.scala | 2 +- .../org/apache/spark/status/api/v1/api.scala | 3 ++ dev/deps/spark-deps-hadoop-2.6 | 16 +++++----- dev/deps/spark-deps-hadoop-2.7 | 16 +++++----- dev/deps/spark-deps-hadoop-3.1 | 16 +++++----- pom.xml | 7 ++--- .../expressions/JsonExpressionsSuite.scala | 7 +++++ .../datasources/json/JsonBenchmarks.scala | 33 +++++++++++--------- 11 files changed, 59 insertions(+), 47 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala index ef5a7e3..97b689c 100644 --- a/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala +++ b/core/src/main/scala/org/apache/spark/deploy/rest/SubmitRestProtocolMessage.scala @@ -36,7 +36,7 @@ import org.apache.spark.util.Utils * (2) the Spark version of the client / server * (3) an optional message */ -@JsonInclude(Include.NON_NULL) +@JsonInclude(Include.NON_ABSENT) @JsonAutoDetect(getterVisibility = Visibility.ANY, setterVisibility = Visibility.ANY) @JsonPropertyOrder(alphabetic = true) private[rest] abstract class SubmitRestProtocolMessage { http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala index 53d69ba..3abb2d8 100644 --- a/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala +++ b/core/src/main/scala/org/apache/spark/rdd/RDDOperationScope.scala @@ -41,7 +41,7 @@ import org.apache.spark.internal.Logging * There is no particular relationship between an operation scope and a stage or a job. * A scope may live inside one stage (e.g. map) or span across multiple jobs (e.g. take). */ -@JsonInclude(Include.NON_NULL) +@JsonInclude(Include.NON_ABSENT) @JsonPropertyOrder(Array("id", "name", "parent")) private[spark] class RDDOperationScope( val name: String, http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/core/src/main/scala/org/apache/spark/status/KVUtils.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/status/KVUtils.scala b/core/src/main/scala/org/apache/spark/status/KVUtils.scala index 99b1843..45348be 100644 --- a/core/src/main/scala/org/apache/spark/status/KVUtils.scala +++ b/core/src/main/scala/org/apache/spark/status/KVUtils.scala @@ -42,7 +42,7 @@ private[spark] object KVUtils extends Logging { private[spark] class KVStoreScalaSerializer extends KVStoreSerializer { mapper.registerModule(DefaultScalaModule) - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) + mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT) } http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala index 4560d30..50a286d 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/JacksonMessageWriter.scala @@ -49,7 +49,7 @@ private[v1] class JacksonMessageWriter extends MessageBodyWriter[Object]{ } mapper.registerModule(com.fasterxml.jackson.module.scala.DefaultScalaModule) mapper.enable(SerializationFeature.INDENT_OUTPUT) - mapper.setSerializationInclusion(JsonInclude.Include.NON_NULL) + mapper.setSerializationInclusion(JsonInclude.Include.NON_ABSENT) mapper.setDateFormat(JacksonMessageWriter.makeISODateFormat) override def isWriteable( http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/core/src/main/scala/org/apache/spark/status/api/v1/api.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala index 77466b6..30afd8b 100644 --- a/core/src/main/scala/org/apache/spark/status/api/v1/api.scala +++ b/core/src/main/scala/org/apache/spark/status/api/v1/api.scala @@ -139,6 +139,9 @@ private[spark] class ExecutorMetricsJsonSerializer jsonGenerator.writeObject(metricsMap) } } + + override def isEmpty(provider: SerializerProvider, value: Option[ExecutorMetrics]): Boolean = + value.isEmpty } class JobData private[spark]( http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/dev/deps/spark-deps-hadoop-2.6 ---------------------------------------------------------------------- diff --git a/dev/deps/spark-deps-hadoop-2.6 b/dev/deps/spark-deps-hadoop-2.6 index 969df4d..2dcab85 100644 --- a/dev/deps/spark-deps-hadoop-2.6 +++ b/dev/deps/spark-deps-hadoop-2.6 @@ -87,16 +87,16 @@ htrace-core-3.0.4.jar httpclient-4.5.6.jar httpcore-4.4.10.jar ivy-2.4.0.jar -jackson-annotations-2.6.7.jar -jackson-core-2.6.7.jar +jackson-annotations-2.9.6.jar +jackson-core-2.9.6.jar jackson-core-asl-1.9.13.jar -jackson-databind-2.6.7.1.jar -jackson-dataformat-yaml-2.6.7.jar +jackson-databind-2.9.6.jar +jackson-dataformat-yaml-2.9.6.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations-2.6.7.jar -jackson-module-paranamer-2.7.9.jar -jackson-module-scala_2.11-2.6.7.1.jar +jackson-module-jaxb-annotations-2.9.6.jar +jackson-module-paranamer-2.9.6.jar +jackson-module-scala_2.11-2.9.6.jar jackson-xc-1.9.13.jar janino-3.0.10.jar javassist-3.18.1-GA.jar @@ -177,7 +177,7 @@ scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar slf4j-log4j12-1.7.16.jar -snakeyaml-1.15.jar +snakeyaml-1.18.jar snappy-0.2.jar snappy-java-1.1.7.1.jar spire-macros_2.11-0.13.0.jar http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/dev/deps/spark-deps-hadoop-2.7 ---------------------------------------------------------------------- diff --git a/dev/deps/spark-deps-hadoop-2.7 b/dev/deps/spark-deps-hadoop-2.7 index e827dc6..d1d695c 100644 --- a/dev/deps/spark-deps-hadoop-2.7 +++ b/dev/deps/spark-deps-hadoop-2.7 @@ -87,16 +87,16 @@ htrace-core-3.1.0-incubating.jar httpclient-4.5.6.jar httpcore-4.4.10.jar ivy-2.4.0.jar -jackson-annotations-2.6.7.jar -jackson-core-2.6.7.jar +jackson-annotations-2.9.6.jar +jackson-core-2.9.6.jar jackson-core-asl-1.9.13.jar -jackson-databind-2.6.7.1.jar -jackson-dataformat-yaml-2.6.7.jar +jackson-databind-2.9.6.jar +jackson-dataformat-yaml-2.9.6.jar jackson-jaxrs-1.9.13.jar jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations-2.6.7.jar -jackson-module-paranamer-2.7.9.jar -jackson-module-scala_2.11-2.6.7.1.jar +jackson-module-jaxb-annotations-2.9.6.jar +jackson-module-paranamer-2.9.6.jar +jackson-module-scala_2.11-2.9.6.jar jackson-xc-1.9.13.jar janino-3.0.10.jar javassist-3.18.1-GA.jar @@ -178,7 +178,7 @@ scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar slf4j-log4j12-1.7.16.jar -snakeyaml-1.15.jar +snakeyaml-1.18.jar snappy-0.2.jar snappy-java-1.1.7.1.jar spire-macros_2.11-0.13.0.jar http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/dev/deps/spark-deps-hadoop-3.1 ---------------------------------------------------------------------- diff --git a/dev/deps/spark-deps-hadoop-3.1 b/dev/deps/spark-deps-hadoop-3.1 index 2b12c35..e9691eb 100644 --- a/dev/deps/spark-deps-hadoop-3.1 +++ b/dev/deps/spark-deps-hadoop-3.1 @@ -86,17 +86,17 @@ htrace-core4-4.1.0-incubating.jar httpclient-4.5.6.jar httpcore-4.4.10.jar ivy-2.4.0.jar -jackson-annotations-2.6.7.jar -jackson-core-2.6.7.jar +jackson-annotations-2.9.6.jar +jackson-core-2.9.6.jar jackson-core-asl-1.9.13.jar -jackson-databind-2.6.7.1.jar -jackson-dataformat-yaml-2.6.7.jar +jackson-databind-2.9.6.jar +jackson-dataformat-yaml-2.9.6.jar jackson-jaxrs-base-2.7.8.jar jackson-jaxrs-json-provider-2.7.8.jar jackson-mapper-asl-1.9.13.jar -jackson-module-jaxb-annotations-2.6.7.jar -jackson-module-paranamer-2.7.9.jar -jackson-module-scala_2.11-2.6.7.1.jar +jackson-module-jaxb-annotations-2.9.6.jar +jackson-module-paranamer-2.9.6.jar +jackson-module-scala_2.11-2.9.6.jar janino-3.0.10.jar javassist-3.18.1-GA.jar javax.annotation-api-1.2.jar @@ -197,7 +197,7 @@ scala-xml_2.11-1.0.5.jar shapeless_2.11-2.3.2.jar slf4j-api-1.7.16.jar slf4j-log4j12-1.7.16.jar -snakeyaml-1.15.jar +snakeyaml-1.18.jar snappy-0.2.jar snappy-java-1.1.7.1.jar spire-macros_2.11-0.13.0.jar http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index cc20c5c..c5eea6a 100644 --- a/pom.xml +++ b/pom.xml @@ -158,8 +158,7 @@ <scala.version>2.11.12</scala.version> <scala.binary.version>2.11</scala.binary.version> <codehaus.jackson.version>1.9.13</codehaus.jackson.version> - <fasterxml.jackson.version>2.6.7</fasterxml.jackson.version> - <fasterxml.jackson.databind.version>2.6.7.1</fasterxml.jackson.databind.version> + <fasterxml.jackson.version>2.9.6</fasterxml.jackson.version> <snappy.version>1.1.7.1</snappy.version> <netlib.java.version>1.1.2</netlib.java.version> <calcite.version>1.2.0-incubating</calcite.version> @@ -629,7 +628,7 @@ <dependency> <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-databind</artifactId> - <version>${fasterxml.jackson.databind.version}</version> + <version>${fasterxml.jackson.version}</version> </dependency> <dependency> <groupId>com.fasterxml.jackson.core</groupId> @@ -641,7 +640,7 @@ <dependency> <groupId>com.fasterxml.jackson.module</groupId> <artifactId>jackson-module-scala_${scala.binary.version}</artifactId> - <version>${fasterxml.jackson.databind.version}</version> + <version>${fasterxml.jackson.version}</version> <exclusions> <exclusion> <groupId>com.google.guava</groupId> http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala ---------------------------------------------------------------------- diff --git a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala index 34fdd0c..81ab7d6 100644 --- a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala +++ b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/JsonExpressionsSuite.scala @@ -244,6 +244,13 @@ class JsonExpressionsSuite extends SparkFunSuite with ExpressionEvalHelper with "1234") } + test("some big value") { + val value = "x" * 3000 + checkEvaluation( + GetJsonObject(NonFoldableLiteral((s"""{"big": "$value"}""")), + NonFoldableLiteral("$.big")), value) + } + val jsonTupleQuery = Literal("f1") :: Literal("f2") :: Literal("f3") :: http://git-wip-us.apache.org/repos/asf/spark/blob/ab1650d2/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala index 368318a..3c4a5ab 100644 --- a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala +++ b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/json/JsonBenchmarks.scala @@ -66,12 +66,13 @@ object JSONBenchmarks extends SQLHelper { } /* - Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz + Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5 + Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz - JSON schema inferring: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative - -------------------------------------------------------------------------------------------- - No encoding 38902 / 39282 2.6 389.0 1.0X - UTF-8 is set 56959 / 57261 1.8 569.6 0.7X + JSON schema inferring: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative + --------------------------------------------------------------------------------------------- + No encoding 45908 / 46480 2.2 459.1 1.0X + UTF-8 is set 68469 / 69762 1.5 684.7 0.7X */ benchmark.run() } @@ -107,12 +108,13 @@ object JSONBenchmarks extends SQLHelper { } /* - Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz + Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5 + Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz - JSON per-line parsing: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative - -------------------------------------------------------------------------------------------- - No encoding 25947 / 26188 3.9 259.5 1.0X - UTF-8 is set 46319 / 46417 2.2 463.2 0.6X + JSON per-line parsing: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative + --------------------------------------------------------------------------------------------- + No encoding 9982 / 10237 10.0 99.8 1.0X + UTF-8 is set 16373 / 16806 6.1 163.7 0.6X */ benchmark.run() } @@ -155,12 +157,13 @@ object JSONBenchmarks extends SQLHelper { } /* - Intel(R) Core(TM) i7-7920HQ CPU @ 3.10GHz + Java HotSpot(TM) 64-Bit Server VM 1.8.0_172-b11 on Mac OS X 10.13.5 + Intel(R) Core(TM) i7-7820HQ CPU @ 2.90GHz - JSON parsing of wide lines: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative - -------------------------------------------------------------------------------------------- - No encoding 45543 / 45660 0.2 4554.3 1.0X - UTF-8 is set 65737 / 65957 0.2 6573.7 0.7X + JSON parsing of wide lines: Best/Avg Time(ms) Rate(M/s) Per Row(ns) Relative + --------------------------------------------------------------------------------------------- + No encoding 26038 / 26386 0.4 2603.8 1.0X + UTF-8 is set 28343 / 28557 0.4 2834.3 0.9X */ benchmark.run() } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
