This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 54a7aa8787d [HUDI-6402] Hudi Spark3.3 and upper version need close
JavaTimeModule For JsonUtils (#9001)
54a7aa8787d is described below
commit 54a7aa8787dcf163a401c9b1d33610e5b8094cdd
Author: xuzifu666 <[email protected]>
AuthorDate: Sat Jun 17 19:20:34 2023 +0800
[HUDI-6402] Hudi Spark3.3 and upper version need close JavaTimeModule For
JsonUtils (#9001)
---
.../src/main/java/org/apache/hudi/common/util/JsonUtils.java | 6 ++----
.../main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala | 6 +++++-
2 files changed, 7 insertions(+), 5 deletions(-)
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/util/JsonUtils.java
b/hudi-common/src/main/java/org/apache/hudi/common/util/JsonUtils.java
index 06b7f7872bf..b35f8a1c18c 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/JsonUtils.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/JsonUtils.java
@@ -36,8 +36,6 @@ public class JsonUtils {
private static final ObjectMapper MAPPER = new ObjectMapper();
static {
- registerModules(MAPPER);
-
MAPPER.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
// We need to exclude custom getters, setters and creators which can use
member fields
// to derive new fields, so that they are not included in the serialization
@@ -61,8 +59,8 @@ public class JsonUtils {
}
}
- private static void registerModules(ObjectMapper mapper) {
+ public static void registerModules() {
// NOTE: Registering [[JavaTimeModule]] is required for Jackson >= 2.11
(Spark >= 3.3)
- mapper.registerModules(new JavaTimeModule());
+ MAPPER.registerModules(new JavaTimeModule());
}
}
diff --git
a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala
b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala
index 763374b36a8..d5a35980526 100644
---
a/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala
+++
b/hudi-spark-datasource/hudi-spark3-common/src/main/scala/org/apache/spark/sql/adapter/BaseSpark3Adapter.scala
@@ -21,8 +21,9 @@ import org.apache.avro.Schema
import org.apache.hadoop.fs.Path
import org.apache.hudi.client.utils.SparkRowSerDe
import org.apache.hudi.common.table.HoodieTableMetaClient
+import org.apache.hudi.common.util.JsonUtils
import org.apache.hudi.spark3.internal.ReflectUtil
-import org.apache.hudi.{AvroConversionUtils, DefaultSource, Spark3RowSerDe}
+import org.apache.hudi.{AvroConversionUtils, DefaultSource, HoodieSparkUtils,
Spark3RowSerDe}
import org.apache.spark.internal.Logging
import org.apache.spark.sql.avro.{HoodieAvroSchemaConverters,
HoodieSparkAvroSchemaConverters}
import org.apache.spark.sql.catalyst.encoders.RowEncoder
@@ -46,6 +47,9 @@ import scala.collection.convert.Wrappers.JConcurrentMapWrapper
*/
abstract class BaseSpark3Adapter extends SparkAdapter with Logging {
+ // JsonUtils for Support Spark Version >= 3.3
+ if (HoodieSparkUtils.gteqSpark3_3) JsonUtils.registerModules()
+
private val cache = JConcurrentMapWrapper(
new ConcurrentHashMap[ZoneId, DateFormatter](1))
