This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 2a7e1e0 [HUDI-1942] Add Default value for
HIVE_AUTO_CREATE_DATABASE_OPT_KEY in HoodieSparkSqlWriter (#3036)
2a7e1e0 is described below
commit 2a7e1e091e69c53acc0a19e3d792ca15a3d7db62
Author: Vinay Patil <[email protected]>
AuthorDate: Sun Jun 6 03:32:26 2021 +0530
[HUDI-1942] Add Default value for HIVE_AUTO_CREATE_DATABASE_OPT_KEY in
HoodieSparkSqlWriter (#3036)
---
.../src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
index abb5f76..17b3cc2 100644
---
a/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
+++
b/hudi-spark-datasource/hudi-spark/src/main/scala/org/apache/hudi/HoodieSparkSqlWriter.scala
@@ -155,7 +155,7 @@ private[hudi] object HoodieSparkSqlWriter {
// Convert to RDD[HoodieRecord]
val genericRecords: RDD[GenericRecord] =
HoodieSparkUtils.createRdd(df, schema, structName, nameSpace)
- val shouldCombine = parameters(INSERT_DROP_DUPS_OPT_KEY).toBoolean
|| operation.equals(WriteOperationType.UPSERT);
+ val shouldCombine = parameters(INSERT_DROP_DUPS_OPT_KEY).toBoolean
|| operation.equals(WriteOperationType.UPSERT)
val hoodieAllIncomingRecords = genericRecords.map(gr => {
val hoodieRecord = if (shouldCombine) {
val orderingVal = HoodieAvroUtils.getNestedFieldVal(gr,
parameters(PRECOMBINE_FIELD_OPT_KEY), false)
@@ -423,7 +423,8 @@ private[hudi] object HoodieSparkSqlWriter {
hiveSyncConfig.verifyMetadataFileListing =
parameters(HoodieMetadataConfig.METADATA_VALIDATE_PROP).toBoolean
hiveSyncConfig.ignoreExceptions =
parameters.get(HIVE_IGNORE_EXCEPTIONS_OPT_KEY).exists(r => r.toBoolean)
hiveSyncConfig.supportTimestamp =
parameters.get(HIVE_SUPPORT_TIMESTAMP).exists(r => r.toBoolean)
- hiveSyncConfig.autoCreateDatabase =
parameters.get(HIVE_AUTO_CREATE_DATABASE_OPT_KEY).exists(r => r.toBoolean)
+ hiveSyncConfig.autoCreateDatabase =
parameters.getOrElse(HIVE_AUTO_CREATE_DATABASE_OPT_KEY,
+ DEFAULT_HIVE_AUTO_CREATE_DATABASE_OPT_KEY).toBoolean
hiveSyncConfig.decodePartition =
parameters.getOrElse(URL_ENCODE_PARTITIONING_OPT_KEY,
DEFAULT_URL_ENCODE_PARTITIONING_OPT_VAL).toBoolean
