This is an automated email from the ASF dual-hosted git repository.
baiyangtx pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/amoro.git
The following commit(s) were added to refs/heads/master by this push:
new 89a700128 [AMORO-2755] Add judgment and reduce duplicate 'put' (#2758)
89a700128 is described below
commit 89a7001289b29b2f6fa1179ceaf8ad6eb5dbe617
Author: Wang Tao <[email protected]>
AuthorDate: Fri Apr 19 18:57:41 2024 +0800
[AMORO-2755] Add judgment and reduce duplicate 'put' (#2758)
* issue_2755 Add judgment and reduce duplicate 'put'
* Update
mixed-format/spark/v3.3/spark/src/main/java/org/apache/amoro/spark/table/ArcticSparkTable.java
Co-authored-by: baiyangtx <[email protected]>
* Update
mixed-format/spark/v3.2/spark/src/main/java/org/apache/amoro/spark/table/ArcticSparkTable.java
Co-authored-by: baiyangtx <[email protected]>
---------
Co-authored-by: baiyangtx <[email protected]>
---
.../org/apache/amoro/table/TableProperties.java | 1 +
.../apache/amoro/spark/table/ArcticSparkTable.java | 25 +++++++++++-----------
.../apache/amoro/spark/table/ArcticSparkTable.java | 25 +++++++++++-----------
3 files changed, 25 insertions(+), 26 deletions(-)
diff --git a/core/src/main/java/org/apache/amoro/table/TableProperties.java
b/core/src/main/java/org/apache/amoro/table/TableProperties.java
index f2b95cc30..fa55bd9a6 100644
--- a/core/src/main/java/org/apache/amoro/table/TableProperties.java
+++ b/core/src/main/java/org/apache/amoro/table/TableProperties.java
@@ -227,6 +227,7 @@ public class TableProperties {
public static final String FILE_FORMAT_ORC = "orc";
public static final String BASE_FILE_FORMAT = "base.write.format";
+ public static final String DELTA_FILE_FORMAT = "delta.write.format";
public static final String BASE_FILE_FORMAT_DEFAULT = FILE_FORMAT_PARQUET;
public static final String CHANGE_FILE_FORMAT = "change.write.format";
diff --git
a/mixed-format/spark/v3.2/spark/src/main/java/org/apache/amoro/spark/table/ArcticSparkTable.java
b/mixed-format/spark/v3.2/spark/src/main/java/org/apache/amoro/spark/table/ArcticSparkTable.java
index 6451b88ed..45363f80c 100644
---
a/mixed-format/spark/v3.2/spark/src/main/java/org/apache/amoro/spark/table/ArcticSparkTable.java
+++
b/mixed-format/spark/v3.2/spark/src/main/java/org/apache/amoro/spark/table/ArcticSparkTable.java
@@ -128,20 +128,19 @@ public class ArcticSparkTable
public Map<String, String> properties() {
ImmutableMap.Builder<String, String> propsBuilder = ImmutableMap.builder();
- String baseFileFormat =
- arcticTable
- .properties()
- .getOrDefault(
- TableProperties.BASE_FILE_FORMAT,
TableProperties.BASE_FILE_FORMAT_DEFAULT);
- String deltaFileFormat =
- arcticTable
- .properties()
- .getOrDefault(
- TableProperties.CHANGE_FILE_FORMAT,
TableProperties.CHANGE_FILE_FORMAT_DEFAULT);
- propsBuilder.put("base.write.format", baseFileFormat);
- propsBuilder.put("delta.write.format", deltaFileFormat);
- propsBuilder.put("provider", "arctic");
+ if
(!arcticTable.properties().containsKey(TableProperties.BASE_FILE_FORMAT)) {
+ propsBuilder.put(TableProperties.BASE_FILE_FORMAT,
TableProperties.BASE_FILE_FORMAT_DEFAULT);
+ }
+ if
(!arcticTable.properties().containsKey(TableProperties.DELTA_FILE_FORMAT)) {
+ propsBuilder.put(
+ TableProperties.DELTA_FILE_FORMAT,
+ arcticTable
+ .properties()
+ .getOrDefault(
+ TableProperties.CHANGE_FILE_FORMAT,
TableProperties.CHANGE_FILE_FORMAT_DEFAULT));
+ }
+ propsBuilder.put("provider", "arctic");
arcticTable.properties().entrySet().stream()
.filter(entry -> !RESERVED_PROPERTIES.contains(entry.getKey()))
.forEach(propsBuilder::put);
diff --git
a/mixed-format/spark/v3.3/spark/src/main/java/org/apache/amoro/spark/table/ArcticSparkTable.java
b/mixed-format/spark/v3.3/spark/src/main/java/org/apache/amoro/spark/table/ArcticSparkTable.java
index 6451b88ed..45363f80c 100644
---
a/mixed-format/spark/v3.3/spark/src/main/java/org/apache/amoro/spark/table/ArcticSparkTable.java
+++
b/mixed-format/spark/v3.3/spark/src/main/java/org/apache/amoro/spark/table/ArcticSparkTable.java
@@ -128,20 +128,19 @@ public class ArcticSparkTable
public Map<String, String> properties() {
ImmutableMap.Builder<String, String> propsBuilder = ImmutableMap.builder();
- String baseFileFormat =
- arcticTable
- .properties()
- .getOrDefault(
- TableProperties.BASE_FILE_FORMAT,
TableProperties.BASE_FILE_FORMAT_DEFAULT);
- String deltaFileFormat =
- arcticTable
- .properties()
- .getOrDefault(
- TableProperties.CHANGE_FILE_FORMAT,
TableProperties.CHANGE_FILE_FORMAT_DEFAULT);
- propsBuilder.put("base.write.format", baseFileFormat);
- propsBuilder.put("delta.write.format", deltaFileFormat);
- propsBuilder.put("provider", "arctic");
+ if
(!arcticTable.properties().containsKey(TableProperties.BASE_FILE_FORMAT)) {
+ propsBuilder.put(TableProperties.BASE_FILE_FORMAT,
TableProperties.BASE_FILE_FORMAT_DEFAULT);
+ }
+ if
(!arcticTable.properties().containsKey(TableProperties.DELTA_FILE_FORMAT)) {
+ propsBuilder.put(
+ TableProperties.DELTA_FILE_FORMAT,
+ arcticTable
+ .properties()
+ .getOrDefault(
+ TableProperties.CHANGE_FILE_FORMAT,
TableProperties.CHANGE_FILE_FORMAT_DEFAULT));
+ }
+ propsBuilder.put("provider", "arctic");
arcticTable.properties().entrySet().stream()
.filter(entry -> !RESERVED_PROPERTIES.contains(entry.getKey()))
.forEach(propsBuilder::put);
