(iceberg) branch main updated: Spark 3.4: Set useCommitCoordinator to false in batch writes (#9028)

[aokolnychyi]

This is an automated email from the ASF dual-hosted git repository.

aokolnychyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new d7f8e91c09 Spark 3.4: Set useCommitCoordinator to false in batch 
writes (#9028)
d7f8e91c09 is described below

commit d7f8e91c0983048768be1cb17893a485088da937
Author: Huaxin Gao <huaxin_...@apple.com>
AuthorDate: Sat Nov 11 10:17:24 2023 -0800

    Spark 3.4: Set useCommitCoordinator to false in batch writes (#9028)
    
    This change cherrypicks PR #9017 to Spark 3.4.
    
    Co-authored-by: Huaxin Gao <huaxin....@apple.com>
---
 .../org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java | 5 +++++
 .../org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java     | 5 +++++
 .../src/main/java/org/apache/iceberg/spark/source/SparkWrite.java    | 5 +++++
 3 files changed, 15 insertions(+)

diff --git 
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java
 
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java
index d0769eaa5f..a397a069ee 100644
--- 
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java
+++ 
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeletesRewrite.java
@@ -136,6 +136,11 @@ public class SparkPositionDeletesRewrite implements Write {
           writeProperties);
     }
 
+    @Override
+    public boolean useCommitCoordinator() {
+      return false;
+    }
+
     @Override
     public void commit(WriterCommitMessage[] messages) {
       PositionDeletesRewriteCoordinator coordinator = 
PositionDeletesRewriteCoordinator.get();
diff --git 
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java
 
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java
index 9fea33948b..791cbd8462 100644
--- 
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java
+++ 
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkPositionDeltaWrite.java
@@ -160,6 +160,11 @@ class SparkPositionDeltaWrite implements DeltaWrite, 
RequiresDistributionAndOrde
       return new PositionDeltaWriteFactory(tableBroadcast, command, context, 
writeProperties);
     }
 
+    @Override
+    public boolean useCommitCoordinator() {
+      return false;
+    }
+
     @Override
     public void commit(WriterCommitMessage[] messages) {
       RowDelta rowDelta = table.newRowDelta();
diff --git 
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
 
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
index 15881098e7..9643217824 100644
--- 
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
+++ 
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
@@ -260,6 +260,11 @@ abstract class SparkWrite implements Write, 
RequiresDistributionAndOrdering {
       return createWriterFactory();
     }
 
+    @Override
+    public boolean useCommitCoordinator() {
+      return false;
+    }
+
     @Override
     public void abort(WriterCommitMessage[] messages) {
       SparkWrite.this.abort(messages);