(iceberg) branch main updated: Spark 3.3, 3.4: Fix file clobbering when Spark reuses query IDs (#9255) (#9399)

[blue]

This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/main by this push:
     new ad423142c4 Spark 3.3, 3.4: Fix file clobbering when Spark reuses query 
IDs (#9255) (#9399)
ad423142c4 is described below

commit ad423142c4d3d41b9423026cd99f8ecf1156a24b
Author: Amogh Jahagirdar <am...@tabular.io>
AuthorDate: Tue Jan 2 12:52:59 2024 -0800

    Spark 3.3, 3.4: Fix file clobbering when Spark reuses query IDs (#9255) 
(#9399)
---
 .../src/main/java/org/apache/iceberg/spark/source/SparkWrite.java     | 4 ++--
 .../src/main/java/org/apache/iceberg/spark/source/SparkWrite.java     | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git 
a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
 
b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
index a080fcead1..59fecd45cc 100644
--- 
a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
+++ 
b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
@@ -643,11 +643,11 @@ abstract class SparkWrite implements Write, 
RequiresDistributionAndOrdering {
       Table table = tableBroadcast.value();
       PartitionSpec spec = table.specs().get(outputSpecId);
       FileIO io = table.io();
-
+      String operationId = queryId + "-" + epochId;
       OutputFileFactory fileFactory =
           OutputFileFactory.builderFor(table, partitionId, taskId)
               .format(format)
-              .operationId(queryId)
+              .operationId(operationId)
               .build();
       SparkFileWriterFactory writerFactory =
           SparkFileWriterFactory.builderFor(table)
diff --git 
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
 
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
index 9643217824..fb49d0bff5 100644
--- 
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
+++ 
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/source/SparkWrite.java
@@ -657,11 +657,11 @@ abstract class SparkWrite implements Write, 
RequiresDistributionAndOrdering {
       Table table = tableBroadcast.value();
       PartitionSpec spec = table.specs().get(outputSpecId);
       FileIO io = table.io();
-
+      String operationId = queryId + "-" + epochId;
       OutputFileFactory fileFactory =
           OutputFileFactory.builderFor(table, partitionId, taskId)
               .format(format)
-              .operationId(queryId)
+              .operationId(operationId)
               .build();
       SparkFileWriterFactory writerFactory =
           SparkFileWriterFactory.builderFor(table)