This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new 2fa8c7d86c Core: Add rewritten delete files to write results (#11203)
2fa8c7d86c is described below
commit 2fa8c7d86c72e1869a97b847eb6b31fed2c4abb3
Author: Anton Okolnychyi <[email protected]>
AuthorDate: Wed Sep 25 08:30:06 2024 -0700
Core: Add rewritten delete files to write results (#11203)
---
.palantir/revapi.yml | 4 ++++
.../apache/iceberg/io/BasePositionDeltaWriter.java | 1 +
.../org/apache/iceberg/io/DeleteWriteResult.java | 18 +++++++++++++++
.../java/org/apache/iceberg/io/WriteResult.java | 27 +++++++++++++++++++---
4 files changed, 47 insertions(+), 3 deletions(-)
diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml
index 9b8017f0be..05ec59226f 100644
--- a/.palantir/revapi.yml
+++ b/.palantir/revapi.yml
@@ -1088,6 +1088,10 @@ acceptedBreaks:
old: "class org.apache.iceberg.GenericManifestFile"
new: "class org.apache.iceberg.GenericManifestFile"
justification: "Serialization across versions is not supported"
+ - code: "java.class.defaultSerializationChanged"
+ old: "class org.apache.iceberg.io.WriteResult"
+ new: "class org.apache.iceberg.io.WriteResult"
+ justification: "Serialization across versions is not supported"
- code: "java.class.removed"
old: "enum org.apache.iceberg.BaseMetastoreTableOperations.CommitStatus"
justification: "Removing deprecated code"
diff --git
a/core/src/main/java/org/apache/iceberg/io/BasePositionDeltaWriter.java
b/core/src/main/java/org/apache/iceberg/io/BasePositionDeltaWriter.java
index e098729ba2..859d339bd9 100644
--- a/core/src/main/java/org/apache/iceberg/io/BasePositionDeltaWriter.java
+++ b/core/src/main/java/org/apache/iceberg/io/BasePositionDeltaWriter.java
@@ -81,6 +81,7 @@ public class BasePositionDeltaWriter<T> implements
PositionDeltaWriter<T> {
.addDataFiles(dataFiles())
.addDeleteFiles(deleteWriteResult.deleteFiles())
.addReferencedDataFiles(deleteWriteResult.referencedDataFiles())
+ .addRewrittenDeleteFiles(deleteWriteResult.rewrittenDeleteFiles())
.build();
}
diff --git a/core/src/main/java/org/apache/iceberg/io/DeleteWriteResult.java
b/core/src/main/java/org/apache/iceberg/io/DeleteWriteResult.java
index 5c8768dcf1..98ac76bc80 100644
--- a/core/src/main/java/org/apache/iceberg/io/DeleteWriteResult.java
+++ b/core/src/main/java/org/apache/iceberg/io/DeleteWriteResult.java
@@ -32,25 +32,39 @@ import org.apache.iceberg.util.CharSequenceSet;
public class DeleteWriteResult {
private final List<DeleteFile> deleteFiles;
private final CharSequenceSet referencedDataFiles;
+ private final List<DeleteFile> rewrittenDeleteFiles;
public DeleteWriteResult(DeleteFile deleteFile) {
this.deleteFiles = Collections.singletonList(deleteFile);
this.referencedDataFiles = CharSequenceSet.empty();
+ this.rewrittenDeleteFiles = Collections.emptyList();
}
public DeleteWriteResult(DeleteFile deleteFile, CharSequenceSet
referencedDataFiles) {
this.deleteFiles = Collections.singletonList(deleteFile);
this.referencedDataFiles = referencedDataFiles;
+ this.rewrittenDeleteFiles = Collections.emptyList();
}
public DeleteWriteResult(List<DeleteFile> deleteFiles) {
this.deleteFiles = deleteFiles;
this.referencedDataFiles = CharSequenceSet.empty();
+ this.rewrittenDeleteFiles = Collections.emptyList();
}
public DeleteWriteResult(List<DeleteFile> deleteFiles, CharSequenceSet
referencedDataFiles) {
this.deleteFiles = deleteFiles;
this.referencedDataFiles = referencedDataFiles;
+ this.rewrittenDeleteFiles = Collections.emptyList();
+ }
+
+ public DeleteWriteResult(
+ List<DeleteFile> deleteFiles,
+ CharSequenceSet referencedDataFiles,
+ List<DeleteFile> rewrittenDeleteFiles) {
+ this.deleteFiles = deleteFiles;
+ this.referencedDataFiles = referencedDataFiles;
+ this.rewrittenDeleteFiles = rewrittenDeleteFiles;
}
public List<DeleteFile> deleteFiles() {
@@ -64,4 +78,8 @@ public class DeleteWriteResult {
public boolean referencesDataFiles() {
return referencedDataFiles != null && !referencedDataFiles.isEmpty();
}
+
+ public List<DeleteFile> rewrittenDeleteFiles() {
+ return rewrittenDeleteFiles;
+ }
}
diff --git a/core/src/main/java/org/apache/iceberg/io/WriteResult.java
b/core/src/main/java/org/apache/iceberg/io/WriteResult.java
index c73b09f46c..39efaec7d3 100644
--- a/core/src/main/java/org/apache/iceberg/io/WriteResult.java
+++ b/core/src/main/java/org/apache/iceberg/io/WriteResult.java
@@ -31,12 +31,17 @@ public class WriteResult implements Serializable {
private DataFile[] dataFiles;
private DeleteFile[] deleteFiles;
private CharSequence[] referencedDataFiles;
+ private DeleteFile[] rewrittenDeleteFiles;
private WriteResult(
- List<DataFile> dataFiles, List<DeleteFile> deleteFiles, CharSequenceSet
referencedDataFiles) {
+ List<DataFile> dataFiles,
+ List<DeleteFile> deleteFiles,
+ CharSequenceSet referencedDataFiles,
+ List<DeleteFile> rewrittenDeleteFiles) {
this.dataFiles = dataFiles.toArray(new DataFile[0]);
this.deleteFiles = deleteFiles.toArray(new DeleteFile[0]);
this.referencedDataFiles = referencedDataFiles.toArray(new
CharSequence[0]);
+ this.rewrittenDeleteFiles = rewrittenDeleteFiles.toArray(new
DeleteFile[0]);
}
public DataFile[] dataFiles() {
@@ -51,6 +56,10 @@ public class WriteResult implements Serializable {
return referencedDataFiles;
}
+ public DeleteFile[] rewrittenDeleteFiles() {
+ return rewrittenDeleteFiles;
+ }
+
public static Builder builder() {
return new Builder();
}
@@ -59,18 +68,20 @@ public class WriteResult implements Serializable {
private final List<DataFile> dataFiles;
private final List<DeleteFile> deleteFiles;
private final CharSequenceSet referencedDataFiles;
+ private final List<DeleteFile> rewrittenDeleteFiles;
private Builder() {
this.dataFiles = Lists.newArrayList();
this.deleteFiles = Lists.newArrayList();
this.referencedDataFiles = CharSequenceSet.empty();
+ this.rewrittenDeleteFiles = Lists.newArrayList();
}
public Builder add(WriteResult result) {
addDataFiles(result.dataFiles);
addDeleteFiles(result.deleteFiles);
addReferencedDataFiles(result.referencedDataFiles);
-
+ addRewrittenDeleteFiles(result.rewrittenDeleteFiles);
return this;
}
@@ -109,8 +120,18 @@ public class WriteResult implements Serializable {
return this;
}
+ public Builder addRewrittenDeleteFiles(DeleteFile... files) {
+ Collections.addAll(rewrittenDeleteFiles, files);
+ return this;
+ }
+
+ public Builder addRewrittenDeleteFiles(Iterable<DeleteFile> files) {
+ Iterables.addAll(rewrittenDeleteFiles, files);
+ return this;
+ }
+
public WriteResult build() {
- return new WriteResult(dataFiles, deleteFiles, referencedDataFiles);
+ return new WriteResult(dataFiles, deleteFiles, referencedDataFiles,
rewrittenDeleteFiles);
}
}
}
