This is an automated email from the ASF dual-hosted git repository.
pvary pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new ded11105fb Spark: Use default cleanExpiredMetadata from Java API
(#13731)
ded11105fb is described below
commit ded11105fb1d038ce03793e2a5c1144e408da260
Author: gaborkaszab <gaborkas...@gmail.com>
AuthorDate: Tue Aug 5 11:07:16 2025 +0200
Spark: Use default cleanExpiredMetadata from Java API (#13731)
---
.../iceberg/spark/actions/ExpireSnapshotsSparkAction.java | 12 +++++++++---
.../iceberg/spark/actions/ExpireSnapshotsSparkAction.java | 12 +++++++++---
.../iceberg/spark/actions/ExpireSnapshotsSparkAction.java | 12 +++++++++---
3 files changed, 27 insertions(+), 9 deletions(-)
diff --git
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
index 8c4fda5b48..5f4d4ec151 100644
---
a/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
+++
b/spark/v3.4/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
@@ -79,7 +79,7 @@ public class ExpireSnapshotsSparkAction extends
BaseSparkAction<ExpireSnapshotsS
private Consumer<String> deleteFunc = null;
private ExecutorService deleteExecutorService = null;
private Dataset<FileInfo> expiredFileDS = null;
- private boolean cleanExpiredMetadata = false;
+ private Boolean cleanExpiredMetadata = null;
ExpireSnapshotsSparkAction(SparkSession spark, Table table) {
super(spark);
@@ -165,7 +165,11 @@ public class ExpireSnapshotsSparkAction extends
BaseSparkAction<ExpireSnapshotsS
expireSnapshots = expireSnapshots.retainLast(retainLastValue);
}
-
expireSnapshots.cleanExpiredMetadata(cleanExpiredMetadata).cleanExpiredFiles(false).commit();
+ if (cleanExpiredMetadata != null) {
+ expireSnapshots.cleanExpiredMetadata(cleanExpiredMetadata);
+ }
+
+ expireSnapshots.cleanExpiredFiles(false).commit();
// fetch valid files after expiration
TableMetadata updatedMetadata = ops.refresh();
@@ -209,7 +213,9 @@ public class ExpireSnapshotsSparkAction extends
BaseSparkAction<ExpireSnapshotsS
}
}
- options.add("clean_expired_metadata=" + cleanExpiredMetadata);
+ if (cleanExpiredMetadata != null) {
+ options.add("clean_expired_metadata=" + cleanExpiredMetadata);
+ }
return String.format("Expiring snapshots (%s) in %s",
COMMA_JOINER.join(options), table.name());
}
diff --git
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
index 8c4fda5b48..5f4d4ec151 100644
---
a/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
+++
b/spark/v3.5/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
@@ -79,7 +79,7 @@ public class ExpireSnapshotsSparkAction extends
BaseSparkAction<ExpireSnapshotsS
private Consumer<String> deleteFunc = null;
private ExecutorService deleteExecutorService = null;
private Dataset<FileInfo> expiredFileDS = null;
- private boolean cleanExpiredMetadata = false;
+ private Boolean cleanExpiredMetadata = null;
ExpireSnapshotsSparkAction(SparkSession spark, Table table) {
super(spark);
@@ -165,7 +165,11 @@ public class ExpireSnapshotsSparkAction extends
BaseSparkAction<ExpireSnapshotsS
expireSnapshots = expireSnapshots.retainLast(retainLastValue);
}
-
expireSnapshots.cleanExpiredMetadata(cleanExpiredMetadata).cleanExpiredFiles(false).commit();
+ if (cleanExpiredMetadata != null) {
+ expireSnapshots.cleanExpiredMetadata(cleanExpiredMetadata);
+ }
+
+ expireSnapshots.cleanExpiredFiles(false).commit();
// fetch valid files after expiration
TableMetadata updatedMetadata = ops.refresh();
@@ -209,7 +213,9 @@ public class ExpireSnapshotsSparkAction extends
BaseSparkAction<ExpireSnapshotsS
}
}
- options.add("clean_expired_metadata=" + cleanExpiredMetadata);
+ if (cleanExpiredMetadata != null) {
+ options.add("clean_expired_metadata=" + cleanExpiredMetadata);
+ }
return String.format("Expiring snapshots (%s) in %s",
COMMA_JOINER.join(options), table.name());
}
diff --git
a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
index 8c4fda5b48..5f4d4ec151 100644
---
a/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
+++
b/spark/v4.0/spark/src/main/java/org/apache/iceberg/spark/actions/ExpireSnapshotsSparkAction.java
@@ -79,7 +79,7 @@ public class ExpireSnapshotsSparkAction extends
BaseSparkAction<ExpireSnapshotsS
private Consumer<String> deleteFunc = null;
private ExecutorService deleteExecutorService = null;
private Dataset<FileInfo> expiredFileDS = null;
- private boolean cleanExpiredMetadata = false;
+ private Boolean cleanExpiredMetadata = null;
ExpireSnapshotsSparkAction(SparkSession spark, Table table) {
super(spark);
@@ -165,7 +165,11 @@ public class ExpireSnapshotsSparkAction extends
BaseSparkAction<ExpireSnapshotsS
expireSnapshots = expireSnapshots.retainLast(retainLastValue);
}
-
expireSnapshots.cleanExpiredMetadata(cleanExpiredMetadata).cleanExpiredFiles(false).commit();
+ if (cleanExpiredMetadata != null) {
+ expireSnapshots.cleanExpiredMetadata(cleanExpiredMetadata);
+ }
+
+ expireSnapshots.cleanExpiredFiles(false).commit();
// fetch valid files after expiration
TableMetadata updatedMetadata = ops.refresh();
@@ -209,7 +213,9 @@ public class ExpireSnapshotsSparkAction extends
BaseSparkAction<ExpireSnapshotsS
}
}
- options.add("clean_expired_metadata=" + cleanExpiredMetadata);
+ if (cleanExpiredMetadata != null) {
+ options.add("clean_expired_metadata=" + cleanExpiredMetadata);
+ }
return String.format("Expiring snapshots (%s) in %s",
COMMA_JOINER.join(options), table.name());
}
