aokolnychyi commented on a change in pull request #4469:
URL: https://github.com/apache/iceberg/pull/4469#discussion_r840971518
##########
File path:
spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteReachableFilesSparkAction.java
##########
@@ -103,53 +87,46 @@ public DeleteReachableFiles io(FileIO fileIO) {
@Override
public DeleteReachableFiles deleteWith(Consumer<String> deleteFunc) {
- this.removeFunc = deleteFunc;
+ this.deleteFunc = deleteFunc;
return this;
-
}
@Override
public DeleteReachableFiles executeDeleteWith(ExecutorService
executorService) {
- this.removeExecutorService = executorService;
+ this.deleteExecutorService = executorService;
return this;
}
@Override
public Result execute() {
- Preconditions.checkArgument(io != null, "File IO cannot be null");
- String msg = String.format("Removing files reachable from %s",
tableMetadata.metadataFileLocation());
- JobGroupInfo info = newJobGroupInfo("REMOVE-FILES", msg);
+ String jobDesc = String.format("Deleting files reachable from %s",
metadataFileLocation);
+ JobGroupInfo info = newJobGroupInfo("DELETE-REACHABLE-FILES", jobDesc);
return withJobGroupInfo(info, this::doExecute);
}
private Result doExecute() {
- boolean streamResults = PropertyUtil.propertyAsBoolean(options(),
STREAM_RESULTS, false);
- Dataset<Row> validFileDF = buildValidFileDF(tableMetadata).distinct();
+ TableMetadata metadata = TableMetadataParser.read(io,
metadataFileLocation);
+
+ ValidationException.check(
+ PropertyUtil.propertyAsBoolean(metadata.properties(), GC_ENABLED,
GC_ENABLED_DEFAULT),
+ "Cannot delete files: GC is disabled (deleting files may corrupt other
tables)");
+
+ Dataset<Row> reachableFileDF = buildReachableFileDF(metadata).distinct();
+
+ boolean streamResults = PropertyUtil.propertyAsBoolean(options(),
STREAM_RESULTS, STREAM_RESULTS_DEFAULT);
if (streamResults) {
- return deleteFiles(validFileDF.toLocalIterator());
+ return deleteFiles(reachableFileDF.toLocalIterator());
} else {
- return deleteFiles(validFileDF.collectAsList().iterator());
+ return deleteFiles(reachableFileDF.collectAsList().iterator());
}
}
- private Dataset<Row> projectFilePathWithType(Dataset<Row> ds, String type) {
Review comment:
There are multiple actions that define the exact same method.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
