This is an automated email from the ASF dual-hosted git repository.
aokolnychyi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 712fe66 API: Use delete instead of remove in action names (#2810)
712fe66 is described below
commit 712fe660dc6b3c8464abca11775f4a19fa2a36ed
Author: Anton Okolnychyi <[email protected]>
AuthorDate: Mon Jul 12 16:20:32 2021 -1000
API: Use delete instead of remove in action names (#2810)
---
.../apache/iceberg/actions/ActionsProvider.java | 12 +++---
...moveOrphanFiles.java => DeleteOrphanFiles.java} | 10 ++---
...achableFiles.java => DeleteReachableFiles.java} | 28 ++++++-------
...java => BaseDeleteOrphanFilesActionResult.java} | 4 +-
...a => BaseDeleteReachableFilesActionResult.java} | 18 ++++-----
.../java/org/apache/iceberg/actions/Actions.java | 4 +-
.../iceberg/actions/RemoveOrphanFilesAction.java | 8 ++--
....java => BaseDeleteOrphanFilesSparkAction.java} | 26 ++++++------
...va => BaseDeleteReachableFilesSparkAction.java} | 28 ++++++-------
.../iceberg/spark/actions/BaseSparkActions.java | 12 +++---
...on.java => TestDeleteReachableFilesAction.java} | 46 +++++++++++-----------
.../actions/TestNewRewriteDataFilesAction.java | 2 +-
....java => TestDeleteReachableFilesAction24.java} | 2 +-
.../procedures/RemoveOrphanFilesProcedure.java | 8 ++--
.../iceberg/actions/TestRemoveFilesAction3.java | 2 +-
15 files changed, 105 insertions(+), 105 deletions(-)
diff --git a/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java
b/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java
index 30adfea..f2564dd 100644
--- a/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java
+++ b/api/src/main/java/org/apache/iceberg/actions/ActionsProvider.java
@@ -41,10 +41,10 @@ public interface ActionsProvider {
}
/**
- * Instantiates an action to remove orphan files.
+ * Instantiates an action to delete orphan files.
*/
- default RemoveOrphanFiles removeOrphanFiles(Table table) {
- throw new UnsupportedOperationException(this.getClass().getName() + " does
not implement removeOrphanFiles");
+ default DeleteOrphanFiles deleteOrphanFiles(Table table) {
+ throw new UnsupportedOperationException(this.getClass().getName() + " does
not implement deleteOrphanFiles");
}
/**
@@ -69,9 +69,9 @@ public interface ActionsProvider {
}
/**
- * Instantiates an action to remove all the files reachable from given
metadata location.
+ * Instantiates an action to delete all the files reachable from given
metadata location.
*/
- default RemoveReachableFiles removeReachableFiles(String metadataLocation) {
- throw new UnsupportedOperationException(this.getClass().getName() + " does
not implement removeReachableFiles");
+ default DeleteReachableFiles deleteReachableFiles(String metadataLocation) {
+ throw new UnsupportedOperationException(this.getClass().getName() + " does
not implement deleteReachableFiles");
}
}
diff --git
a/api/src/main/java/org/apache/iceberg/actions/RemoveOrphanFiles.java
b/api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java
similarity index 89%
rename from api/src/main/java/org/apache/iceberg/actions/RemoveOrphanFiles.java
rename to api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java
index 765eba1..b841edb 100644
--- a/api/src/main/java/org/apache/iceberg/actions/RemoveOrphanFiles.java
+++ b/api/src/main/java/org/apache/iceberg/actions/DeleteOrphanFiles.java
@@ -22,13 +22,13 @@ package org.apache.iceberg.actions;
import java.util.function.Consumer;
/**
- * An action that removes orphan files in a table.
+ * An action that deletes orphan files in a table.
* <p>
* A metadata or data file is considered orphan if it is not reachable by any
valid snapshot.
* The set of actual files is built by listing the underlying storage which
makes this operation
* expensive.
*/
-public interface RemoveOrphanFiles extends Action<RemoveOrphanFiles,
RemoveOrphanFiles.Result> {
+public interface DeleteOrphanFiles extends Action<DeleteOrphanFiles,
DeleteOrphanFiles.Result> {
/**
* Passes a location which should be scanned for orphan files.
* <p>
@@ -38,7 +38,7 @@ public interface RemoveOrphanFiles extends
Action<RemoveOrphanFiles, RemoveOrpha
* @param location the location where to look for orphan files
* @return this for method chaining
*/
- RemoveOrphanFiles location(String location);
+ DeleteOrphanFiles location(String location);
/**
* Removes orphan files only if they are older than the given timestamp.
@@ -52,7 +52,7 @@ public interface RemoveOrphanFiles extends
Action<RemoveOrphanFiles, RemoveOrpha
* @param olderThanTimestamp a long timestamp, as returned by {@link
System#currentTimeMillis()}
* @return this for method chaining
*/
- RemoveOrphanFiles olderThan(long olderThanTimestamp);
+ DeleteOrphanFiles olderThan(long olderThanTimestamp);
/**
* Passes an alternative delete implementation that will be used for orphan
files.
@@ -65,7 +65,7 @@ public interface RemoveOrphanFiles extends
Action<RemoveOrphanFiles, RemoveOrpha
* @param deleteFunc a function that will be called to delete files
* @return this for method chaining
*/
- RemoveOrphanFiles deleteWith(Consumer<String> deleteFunc);
+ DeleteOrphanFiles deleteWith(Consumer<String> deleteFunc);
/**
* The action result that contains a summary of the execution.
diff --git
a/api/src/main/java/org/apache/iceberg/actions/RemoveReachableFiles.java
b/api/src/main/java/org/apache/iceberg/actions/DeleteReachableFiles.java
similarity index 73%
rename from
api/src/main/java/org/apache/iceberg/actions/RemoveReachableFiles.java
rename to api/src/main/java/org/apache/iceberg/actions/DeleteReachableFiles.java
index c643f2e..3b0eb96 100644
--- a/api/src/main/java/org/apache/iceberg/actions/RemoveReachableFiles.java
+++ b/api/src/main/java/org/apache/iceberg/actions/DeleteReachableFiles.java
@@ -24,7 +24,7 @@ import java.util.function.Consumer;
import org.apache.iceberg.io.FileIO;
/**
- * An action that removes all files referenced by a table metadata file.
+ * An action that deletes all files referenced by a table metadata file.
* <p>
* This action will irreversibly delete all reachable files such as data
files, manifests,
* manifest lists and should be used to clean up the underlying storage once a
table is dropped
@@ -32,16 +32,16 @@ import org.apache.iceberg.io.FileIO;
* <p>
* Implementations may use a query engine to distribute parts of work.
*/
-public interface RemoveReachableFiles extends Action<RemoveReachableFiles,
RemoveReachableFiles.Result> {
+public interface DeleteReachableFiles extends Action<DeleteReachableFiles,
DeleteReachableFiles.Result> {
/**
* Passes an alternative delete implementation that will be used for files.
*
- * @param removeFunc a function that will be called to delete files.
+ * @param deleteFunc a function that will be called to delete files.
* The function accepts path to file as an argument.
* @return this for method chaining
*/
- RemoveReachableFiles deleteWith(Consumer<String> removeFunc);
+ DeleteReachableFiles deleteWith(Consumer<String> deleteFunc);
/**
* Passes an alternative executor service that will be used for files
removal.
@@ -51,7 +51,7 @@ public interface RemoveReachableFiles extends
Action<RemoveReachableFiles, Remov
* @param executorService the service to use
* @return this for method chaining
*/
- RemoveReachableFiles executeDeleteWith(ExecutorService executorService);
+ DeleteReachableFiles executeDeleteWith(ExecutorService executorService);
/**
* Set the {@link FileIO} to be used for files removal
@@ -59,7 +59,7 @@ public interface RemoveReachableFiles extends
Action<RemoveReachableFiles, Remov
* @param io FileIO to use for files removal
* @return this for method chaining
*/
- RemoveReachableFiles io(FileIO io);
+ DeleteReachableFiles io(FileIO io);
/**
* The action result that contains a summary of the execution.
@@ -67,23 +67,23 @@ public interface RemoveReachableFiles extends
Action<RemoveReachableFiles, Remov
interface Result {
/**
- * Returns the number of data files removed.
+ * Returns the number of deleted data files.
*/
- long removedDataFilesCount();
+ long deletedDataFilesCount();
/**
- * Returns the number of manifests removed.
+ * Returns the number of deleted manifests.
*/
- long removedManifestsCount();
+ long deletedManifestsCount();
/**
- * Returns the number of manifest lists removed.
+ * Returns the number of deleted manifest lists.
*/
- long removedManifestListsCount();
+ long deletedManifestListsCount();
/**
- * Returns the number of metadata json, version hint files removed.
+ * Returns the number of deleted metadata json, version hint files.
*/
- long otherRemovedFilesCount();
+ long deletedOtherFilesCount();
}
}
diff --git
a/core/src/main/java/org/apache/iceberg/actions/BaseRemoveOrphanFilesActionResult.java
b/core/src/main/java/org/apache/iceberg/actions/BaseDeleteOrphanFilesActionResult.java
similarity index 89%
rename from
core/src/main/java/org/apache/iceberg/actions/BaseRemoveOrphanFilesActionResult.java
rename to
core/src/main/java/org/apache/iceberg/actions/BaseDeleteOrphanFilesActionResult.java
index 7122943..881e908 100644
---
a/core/src/main/java/org/apache/iceberg/actions/BaseRemoveOrphanFilesActionResult.java
+++
b/core/src/main/java/org/apache/iceberg/actions/BaseDeleteOrphanFilesActionResult.java
@@ -19,11 +19,11 @@
package org.apache.iceberg.actions;
-public class BaseRemoveOrphanFilesActionResult implements
RemoveOrphanFiles.Result {
+public class BaseDeleteOrphanFilesActionResult implements
DeleteOrphanFiles.Result {
private final Iterable<String> orphanFileLocations;
- public BaseRemoveOrphanFilesActionResult(Iterable<String>
orphanFileLocations) {
+ public BaseDeleteOrphanFilesActionResult(Iterable<String>
orphanFileLocations) {
this.orphanFileLocations = orphanFileLocations;
}
diff --git
a/core/src/main/java/org/apache/iceberg/actions/BaseRemoveFilesActionResult.java
b/core/src/main/java/org/apache/iceberg/actions/BaseDeleteReachableFilesActionResult.java
similarity index 72%
rename from
core/src/main/java/org/apache/iceberg/actions/BaseRemoveFilesActionResult.java
rename to
core/src/main/java/org/apache/iceberg/actions/BaseDeleteReachableFilesActionResult.java
index 8707975..61abd43 100644
---
a/core/src/main/java/org/apache/iceberg/actions/BaseRemoveFilesActionResult.java
+++
b/core/src/main/java/org/apache/iceberg/actions/BaseDeleteReachableFilesActionResult.java
@@ -19,17 +19,17 @@
package org.apache.iceberg.actions;
-public class BaseRemoveFilesActionResult implements
RemoveReachableFiles.Result {
+public class BaseDeleteReachableFilesActionResult implements
DeleteReachableFiles.Result {
private final long deletedDataFilesCount;
private final long deletedManifestsCount;
private final long deletedManifestListsCount;
private final long deletedOtherFilesCount;
- public BaseRemoveFilesActionResult(long deletedDataFilesCount,
- long deletedManifestsCount,
- long deletedManifestListsCount,
- long otherDeletedFilesCount) {
+ public BaseDeleteReachableFilesActionResult(long deletedDataFilesCount,
+ long deletedManifestsCount,
+ long deletedManifestListsCount,
+ long otherDeletedFilesCount) {
this.deletedDataFilesCount = deletedDataFilesCount;
this.deletedManifestsCount = deletedManifestsCount;
this.deletedManifestListsCount = deletedManifestListsCount;
@@ -37,22 +37,22 @@ public class BaseRemoveFilesActionResult implements
RemoveReachableFiles.Result
}
@Override
- public long removedDataFilesCount() {
+ public long deletedDataFilesCount() {
return deletedDataFilesCount;
}
@Override
- public long removedManifestsCount() {
+ public long deletedManifestsCount() {
return deletedManifestsCount;
}
@Override
- public long removedManifestListsCount() {
+ public long deletedManifestListsCount() {
return deletedManifestListsCount;
}
@Override
- public long otherRemovedFilesCount() {
+ public long deletedOtherFilesCount() {
return deletedOtherFilesCount;
}
}
diff --git a/spark/src/main/java/org/apache/iceberg/actions/Actions.java
b/spark/src/main/java/org/apache/iceberg/actions/Actions.java
index d1b7d77..8028db9 100644
--- a/spark/src/main/java/org/apache/iceberg/actions/Actions.java
+++ b/spark/src/main/java/org/apache/iceberg/actions/Actions.java
@@ -22,8 +22,8 @@ package org.apache.iceberg.actions;
import org.apache.iceberg.Table;
import org.apache.iceberg.common.DynConstructors;
import org.apache.iceberg.common.DynMethods;
+import org.apache.iceberg.spark.actions.BaseDeleteOrphanFilesSparkAction;
import org.apache.iceberg.spark.actions.BaseExpireSnapshotsSparkAction;
-import org.apache.iceberg.spark.actions.BaseRemoveOrphanFilesSparkAction;
import org.apache.iceberg.spark.actions.BaseRewriteManifestsSparkAction;
import org.apache.spark.sql.SparkSession;
@@ -70,7 +70,7 @@ public class Actions {
}
public RemoveOrphanFilesAction removeOrphanFiles() {
- BaseRemoveOrphanFilesSparkAction delegate = new
BaseRemoveOrphanFilesSparkAction(spark, table);
+ BaseDeleteOrphanFilesSparkAction delegate = new
BaseDeleteOrphanFilesSparkAction(spark, table);
return new RemoveOrphanFilesAction(delegate);
}
diff --git
a/spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java
b/spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java
index a3298ac..ac03b5f 100644
---
a/spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java
+++
b/spark/src/main/java/org/apache/iceberg/actions/RemoveOrphanFilesAction.java
@@ -39,13 +39,13 @@ import
org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
* <em>Note:</em> It is dangerous to call this action with a short retention
interval as it might corrupt
* the state of the table if another operation is writing at the same time.
*
- * @deprecated since 0.12.0, will be removed in 0.13.0; use {@link
RemoveOrphanFiles} instead.
+ * @deprecated since 0.12.0, will be removed in 0.13.0; use {@link
DeleteOrphanFiles} instead.
*/
@Deprecated
public class RemoveOrphanFilesAction implements
Action<RemoveOrphanFilesAction, List<String>> {
- private final RemoveOrphanFiles delegate;
+ private final DeleteOrphanFiles delegate;
- RemoveOrphanFilesAction(RemoveOrphanFiles delegate) {
+ RemoveOrphanFilesAction(DeleteOrphanFiles delegate) {
this.delegate = delegate;
}
@@ -84,7 +84,7 @@ public class RemoveOrphanFilesAction implements
Action<RemoveOrphanFilesAction,
@Override
public List<String> execute() {
- RemoveOrphanFiles.Result result = delegate.execute();
+ DeleteOrphanFiles.Result result = delegate.execute();
return ImmutableList.copyOf(result.orphanFileLocations());
}
}
diff --git
a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRemoveOrphanFilesSparkAction.java
b/spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteOrphanFilesSparkAction.java
similarity index 92%
rename from
spark/src/main/java/org/apache/iceberg/spark/actions/BaseRemoveOrphanFilesSparkAction.java
rename to
spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteOrphanFilesSparkAction.java
index 844dfd0..daa96b7 100644
---
a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRemoveOrphanFilesSparkAction.java
+++
b/spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteOrphanFilesSparkAction.java
@@ -31,8 +31,8 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.iceberg.Table;
-import org.apache.iceberg.actions.BaseRemoveOrphanFilesActionResult;
-import org.apache.iceberg.actions.RemoveOrphanFiles;
+import org.apache.iceberg.actions.BaseDeleteOrphanFilesActionResult;
+import org.apache.iceberg.actions.DeleteOrphanFiles;
import org.apache.iceberg.exceptions.RuntimeIOException;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.hadoop.HiddenPathFilter;
@@ -73,10 +73,10 @@ import static
org.apache.iceberg.TableProperties.GC_ENABLED_DEFAULT;
* <em>Note:</em> It is dangerous to call this action with a short retention
interval as it might corrupt
* the state of the table if another operation is writing at the same time.
*/
-public class BaseRemoveOrphanFilesSparkAction
- extends BaseSparkAction<RemoveOrphanFiles, RemoveOrphanFiles.Result>
implements RemoveOrphanFiles {
+public class BaseDeleteOrphanFilesSparkAction
+ extends BaseSparkAction<DeleteOrphanFiles, DeleteOrphanFiles.Result>
implements DeleteOrphanFiles {
- private static final Logger LOG =
LoggerFactory.getLogger(BaseRemoveOrphanFilesSparkAction.class);
+ private static final Logger LOG =
LoggerFactory.getLogger(BaseDeleteOrphanFilesSparkAction.class);
private static final UserDefinedFunction filenameUDF = functions.udf((String
path) -> {
int lastIndex = path.lastIndexOf(File.separator);
if (lastIndex == -1) {
@@ -99,7 +99,7 @@ public class BaseRemoveOrphanFilesSparkAction
}
};
- public BaseRemoveOrphanFilesSparkAction(SparkSession spark, Table table) {
+ public BaseDeleteOrphanFilesSparkAction(SparkSession spark, Table table) {
super(spark);
this.hadoopConf = new
SerializableConfiguration(spark.sessionState().newHadoopConf());
@@ -113,30 +113,30 @@ public class BaseRemoveOrphanFilesSparkAction
}
@Override
- protected RemoveOrphanFiles self() {
+ protected DeleteOrphanFiles self() {
return this;
}
@Override
- public BaseRemoveOrphanFilesSparkAction location(String newLocation) {
+ public BaseDeleteOrphanFilesSparkAction location(String newLocation) {
this.location = newLocation;
return this;
}
@Override
- public BaseRemoveOrphanFilesSparkAction olderThan(long
newOlderThanTimestamp) {
+ public BaseDeleteOrphanFilesSparkAction olderThan(long
newOlderThanTimestamp) {
this.olderThanTimestamp = newOlderThanTimestamp;
return this;
}
@Override
- public BaseRemoveOrphanFilesSparkAction deleteWith(Consumer<String>
newDeleteFunc) {
+ public BaseDeleteOrphanFilesSparkAction deleteWith(Consumer<String>
newDeleteFunc) {
this.deleteFunc = newDeleteFunc;
return this;
}
@Override
- public RemoveOrphanFiles.Result execute() {
+ public DeleteOrphanFiles.Result execute() {
JobGroupInfo info = newJobGroupInfo("REMOVE-ORPHAN-FILES", jobDesc());
return withJobGroupInfo(info, this::doExecute);
}
@@ -150,7 +150,7 @@ public class BaseRemoveOrphanFilesSparkAction
return String.format("Removing orphan files (%s) from %s",
Joiner.on(',').join(options), table.name());
}
- private RemoveOrphanFiles.Result doExecute() {
+ private DeleteOrphanFiles.Result doExecute() {
Dataset<Row> validDataFileDF = buildValidDataFileDF(table);
Dataset<Row> validMetadataFileDF = buildValidMetadataFileDF(table);
Dataset<Row> validFileDF = validDataFileDF.union(validMetadataFileDF);
@@ -171,7 +171,7 @@ public class BaseRemoveOrphanFilesSparkAction
.onFailure((file, exc) -> LOG.warn("Failed to delete file: {}", file,
exc))
.run(deleteFunc::accept);
- return new BaseRemoveOrphanFilesActionResult(orphanFiles);
+ return new BaseDeleteOrphanFilesActionResult(orphanFiles);
}
private Dataset<Row> buildActualFileDF() {
diff --git
a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRemoveReachableFilesSparkAction.java
b/spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteReachableFilesSparkAction.java
similarity index 88%
rename from
spark/src/main/java/org/apache/iceberg/spark/actions/BaseRemoveReachableFilesSparkAction.java
rename to
spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteReachableFilesSparkAction.java
index dcbfe6a..6534617 100644
---
a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseRemoveReachableFilesSparkAction.java
+++
b/spark/src/main/java/org/apache/iceberg/spark/actions/BaseDeleteReachableFilesSparkAction.java
@@ -28,8 +28,8 @@ import org.apache.iceberg.ReachableFileUtil;
import org.apache.iceberg.Table;
import org.apache.iceberg.TableMetadata;
import org.apache.iceberg.TableMetadataParser;
-import org.apache.iceberg.actions.BaseRemoveFilesActionResult;
-import org.apache.iceberg.actions.RemoveReachableFiles;
+import org.apache.iceberg.actions.BaseDeleteReachableFilesActionResult;
+import org.apache.iceberg.actions.DeleteReachableFiles;
import org.apache.iceberg.exceptions.NotFoundException;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.hadoop.HadoopFileIO;
@@ -51,13 +51,13 @@ import static org.apache.iceberg.TableProperties.GC_ENABLED;
import static org.apache.iceberg.TableProperties.GC_ENABLED_DEFAULT;
/**
- * An implementation of {@link RemoveReachableFiles} that uses metadata tables
in Spark
+ * An implementation of {@link DeleteReachableFiles} that uses metadata tables
in Spark
* to determine which files should be deleted.
*/
@SuppressWarnings("UnnecessaryAnonymousClass")
-public class BaseRemoveReachableFilesSparkAction
- extends BaseSparkAction<RemoveReachableFiles, RemoveReachableFiles.Result>
implements RemoveReachableFiles {
- private static final Logger LOG =
LoggerFactory.getLogger(BaseRemoveReachableFilesSparkAction.class);
+public class BaseDeleteReachableFilesSparkAction
+ extends BaseSparkAction<DeleteReachableFiles, DeleteReachableFiles.Result>
implements DeleteReachableFiles {
+ private static final Logger LOG =
LoggerFactory.getLogger(BaseDeleteReachableFilesSparkAction.class);
private static final String DATA_FILE = "Data File";
private static final String MANIFEST = "Manifest";
@@ -82,7 +82,7 @@ public class BaseRemoveReachableFilesSparkAction
private ExecutorService removeExecutorService =
DEFAULT_DELETE_EXECUTOR_SERVICE;
private FileIO io = new HadoopFileIO(spark().sessionState().newHadoopConf());
- public BaseRemoveReachableFilesSparkAction(SparkSession spark, String
metadataLocation) {
+ public BaseDeleteReachableFilesSparkAction(SparkSession spark, String
metadataLocation) {
super(spark);
this.tableMetadata = TableMetadataParser.read(io, metadataLocation);
ValidationException.check(
@@ -91,25 +91,25 @@ public class BaseRemoveReachableFilesSparkAction
}
@Override
- protected RemoveReachableFiles self() {
+ protected DeleteReachableFiles self() {
return this;
}
@Override
- public RemoveReachableFiles io(FileIO fileIO) {
+ public DeleteReachableFiles io(FileIO fileIO) {
this.io = fileIO;
return this;
}
@Override
- public RemoveReachableFiles deleteWith(Consumer<String> removeFn) {
- this.removeFunc = removeFn;
+ public DeleteReachableFiles deleteWith(Consumer<String> deleteFunc) {
+ this.removeFunc = deleteFunc;
return this;
}
@Override
- public RemoveReachableFiles executeDeleteWith(ExecutorService
executorService) {
+ public DeleteReachableFiles executeDeleteWith(ExecutorService
executorService) {
this.removeExecutorService = executorService;
return this;
}
@@ -158,7 +158,7 @@ public class BaseRemoveReachableFilesSparkAction
* @param deleted an Iterator of Spark Rows of the structure (path: String,
type: String)
* @return Statistics on which files were deleted
*/
- private BaseRemoveFilesActionResult deleteFiles(Iterator<Row> deleted) {
+ private BaseDeleteReachableFilesActionResult deleteFiles(Iterator<Row>
deleted) {
AtomicLong dataFileCount = new AtomicLong(0L);
AtomicLong manifestCount = new AtomicLong(0L);
AtomicLong manifestListCount = new AtomicLong(0L);
@@ -198,7 +198,7 @@ public class BaseRemoveReachableFilesSparkAction
long filesCount = dataFileCount.get() + manifestCount.get() +
manifestListCount.get() + otherFilesCount.get();
LOG.info("Total files removed: {}", filesCount);
- return new BaseRemoveFilesActionResult(dataFileCount.get(),
manifestCount.get(), manifestListCount.get(),
+ return new BaseDeleteReachableFilesActionResult(dataFileCount.get(),
manifestCount.get(), manifestListCount.get(),
otherFilesCount.get());
}
}
diff --git
a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseSparkActions.java
b/spark/src/main/java/org/apache/iceberg/spark/actions/BaseSparkActions.java
index 65b69b7..58b5717 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/actions/BaseSparkActions.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/actions/BaseSparkActions.java
@@ -21,9 +21,9 @@ package org.apache.iceberg.spark.actions;
import org.apache.iceberg.Table;
import org.apache.iceberg.actions.ActionsProvider;
+import org.apache.iceberg.actions.DeleteOrphanFiles;
+import org.apache.iceberg.actions.DeleteReachableFiles;
import org.apache.iceberg.actions.ExpireSnapshots;
-import org.apache.iceberg.actions.RemoveOrphanFiles;
-import org.apache.iceberg.actions.RemoveReachableFiles;
import org.apache.iceberg.actions.RewriteManifests;
import org.apache.spark.sql.SparkSession;
@@ -40,8 +40,8 @@ abstract class BaseSparkActions implements ActionsProvider {
}
@Override
- public RemoveOrphanFiles removeOrphanFiles(Table table) {
- return new BaseRemoveOrphanFilesSparkAction(spark, table);
+ public DeleteOrphanFiles deleteOrphanFiles(Table table) {
+ return new BaseDeleteOrphanFilesSparkAction(spark, table);
}
@Override
@@ -55,7 +55,7 @@ abstract class BaseSparkActions implements ActionsProvider {
}
@Override
- public RemoveReachableFiles removeReachableFiles(String metadataLocation) {
- return new BaseRemoveReachableFilesSparkAction(spark, metadataLocation);
+ public DeleteReachableFiles deleteReachableFiles(String metadataLocation) {
+ return new BaseDeleteReachableFilesSparkAction(spark, metadataLocation);
}
}
diff --git
a/spark/src/test/java/org/apache/iceberg/actions/TestRemoveReachableFilesAction.java
b/spark/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction.java
similarity index 87%
rename from
spark/src/test/java/org/apache/iceberg/actions/TestRemoveReachableFilesAction.java
rename to
spark/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction.java
index 39d44ee..51db4ad 100644
---
a/spark/src/test/java/org/apache/iceberg/actions/TestRemoveReachableFilesAction.java
+++
b/spark/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction.java
@@ -51,7 +51,7 @@ import org.junit.rules.TemporaryFolder;
import static org.apache.iceberg.types.Types.NestedField.optional;
-public abstract class TestRemoveReachableFilesAction extends SparkTestBase {
+public abstract class TestDeleteReachableFilesAction extends SparkTestBase {
private static final HadoopTables TABLES = new HadoopTables(new
Configuration());
private static final Schema SCHEMA = new Schema(
optional(1, "c1", Types.IntegerType.get()),
@@ -102,15 +102,15 @@ public abstract class TestRemoveReachableFilesAction
extends SparkTestBase {
private void checkRemoveFilesResults(long expectedDatafiles, long
expectedManifestsDeleted,
long expectedManifestListsDeleted, long
expectedOtherFilesDeleted,
- RemoveReachableFiles.Result results) {
+ DeleteReachableFiles.Result results) {
Assert.assertEquals("Incorrect number of manifest files deleted",
- expectedManifestsDeleted, results.removedManifestsCount());
+ expectedManifestsDeleted, results.deletedManifestsCount());
Assert.assertEquals("Incorrect number of datafiles deleted",
- expectedDatafiles, results.removedDataFilesCount());
+ expectedDatafiles, results.deletedDataFilesCount());
Assert.assertEquals("Incorrect number of manifest lists deleted",
- expectedManifestListsDeleted, results.removedManifestListsCount());
+ expectedManifestListsDeleted, results.deletedManifestListsCount());
Assert.assertEquals("Incorrect number of other lists deleted",
- expectedOtherFilesDeleted, results.otherRemovedFilesCount());
+ expectedOtherFilesDeleted, results.deletedOtherFilesCount());
}
@Test
@@ -135,7 +135,7 @@ public abstract class TestRemoveReachableFilesAction
extends SparkTestBase {
Set<String> deleteThreads = ConcurrentHashMap.newKeySet();
AtomicInteger deleteThreadsIndex = new AtomicInteger(0);
- RemoveReachableFiles.Result result =
sparkActions().removeReachableFiles(metadataLocation(table))
+ DeleteReachableFiles.Result result =
sparkActions().deleteReachableFiles(metadataLocation(table))
.io(table.io())
.executeDeleteWith(Executors.newFixedThreadPool(4, runnable -> {
Thread thread = new Thread(runnable);
@@ -178,7 +178,7 @@ public abstract class TestRemoveReachableFilesAction
extends SparkTestBase {
.appendFile(FILE_C)
.commit();
- RemoveReachableFiles.Result result =
sparkActions().removeReachableFiles(metadataLocation(table))
+ DeleteReachableFiles.Result result =
sparkActions().deleteReachableFiles(metadataLocation(table))
.io(table.io())
.execute();
@@ -187,7 +187,7 @@ public abstract class TestRemoveReachableFilesAction
extends SparkTestBase {
@Test
public void testRemoveFileActionOnEmptyTable() {
- RemoveReachableFiles.Result result =
sparkActions().removeReachableFiles(metadataLocation(table))
+ DeleteReachableFiles.Result result =
sparkActions().deleteReachableFiles(metadataLocation(table))
.io(table.io())
.execute();
@@ -218,10 +218,10 @@ public abstract class TestRemoveReachableFilesAction
extends SparkTestBase {
.appendFile(FILE_D)
.commit();
- RemoveReachableFiles baseRemoveFilesSparkAction = sparkActions()
- .removeReachableFiles(metadataLocation(table))
+ DeleteReachableFiles baseRemoveFilesSparkAction = sparkActions()
+ .deleteReachableFiles(metadataLocation(table))
.io(table.io());
- RemoveReachableFiles.Result result = baseRemoveFilesSparkAction.execute();
+ DeleteReachableFiles.Result result = baseRemoveFilesSparkAction.execute();
checkRemoveFilesResults(4, 5, 5, 8, result);
}
@@ -236,8 +236,8 @@ public abstract class TestRemoveReachableFilesAction
extends SparkTestBase {
.appendFile(FILE_B)
.commit();
- RemoveReachableFiles baseRemoveFilesSparkAction = sparkActions()
- .removeReachableFiles(metadataLocation(table))
+ DeleteReachableFiles baseRemoveFilesSparkAction = sparkActions()
+ .deleteReachableFiles(metadataLocation(table))
.io(table.io());
checkRemoveFilesResults(2, 2, 2, 4, baseRemoveFilesSparkAction.execute());
}
@@ -254,8 +254,8 @@ public abstract class TestRemoveReachableFilesAction
extends SparkTestBase {
// IO not set explicitly on removeReachableFiles action
// IO defaults to HadoopFileIO
- RemoveReachableFiles baseRemoveFilesSparkAction = sparkActions()
- .removeReachableFiles(metadataLocation(table));
+ DeleteReachableFiles baseRemoveFilesSparkAction = sparkActions()
+ .deleteReachableFiles(metadataLocation(table));
checkRemoveFilesResults(2, 2, 2, 4, baseRemoveFilesSparkAction.execute());
}
@@ -276,7 +276,7 @@ public abstract class TestRemoveReachableFilesAction
extends SparkTestBase {
int jobsBefore = spark.sparkContext().dagScheduler().nextJobId().get();
- RemoveReachableFiles.Result results =
sparkActions().removeReachableFiles(metadataLocation(table))
+ DeleteReachableFiles.Result results =
sparkActions().deleteReachableFiles(metadataLocation(table))
.io(table.io())
.option("stream-results", "true").execute();
@@ -305,18 +305,18 @@ public abstract class TestRemoveReachableFilesAction
extends SparkTestBase {
Assert.assertEquals("Should delete 1 file", 1, result.size());
Assert.assertTrue("Should remove v1 file",
result.get(0).contains("v1.metadata.json"));
- RemoveReachableFiles baseRemoveFilesSparkAction = sparkActions()
- .removeReachableFiles(metadataLocation(table))
+ DeleteReachableFiles baseRemoveFilesSparkAction = sparkActions()
+ .deleteReachableFiles(metadataLocation(table))
.io(table.io());
- RemoveReachableFiles.Result res = baseRemoveFilesSparkAction.execute();
+ DeleteReachableFiles.Result res = baseRemoveFilesSparkAction.execute();
checkRemoveFilesResults(1, 1, 1, 4, res);
}
@Test
public void testEmptyIOThrowsException() {
- RemoveReachableFiles baseRemoveFilesSparkAction = sparkActions()
- .removeReachableFiles(metadataLocation(table))
+ DeleteReachableFiles baseRemoveFilesSparkAction = sparkActions()
+ .deleteReachableFiles(metadataLocation(table))
.io(null);
AssertHelpers.assertThrows("FileIO needs to be set to use RemoveFiles
action",
IllegalArgumentException.class, "File IO cannot be null",
@@ -331,7 +331,7 @@ public abstract class TestRemoveReachableFilesAction
extends SparkTestBase {
AssertHelpers.assertThrows("Should complain about removing files when GC
is disabled",
ValidationException.class, "Cannot remove files: GC is disabled
(deleting files may corrupt other tables)",
- () -> sparkActions().removeReachableFiles(metadataLocation(table)));
+ () -> sparkActions().deleteReachableFiles(metadataLocation(table)));
}
private String metadataLocation(Table tbl) {
diff --git
a/spark/src/test/java/org/apache/iceberg/spark/actions/TestNewRewriteDataFilesAction.java
b/spark/src/test/java/org/apache/iceberg/spark/actions/TestNewRewriteDataFilesAction.java
index 4069d01..0631d03 100644
---
a/spark/src/test/java/org/apache/iceberg/spark/actions/TestNewRewriteDataFilesAction.java
+++
b/spark/src/test/java/org/apache/iceberg/spark/actions/TestNewRewriteDataFilesAction.java
@@ -644,7 +644,7 @@ public abstract class TestNewRewriteDataFilesAction extends
SparkTestBase {
protected void shouldHaveNoOrphans(Table table) {
Assert.assertEquals("Should not have found any orphan files",
ImmutableList.of(),
- actions().removeOrphanFiles(table)
+ actions().deleteOrphanFiles(table)
.olderThan(System.currentTimeMillis())
.execute()
.orphanFileLocations());
diff --git
a/spark2/src/test/java/org/apache/iceberg/actions/TestRemoveReachableFilesAction24.java
b/spark2/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java
similarity index 94%
rename from
spark2/src/test/java/org/apache/iceberg/actions/TestRemoveReachableFilesAction24.java
rename to
spark2/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java
index 134a64e..80c31e9 100644
---
a/spark2/src/test/java/org/apache/iceberg/actions/TestRemoveReachableFilesAction24.java
+++
b/spark2/src/test/java/org/apache/iceberg/actions/TestDeleteReachableFilesAction24.java
@@ -21,7 +21,7 @@ package org.apache.iceberg.actions;
import org.apache.iceberg.spark.actions.SparkActions;
-public class TestRemoveReachableFilesAction24 extends
TestRemoveReachableFilesAction {
+public class TestDeleteReachableFilesAction24 extends
TestDeleteReachableFilesAction {
@Override
ActionsProvider sparkActions() {
diff --git
a/spark3/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java
b/spark3/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java
index f1dcf31..84703db 100644
---
a/spark3/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java
+++
b/spark3/src/main/java/org/apache/iceberg/spark/procedures/RemoveOrphanFilesProcedure.java
@@ -21,7 +21,7 @@ package org.apache.iceberg.spark.procedures;
import java.util.concurrent.TimeUnit;
import org.apache.iceberg.actions.Actions;
-import org.apache.iceberg.actions.RemoveOrphanFiles;
+import org.apache.iceberg.actions.DeleteOrphanFiles;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.spark.procedures.SparkProcedures.ProcedureBuilder;
import org.apache.iceberg.util.DateTimeUtil;
@@ -84,7 +84,7 @@ public class RemoveOrphanFilesProcedure extends BaseProcedure
{
boolean dryRun = args.isNullAt(3) ? false : args.getBoolean(3);
return withIcebergTable(tableIdent, table -> {
- RemoveOrphanFiles action = actions().removeOrphanFiles(table);
+ DeleteOrphanFiles action = actions().deleteOrphanFiles(table);
if (olderThanMillis != null) {
boolean isTesting =
Boolean.parseBoolean(spark().conf().get("spark.testing", "false"));
@@ -102,13 +102,13 @@ public class RemoveOrphanFilesProcedure extends
BaseProcedure {
action.deleteWith(file -> { });
}
- RemoveOrphanFiles.Result result = action.execute();
+ DeleteOrphanFiles.Result result = action.execute();
return toOutputRows(result);
});
}
- private InternalRow[] toOutputRows(RemoveOrphanFiles.Result result) {
+ private InternalRow[] toOutputRows(DeleteOrphanFiles.Result result) {
Iterable<String> orphanFileLocations = result.orphanFileLocations();
int orphanFileLocationsCount = Iterables.size(orphanFileLocations);
diff --git
a/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java
b/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java
index a7b4203..838741a 100644
---
a/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java
+++
b/spark3/src/test/java/org/apache/iceberg/actions/TestRemoveFilesAction3.java
@@ -21,7 +21,7 @@ package org.apache.iceberg.actions;
import org.apache.iceberg.spark.actions.SparkActions;
-public class TestRemoveFilesAction3 extends TestRemoveReachableFilesAction {
+public class TestRemoveFilesAction3 extends TestDeleteReachableFilesAction {
@Override
ActionsProvider sparkActions() {
return SparkActions.get();
