steveloughran commented on code in PR #15111:
URL: https://github.com/apache/iceberg/pull/15111#discussion_r2823722067
##########
core/src/main/java/org/apache/iceberg/hadoop/HadoopFileIO.java:
##########
@@ -214,14 +215,60 @@ private ExecutorService executorService() {
return executorService;
}
+ /**
+ * Is a path in a schema of a filesystem where the hadoop trash policy
should be used to move to
+ * it to trash?
+ *
+ * @param toDelete path to delete
+ * @return true if the path is in the list of schemas for which the
+ */
+ @VisibleForTesting
+ boolean isTrashSchema(Path toDelete) {
+ final String scheme = toDelete.toUri().getScheme();
+ for (String s : getConf().getTrimmedStrings(DELETE_TRASH_SCHEMAS,
DEFAULT_TRASH_SCHEMAS)) {
+ if (s.equalsIgnoreCase(scheme)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Delete a path.
+ *
+ * <p>If the filesystem is in the trash schemas, it will attempt to move it
to trash. If there is
+ * any failure to move to trash then the fallback is to delete the path. As
a result: when this
+ * operation returns there will not be a file/dir at the target path.
+ *
+ * @param fs target filesystem.
+ * @param toDelete path to delete/move
+ * @param recursive should the delete operation be recursive?
+ * @throws IOException on a delete failure.
+ */
private void deletePath(FileSystem fs, Path toDelete, boolean recursive)
throws IOException {
- Trash trash = new Trash(fs, getConf());
- if ((fs instanceof LocalFileSystem || fs instanceof DistributedFileSystem)
- && trash.isEnabled()) {
- trash.moveToTrash(toDelete);
- } else {
- fs.delete(toDelete, recursive);
+ if (isTrashSchema(toDelete)) {
+ try {
+ // use moveToAppropriateTrash() which not only resolves through
mounted filesystems like
+ // viewfs,
+ // it will query the resolved filesystem for its trash policy.
+ // The HDFS client will ask the remote server for its configuration,
rather than
+ // what the client is configured with.
+ if (Trash.moveToAppropriateTrash(fs, toDelete, fs.getConf())) {
+ // trash enabled and operation successful.
+ return;
+ }
+ } catch (FileNotFoundException e) {
+ // the source file is missing. Nothing to do.
+ return;
+ } catch (IOException e) {
+ // other failure (failure to get remote server trash policy,
rename,...)
+ // log one line at info, full stack at debug, so a failure to move
many files to trash
+ // doesn't flood the log
+ LOG.info("Failed to move {} to trash: {}", toDelete, e.toString());
+ LOG.debug("Trash.moveToAppropriateTrash failure", e);
Review Comment:
could be noisy if something is playing up, but OK
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
