amogh-jahagirdar commented on code in PR #5379:
URL: https://github.com/apache/iceberg/pull/5379#discussion_r942912194
##########
aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIO.java:
##########
@@ -182,41 +184,55 @@ public void deleteFiles(Iterable<String> paths) throws
BulkDeletionFailureExcept
.run(path -> tagFileToDelete(path, awsProperties.s3DeleteTags()));
}
- if (!awsProperties.isS3DeleteEnabled()) {
- return;
- }
+ if (awsProperties.isS3DeleteEnabled()) {
+ SetMultimap<String, String> bucketToObjects =
+ Multimaps.newSetMultimap(Maps.newHashMap(), Sets::newHashSet);
+ List<Future<List<String>>> deletionTasks = Lists.newArrayList();
+ for (String path : paths) {
+ S3URI location = new S3URI(path,
awsProperties.s3BucketToAccessPointMapping());
+ String bucket = location.bucket();
+ String objectKey = location.key();
+ bucketToObjects.get(bucket).add(objectKey);
+ if (bucketToObjects.get(bucket).size() ==
awsProperties.s3FileIoDeleteBatchSize()) {
+ Set<String> keys = Sets.newHashSet(bucketToObjects.get(bucket));
+ Future<List<String>> deletionTask =
+ executorService().submit(() -> deleteBatch(bucket, keys));
+ deletionTasks.add(deletionTask);
+ bucketToObjects.removeAll(bucket);
+ }
+ }
- SetMultimap<String, String> bucketToObjects =
- Multimaps.newSetMultimap(Maps.newHashMap(), Sets::newHashSet);
- int numberOfFailedDeletions = 0;
- for (String path : paths) {
- S3URI location = new S3URI(path,
awsProperties.s3BucketToAccessPointMapping());
- String bucket = location.bucket();
- String objectKey = location.key();
- Set<String> objectsInBucket = bucketToObjects.get(bucket);
- if (objectsInBucket.size() == awsProperties.s3FileIoDeleteBatchSize()) {
- List<String> failedDeletionsForBatch = deleteObjectsInBucket(bucket,
objectsInBucket);
- numberOfFailedDeletions += failedDeletionsForBatch.size();
- failedDeletionsForBatch.forEach(
- failedPath -> LOG.warn("Failed to delete object at path {}",
failedPath));
- bucketToObjects.removeAll(bucket);
+ // Delete the remainder
+ for (Map.Entry<String, Collection<String>> bucketToObjectsEntry :
+ bucketToObjects.asMap().entrySet()) {
+ Future<List<String>> deletionTask =
+ executorService()
+ .submit(
+ () ->
+ deleteBatch(
+ bucketToObjectsEntry.getKey(),
bucketToObjectsEntry.getValue()));
+ deletionTasks.add(deletionTask);
}
- bucketToObjects.get(bucket).add(objectKey);
- }
- // Delete the remainder
- for (Map.Entry<String, Collection<String>> bucketToObjectsEntry :
- bucketToObjects.asMap().entrySet()) {
- final String bucket = bucketToObjectsEntry.getKey();
- final Collection<String> objects = bucketToObjectsEntry.getValue();
- List<String> failedDeletions = deleteObjectsInBucket(bucket, objects);
- failedDeletions.forEach(
- failedPath -> LOG.warn("Failed to delete object at path {}",
failedPath));
- numberOfFailedDeletions += failedDeletions.size();
- }
+ int totalFailedDeletions = 0;
+
+ for (Future<List<String>> deletionTask : deletionTasks) {
+ try {
+ List<String> failedDeletions = deletionTask.get();
+ failedDeletions.forEach(
+ failedPath -> LOG.warn("Failed to delete object at path {}",
failedPath));
+ totalFailedDeletions += failedDeletions.size();
+ } catch (ExecutionException e) {
+ LOG.warn("Caught unexpected exception during batch deletion: ",
e.getCause());
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted when waiting for deletions
to complete", e);
Review Comment:
Thanks for the pointer @aokolnychyi , this is a pattern I will take a look
at and clean it up in a subsequent PR if it applies!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
