amogh-jahagirdar commented on code in PR #5379:
URL: https://github.com/apache/iceberg/pull/5379#discussion_r939451203
##########
aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIO.java:
##########
@@ -182,41 +184,55 @@ public void deleteFiles(Iterable<String> paths) throws
BulkDeletionFailureExcept
.run(path -> tagFileToDelete(path, awsProperties.s3DeleteTags()));
}
- if (!awsProperties.isS3DeleteEnabled()) {
- return;
- }
+ if (awsProperties.isS3DeleteEnabled()) {
+ SetMultimap<String, String> bucketToObjects =
+ Multimaps.newSetMultimap(Maps.newHashMap(), Sets::newHashSet);
+ List<Future<List<String>>> deletionTasks = Lists.newArrayList();
+ for (String path : paths) {
+ S3URI location = new S3URI(path,
awsProperties.s3BucketToAccessPointMapping());
+ String bucket = location.bucket();
+ String objectKey = location.key();
+ bucketToObjects.get(bucket).add(objectKey);
+ if (bucketToObjects.get(bucket).size() ==
awsProperties.s3FileIoDeleteBatchSize()) {
+ Set<String> keys = Sets.newHashSet(bucketToObjects.get(bucket));
+ Future<List<String>> deletionTask =
+ executorService().submit(() -> deleteBatch(bucket, keys));
+ deletionTasks.add(deletionTask);
+ bucketToObjects.removeAll(bucket);
+ }
+ }
- SetMultimap<String, String> bucketToObjects =
- Multimaps.newSetMultimap(Maps.newHashMap(), Sets::newHashSet);
- int numberOfFailedDeletions = 0;
- for (String path : paths) {
- S3URI location = new S3URI(path,
awsProperties.s3BucketToAccessPointMapping());
- String bucket = location.bucket();
- String objectKey = location.key();
- Set<String> objectsInBucket = bucketToObjects.get(bucket);
- if (objectsInBucket.size() == awsProperties.s3FileIoDeleteBatchSize()) {
- List<String> failedDeletionsForBatch = deleteObjectsInBucket(bucket,
objectsInBucket);
- numberOfFailedDeletions += failedDeletionsForBatch.size();
- failedDeletionsForBatch.forEach(
- failedPath -> LOG.warn("Failed to delete object at path {}",
failedPath));
- bucketToObjects.removeAll(bucket);
+ // Delete the remainder
+ for (Map.Entry<String, Collection<String>> bucketToObjectsEntry :
+ bucketToObjects.asMap().entrySet()) {
+ Future<List<String>> deletionTask =
+ executorService()
+ .submit(
+ () ->
+ deleteBatch(
+ bucketToObjectsEntry.getKey(),
bucketToObjectsEntry.getValue()));
+ deletionTasks.add(deletionTask);
}
- bucketToObjects.get(bucket).add(objectKey);
- }
- // Delete the remainder
- for (Map.Entry<String, Collection<String>> bucketToObjectsEntry :
- bucketToObjects.asMap().entrySet()) {
- final String bucket = bucketToObjectsEntry.getKey();
- final Collection<String> objects = bucketToObjectsEntry.getValue();
- List<String> failedDeletions = deleteObjectsInBucket(bucket, objects);
- failedDeletions.forEach(
- failedPath -> LOG.warn("Failed to delete object at path {}",
failedPath));
- numberOfFailedDeletions += failedDeletions.size();
- }
+ int numberFailedDeletions = 0;
Review Comment:
failedDeletions is already used for the task level failed deletions. I
renamed the overall counter to totalFailedDeletions so it's more obvious, since
it represents the total number of failures across all tasks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
