jackye1995 commented on a change in pull request #4342:
URL: https://github.com/apache/iceberg/pull/4342#discussion_r828374563
##########
File path: aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIO.java
##########
@@ -128,36 +137,65 @@ public void deleteFile(String path) {
*/
@Override
public void deleteFiles(Iterable<String> paths) throws
BulkDeletionFailureException {
- SetMultimap<String, String> bucketToObjects =
Multimaps.newSetMultimap(Maps.newHashMap(), Sets::newHashSet);
- int numberOfFailedDeletions = 0;
- for (String path : paths) {
- S3URI location = new S3URI(path);
- String bucket = location.bucket();
- String objectKey = location.key();
- Set<String> objectsInBucket = bucketToObjects.get(bucket);
- if (objectsInBucket.size() == awsProperties.s3FileIoDeleteBatchSize()) {
- List<String> failedDeletionsForBatch = deleteObjectsInBucket(bucket,
objectsInBucket);
- numberOfFailedDeletions += failedDeletionsForBatch.size();
- failedDeletionsForBatch.forEach(failedPath -> LOG.warn("Failed to
delete object at path {}", failedPath));
- bucketToObjects.removeAll(bucket);
+ if (deleteTags.isEmpty()) {
+ SetMultimap<String, String> bucketToObjects = Multimaps
+ .newSetMultimap(Maps.newHashMap(), Sets::newHashSet);
+ int numberOfFailedDeletions = 0;
+ for (String path : paths) {
+ S3URI location = new S3URI(path);
+ String bucket = location.bucket();
+ String objectKey = location.key();
+ Set<String> objectsInBucket = bucketToObjects.get(bucket);
+ if (objectsInBucket.size() == awsProperties.s3FileIoDeleteBatchSize())
{
+ List<String> failedDeletionsForBatch = deleteObjectsInBucket(bucket,
objectsInBucket);
+ numberOfFailedDeletions += failedDeletionsForBatch.size();
+ failedDeletionsForBatch
+ .forEach(failedPath -> LOG.warn("Failed to delete object at path
{}", failedPath));
+ bucketToObjects.removeAll(bucket);
+ }
+ bucketToObjects.get(bucket).add(objectKey);
}
- bucketToObjects.get(bucket).add(objectKey);
- }
- // Delete the remainder
- for (Map.Entry<String, Collection<String>> bucketToObjectsEntry :
bucketToObjects.asMap().entrySet()) {
- final String bucket = bucketToObjectsEntry.getKey();
- final Collection<String> objects = bucketToObjectsEntry.getValue();
- List<String> failedDeletions = deleteObjectsInBucket(bucket, objects);
- failedDeletions.forEach(failedPath -> LOG.warn("Failed to delete object
at path {}", failedPath));
- numberOfFailedDeletions += failedDeletions.size();
- }
+ // Delete the remainder
+ for (Map.Entry<String, Collection<String>> bucketToObjectsEntry :
bucketToObjects.asMap()
+ .entrySet()) {
+ final String bucket = bucketToObjectsEntry.getKey();
+ final Collection<String> objects = bucketToObjectsEntry.getValue();
+ List<String> failedDeletions = deleteObjectsInBucket(bucket, objects);
+ failedDeletions
+ .forEach(failedPath -> LOG.warn("Failed to delete object at path
{}", failedPath));
+ numberOfFailedDeletions += failedDeletions.size();
+ }
- if (numberOfFailedDeletions > 0) {
- throw new BulkDeletionFailureException(numberOfFailedDeletions);
+ if (numberOfFailedDeletions > 0) {
+ throw new BulkDeletionFailureException(numberOfFailedDeletions);
+ }
+ } else {
+ paths.forEach(this::doSoftDelete);
}
}
+ private void doSoftDelete(String path) {
+ S3URI location = new S3URI(path);
+ String bucket = location.bucket();
+ String objectKey = location.key();
+ GetObjectTaggingRequest getObjectTaggingRequest =
GetObjectTaggingRequest.builder()
+ .bucket(bucket)
+ .key(objectKey)
+ .build();
+ GetObjectTaggingResponse getObjectTaggingResponse = client()
+ .getObjectTagging(getObjectTaggingRequest);
+ // Get existing tags, if any and then add the delete tags
+ Set<Tag> tags = Sets.newHashSet(getObjectTaggingResponse.tagSet());
+ tags.addAll(deleteTags);
+ PutObjectTaggingRequest putObjectTaggingRequest =
PutObjectTaggingRequest.builder()
Review comment:
this might throw limit exceeded exception because an object can have max
10 tags. But I guess that's fine, we are not catching any delete failures in
the original `s3.deleteObject`, caller needs to catch `S3Exception` in general.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
