jackye1995 commented on a change in pull request #4342:
URL: https://github.com/apache/iceberg/pull/4342#discussion_r828370707
##########
File path: aws/src/main/java/org/apache/iceberg/aws/s3/S3FileIO.java
##########
@@ -128,36 +137,65 @@ public void deleteFile(String path) {
*/
@Override
public void deleteFiles(Iterable<String> paths) throws
BulkDeletionFailureException {
- SetMultimap<String, String> bucketToObjects =
Multimaps.newSetMultimap(Maps.newHashMap(), Sets::newHashSet);
- int numberOfFailedDeletions = 0;
- for (String path : paths) {
- S3URI location = new S3URI(path);
- String bucket = location.bucket();
- String objectKey = location.key();
- Set<String> objectsInBucket = bucketToObjects.get(bucket);
- if (objectsInBucket.size() == awsProperties.s3FileIoDeleteBatchSize()) {
- List<String> failedDeletionsForBatch = deleteObjectsInBucket(bucket,
objectsInBucket);
- numberOfFailedDeletions += failedDeletionsForBatch.size();
- failedDeletionsForBatch.forEach(failedPath -> LOG.warn("Failed to
delete object at path {}", failedPath));
- bucketToObjects.removeAll(bucket);
+ if (deleteTags.isEmpty()) {
+ SetMultimap<String, String> bucketToObjects = Multimaps
+ .newSetMultimap(Maps.newHashMap(), Sets::newHashSet);
+ int numberOfFailedDeletions = 0;
+ for (String path : paths) {
+ S3URI location = new S3URI(path);
+ String bucket = location.bucket();
+ String objectKey = location.key();
+ Set<String> objectsInBucket = bucketToObjects.get(bucket);
+ if (objectsInBucket.size() == awsProperties.s3FileIoDeleteBatchSize())
{
+ List<String> failedDeletionsForBatch = deleteObjectsInBucket(bucket,
objectsInBucket);
+ numberOfFailedDeletions += failedDeletionsForBatch.size();
+ failedDeletionsForBatch
+ .forEach(failedPath -> LOG.warn("Failed to delete object at path
{}", failedPath));
+ bucketToObjects.removeAll(bucket);
+ }
+ bucketToObjects.get(bucket).add(objectKey);
}
- bucketToObjects.get(bucket).add(objectKey);
- }
- // Delete the remainder
- for (Map.Entry<String, Collection<String>> bucketToObjectsEntry :
bucketToObjects.asMap().entrySet()) {
- final String bucket = bucketToObjectsEntry.getKey();
- final Collection<String> objects = bucketToObjectsEntry.getValue();
- List<String> failedDeletions = deleteObjectsInBucket(bucket, objects);
- failedDeletions.forEach(failedPath -> LOG.warn("Failed to delete object
at path {}", failedPath));
- numberOfFailedDeletions += failedDeletions.size();
- }
+ // Delete the remainder
+ for (Map.Entry<String, Collection<String>> bucketToObjectsEntry :
bucketToObjects.asMap()
+ .entrySet()) {
+ final String bucket = bucketToObjectsEntry.getKey();
+ final Collection<String> objects = bucketToObjectsEntry.getValue();
+ List<String> failedDeletions = deleteObjectsInBucket(bucket, objects);
+ failedDeletions
+ .forEach(failedPath -> LOG.warn("Failed to delete object at path
{}", failedPath));
+ numberOfFailedDeletions += failedDeletions.size();
+ }
- if (numberOfFailedDeletions > 0) {
- throw new BulkDeletionFailureException(numberOfFailedDeletions);
+ if (numberOfFailedDeletions > 0) {
+ throw new BulkDeletionFailureException(numberOfFailedDeletions);
+ }
+ } else {
+ paths.forEach(this::doSoftDelete);
}
}
+ private void doSoftDelete(String path) {
+ S3URI location = new S3URI(path);
+ String bucket = location.bucket();
+ String objectKey = location.key();
+ GetObjectTaggingRequest getObjectTaggingRequest =
GetObjectTaggingRequest.builder()
+ .bucket(bucket)
+ .key(objectKey)
+ .build();
+ GetObjectTaggingResponse getObjectTaggingResponse = client()
+ .getObjectTagging(getObjectTaggingRequest);
+ // Get existing tags, if any and then add the delete tags
+ Set<Tag> tags = Sets.newHashSet(getObjectTaggingResponse.tagSet());
Review comment:
we can use the method `response.hasTagSet()` to avoid `tagSet`
potentially being null
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
