zachjsh commented on a change in pull request #9459: Ability to Delete task
logs and segments from S3
URL: https://github.com/apache/druid/pull/9459#discussion_r390538553
##########
File path:
extensions-core/s3-extensions/src/main/java/org/apache/druid/storage/s3/S3Utils.java
##########
@@ -200,6 +204,54 @@ public static S3ObjectSummary
getSingleObjectSummary(ServerSideEncryptingAmazonS
return objectSummary;
}
+ public static void deleteObjectsInPath(
+ ServerSideEncryptingAmazonS3 s3Client,
+ S3InputDataConfig config,
+ String bucket,
+ String prefix,
+ Predicate<S3ObjectSummary> filter
+ )
+ throws Exception
+ {
+ final List<DeleteObjectsRequest.KeyVersion> keysToDelete = new
ArrayList<>(config.getMaxListingLength());
+ final ObjectSummaryIterator iterator = new ObjectSummaryIterator(
+ s3Client,
+ ImmutableList.of(new CloudObjectLocation(bucket, prefix).toUri("s3")),
+ config.getMaxListingLength()
+ );
+
+ while (iterator.hasNext()) {
+ final S3ObjectSummary nextObject = iterator.next();
+ if (filter.apply(nextObject)) {
+ keysToDelete.add(new
DeleteObjectsRequest.KeyVersion(nextObject.getKey()));
+ if (keysToDelete.size() == config.getMaxListingLength()) {
+ deleteBucketKeys(s3Client, bucket, keysToDelete);
+ log.info("Deleted %d files", keysToDelete.size());
+ keysToDelete.clear();
+ }
+ }
+ }
+
+ if (keysToDelete.size() > 0) {
+ deleteBucketKeys(s3Client, bucket, keysToDelete);
+ log.info("Deleted %d files", keysToDelete.size());
+ }
+ }
+
+ public static void deleteBucketKeys(
Review comment:
I'll get this in the next change which should be coming shortly.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
