JackDrogon commented on code in PR #25847:
URL: https://github.com/apache/doris/pull/25847#discussion_r1374133116
##########
fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java:
##########
@@ -223,6 +228,52 @@ public Status deleteObject(String remotePath) {
}
}
+ @Override
+ public Status deleteObjects(String absolutePath) {
+ try {
+ S3URI baseUri = S3URI.create(absolutePath, forceHostedStyle);
+ String continuationToken = "";
+ boolean isTruncated = false;
+ long totalObjects = 0;
+ do {
+ RemoteObjects objects = listObjects(absolutePath,
continuationToken);
+ List<RemoteObject> objectList = objects.getObjectList();
+ if (!objectList.isEmpty()) {
+ Delete delete = Delete.builder()
+ .objects(objectList.stream()
+ .map(RemoteObject::getKey)
+ .map(k ->
ObjectIdentifier.builder().key(k).build())
+ .collect(Collectors.toList()))
+ .build();
+ DeleteObjectsRequest req = DeleteObjectsRequest.builder()
+ .bucket(baseUri.getBucket())
+ .delete(delete)
+ .build();
+
+ DeleteObjectsResponse resp =
getClient(baseUri.getVirtualBucket()).deleteObjects(req);
+ if (resp.errors().size() > 0) {
+ LOG.warn("{} errors returned while deleting {} objects
for dir {}",
+ resp.errors().size(), objectList.size(),
absolutePath);
+ }
+ LOG.info("{} of {} objects deleted for dir {}",
+ resp.deleted().size(), objectList.size(),
absolutePath);
+ totalObjects += objectList.size();
+ }
+
+ isTruncated = objects.isTruncated();
+ continuationToken = objects.getContinuationToken();
+ } while (isTruncated);
+ LOG.info("total delete {} objects for dir {}", totalObjects,
absolutePath);
+ return Status.OK;
+ } catch (DdlException e) {
+ return new Status(Status.ErrCode.COMMON_ERROR, "list objects for
delete objects failed: " + e.getMessage());
+ } catch (Exception e) {
Review Comment:
what difference between DdlException and other Exceptions
##########
fe/fe-core/src/main/java/org/apache/doris/fs/obj/S3ObjStorage.java:
##########
@@ -249,9 +300,26 @@ public Status copyObject(String origFilePath, String
destFilePath) {
public RemoteObjects listObjects(String absolutePath, String
continuationToken) throws DdlException {
try {
S3URI uri = S3URI.create(absolutePath, forceHostedStyle);
+ String bucket = uri.getBucket();
String prefix = uri.getKey();
- ListObjectsV2Request.Builder requestBuilder =
ListObjectsV2Request.builder().bucket(uri.getBucket())
- .prefix(normalizePrefix(prefix));
+ if (!StringUtils.isEmpty(uri.getVirtualBucket())) {
+ // Support s3 compatible service. The generated HTTP request
for list objects likes:
+ //
+ // GET /<bucket-name>?list-type=2&prefix=<prefix>
+ prefix = bucket + "/" + prefix;
+ String endpoint = properties.get(S3Properties.ENDPOINT);
+ if (endpoint.contains("cos.")) {
+ bucket = "/";
+ } else if (endpoint.contains("oss-")) {
+ bucket = uri.getVirtualBucket();
+ } else if (endpoint.contains("obs.")) {
Review Comment:
maybe throw an exception
##########
fe/fe-core/src/main/java/org/apache/doris/backup/Repository.java:
##########
@@ -215,6 +216,26 @@ public Status initRepository() {
if (FeConstants.runningUnitTest) {
return Status.OK;
}
+
+ // A temporary solution is to delete all stale snapshots before
creating an S3 repository
+ // so that we can add regression tests about backup/restore.
+ //
+ // TODO: support hdfs/brokers
+ if (fileSystem instanceof S3FileSystem) {
+ String deleteStaledSnapshots =
fileSystem.getProperties().getOrDefault(PROPERTY_DELETE_IF_EXISTS, "false");
+ if (deleteStaledSnapshots.equalsIgnoreCase("true")) {
Review Comment:
check "true" as true/false in analyze
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
