This is an automated email from the ASF dual-hosted git repository.
sadanand48 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 2b5874b743b HDDS-16226. Tool to obtain snapshot trapped bytes for a
bucket. (#11122)
2b5874b743b is described below
commit 2b5874b743b0faba5c4cc4f125030ca4c3af1469
Author: Sadanand Shenoy <[email protected]>
AuthorDate: Wed Sep 30 13:55:18 2026 +0530
HDDS-16226. Tool to obtain snapshot trapped bytes for a bucket. (#11122)
---
.../admin/om/BucketDeletedBytesSubCommand.java | 101 ++++++
.../org/apache/hadoop/ozone/admin/om/OMAdmin.java | 1 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 2 +
.../ozone/om/helpers/BucketDeletedBytes.java | 79 +++++
.../ozone/om/protocol/OzoneManagerProtocol.java | 8 +
...OzoneManagerProtocolClientSideTranslatorPB.java | 25 ++
.../java/org/apache/hadoop/ozone/TestOmUtils.java | 10 +
.../src/main/proto/OmClientProtocol.proto | 16 +
.../org/apache/hadoop/ozone/om/OzoneManager.java | 43 +++
.../trapped/BucketDeletedDataCalculator.java | 373 +++++++++++++++++++++
.../ozone/om/snapshot/trapped/package-info.java | 21 ++
.../protocolPB/OzoneManagerRequestHandler.java | 23 ++
.../trapped/TestBucketDeletedDataCalculator.java | 243 ++++++++++++++
13 files changed, 945 insertions(+)
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/BucketDeletedBytesSubCommand.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/BucketDeletedBytesSubCommand.java
new file mode 100644
index 00000000000..db1360b03db
--- /dev/null
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/BucketDeletedBytesSubCommand.java
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.admin.om;
+
+import java.util.LinkedHashMap;
+import java.util.Map;
+import java.util.concurrent.Callable;
+import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.server.JsonUtils;
+import org.apache.hadoop.ozone.om.helpers.BucketDeletedBytes;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import picocli.CommandLine;
+
+/**
+ * Handler of ozone admin om bucket-deleted-bytes command.
+ */
[email protected](
+ name = "bucket-deleted-bytes",
+ aliases = {"bdb"},
+ description = "Shows snapshot-trapped vs purgeable deleted bytes for a
bucket.",
+ mixinStandardHelpOptions = true,
+ versionProvider = HddsVersionProvider.class)
+public class BucketDeletedBytesSubCommand implements Callable<Void> {
+
+ @CommandLine.Mixin
+ private OmAddressOptions.OptionalServiceIdOrHostMixin omAddressOptions;
+
+ @CommandLine.Parameters(index = "0", arity = "1",
+ description = "Bucket path in volume/bucket format")
+ private String bucketPath;
+
+ @CommandLine.Option(names = { "--json" }, defaultValue = "false",
+ description = "Print output in JSON format")
+ private boolean json;
+
+ @Override
+ public Void call() throws Exception {
+ try (OzoneManagerProtocol omClient = omAddressOptions.newClient()) {
+ BucketDeletedBytes result = omClient.getBucketDeletedBytes(bucketPath);
+ if (json) {
+ printJson(result);
+ } else {
+ printText(result);
+ }
+ }
+ return null;
+ }
+
+ private void printText(BucketDeletedBytes result) {
+ long totalDeletedBytes =
+ result.getSnapshotTrappedBytes() + result.getPurgeableBytes();
+ long totalDeletedKeys =
+ result.getSnapshotTrappedKeys() + result.getPurgeableKeys();
+ long totalDeletedDirs =
+ result.getSnapshotTrappedDirs() + result.getPurgeableDirs();
+ System.out.println("Bucket: " + bucketPath);
+ System.out.println("SnapshotTrappedBytes: " +
result.getSnapshotTrappedBytes());
+ System.out.println("PurgeableBytes: " + result.getPurgeableBytes());
+ System.out.println("SnapshotTrappedKeys: " +
result.getSnapshotTrappedKeys());
+ System.out.println("PurgeableKeys: " + result.getPurgeableKeys());
+ System.out.println("SnapshotTrappedDirs: " +
result.getSnapshotTrappedDirs());
+ System.out.println("PurgeableDirs: " + result.getPurgeableDirs());
+ System.out.println("TotalDeletedBytes: " + totalDeletedBytes);
+ System.out.println("TotalDeletedKeys: " + totalDeletedKeys);
+ System.out.println("TotalDeletedDirs: " + totalDeletedDirs);
+ }
+
+ private void printJson(BucketDeletedBytes result) throws Exception {
+ Map<String, Object> output = new LinkedHashMap<>();
+ output.put("bucketPath", bucketPath);
+ output.put("snapshotTrappedBytes", result.getSnapshotTrappedBytes());
+ output.put("purgeableBytes", result.getPurgeableBytes());
+ output.put("snapshotTrappedKeys", result.getSnapshotTrappedKeys());
+ output.put("purgeableKeys", result.getPurgeableKeys());
+ output.put("snapshotTrappedDirs", result.getSnapshotTrappedDirs());
+ output.put("purgeableDirs", result.getPurgeableDirs());
+ output.put("totalDeletedBytes",
+ result.getSnapshotTrappedBytes() + result.getPurgeableBytes());
+ output.put("totalDeletedKeys",
+ result.getSnapshotTrappedKeys() + result.getPurgeableKeys());
+ output.put("totalDeletedDirs",
+ result.getSnapshotTrappedDirs() + result.getPurgeableDirs());
+ System.out.println(JsonUtils.toJsonStringWithDefaultPrettyPrinter(output));
+ }
+}
+
diff --git
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java
index c31477e3d35..5eae7fde42a 100644
---
a/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java
+++
b/hadoop-ozone/cli-admin/src/main/java/org/apache/hadoop/ozone/admin/om/OMAdmin.java
@@ -50,6 +50,7 @@
mixinStandardHelpOptions = true,
versionProvider = HddsVersionProvider.class,
subcommands = {
+ BucketDeletedBytesSubCommand.class,
FinalizeUpgradeSubCommand.class,
ListOpenFilesSubCommand.class,
GetServiceRolesSubcommand.class,
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 617e1636440..e118ef68b41 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -264,6 +264,7 @@ public static boolean isReadOnly(OMRequest omRequest) {
case GetSnapshotInfo:
case GetObjectTagging:
case GetBucketTagging:
+ case GetBucketDeletedBytes:
return true;
case GetQuotaRepairStatus:
case StartQuotaRepair:
@@ -398,6 +399,7 @@ public static boolean shouldSendToFollower(OMRequest
omRequest) {
case GetObjectTagging:
case GetLifecycleConfiguration:
case GetLifecycleServiceStatus:
+ case GetBucketDeletedBytes:
return true;
case GetBucketTagging:
return true;
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java
new file mode 100644
index 00000000000..36967b25eca
--- /dev/null
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/BucketDeletedBytes.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.helpers;
+
+/**
+ * Bucket-level deleted-data split for a bucket at query time.
+ * <p>
+ * {@code snapshotTrappedBytes + purgeableBytes} represents the replicated size
+ * currently visible to the on-demand calculator for deleted keys and files
+ * discovered in deleted directory traversal for that bucket.
+ * <p>
+ * These values are not the same as OM global metrics such as
+ * {@code totalPendingDeletionBytes} / {@code totalPendingDeletionKeyCount}
+ * because those are cluster-wide service counters with different timing and
+ * accounting scope.
+ */
+public final class BucketDeletedBytes {
+ private final long snapshotTrappedBytes;
+ private final long purgeableBytes;
+ private final long snapshotTrappedKeys;
+ private final long purgeableKeys;
+ private final long snapshotTrappedDirs;
+ private final long purgeableDirs;
+
+ public BucketDeletedBytes(
+ long snapshotTrappedBytes,
+ long purgeableBytes,
+ long snapshotTrappedKeys,
+ long purgeableKeys,
+ long snapshotTrappedDirs,
+ long purgeableDirs) {
+ this.snapshotTrappedBytes = snapshotTrappedBytes;
+ this.purgeableBytes = purgeableBytes;
+ this.snapshotTrappedKeys = snapshotTrappedKeys;
+ this.purgeableKeys = purgeableKeys;
+ this.snapshotTrappedDirs = snapshotTrappedDirs;
+ this.purgeableDirs = purgeableDirs;
+ }
+
+ public long getSnapshotTrappedBytes() {
+ return snapshotTrappedBytes;
+ }
+
+ public long getPurgeableBytes() {
+ return purgeableBytes;
+ }
+
+ public long getSnapshotTrappedKeys() {
+ return snapshotTrappedKeys;
+ }
+
+ public long getPurgeableKeys() {
+ return purgeableKeys;
+ }
+
+ public long getSnapshotTrappedDirs() {
+ return snapshotTrappedDirs;
+ }
+
+ public long getPurgeableDirs() {
+ return purgeableDirs;
+ }
+}
+
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
index 71c2d6b2294..e76ec341e77 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocol/OzoneManagerProtocol.java
@@ -30,6 +30,7 @@
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo;
+import org.apache.hadoop.ozone.om.helpers.BucketDeletedBytes;
import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo;
import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.DeleteTenantState;
@@ -447,6 +448,13 @@ List<OmBucketInfo> listBuckets(String volumeName,
ListOpenFilesResult listOpenFiles(String path, int maxKeys, String contToken)
throws IOException;
+ /**
+ * Calculate bucket-level deleted bytes split by reclaimability.
+ * @param bucketPath bucket path in volume/bucket format.
+ */
+ BucketDeletedBytes getBucketDeletedBytes(String bucketPath)
+ throws IOException;
+
/**
* Transfer the raft leadership.
*
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
index a6f8c7053c1..835b5010a2f 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/OzoneManagerProtocolClientSideTranslatorPB.java
@@ -61,6 +61,7 @@
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.AssumeRoleResponseInfo;
import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.BucketDeletedBytes;
import org.apache.hadoop.ozone.om.helpers.CallerIdentityInfo;
import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.DeleteTenantState;
@@ -143,6 +144,8 @@
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetAclRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetAclResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketDeletedBytesRequest;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketDeletedBytesResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketTaggingRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketTaggingResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetDelegationTokenResponseProto;
@@ -2067,6 +2070,28 @@ public ListOpenFilesResult listOpenFiles(String path,
resp.getKeyInfoList());
}
+ @Override
+ public BucketDeletedBytes getBucketDeletedBytes(String bucketPath)
+ throws IOException {
+ GetBucketDeletedBytesRequest request = GetBucketDeletedBytesRequest
+ .newBuilder()
+ .setBucketPath(bucketPath)
+ .build();
+ OMRequest omRequest = createOMRequest(Type.GetBucketDeletedBytes)
+ .setGetBucketDeletedBytesRequest(request)
+ .build();
+
+ GetBucketDeletedBytesResponse response =
handleError(submitRequest(omRequest))
+ .getGetBucketDeletedBytesResponse();
+ return new BucketDeletedBytes(
+ response.getSnapshotTrappedBytes(),
+ response.getPurgeableBytes(),
+ response.getSnapshotTrappedKeys(),
+ response.getPurgeableKeys(),
+ response.getSnapshotTrappedDirs(),
+ response.getPurgeableDirs());
+ }
+
@Override
public void transferLeadership(String newLeaderId)
throws IOException {
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
index 1765ad55523..8abfba48a13 100644
--- a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
+++ b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/TestOmUtils.java
@@ -437,6 +437,16 @@ public void testShouldSendToFollowerImpliesIsReadOnly() {
}
}
+ @Test
+ public void
testGetBucketDeletedBytesIsClassifiedAsReadOnlyAndFollowerEligible() {
+ OMRequest request = OMRequest.newBuilder()
+ .setCmdType(OzoneManagerProtocolProtos.Type.GetBucketDeletedBytes)
+ .setClientId(UUID.randomUUID().toString())
+ .build();
+ assertTrue(OmUtils.isReadOnly(request));
+ assertTrue(OmUtils.shouldSendToFollower(request));
+ }
+
@Test
void testOmAddressesBracketIPv6Literals() {
OzoneConfiguration conf = new OzoneConfiguration();
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index 9af165bbcd0..ddc4cd80aab 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -171,6 +171,7 @@ enum Type {
DeleteRevokedSTSTokens = 155;
GetCallerIdentity = 156;
SnapshotCount = 157;
+ GetBucketDeletedBytes = 158;
}
enum SafeMode {
@@ -337,6 +338,7 @@ message OMRequest {
optional UpdateAssumeRoleRequest updateAssumeRoleRequest =
157;
optional GetCallerIdentityRequest getCallerIdentityRequest =
158;
optional SnapshotCountRequest snapshotCountRequest = 159;
+ optional GetBucketDeletedBytesRequest getBucketDeletedBytesRequest =
160;
}
message OMResponse {
@@ -487,6 +489,7 @@ message OMResponse {
optional DeleteRevokedSTSTokensResponse deleteRevokedSTSTokensResponse =
155;
optional GetCallerIdentityResponse getCallerIdentityResponse =
156;
optional SnapshotCountResponse snapshotCountResponse = 157;
+ optional GetBucketDeletedBytesResponse getBucketDeletedBytesResponse =
158;
}
enum Status {
@@ -1767,6 +1770,19 @@ message ListOpenFilesResponse {
repeated KeyInfo keyInfo = 5;
}
+message GetBucketDeletedBytesRequest {
+ required string bucketPath = 1;
+}
+
+message GetBucketDeletedBytesResponse {
+ required uint64 snapshotTrappedBytes = 1;
+ required uint64 purgeableBytes = 2;
+ optional uint64 snapshotTrappedKeys = 3;
+ optional uint64 purgeableKeys = 4;
+ optional uint64 snapshotTrappedDirs = 5;
+ optional uint64 purgeableDirs = 6;
+}
+
message ServicePort {
enum Type {
RPC = 1;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
index fdc6f95b48d..be952b3f6be 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java
@@ -261,6 +261,7 @@
import org.apache.hadoop.ozone.om.ha.OMHANodeDetails;
import org.apache.hadoop.ozone.om.ha.OMServiceManager;
import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.BucketDeletedBytes;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
@@ -312,6 +313,7 @@
import org.apache.hadoop.ozone.om.service.QuotaRepairTask;
import org.apache.hadoop.ozone.om.service.RevokedSTSTokenCleanupService;
import org.apache.hadoop.ozone.om.snapshot.defrag.SnapshotDefragService;
+import org.apache.hadoop.ozone.om.snapshot.trapped.BucketDeletedDataCalculator;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutFeature;
import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager;
import org.apache.hadoop.ozone.om.upgrade.OMUpgradeFinalizer;
@@ -1924,6 +1926,47 @@ public OmSnapshotManager getOmSnapshotManager() {
return omSnapshotManager;
}
+ /**
+ * Compute deleted bytes split for a bucket.
+ *
+ * @return stats containing snapshot-trapped bytes and purgeable bytes.
+ */
+ public BucketDeletedDataCalculator.BucketDeletedBytesStats
+ calculateDeletedBytesForBucket(String volume, String bucket)
+ throws IOException {
+ return new BucketDeletedDataCalculator(this).calculate(volume, bucket);
+ }
+
+ /**
+ * Compute deleted bytes split for a bucket path in {@code volume/bucket}
+ * format.
+ */
+ public BucketDeletedDataCalculator.BucketDeletedBytesStats
+ calculateDeletedBytesForBucket(String bucketPath)
+ throws IOException {
+ String[] tokens = bucketPath.split("/", 2);
+ if (tokens.length != 2 || tokens[0].isEmpty() || tokens[1].isEmpty()) {
+ throw new OMException("bucketPath must be in volume/bucket format: " +
bucketPath, INVALID_PATH);
+ }
+ ResolvedBucket resolvedBucket = resolveBucketLink(Pair.of(tokens[0],
tokens[1]));
+ return calculateDeletedBytesForBucket(resolvedBucket.realVolume(),
resolvedBucket.realBucket());
+ }
+
+ @Override
+ public BucketDeletedBytes getBucketDeletedBytes(String bucketPath)
+ throws IOException {
+ checkAdminUserPrivilege("get bucket deleted bytes.");
+ BucketDeletedDataCalculator.BucketDeletedBytesStats stats =
+ calculateDeletedBytesForBucket(bucketPath);
+ return new BucketDeletedBytes(
+ stats.getSnapshotTrappedBytes(),
+ stats.getPurgeableBytes(),
+ stats.getSnapshotTrappedKeys(),
+ stats.getPurgeableKeys(),
+ stats.getSnapshotTrappedDirs(),
+ stats.getPurgeableDirs());
+ }
+
/**
* Get metadata manager.
*
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java
new file mode 100644
index 00000000000..5fc2b144bee
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/BucketDeletedDataCalculator.java
@@ -0,0 +1,373 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.trapped;
+
+import java.io.IOException;
+import java.util.ArrayDeque;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.Queue;
+import java.util.Set;
+import java.util.UUID;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.Table.KeyValue;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.ozone.om.KeyManager;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.OmSnapshotManager;
+import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.om.SnapshotChainManager;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.snapshot.SnapshotUtils;
+import org.apache.hadoop.ozone.om.snapshot.filter.ReclaimableDirFilter;
+import org.apache.hadoop.ozone.om.snapshot.filter.ReclaimableKeyFilter;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+
+/**
+ * Computes bucket-level deleted bytes split into purgeable and
snapshot-trapped.
+ * <p>
+ * This calculator follows KDS/DDS reclaimability semantics across active OM DB
+ * and all active snapshots in the bucket chain.
+ */
+public class BucketDeletedDataCalculator {
+
+ private final OzoneManager ozoneManager;
+ private final OmSnapshotManager omSnapshotManager;
+ private final SnapshotChainManager snapshotChainManager;
+
+ public BucketDeletedDataCalculator(OzoneManager ozoneManager) {
+ this.ozoneManager = ozoneManager;
+ this.omSnapshotManager = ozoneManager.getOmSnapshotManager();
+ this.snapshotChainManager =
+ ((OmMetadataManagerImpl) ozoneManager.getMetadataManager())
+ .getSnapshotChainManager();
+ }
+
+ public BucketDeletedBytesStats calculate(String volume, String bucket)
+ throws IOException {
+ BucketDeletedBytesStats totals = new BucketDeletedBytesStats();
+ processStore(volume, bucket, null,
+ requireKeyManager(ozoneManager.getKeyManager(), "active store"),
+ totals, false);
+
+ Iterator<UUID> iterator = snapshotChainManager.iterator(true);
+ while (iterator.hasNext()) {
+ UUID snapshotId = iterator.next();
+ SnapshotInfo snapshotInfo = SnapshotUtils.getSnapshotInfo(
+ ozoneManager, snapshotChainManager, snapshotId);
+ if (snapshotInfo == null
+ || snapshotInfo.getSnapshotStatus() !=
SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE
+ || !snapshotInfo.getVolumeName().equals(volume)
+ || !snapshotInfo.getBucketName().equals(bucket)
+ || !OmSnapshotManager.areSnapshotChangesFlushedToDB(
+ ozoneManager.getMetadataManager(), snapshotInfo)) {
+ continue;
+ }
+ // Not deep-cleaned snapshots are still expanded and counted, but treated
+ // as trapped because reclaimability is not final yet.
+ boolean forceSnapshotTrapped = !snapshotInfo.isDeepCleanedDeletedDir();
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshot =
+ omSnapshotManager.getActiveSnapshot(volume, bucket,
snapshotInfo.getName())) {
+ processStore(volume, bucket, snapshotInfo,
+ requireKeyManager(snapshot.get().getKeyManager(),
+ "snapshot " + snapshotInfo.getTableKey()),
+ totals, forceSnapshotTrapped);
+ }
+ }
+ return totals;
+ }
+
+ private static KeyManager requireKeyManager(KeyManager keyManager, String
scope)
+ throws IOException {
+ if (keyManager == null) {
+ throw new IOException("KeyManager is not available for " + scope
+ + ". Ensure OM is fully initialized before running the
calculation.");
+ }
+ return keyManager;
+ }
+
+ private void processStore(
+ String volume,
+ String bucket,
+ SnapshotInfo currentSnapshotInfo,
+ KeyManager keyManager,
+ BucketDeletedBytesStats totals,
+ boolean forceSnapshotTrapped) throws IOException {
+ if (forceSnapshotTrapped) {
+ processDeletedKeysAsTrapped(volume, bucket, keyManager, totals);
+ processDeletedDirsAsTrapped(volume, bucket, keyManager, totals);
+ return;
+ }
+ try (ReclaimableKeyFilter reclaimableKeyFilter = new ReclaimableKeyFilter(
+ ozoneManager,
+ omSnapshotManager,
+ snapshotChainManager,
+ currentSnapshotInfo,
+ keyManager,
+ ozoneManager.getMetadataManager().getLock());
+ ReclaimableDirFilter reclaimableDirFilter = new ReclaimableDirFilter(
+ ozoneManager,
+ omSnapshotManager,
+ snapshotChainManager,
+ currentSnapshotInfo,
+ keyManager,
+ ozoneManager.getMetadataManager().getLock())) {
+
+ processDeletedKeys(volume, bucket, keyManager, reclaimableKeyFilter,
totals);
+ processDeletedDirs(volume, bucket, keyManager, reclaimableKeyFilter,
+ reclaimableDirFilter, totals);
+ }
+ }
+
+ private void processDeletedKeysAsTrapped(
+ String volume,
+ String bucket,
+ KeyManager keyManager,
+ BucketDeletedBytesStats totals) throws IOException {
+ OMMetadataManager metadataManager = keyManager.getMetadataManager();
+ Table<String, RepeatedOmKeyInfo> deletedTable =
metadataManager.getDeletedTable();
+ String deletedTablePrefix = metadataManager.getTableBucketPrefix(
+ deletedTable.getName(), volume, bucket);
+ try (TableIterator<String, ? extends KeyValue<String, RepeatedOmKeyInfo>>
itr =
+ deletedTable.iterator(deletedTablePrefix)) {
+ while (itr.hasNext()) {
+ KeyValue<String, ? extends RepeatedOmKeyInfo> entry = itr.next();
+ for (OmKeyInfo omKeyInfo : entry.getValue().getOmKeyInfoList()) {
+ totals.addSnapshotTrappedKey(omKeyInfo.getReplicatedSize());
+ }
+ }
+ }
+ }
+
+ private void processDeletedKeys(
+ String volume,
+ String bucket,
+ KeyManager keyManager,
+ ReclaimableKeyFilter reclaimableKeyFilter,
+ BucketDeletedBytesStats totals) throws IOException {
+ OMMetadataManager metadataManager = keyManager.getMetadataManager();
+ Table<String, RepeatedOmKeyInfo> deletedTable =
metadataManager.getDeletedTable();
+ String deletedTablePrefix = metadataManager.getTableBucketPrefix(
+ deletedTable.getName(), volume, bucket);
+ try (TableIterator<String, ? extends KeyValue<String, RepeatedOmKeyInfo>>
itr =
+ deletedTable.iterator(deletedTablePrefix)) {
+ while (itr.hasNext()) {
+ KeyValue<String, ? extends RepeatedOmKeyInfo> entry = itr.next();
+ for (OmKeyInfo omKeyInfo : entry.getValue().getOmKeyInfoList()) {
+ if (reclaimableKeyFilter.apply(Table.newKeyValue(entry.getKey(),
omKeyInfo))) {
+ totals.addPurgeableKey(omKeyInfo.getReplicatedSize());
+ } else {
+ totals.addSnapshotTrappedKey(omKeyInfo.getReplicatedSize());
+ }
+ }
+ }
+ }
+ }
+
+ private void processDeletedDirs(
+ String volume,
+ String bucket,
+ KeyManager keyManager,
+ ReclaimableKeyFilter reclaimableKeyFilter,
+ ReclaimableDirFilter reclaimableDirFilter,
+ BucketDeletedBytesStats totals) throws IOException {
+ OMMetadataManager metadataManager = ozoneManager.getMetadataManager();
+ long volumeId = metadataManager.getVolumeId(volume);
+ long bucketId = metadataManager.getBucketId(volume, bucket);
+
+ try (TableIterator<String, Table.KeyValue<String, OmKeyInfo>>
deletedDirItr =
+ keyManager.getDeletedDirEntries(volume, bucket)) {
+ while (deletedDirItr.hasNext()) {
+ Table.KeyValue<String, OmKeyInfo> rootEntry = deletedDirItr.next();
+ traverseDeletedDirSubtree(
+ keyManager, reclaimableKeyFilter, reclaimableDirFilter,
+ volumeId, bucketId, rootEntry, totals);
+ }
+ }
+ }
+
+ private void processDeletedDirsAsTrapped(
+ String volume,
+ String bucket,
+ KeyManager keyManager,
+ BucketDeletedBytesStats totals) throws IOException {
+ OMMetadataManager metadataManager = ozoneManager.getMetadataManager();
+ long volumeId = metadataManager.getVolumeId(volume);
+ long bucketId = metadataManager.getBucketId(volume, bucket);
+
+ try (TableIterator<String, Table.KeyValue<String, OmKeyInfo>>
deletedDirItr =
+ keyManager.getDeletedDirEntries(volume, bucket)) {
+ while (deletedDirItr.hasNext()) {
+ Table.KeyValue<String, OmKeyInfo> rootEntry = deletedDirItr.next();
+ traverseDeletedDirSubtreeAsTrapped(
+ keyManager, volumeId, bucketId, rootEntry, totals);
+ }
+ }
+ }
+
+ private void traverseDeletedDirSubtree(
+ KeyManager keyManager,
+ ReclaimableKeyFilter reclaimableKeyFilter,
+ ReclaimableDirFilter reclaimableDirFilter,
+ long volumeId,
+ long bucketId,
+ Table.KeyValue<String, OmKeyInfo> rootEntry,
+ BucketDeletedBytesStats totals) throws IOException {
+ Queue<OmKeyInfo> queue = new ArrayDeque<>();
+ Set<Long> visitedDirIds = new HashSet<>();
+ Set<Long> visitedFileIds = new HashSet<>();
+
+ queue.add(rootEntry.getValue());
+ visitedDirIds.add(rootEntry.getValue().getObjectID());
+
+ while (!queue.isEmpty()) {
+ OmKeyInfo currentDir = queue.poll();
+ boolean currentDirReclaimable =
+ reclaimableDirFilter.apply(Table.newKeyValue("", currentDir));
+ if (currentDirReclaimable) {
+ totals.addPurgeableDir();
+ } else {
+ totals.addSnapshotTrappedDir();
+ }
+
+ for (OmKeyInfo subFile : keyManager.getPendingDeletionSubFiles(
+ volumeId, bucketId, currentDir, keyValue -> true,
Integer.MAX_VALUE).getKeysToDelete()) {
+ if (!visitedFileIds.add(subFile.getObjectID())) {
+ continue;
+ }
+ // Keep parity with DirectoryDeletingService where purgeable parent
+ // directory implies purgeable descendant files.
+ boolean fileReclaimable = currentDirReclaimable
+ || reclaimableKeyFilter.apply(Table.newKeyValue("", subFile));
+ if (fileReclaimable) {
+ totals.addPurgeableKey(subFile.getReplicatedSize());
+ } else {
+ totals.addSnapshotTrappedKey(subFile.getReplicatedSize());
+ }
+ }
+
+ for (OmKeyInfo subDir : keyManager.getPendingDeletionSubDirs(
+ volumeId, bucketId, currentDir, keyValue -> true,
Integer.MAX_VALUE).getKeysToDelete()) {
+ if (visitedDirIds.add(subDir.getObjectID())) {
+ queue.add(subDir);
+ }
+ }
+ }
+ }
+
+ private void traverseDeletedDirSubtreeAsTrapped(
+ KeyManager keyManager,
+ long volumeId,
+ long bucketId,
+ Table.KeyValue<String, OmKeyInfo> rootEntry,
+ BucketDeletedBytesStats totals) throws IOException {
+ Queue<OmKeyInfo> queue = new ArrayDeque<>();
+ Set<Long> visitedDirIds = new HashSet<>();
+ Set<Long> visitedFileIds = new HashSet<>();
+
+ queue.add(rootEntry.getValue());
+ visitedDirIds.add(rootEntry.getValue().getObjectID());
+
+ while (!queue.isEmpty()) {
+ OmKeyInfo currentDir = queue.poll();
+ totals.addSnapshotTrappedDir();
+
+ for (OmKeyInfo subFile : keyManager.getPendingDeletionSubFiles(
+ volumeId, bucketId, currentDir, keyValue -> true,
Integer.MAX_VALUE).getKeysToDelete()) {
+ if (!visitedFileIds.add(subFile.getObjectID())) {
+ continue;
+ }
+ totals.addSnapshotTrappedKey(subFile.getReplicatedSize());
+ }
+
+ for (OmKeyInfo subDir : keyManager.getPendingDeletionSubDirs(
+ volumeId, bucketId, currentDir, keyValue -> true,
Integer.MAX_VALUE).getKeysToDelete()) {
+ if (visitedDirIds.add(subDir.getObjectID())) {
+ queue.add(subDir);
+ }
+ }
+ }
+ }
+
+ /**
+ * Aggregated on-demand deleted-bytes view for one bucket.
+ */
+ public static final class BucketDeletedBytesStats {
+ private long snapshotTrappedBytes;
+ private long purgeableBytes;
+ private long snapshotTrappedKeys;
+ private long purgeableKeys;
+ private long snapshotTrappedDirs;
+ private long purgeableDirs;
+
+ public long getSnapshotTrappedBytes() {
+ return snapshotTrappedBytes;
+ }
+
+ public long getPurgeableBytes() {
+ return purgeableBytes;
+ }
+
+ public long getSnapshotTrappedKeys() {
+ return snapshotTrappedKeys;
+ }
+
+ public long getPurgeableKeys() {
+ return purgeableKeys;
+ }
+
+ public long getSnapshotTrappedDirs() {
+ return snapshotTrappedDirs;
+ }
+
+ public long getPurgeableDirs() {
+ return purgeableDirs;
+ }
+
+ public void addSnapshotTrappedBytes(long value) {
+ snapshotTrappedBytes += value;
+ }
+
+ public void addPurgeableBytes(long value) {
+ purgeableBytes += value;
+ }
+
+ public void addSnapshotTrappedKey(long bytes) {
+ addSnapshotTrappedBytes(bytes);
+ snapshotTrappedKeys++;
+ }
+
+ public void addPurgeableKey(long bytes) {
+ addPurgeableBytes(bytes);
+ purgeableKeys++;
+ }
+
+ public void addSnapshotTrappedDir() {
+ snapshotTrappedDirs++;
+ }
+
+ public void addPurgeableDir() {
+ purgeableDirs++;
+ }
+ }
+}
+
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/package-info.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/package-info.java
new file mode 100644
index 00000000000..b4fb62c8cd8
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/snapshot/trapped/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Helpers for bucket-level deleted-data split and snapshot-trapped accounting.
+ */
+package org.apache.hadoop.ozone.om.snapshot.trapped;
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
index 2fc6000abe6..2e234ac76ec 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java
@@ -68,6 +68,7 @@
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.execution.flowcontrol.ExecutionContext;
import org.apache.hadoop.ozone.om.helpers.BasicOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.BucketDeletedBytes;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.DBUpdates;
import org.apache.hadoop.ozone.om.helpers.KeyInfoWithVolumeContext;
@@ -114,6 +115,8 @@
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.EchoRPCResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeProgressRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.FinalizeUpgradeProgressResponse;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketDeletedBytesRequest;
+import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketDeletedBytesResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketTaggingRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetBucketTaggingResponse;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetCallerIdentityResponse;
@@ -267,6 +270,12 @@ public OMResponse handleReadRequest(OMRequest request) {
request.getListOpenFilesRequest(), request.getVersion());
responseBuilder.setListOpenFilesResponse(listOpenFilesResponse);
break;
+ case GetBucketDeletedBytes:
+ GetBucketDeletedBytesResponse getBucketDeletedBytesResponse =
+ getBucketDeletedBytes(request.getGetBucketDeletedBytesRequest());
+ responseBuilder.setGetBucketDeletedBytesResponse(
+ getBucketDeletedBytesResponse);
+ break;
case ServiceList:
ServiceListResponse serviceListResponse = getServiceList(
request.getServiceListRequest());
@@ -989,6 +998,20 @@ private ListOpenFilesResponse
listOpenFiles(ListOpenFilesRequest req,
return resp.build();
}
+ private GetBucketDeletedBytesResponse getBucketDeletedBytes(
+ GetBucketDeletedBytesRequest request) throws IOException {
+ BucketDeletedBytes bytes = impl.getBucketDeletedBytes(
+ request.getBucketPath());
+ return GetBucketDeletedBytesResponse.newBuilder()
+ .setSnapshotTrappedBytes(bytes.getSnapshotTrappedBytes())
+ .setPurgeableBytes(bytes.getPurgeableBytes())
+ .setSnapshotTrappedKeys(bytes.getSnapshotTrappedKeys())
+ .setPurgeableKeys(bytes.getPurgeableKeys())
+ .setSnapshotTrappedDirs(bytes.getSnapshotTrappedDirs())
+ .setPurgeableDirs(bytes.getPurgeableDirs())
+ .build();
+ }
+
private ServiceListResponse getServiceList(ServiceListRequest request)
throws IOException {
ServiceListResponse.Builder resp = ServiceListResponse.newBuilder();
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java
new file mode 100644
index 00000000000..a3b8e1334b7
--- /dev/null
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/snapshot/trapped/TestBucketDeletedDataCalculator.java
@@ -0,0 +1,243 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.om.snapshot.trapped;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyInt;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Collections;
+import org.apache.hadoop.hdds.client.RatisReplicationConfig;
+import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
+import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.ozone.om.BucketManager;
+import org.apache.hadoop.ozone.om.DeleteKeysResult;
+import org.apache.hadoop.ozone.om.KeyManager;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.OmSnapshot;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
+import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
+import org.apache.hadoop.ozone.om.snapshot.SnapshotRequestAndResponseTests;
+import org.apache.ratis.util.function.UncheckedAutoCloseableSupplier;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link BucketDeletedDataCalculator}.
+ */
+public class TestBucketDeletedDataCalculator extends
SnapshotRequestAndResponseTests {
+
+ private KeyManager mockActiveKeyManager(String volume, String bucket) throws
IOException {
+ KeyManager keyManager = mock(KeyManager.class);
+ when(keyManager.getMetadataManager()).thenReturn(getOmMetadataManager());
+ when(keyManager.getDeletedDirEntries(volume,
bucket)).thenAnswer(invocation -> {
+ Table<String, OmKeyInfo> deletedDirTable =
getOmMetadataManager().getDeletedDirTable();
+ String prefix = getOmMetadataManager().getTableBucketPrefix(
+ deletedDirTable.getName(), volume, bucket);
+ return deletedDirTable.iterator(prefix);
+ });
+ when(getOzoneManager().getKeyManager()).thenReturn(keyManager);
+ return keyManager;
+ }
+
+ private void mockBucketManager(String volume, String bucket) throws
IOException {
+ BucketManager bucketManager = mock(BucketManager.class);
+ String bucketDbKey = getOmMetadataManager().getBucketKey(volume, bucket);
+ OmBucketInfo bucketInfo =
getOmMetadataManager().getBucketTable().get(bucketDbKey);
+ when(bucketManager.getBucketInfo(volume, bucket)).thenReturn(bucketInfo);
+ when(getOzoneManager().getBucketManager()).thenReturn(bucketManager);
+ }
+
+ private OmKeyInfo newOmKeyInfo(String volume, String bucket, String keyName,
long objectId) {
+ return OMRequestTestUtils.createOmKeyInfo(
+ volume, bucket, keyName,
+
RatisReplicationConfig.getInstance(HddsProtos.ReplicationFactor.THREE))
+ .setObjectID(objectId)
+ .setUpdateID(objectId)
+ .build();
+ }
+
+ private void putDeletedKey(
+ OMMetadataManager metadataManager, String volume, String bucket,
OmKeyInfo keyInfo) throws Exception {
+ long bucketId = getOmMetadataManager().getBucketId(volume, bucket);
+ RepeatedOmKeyInfo repeated = new RepeatedOmKeyInfo(keyInfo, bucketId);
+ String deletedDbKey = metadataManager.getOzoneKey(volume, bucket,
keyInfo.getKeyName());
+ try (BatchOperation batch =
metadataManager.getStore().initBatchOperation()) {
+ metadataManager.getDeletedTable().putWithBatch(batch, deletedDbKey,
repeated);
+ metadataManager.getStore().commitBatchOperation(batch);
+ }
+ }
+
+ private SnapshotInfo setDeepCleanedDeletedDir(
+ String volume, String bucket, String snapshotName, boolean
deepCleanedDeletedDir) throws Exception {
+ String snapshotTableKey = SnapshotInfo.getTableKey(volume, bucket,
snapshotName);
+ SnapshotInfo snapshotInfo =
getOmMetadataManager().getSnapshotInfoTable().get(snapshotTableKey);
+ snapshotInfo.setDeepCleanedDeletedDir(deepCleanedDeletedDir);
+ getOmMetadataManager().getSnapshotInfoTable()
+ .addCacheEntry(snapshotTableKey, snapshotInfo,
System.currentTimeMillis());
+ return snapshotInfo;
+ }
+
+ private void ensureBucketContextInSnapshot(
+ OMMetadataManager snapshotMetadataManager, String volume, String bucket)
throws Exception {
+ OmVolumeArgs volumeArgs =
getOmMetadataManager().getVolumeTable().get(getOmMetadataManager().getVolumeKey(volume));
+ OmBucketInfo bucketInfo = getOmMetadataManager().getBucketTable()
+ .get(getOmMetadataManager().getBucketKey(volume, bucket));
+ if (volumeArgs != null) {
+
snapshotMetadataManager.getVolumeTable().put(snapshotMetadataManager.getVolumeKey(volume),
volumeArgs);
+ }
+ if (bucketInfo != null) {
+
snapshotMetadataManager.getBucketTable().put(snapshotMetadataManager.getBucketKey(volume,
bucket), bucketInfo);
+ }
+ }
+
+ @Test
+ public void testAosDeletedKeyReportedAsPurgeable() throws Exception {
+ String volume = getVolumeName();
+ String bucket = getBucketName();
+ mockActiveKeyManager(volume, bucket);
+ mockBucketManager(volume, bucket);
+ OmKeyInfo keyInfo = newOmKeyInfo(volume, bucket, "key-a", 101L);
+ putDeletedKey(getOmMetadataManager(), volume, bucket, keyInfo);
+
+ BucketDeletedDataCalculator.BucketDeletedBytesStats stats =
+ new BucketDeletedDataCalculator(getOzoneManager())
+ .calculate(volume, bucket);
+
+ assertEquals(0L, stats.getSnapshotTrappedBytes());
+ assertEquals(0L, stats.getSnapshotTrappedKeys());
+ assertEquals(0L, stats.getSnapshotTrappedDirs());
+ assertTrue(stats.getPurgeableBytes() > 0L);
+ assertEquals(1L, stats.getPurgeableKeys());
+ assertEquals(0L, stats.getPurgeableDirs());
+ }
+
+ @Test
+ public void
testSnapshotDeletedKeysCountedAsTrappedUntilDeletedDirDeepClean() throws
Exception {
+ String volume = getVolumeName();
+ String bucket = getBucketName();
+ mockActiveKeyManager(volume, bucket);
+ mockBucketManager(volume, bucket);
+
+ String snapshotName = "snap-skip";
+ createSnapshotCheckpoint(volume, bucket, snapshotName);
+ setDeepCleanedDeletedDir(volume, bucket, snapshotName, false);
+
+ OmKeyInfo keyInfo = newOmKeyInfo(volume, bucket, "key-in-snapshot", 201L);
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshot =
+ getOmSnapshotManager().getActiveSnapshot(volume, bucket,
snapshotName)) {
+ OMMetadataManager snapshotMetadataManager =
snapshot.get().getMetadataManager();
+ ensureBucketContextInSnapshot(snapshotMetadataManager, volume, bucket);
+ putDeletedKey(snapshotMetadataManager, volume, bucket, keyInfo);
+ }
+
+ BucketDeletedDataCalculator.BucketDeletedBytesStats stats =
+ new BucketDeletedDataCalculator(getOzoneManager()).calculate(volume,
bucket);
+
+ assertEquals(keyInfo.getReplicatedSize(), stats.getSnapshotTrappedBytes());
+ assertEquals(0L, stats.getPurgeableBytes());
+ assertEquals(1L, stats.getSnapshotTrappedKeys());
+ assertEquals(0L, stats.getPurgeableKeys());
+ assertEquals(0L, stats.getSnapshotTrappedDirs());
+ assertEquals(0L, stats.getPurgeableDirs());
+ }
+
+ @Test
+ public void testSnapshotDeletedKeyReportedAsSnapshotTrapped() throws
Exception {
+ String volume = getVolumeName();
+ String bucket = getBucketName();
+ mockActiveKeyManager(volume, bucket);
+ mockBucketManager(volume, bucket);
+
+ String snapshotOne = "snap-1";
+ String snapshotTwo = "snap-2";
+ createSnapshotCheckpoint(volume, bucket, snapshotOne);
+ createSnapshotCheckpoint(volume, bucket, snapshotTwo);
+ setDeepCleanedDeletedDir(volume, bucket, snapshotOne, true);
+ setDeepCleanedDeletedDir(volume, bucket, snapshotTwo, true);
+
+ OmKeyInfo keyInfo = newOmKeyInfo(volume, bucket, "key-trapped", 301L);
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshot =
+ getOmSnapshotManager().getActiveSnapshot(volume, bucket,
snapshotOne)) {
+ OMMetadataManager snapshotMetadataManager =
snapshot.get().getMetadataManager();
+ ensureBucketContextInSnapshot(snapshotMetadataManager, volume, bucket);
+ String keyDbKey = snapshotMetadataManager.getOzoneKey(volume, bucket,
keyInfo.getKeyName());
+ snapshotMetadataManager.getKeyTable(BucketLayout.LEGACY).put(keyDbKey,
keyInfo);
+ }
+ try (UncheckedAutoCloseableSupplier<OmSnapshot> snapshot =
+ getOmSnapshotManager().getActiveSnapshot(volume, bucket,
snapshotTwo)) {
+ OMMetadataManager snapshotMetadataManager =
snapshot.get().getMetadataManager();
+ ensureBucketContextInSnapshot(snapshotMetadataManager, volume, bucket);
+ putDeletedKey(snapshotMetadataManager, volume, bucket, keyInfo);
+ }
+
+ BucketDeletedDataCalculator.BucketDeletedBytesStats stats =
+ new BucketDeletedDataCalculator(getOzoneManager()).calculate(volume,
bucket);
+
+ assertEquals(keyInfo.getReplicatedSize(), stats.getSnapshotTrappedBytes());
+ assertEquals(0L, stats.getPurgeableBytes());
+ assertEquals(1L, stats.getSnapshotTrappedKeys());
+ assertEquals(0L, stats.getPurgeableKeys());
+ assertEquals(0L, stats.getSnapshotTrappedDirs());
+ assertEquals(0L, stats.getPurgeableDirs());
+ }
+
+ @Test
+ public void testAosDeletedDirTraversalReportedAsPurgeable() throws Exception
{
+ String volume = getVolumeName();
+ String bucket = getBucketName();
+ KeyManager keyManager = mockActiveKeyManager(volume, bucket);
+ mockBucketManager(volume, bucket);
+
+ Table<String, OmKeyInfo> deletedDirTable =
getOmMetadataManager().getDeletedDirTable();
+ String deletedDirPrefix = getOmMetadataManager().getTableBucketPrefix(
+ deletedDirTable.getName(), volume, bucket);
+ OmKeyInfo rootDirInfo = newOmKeyInfo(volume, bucket, "dir-root", 401L);
+ try (BatchOperation batch =
getOmMetadataManager().getStore().initBatchOperation()) {
+ deletedDirTable.putWithBatch(batch, deletedDirPrefix + "/dir-root",
rootDirInfo);
+ getOmMetadataManager().getStore().commitBatchOperation(batch);
+ }
+
+ OmKeyInfo subFileInfo = newOmKeyInfo(volume, bucket, "dir-root/file-a",
402L);
+ when(keyManager.getPendingDeletionSubFiles(anyLong(), anyLong(),
any(OmKeyInfo.class), any(), anyInt()))
+ .thenReturn(new
DeleteKeysResult(Collections.singletonList(subFileInfo), true));
+ when(keyManager.getPendingDeletionSubDirs(anyLong(), anyLong(),
any(OmKeyInfo.class), any(), anyInt()))
+ .thenReturn(new DeleteKeysResult(Collections.emptyList(), true));
+
+ BucketDeletedDataCalculator.BucketDeletedBytesStats stats =
+ new BucketDeletedDataCalculator(getOzoneManager()).calculate(volume,
bucket);
+
+ assertEquals(0L, stats.getSnapshotTrappedBytes());
+ assertEquals(subFileInfo.getReplicatedSize(), stats.getPurgeableBytes());
+ assertEquals(0L, stats.getSnapshotTrappedKeys());
+ assertEquals(1L, stats.getPurgeableKeys());
+ assertEquals(0L, stats.getSnapshotTrappedDirs());
+ assertEquals(1L, stats.getPurgeableDirs());
+ }
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
