This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 4066b0e8e23 HDDS-15991. Speed up deleted table scans in quota repair
(#11386)
4066b0e8e23 is described below
commit 4066b0e8e237375196664599e50b5b55d7b48b83
Author: Chi-Hsuan Huang <[email protected]>
AuthorDate: Mon Oct 5 23:25:55 2026 +0800
HDDS-15991. Speed up deleted table scans in quota repair (#11386)
Generated-by: Claude Code (Claude Opus 5.5)
---
.../hadoop/ozone/om/service/QuotaRepairTask.java | 65 ++++++++---
.../ozone/om/service/TestQuotaRepairTask.java | 119 +++++++++++++++++++++
2 files changed, 168 insertions(+), 16 deletions(-)
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/QuotaRepairTask.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/QuotaRepairTask.java
index bc800b5a5b2..10c9a4ca864 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/QuotaRepairTask.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/service/QuotaRepairTask.java
@@ -19,6 +19,7 @@
import static org.apache.hadoop.hdds.utils.db.IteratorType.KEY_AND_VALUE;
import static org.apache.hadoop.hdds.utils.db.IteratorType.KEY_ONLY;
+import static org.apache.hadoop.hdds.utils.db.IteratorType.VALUE_ONLY;
import static org.apache.hadoop.ozone.OzoneConsts.OLD_QUOTA_DEFAULT;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static
org.apache.hadoop.ozone.om.helpers.SnapshotInfo.SnapshotStatus.SNAPSHOT_ACTIVE;
@@ -55,9 +56,13 @@
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.server.JsonUtils;
+import org.apache.hadoop.hdds.utils.db.ByteArrayCodec;
+import org.apache.hadoop.hdds.utils.db.Codec;
import org.apache.hadoop.hdds.utils.db.DBCheckpoint;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
+import org.apache.hadoop.hdds.utils.db.cache.TableCache.CacheType;
import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.OmMetadataManagerImpl;
import org.apache.hadoop.ozone.om.OmSnapshot;
@@ -65,6 +70,7 @@
import org.apache.hadoop.ozone.om.OzoneManager;
import org.apache.hadoop.ozone.om.SnapshotChainInfo;
import org.apache.hadoop.ozone.om.SnapshotChainManager;
+import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
@@ -362,9 +368,8 @@ private void repairCount(
Map<Long, OmBucketInfo> bucketById =
buildBucketByObjectId(idBucketInfoMap);
- tasks.add(executor.submit(() -> recalculateDeletedKeyUsages(
- metadataManager.getDeletedTable(), bucketById, snapshotDeletedKeyMap,
- "active DB checkpoint")));
+ tasks.add(executor.submit(() -> recalculateActiveDeletedKeyUsages(
+ metadataManager, bucketById, snapshotDeletedKeyMap)));
tasks.add(executor.submit(() -> recalculateDeletedDirNamespace(
metadataManager.getDeletedDirTable(), snapshotDeletedDirMap,
"active DB checkpoint")));
@@ -503,6 +508,28 @@ private void scanDeletedTables(
snapshotDeletedDirMap, sourceLabel);
}
+ private void recalculateActiveDeletedKeyUsages(
+ OMMetadataManager metadataManager,
+ Map<Long, OmBucketInfo> bucketById,
+ Map<String, CountPair> snapshotCountByBucketNameKey)
+ throws UncheckedIOException, UncheckedExecutionException {
+ Codec<RepeatedOmKeyInfo> codec =
OMDBDefinition.DELETED_TABLE_DEF.getValueCodec();
+ // a typed iterator decodes on this thread, so keep values raw and let the
workers decode
+ try (Table.KeyValueIterator<String, byte[]> keyIter =
metadataManager.getStore()
+ .getTable(OMDBDefinition.DELETED_TABLE, StringCodec.get(),
ByteArrayCodec.get(), CacheType.NO_CACHE)
+ .iterator(null, VALUE_ONLY)) {
+ scanTableInBatches(executor, keyIter, "snapshot usages from deletedTable
(active DB checkpoint)", kv -> {
+ try {
+ countDeletedKey(codec.fromPersistedFormat(kv.getValue()),
bucketById, snapshotCountByBucketNameKey);
+ } catch (IOException ex) {
+ throw new UncheckedIOException(ex);
+ }
+ });
+ } catch (IOException ex) {
+ throw new UncheckedIOException(ex);
+ }
+ }
+
private void recalculateDeletedKeyUsages(
Table<String, RepeatedOmKeyInfo> deletedTable,
Map<Long, OmBucketInfo> bucketById,
@@ -518,18 +545,7 @@ private void recalculateDeletedKeyUsages(
while (keyIter.hasNext()) {
Table.KeyValue<String, RepeatedOmKeyInfo> kv = keyIter.next();
count++;
- RepeatedOmKeyInfo val = kv.getValue();
- OmBucketInfo bucket = bucketById.get(val.getBucketId());
- if (bucket == null) {
- continue;
- }
- String nameKey = buildNamePath(bucket.getVolumeName(),
bucket.getBucketName());
- CountPair usage = snapshotCountByBucketNameKey.get(nameKey);
- if (usage == null) {
- continue;
- }
- usage.incrSpace(val.getTotalSize().getRight());
- usage.incrNamespace(val.getOmKeyInfoList().size());
+ countDeletedKey(kv.getValue(), bucketById,
snapshotCountByBucketNameKey);
}
LOG.info("Recalculate snapshot usages from deletedTable ({}) completed,
count {} time {}ms",
sourceLabel, count, (Time.monotonicNow() - startTime));
@@ -538,6 +554,23 @@ private void recalculateDeletedKeyUsages(
}
}
+ private static void countDeletedKey(
+ RepeatedOmKeyInfo val,
+ Map<Long, OmBucketInfo> bucketById,
+ Map<String, CountPair> snapshotCountByBucketNameKey) {
+ OmBucketInfo bucket = bucketById.get(val.getBucketId());
+ if (bucket == null) {
+ return;
+ }
+ String nameKey = buildNamePath(bucket.getVolumeName(),
bucket.getBucketName());
+ CountPair usage = snapshotCountByBucketNameKey.get(nameKey);
+ if (usage == null) {
+ return;
+ }
+ usage.incrSpace(val.getTotalSize().getRight());
+ usage.incrNamespace(val.getOmKeyInfoList().size());
+ }
+
private void recalculateDeletedDirNamespace(
Table<String, OmKeyInfo> deletedDirTable,
Map<String, CountPair> snapshotDirNsByIdPrefix,
@@ -549,7 +582,7 @@ private void recalculateDeletedDirNamespace(
int count = 0;
long startTime = Time.monotonicNow();
try (Table.KeyValueIterator<String, OmKeyInfo> keyIter
- = deletedDirTable.iterator()) {
+ = deletedDirTable.iterator(null, KEY_ONLY)) {
while (keyIter.hasNext()) {
Table.KeyValue<String, OmKeyInfo> kv = keyIter.next();
count++;
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestQuotaRepairTask.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestQuotaRepairTask.java
index 3565484b4c5..3f47ed06bd9 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestQuotaRepairTask.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestQuotaRepairTask.java
@@ -19,6 +19,7 @@
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
@@ -52,11 +53,14 @@
import org.apache.hadoop.hdds.client.ReplicationConfig;
import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
+import org.apache.hadoop.hdds.utils.db.ByteArrayCodec;
+import org.apache.hadoop.hdds.utils.db.StringCodec;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.cache.CacheKey;
import org.apache.hadoop.hdds.utils.db.cache.CacheValue;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMMetadataManager;
+import org.apache.hadoop.ozone.om.codec.OMDBDefinition;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
@@ -355,6 +359,99 @@ public void testQuotaRepairSnapshotDbDeletedTableQuota()
throws Exception {
assertEquals(1, repaired.getSnapshotUsedNamespace());
}
+ @Test
+ public void testQuotaRepairCountsDeletedDirectories() throws Exception {
+ AtomicReference<OzoneManagerProtocolProtos.OMRequest> request =
mockQuotaRepairRequest();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ long volumeId = omMetadataManager.getVolumeId(volumeName);
+ long bucketId = omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName,
bucketName)).getObjectID();
+
+ int dirs = 3;
+ for (int i = 0; i < dirs; i++) {
+ String dirName = "dir" + i;
+ long objectId = 100L + i;
+ OmKeyInfo dirInfo = OMRequestTestUtils.createOmKeyInfo(volumeName,
bucketName, dirName, RATIS_ONE)
+ .setObjectID(objectId).setParentObjectID(bucketId).build();
+
omMetadataManager.getDeletedDirTable().put(omMetadataManager.getOzoneDeletePathKey(objectId,
+ omMetadataManager.getOzonePathKey(volumeId, bucketId, bucketId,
dirName)), dirInfo);
+ }
+ // the scan reads keys only, so an undecodable value is still counted
+ omMetadataManager.getStore().getTable(OMDBDefinition.DELETED_DIR_TABLE,
StringCodec.get(), ByteArrayCodec.get())
+ .put(omMetadataManager.getOzoneDeletePathKey(200L,
+ omMetadataManager.getOzonePathKey(volumeId, bucketId, bucketId,
"badDir")), new byte[] {0});
+ String bucketKey = corruptSnapshotUsage(bucketName, 1L);
+
+ applyQuotaRepair(request, 2L, bucketKey);
+
+ OmBucketInfo repaired = omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals(0, repaired.getSnapshotUsedBytes());
+ assertEquals(dirs + 1, repaired.getSnapshotUsedNamespace());
+ }
+
+ @Test
+ public void testQuotaRepairCountsDeletedKeysAcrossBatches() throws Exception
{
+ AtomicReference<OzoneManagerProtocolProtos.OMRequest> request =
mockQuotaRepairRequest();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ long bucketId = omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName,
bucketName)).getObjectID();
+
+ // a partial last batch, so a dropped tail batch would show up in the
totals
+ int rows = QuotaRepairTask.BATCH_SIZE * 2 + 1;
+ for (int i = 0; i < rows; i++) {
+ putDeletedKey(bucketName, bucketId, "key" + i, i == 0 ? 2 : 1,
RATIS_ONE);
+ }
+ // replicated size differs from data size only with more than one replica
+ putDeletedKey(bucketName, bucketId, "threeReplicaKey", 1,
RatisReplicationConfig.getInstance(THREE));
+ String bucketKey = corruptSnapshotUsage(bucketName, 1L);
+
+ applyQuotaRepair(request, 2L, bucketKey);
+
+ OmBucketInfo repaired = omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals((rows + 1) * 1000L + 3000L, repaired.getSnapshotUsedBytes());
+ assertEquals(rows + 2, repaired.getSnapshotUsedNamespace());
+ }
+
+ @Test
+ public void testQuotaRepairSkipsDeletedKeysOfRecreatedBucket() throws
Exception {
+ AtomicReference<OzoneManagerProtocolProtos.OMRequest> request =
mockQuotaRepairRequest();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ long bucketId = omMetadataManager.getBucketTable()
+ .get(omMetadataManager.getBucketKey(volumeName,
bucketName)).getObjectID();
+
+ putDeletedKey(bucketName, bucketId, "liveKey", 1, RATIS_ONE);
+ // left by an earlier bucket with the same name: the key prefix matches,
the bucket id does not
+ putDeletedKey(bucketName, bucketId + 1000, "staleKey", 1, RATIS_ONE);
+ // written before deletedTable rows carried a bucket id, so it decodes to 0
+ putDeletedKey(bucketName, 0, "legacyKey", 1, RATIS_ONE);
+ String bucketKey = corruptSnapshotUsage(bucketName, 1L);
+
+ applyQuotaRepair(request, 2L, bucketKey);
+
+ OmBucketInfo repaired = omMetadataManager.getBucketTable().get(bucketKey);
+ assertEquals(1000L, repaired.getSnapshotUsedBytes());
+ assertEquals(1, repaired.getSnapshotUsedNamespace());
+ }
+
+ @Test
+ public void testQuotaRepairFailsOnUndecodableDeletedKey() throws Exception {
+ AtomicReference<OzoneManagerProtocolProtos.OMRequest> request =
mockQuotaRepairRequest();
+ OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
+ omMetadataManager, BucketLayout.OBJECT_STORE);
+ // a zero tag is never valid protobuf
+ String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucketName,
"badKey");
+ omMetadataManager.getStore().getTable(OMDBDefinition.DELETED_TABLE,
StringCodec.get(), ByteArrayCodec.get())
+ .put(omMetadataManager.getOzoneDeletePathKey(1L, ozoneKey), new byte[]
{0});
+ corruptSnapshotUsage(bucketName, 1L);
+
+ assertFalse(awaitRepair(new QuotaRepairTask(ozoneManager).repair()));
+ assertThat(QuotaRepairTask.getStatus()).contains("CodecException");
+ assertNull(request.get());
+ }
+
private void zeroOutBucketUsedBytes(String volumeName, String bucketName,
long trxnLogIndex)
throws IOException {
@@ -706,6 +803,28 @@ private String corruptBucketUsage(String bucket, long
usedBytes, long usedNamesp
return bucketKey;
}
+ private String corruptSnapshotUsage(String bucket, long transactionIndex)
throws IOException {
+ String bucketKey = omMetadataManager.getBucketKey(volumeName, bucket);
+ OmBucketInfo corrupted =
omMetadataManager.getBucketTable().get(bucketKey).toBuilder()
+ .setSnapshotUsedBytes(7L).setSnapshotUsedNamespace(99L).build();
+ omMetadataManager.getBucketTable().put(bucketKey, corrupted);
+ omMetadataManager.getBucketTable().addCacheEntry(
+ new CacheKey<>(bucketKey), CacheValue.get(transactionIndex,
corrupted));
+ return bucketKey;
+ }
+
+ // repair scans a RocksDB checkpoint, so rows must be written to the table,
not the cache
+ private void putDeletedKey(String bucket, long bucketId, String keyName, int
versions,
+ ReplicationConfig replicationConfig) throws IOException {
+ RepeatedOmKeyInfo row = new RepeatedOmKeyInfo(bucketId);
+ for (int i = 0; i < versions; i++) {
+ row.addOmKeyInfo(OMRequestTestUtils.createOmKeyInfo(volumeName, bucket,
keyName, replicationConfig)
+ .setObjectID(i + 1L).build());
+ }
+ String ozoneKey = omMetadataManager.getOzoneKey(volumeName, bucket,
keyName);
+
omMetadataManager.getDeletedTable().put(omMetadataManager.getOzoneDeletePathKey(1L,
ozoneKey), row);
+ }
+
private void
applyQuotaRepair(AtomicReference<OzoneManagerProtocolProtos.OMRequest> request,
long transactionIndex, String... buckets) throws Exception {
QuotaRepairTask quotaRepairTask = new QuotaRepairTask(ozoneManager);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
