This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch HDDS-3630
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3630 by this push:
new acdc4ca HDDS-6404. Format table key according to schema in
KeyValueContainerData. (#3172)
acdc4ca is described below
commit acdc4ca5693d9af90b40fc4b33e3e566a86bae50
Author: Gui Hecheng <[email protected]>
AuthorDate: Tue Mar 15 21:42:32 2022 +0800
HDDS-6404. Format table key according to schema in KeyValueContainerData.
(#3172)
---
.../commandhandler/DeleteBlocksCommandHandler.java | 19 ++--
.../container/keyvalue/KeyValueContainerCheck.java | 9 +-
.../container/keyvalue/KeyValueContainerData.java | 89 ++++++++++++++++-
.../KeyValueContainerMetadataInspector.java | 14 +--
.../keyvalue/helpers/KeyValueContainerUtil.java | 17 ++--
.../container/keyvalue/impl/BlockManagerImpl.java | 44 ++++-----
.../background/BlockDeletingService.java | 13 ++-
.../container/common/TestBlockDeletingService.java | 32 +++---
.../TestSchemaOneBackwardsCompatibility.java | 109 ++++++++++++---------
.../keyvalue/TestKeyValueBlockIterator.java | 11 ++-
.../container/keyvalue/TestKeyValueContainer.java | 9 +-
.../TestKeyValueContainerIntegrityChecks.java | 5 +-
.../TestKeyValueContainerMetadataInspector.java | 8 +-
.../container/ozoneimpl/TestContainerReader.java | 22 ++---
.../container/ozoneimpl/TestOzoneContainer.java | 55 ++++++-----
.../hadoop/hdds/utils/MetadataKeyFilters.java | 1 +
.../ozone/TestStorageContainerManagerHelper.java | 31 +++---
.../client/rpc/TestFailureHandlingByClient.java | 6 +-
.../client/rpc/TestOzoneRpcClientAbstract.java | 9 +-
.../client/rpc/TestValidateBCSIDOnRestart.java | 3 +-
.../commandhandler/TestBlockDeletion.java | 23 ++---
21 files changed, 309 insertions(+), 220 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
index 314ca7d..bd93a41 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/DeleteBlocksCommandHandler.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.hdds.protocol.proto
import org.apache.hadoop.hdds.protocol.proto
.StorageContainerDatanodeProtocolProtos.DeletedBlocksTransaction;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfoList;
import org.apache.hadoop.ozone.container.common.helpers
@@ -366,16 +365,15 @@ public class DeleteBlocksCommandHandler implements
CommandHandler {
try (BatchOperation batch = containerDB.getStore().getBatchHandler()
.initBatchOperation()) {
for (Long blkLong : delTX.getLocalIDList()) {
- String blk = blkLong.toString();
+ String blk = containerData.blockKey(blkLong);
BlockData blkInfo = blockDataTable.get(blk);
if (blkInfo != null) {
- String deletingKey = OzoneConsts.DELETING_KEY_PREFIX + blk;
+ String deletingKey = containerData.deletingBlockKey(blkLong);
if (blockDataTable.get(deletingKey) != null
|| deletedBlocksTable.get(blk) != null) {
if (LOG.isDebugEnabled()) {
- LOG.debug(String.format(
- "Ignoring delete for block %s in container %d."
- + " Entry already added.", blk, containerId));
+ LOG.debug("Ignoring delete for block {} in container {}."
+ + " Entry already added.", blkLong, containerId);
}
continue;
}
@@ -386,12 +384,12 @@ public class DeleteBlocksCommandHandler implements
CommandHandler {
newDeletionBlocks++;
if (LOG.isDebugEnabled()) {
LOG.debug("Transited Block {} to DELETING state in container {}",
- blk, containerId);
+ blkLong, containerId);
}
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Block {} not found or already under deletion in"
- + " container {}, skip deleting it.", blk, containerId);
+ + " container {}, skip deleting it.", blkLong, containerId);
}
}
}
@@ -422,14 +420,15 @@ public class DeleteBlocksCommandHandler implements
CommandHandler {
if (delTX.getTxID() > containerData.getDeleteTransactionId()) {
// Update in DB pending delete key count and delete transaction ID.
metadataTable
- .putWithBatch(batchOperation, OzoneConsts.DELETE_TRANSACTION_KEY,
+ .putWithBatch(batchOperation, containerData.latestDeleteTxnKey(),
delTX.getTxID());
}
long pendingDeleteBlocks =
containerData.getNumPendingDeletionBlocks() + newDeletionBlocks;
metadataTable
- .putWithBatch(batchOperation, OzoneConsts.PENDING_DELETE_BLOCK_COUNT,
+ .putWithBatch(batchOperation,
+ containerData.pendingDeleteBlockCountKey(),
pendingDeleteBlocks);
// update pending deletion blocks count and delete transaction ID in
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
index 1c1e558..5f3f1f8 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerCheck.java
@@ -232,7 +232,8 @@ public class KeyValueContainerCheck {
try (DBHandle db = BlockUtils.getDB(onDiskContainerData, checkConfig);
BlockIterator<BlockData> kvIter = db.getStore().getBlockIterator(
- onDiskContainerData.getContainerID())) {
+ onDiskContainerData.getContainerID(),
+ onDiskContainerData.getUnprefixedKeyFilter())) {
while (kvIter.hasNext()) {
BlockData block = kvIter.nextBlock();
@@ -242,11 +243,11 @@ public class KeyValueContainerCheck {
if (!chunkFile.exists()) {
// concurrent mutation in Block DB? lookup the block again.
- String localBlockID =
- Long.toString(block.getBlockID().getLocalID());
+ String blockKey =
+ onDiskContainerData.blockKey(block.getBlockID().getLocalID());
BlockData bdata = db.getStore()
.getBlockDataTable()
- .get(localBlockID);
+ .get(blockKey);
if (bdata != null) {
throw new IOException("Missing chunk file "
+ chunkFile.getAbsolutePath());
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
index 594f3b2..55d8f6d 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerData.java
@@ -28,6 +28,7 @@ import java.util.Collections;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos
.ContainerDataProto;
+import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
@@ -42,9 +43,12 @@ import java.util.Map;
import java.util.concurrent.atomic.AtomicLong;
import static java.lang.Math.max;
+import static org.apache.hadoop.ozone.OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE_ROCKSDB;
import static org.apache.hadoop.ozone.OzoneConsts.CHUNKS_PATH;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_DB_TYPE;
+import static org.apache.hadoop.ozone.OzoneConsts.DELETE_TRANSACTION_KEY;
+import static org.apache.hadoop.ozone.OzoneConsts.DELETING_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.METADATA_PATH;
import static org.apache.hadoop.ozone.OzoneConsts.SCHEMA_VERSION;
import static org.apache.hadoop.ozone.OzoneConsts.CONTAINER_BYTES_USED;
@@ -295,13 +299,90 @@ public class KeyValueContainerData extends ContainerData {
Table<String, Long> metadataTable = db.getStore().getMetadataTable();
// Set Bytes used and block count key.
- metadataTable.putWithBatch(batchOperation, CONTAINER_BYTES_USED,
+ metadataTable.putWithBatch(batchOperation, bytesUsedKey(),
getBytesUsed() - releasedBytes);
- metadataTable.putWithBatch(batchOperation, BLOCK_COUNT,
+ metadataTable.putWithBatch(batchOperation, blockCountKey(),
getBlockCount() - deletedBlockCount);
- metadataTable.putWithBatch(batchOperation, PENDING_DELETE_BLOCK_COUNT,
- (long)(getNumPendingDeletionBlocks() - deletedBlockCount));
+ metadataTable.putWithBatch(batchOperation, pendingDeleteBlockCountKey(),
+ getNumPendingDeletionBlocks() - deletedBlockCount);
db.getStore().getBatchHandler().commitBatchOperation(batchOperation);
}
+
+ // NOTE: Below are some helper functions to format keys according
+ // to container schemas, we should use them instead of using
+ // raw const variables defined.
+
+ public String blockKey(long localID) {
+ return formatKey(Long.toString(localID));
+ }
+
+ public String deletingBlockKey(long localID) {
+ return formatKey(DELETING_KEY_PREFIX + localID);
+ }
+
+ public String deleteTxnKey(long txnID) {
+ return formatKey(Long.toString(txnID));
+ }
+
+ public String latestDeleteTxnKey() {
+ return formatKey(DELETE_TRANSACTION_KEY);
+ }
+
+ public String bcsIdKey() {
+ return formatKey(BLOCK_COMMIT_SEQUENCE_ID);
+ }
+
+ public String blockCountKey() {
+ return formatKey(BLOCK_COUNT);
+ }
+
+ public String bytesUsedKey() {
+ return formatKey(CONTAINER_BYTES_USED);
+ }
+
+ public String pendingDeleteBlockCountKey() {
+ return formatKey(PENDING_DELETE_BLOCK_COUNT);
+ }
+
+ public String deletingBlockKeyPrefix() {
+ return formatKey(DELETING_KEY_PREFIX);
+ }
+
+ public KeyPrefixFilter getUnprefixedKeyFilter() {
+ return new KeyPrefixFilter().addFilter(containerPrefix() + "#", true);
+ }
+
+ public KeyPrefixFilter getDeletingBlockKeyFilter() {
+ return new KeyPrefixFilter().addFilter(deletingBlockKeyPrefix());
+ }
+
+ /**
+ * Schema v3 use a prefix as startKey,
+ * for other schemas just return null.
+ * @return
+ */
+ public String startKeyEmpty() {
+ return null;
+ }
+
+ /**
+ * Schema v3 use containerID as key prefix,
+ * for other schemas just return empty.
+ * @return
+ */
+ public String containerPrefix() {
+ return "";
+ }
+
+ /**
+ * Format the raw key to a schema specific format key.
+ * Schema v3 use container ID as key prefix,
+ * for other schemas just return the raw key.
+ * @param key raw key
+ * @return formatted key
+ */
+ private String formatKey(String key) {
+ return key;
+ }
}
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
index bb7fdc6..5dd7da3 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/KeyValueContainerMetadataInspector.java
@@ -25,7 +25,6 @@ import com.google.gson.JsonObject;
import com.google.gson.JsonPrimitive;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -209,7 +208,7 @@ public class KeyValueContainerMetadataInspector implements
ContainerInspector {
// Build aggregate values.
JsonObject aggregates = getAggregateValues(store,
- containerData.getContainerID(), schemaVersion);
+ containerData, schemaVersion);
containerJson.add("aggregates", aggregates);
// Build info about chunks directory.
@@ -243,7 +242,8 @@ public class KeyValueContainerMetadataInspector implements
ContainerInspector {
}
private JsonObject getAggregateValues(DatanodeStore store,
- long containerID, String schemaVersion) throws IOException {
+ KeyValueContainerData containerData, String schemaVersion)
+ throws IOException {
JsonObject aggregates = new JsonObject();
long usedBytesTotal = 0;
@@ -251,8 +251,8 @@ public class KeyValueContainerMetadataInspector implements
ContainerInspector {
long pendingDeleteBlockCountTotal = 0;
// Count normal blocks.
try (BlockIterator<BlockData> blockIter =
- store.getBlockIterator(containerID,
- MetadataKeyFilters.getUnprefixedKeyFilter())) {
+ store.getBlockIterator(containerData.getContainerID(),
+ containerData.getUnprefixedKeyFilter())) {
while (blockIter.hasNext()) {
blockCountTotal++;
@@ -263,8 +263,8 @@ public class KeyValueContainerMetadataInspector implements
ContainerInspector {
// Count pending delete blocks.
if (schemaVersion.equals(OzoneConsts.SCHEMA_V1)) {
try (BlockIterator<BlockData> blockIter =
- store.getBlockIterator(containerID,
- MetadataKeyFilters.getDeletingKeyFilter())) {
+ store.getBlockIterator(containerData.getContainerID(),
+ containerData.getDeletingBlockKeyFilter())) {
while (blockIter.hasNext()) {
blockCountTotal++;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
index 1958a3e..365b44e 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/helpers/KeyValueContainerUtil.java
@@ -207,14 +207,14 @@ public final class KeyValueContainerUtil {
// Set pending deleted block count.
Long pendingDeleteBlockCount =
- metadataTable.get(OzoneConsts.PENDING_DELETE_BLOCK_COUNT);
+ metadataTable.get(kvContainerData.pendingDeleteBlockCountKey());
if (pendingDeleteBlockCount != null) {
kvContainerData.incrPendingDeletionBlocks(
pendingDeleteBlockCount);
} else {
// Set pending deleted block count.
MetadataKeyFilters.KeyPrefixFilter filter =
- MetadataKeyFilters.getDeletingKeyFilter();
+ kvContainerData.getDeletingBlockKeyFilter();
int numPendingDeletionBlocks =
store.getBlockDataTable()
.getSequentialRangeKVs(null, Integer.MAX_VALUE, filter)
@@ -224,15 +224,14 @@ public final class KeyValueContainerUtil {
// Set delete transaction id.
Long delTxnId =
- metadataTable.get(OzoneConsts.DELETE_TRANSACTION_KEY);
+ metadataTable.get(kvContainerData.latestDeleteTxnKey());
if (delTxnId != null) {
kvContainerData
.updateDeleteTransactionId(delTxnId);
}
// Set BlockCommitSequenceId.
- Long bcsId = metadataTable.get(
- OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID);
+ Long bcsId = metadataTable.get(kvContainerData.bcsIdKey());
if (bcsId != null) {
kvContainerData
.updateBlockCommitSequenceId(bcsId);
@@ -241,14 +240,14 @@ public final class KeyValueContainerUtil {
// Set bytes used.
// commitSpace for Open Containers relies on usedBytes
Long bytesUsed =
- metadataTable.get(OzoneConsts.CONTAINER_BYTES_USED);
+ metadataTable.get(kvContainerData.bytesUsedKey());
if (bytesUsed != null) {
isBlockMetadataSet = true;
kvContainerData.setBytesUsed(bytesUsed);
}
// Set block count.
- Long blockCount = metadataTable.get(OzoneConsts.BLOCK_COUNT);
+ Long blockCount = metadataTable.get(kvContainerData.blockCountKey());
if (blockCount != null) {
isBlockMetadataSet = true;
kvContainerData.setBlockCount(blockCount);
@@ -301,7 +300,7 @@ public final class KeyValueContainerUtil {
try (BlockIterator<BlockData> blockIter =
store.getBlockIterator(kvData.getContainerID(),
- MetadataKeyFilters.getUnprefixedKeyFilter())) {
+ kvData.getUnprefixedKeyFilter())) {
while (blockIter.hasNext()) {
blockCount++;
@@ -316,7 +315,7 @@ public final class KeyValueContainerUtil {
// Count all deleting blocks.
try (BlockIterator<BlockData> blockIter =
store.getBlockIterator(kvData.getContainerID(),
- MetadataKeyFilters.getDeletingKeyFilter())) {
+ kvData.getDeletingBlockKeyFilter())) {
while (blockIter.hasNext()) {
blockCount++;
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
index d8407e6..80e48e0 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/impl/BlockManagerImpl.java
@@ -27,10 +27,8 @@ import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import org.apache.hadoop.ozone.container.common.interfaces.DBHandle;
@@ -117,17 +115,18 @@ public class BlockManagerImpl implements BlockManager {
"operation.");
Preconditions.checkState(data.getContainerID() >= 0, "Container Id " +
"cannot be negative");
+
+ KeyValueContainerData containerData = container.getContainerData();
+
// We are not locking the key manager since LevelDb serializes all actions
// against a single DB. We rely on DB level locking to avoid conflicts.
- try (DBHandle db = BlockUtils.
- getDB(container.getContainerData(), config)) {
+ try (DBHandle db = BlockUtils.getDB(containerData, config)) {
// This is a post condition that acts as a hint to the user.
// Should never fail.
Preconditions.checkNotNull(db, DB_NULL_ERR_MSG);
long bcsId = data.getBlockCommitSequenceId();
- long containerBCSId = container.
- getContainerData().getBlockCommitSequenceId();
+ long containerBCSId = containerData.getBlockCommitSequenceId();
// default blockCommitSequenceId for any block is 0. It the putBlock
// request is not coming via Ratis(for test scenarios), it will be 0.
@@ -160,7 +159,7 @@ public class BlockManagerImpl implements BlockManager {
// If block exists in cache, blockCount should not be incremented.
if (!isBlockInCache) {
if (db.getStore().getBlockDataTable().get(
- Long.toString(localID)) == null) {
+ containerData.blockKey(localID)) == null) {
// Block does not exist in DB => blockCount needs to be
// incremented when the block is added into DB.
incrBlockCount = true;
@@ -168,10 +167,10 @@ public class BlockManagerImpl implements BlockManager {
}
db.getStore().getBlockDataTable().putWithBatch(
- batch, Long.toString(localID), data);
+ batch, containerData.blockKey(localID), data);
if (bcsId != 0) {
db.getStore().getMetadataTable().putWithBatch(
- batch, OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID, bcsId);
+ batch, containerData.bcsIdKey(), bcsId);
}
// Set Bytes used, this bytes used will be updated for every write and
@@ -181,14 +180,14 @@ public class BlockManagerImpl implements BlockManager {
// is only used to compute the bytes used. This is done to keep the
// current behavior and avoid DB write during write chunk operation.
db.getStore().getMetadataTable().putWithBatch(
- batch, OzoneConsts.CONTAINER_BYTES_USED,
- container.getContainerData().getBytesUsed());
+ batch, containerData.bytesUsedKey(),
+ containerData.getBytesUsed());
// Set Block Count for a container.
if (incrBlockCount) {
db.getStore().getMetadataTable().putWithBatch(
- batch, OzoneConsts.BLOCK_COUNT,
- container.getContainerData().getBlockCount() + 1);
+ batch, containerData.blockCountKey(),
+ containerData.getBlockCount() + 1);
}
db.getStore().getBatchHandler().commitBatchOperation(batch);
@@ -201,7 +200,7 @@ public class BlockManagerImpl implements BlockManager {
// Increment block count and add block to pendingPutBlockCache
// in-memory after the DB update.
if (incrBlockCount) {
- container.getContainerData().incrBlockCount();
+ containerData.incrBlockCount();
}
// If the Block is not in PendingPutBlockCache (and it is not
endOfBlock),
@@ -247,7 +246,7 @@ public class BlockManagerImpl implements BlockManager {
if (containerBCSId < bcsId) {
throw new StorageContainerException(
"Unable to find the block with bcsID " + bcsId + " .Container "
- + container.getContainerData().getContainerID() + " bcsId is "
+ + containerData.getContainerID() + " bcsId is "
+ containerBCSId + ".", UNKNOWN_BCSID);
}
@@ -255,7 +254,7 @@ public class BlockManagerImpl implements BlockManager {
// This is a post condition that acts as a hint to the user.
// Should never fail.
Preconditions.checkNotNull(db, DB_NULL_ERR_MSG);
- BlockData blockData = getBlockByID(db, blockID);
+ BlockData blockData = getBlockByID(db, blockID, containerData);
long id = blockData.getBlockID().getBlockCommitSequenceId();
if (id < bcsId) {
throw new StorageContainerException(
@@ -283,7 +282,7 @@ public class BlockManagerImpl implements BlockManager {
// This is a post condition that acts as a hint to the user.
// Should never fail.
Preconditions.checkNotNull(db, DB_NULL_ERR_MSG);
- BlockData blockData = getBlockByID(db, blockID);
+ BlockData blockData = getBlockByID(db, blockID, containerData);
return blockData.getSize();
}
}
@@ -328,10 +327,11 @@ public class BlockManagerImpl implements BlockManager {
(KeyValueContainerData) container.getContainerData();
try (DBHandle db = BlockUtils.getDB(cData, config)) {
result = new ArrayList<>();
+ String startKey = cData.blockKey(startLocalID);
List<? extends Table.KeyValue<String, BlockData>> range =
db.getStore().getBlockDataTable()
- .getSequentialRangeKVs(Long.toString(startLocalID), count,
- MetadataKeyFilters.getUnprefixedKeyFilter());
+ .getSequentialRangeKVs(startKey, count,
+ cData.getUnprefixedKeyFilter());
for (Table.KeyValue<String, BlockData> entry : range) {
BlockData data = new BlockData(entry.getValue().getBlockID());
result.add(data);
@@ -351,9 +351,9 @@ public class BlockManagerImpl implements BlockManager {
BlockUtils.shutdownCache(ContainerCache.getInstance(config));
}
- private BlockData getBlockByID(DBHandle db, BlockID blockID)
- throws IOException {
- String blockKey = Long.toString(blockID.getLocalID());
+ private BlockData getBlockByID(DBHandle db, BlockID blockID,
+ KeyValueContainerData containerData) throws IOException {
+ String blockKey = containerData.blockKey(blockID.getLocalID());
BlockData blockData = db.getStore().getBlockDataTable().get(blockKey);
if (blockData == null) {
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index 8206c29..36294d3 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -36,7 +36,6 @@ import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerExcep
import org.apache.hadoop.hdds.scm.pipeline.PipelineID;
import org.apache.hadoop.hdds.utils.BackgroundTaskResult;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
import org.apache.hadoop.hdds.utils.BackgroundTaskQueue;
import org.apache.hadoop.hdds.utils.BackgroundService;
import org.apache.hadoop.hdds.utils.BackgroundTask;
@@ -315,7 +314,7 @@ public class BlockDeletingService extends BackgroundService
{
meta.getStore().getBlockDataTable();
// # of blocks to delete is throttled
- KeyPrefixFilter filter = MetadataKeyFilters.getDeletingKeyFilter();
+ KeyPrefixFilter filter = containerData.getDeletingBlockKeyFilter();
List<? extends Table.KeyValue<String, BlockData>> toDeleteBlocks =
blockDataTable
.getSequentialRangeKVs(null, (int) blocksToDelete, filter);
@@ -487,13 +486,13 @@ public class BlockDeletingService extends
BackgroundService {
long bytesReleased = 0;
for (DeletedBlocksTransaction entry : delBlocks) {
for (Long blkLong : entry.getLocalIDList()) {
- String blk = blkLong.toString();
+ String blk = containerData.blockKey(blkLong);
BlockData blkInfo = blockDataTable.get(blk);
- LOG.debug("Deleting block {}", blk);
+ LOG.debug("Deleting block {}", blkLong);
if (blkInfo == null) {
LOG.warn("Missing delete block(Container = " +
container.getContainerData().getContainerID() + ", Block = " +
- blk);
+ blkLong);
continue;
}
try {
@@ -501,9 +500,9 @@ public class BlockDeletingService extends BackgroundService
{
blocksDeleted++;
bytesReleased += KeyValueContainerUtil.getBlockLength(blkInfo);
} catch (InvalidProtocolBufferException e) {
- LOG.error("Failed to parse block info for block {}", blk, e);
+ LOG.error("Failed to parse block info for block {}", blkLong, e);
} catch (IOException e) {
- LOG.error("Failed to delete files for block {}", blk, e);
+ LOG.error("Failed to delete files for block {}", blkLong, e);
}
}
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
index ce644ce..6d2e4ac 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestBlockDeletingService.java
@@ -39,11 +39,9 @@ import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerDatanodeProtocolProtos;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.utils.BackgroundService;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
import org.apache.hadoop.hdds.utils.db.BatchOperation;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.hdds.utils.db.TableIterator;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.common.ChunkBuffer;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
@@ -239,8 +237,7 @@ public class TestBlockDeletingService {
try (DBHandle metadata = BlockUtils.getDB(data, conf)) {
for (int j = 0; j < numOfBlocksPerContainer; j++) {
blockID = ContainerTestHelper.getTestBlockID(containerID);
- String deleteStateName =
- OzoneConsts.DELETING_KEY_PREFIX + blockID.getLocalID();
+ String deleteStateName = data.deletingBlockKey(blockID.getLocalID());
BlockData kd = new BlockData(blockID);
List<ContainerProtos.ChunkInfo> chunks = Lists.newArrayList();
putChunksInBlock(numOfChunksPerBlock, j, chunks, buffer, chunkManager,
@@ -271,13 +268,13 @@ public class TestBlockDeletingService {
putChunksInBlock(numOfChunksPerBlock, i, chunks, buffer, chunkManager,
container, blockID);
kd.setChunks(chunks);
- String bID = null;
try (DBHandle metadata = BlockUtils.getDB(data, conf)) {
- bID = blockID.getLocalID() + "";
- metadata.getStore().getBlockDataTable().put(bID, kd);
+ String blockKey = data.blockKey(blockID.getLocalID());
+ metadata.getStore().getBlockDataTable().put(blockKey, kd);
} catch (IOException exception) {
LOG.info("Exception = " + exception);
- LOG.warn("Failed to put block: " + bID + " in BlockDataTable.");
+ LOG.warn("Failed to put block: " + blockID.getLocalID()
+ + " in BlockDataTable.");
}
container.getContainerData().incrPendingDeletionBlocks(1);
@@ -348,12 +345,12 @@ public class TestBlockDeletingService {
container.getContainerData().setBlockCount(numOfBlocksPerContainer);
// Set block count, bytes used and pending delete block count.
metadata.getStore().getMetadataTable()
- .put(OzoneConsts.BLOCK_COUNT, (long) numOfBlocksPerContainer);
+ .put(data.blockCountKey(), (long) numOfBlocksPerContainer);
metadata.getStore().getMetadataTable()
- .put(OzoneConsts.CONTAINER_BYTES_USED,
+ .put(data.bytesUsedKey(),
chunkLength * numOfChunksPerBlock * numOfBlocksPerContainer);
metadata.getStore().getMetadataTable()
- .put(OzoneConsts.PENDING_DELETE_BLOCK_COUNT,
+ .put(data.pendingDeleteBlockCountKey(),
(long) numOfBlocksPerContainer);
} catch (IOException exception) {
LOG.warn("Meta Data update was not successful for container: "
@@ -379,7 +376,7 @@ public class TestBlockDeletingService {
KeyValueContainerData data) throws IOException {
if (data.getSchemaVersion().equals(SCHEMA_V1)) {
return meta.getStore().getBlockDataTable()
- .getRangeKVs(null, 100, MetadataKeyFilters.getDeletingKeyFilter())
+ .getRangeKVs(null, 100, data.getDeletingBlockKeyFilter())
.size();
} else if (data.getSchemaVersion().equals(SCHEMA_V2)) {
int pendingBlocks = 0;
@@ -425,11 +422,10 @@ public class TestBlockDeletingService {
// Ensure 1 container was created
List<ContainerData> containerData = Lists.newArrayList();
containerSet.listContainer(0L, 1, containerData);
- KeyValueContainerData data = (KeyValueContainerData) containerData.get(0);
Assert.assertEquals(1, containerData.size());
+ KeyValueContainerData data = (KeyValueContainerData) containerData.get(0);
- try (DBHandle meta = BlockUtils.getDB(
- (KeyValueContainerData) containerData.get(0), conf)) {
+ try (DBHandle meta = BlockUtils.getDB(data, conf)) {
Map<Long, Container<?>> containerMap =
containerSet.getContainerMapCopy();
// NOTE: this test assumes that all the container is KetValueContainer
and
// have DeleteTransactionId in KetValueContainerData. If other
@@ -447,7 +443,7 @@ public class TestBlockDeletingService {
// Ensure there are 3 blocks under deletion and 0 deleted blocks
Assert.assertEquals(3, getUnderDeletionBlocksCount(meta, data));
Assert.assertEquals(3, meta.getStore().getMetadataTable()
- .get(OzoneConsts.PENDING_DELETE_BLOCK_COUNT).longValue());
+ .get(data.pendingDeleteBlockCountKey()).longValue());
// Container contains 3 blocks. So, space used by the container
// should be greater than zero.
@@ -474,9 +470,9 @@ public class TestBlockDeletingService {
// Check finally DB counters.
// Not checking bytes used, as handler is a mock call.
Assert.assertEquals(0, meta.getStore().getMetadataTable()
- .get(OzoneConsts.PENDING_DELETE_BLOCK_COUNT).longValue());
+ .get(data.pendingDeleteBlockCountKey()).longValue());
Assert.assertEquals(0,
- meta.getStore().getMetadataTable().get(OzoneConsts.BLOCK_COUNT)
+ meta.getStore().getMetadataTable().get(data.blockCountKey())
.longValue());
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
index ec46bd5..ff013da 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestSchemaOneBackwardsCompatibility.java
@@ -22,7 +22,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.hadoop.hdds.HddsConfigKeys;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
+import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
@@ -158,14 +158,16 @@ public class TestSchemaOneBackwardsCompatibility {
*/
@Test
public void testBlockIteration() throws IOException {
- try (DBHandle refCountedDB = BlockUtils.getDB(newKvData(), conf)) {
- assertEquals(TestDB.NUM_DELETED_BLOCKS,
countDeletedBlocks(refCountedDB));
+ KeyValueContainerData cData = newKvData();
+ try (DBHandle refCountedDB = BlockUtils.getDB(cData, conf)) {
+ assertEquals(TestDB.NUM_DELETED_BLOCKS,
+ countDeletedBlocks(refCountedDB, cData));
assertEquals(TestDB.NUM_PENDING_DELETION_BLOCKS,
- countDeletingBlocks(refCountedDB));
+ countDeletingBlocks(refCountedDB, cData));
assertEquals(TestDB.KEY_COUNT - TestDB.NUM_PENDING_DELETION_BLOCKS,
- countUnprefixedBlocks(refCountedDB));
+ countUnprefixedBlocks(refCountedDB, cData));
// Test that deleted block keys do not have a visible prefix when
// iterating.
@@ -175,14 +177,15 @@ public class TestSchemaOneBackwardsCompatibility {
// Test rangeKVs.
List<? extends Table.KeyValue<String, ChunkInfoList>> deletedBlocks =
- deletedBlocksTable.getRangeKVs(null, 100);
+ deletedBlocksTable.getRangeKVs(cData.startKeyEmpty(), 100);
for (Table.KeyValue<String, ChunkInfoList> kv: deletedBlocks) {
assertFalse(kv.getKey().contains(prefix));
}
// Test sequentialRangeKVs.
- deletedBlocks = deletedBlocksTable.getRangeKVs(null, 100);
+ deletedBlocks = deletedBlocksTable.getRangeKVs(cData.startKeyEmpty(),
+ 100);
for (Table.KeyValue<String, ChunkInfoList> kv: deletedBlocks) {
assertFalse(kv.getKey().contains(prefix));
@@ -215,17 +218,18 @@ public class TestSchemaOneBackwardsCompatibility {
public void testReadWithoutMetadata() throws Exception {
// Delete metadata keys from our copy of the DB.
// This simulates them not being there to start with.
- try (DBHandle db = BlockUtils.getDB(newKvData(), conf)) {
+ KeyValueContainerData cData = newKvData();
+ try (DBHandle db = BlockUtils.getDB(cData, conf)) {
Table<String, Long> metadataTable = db.getStore().getMetadataTable();
- metadataTable.delete(OzoneConsts.BLOCK_COUNT);
- assertNull(metadataTable.get(OzoneConsts.BLOCK_COUNT));
+ metadataTable.delete(cData.blockCountKey());
+ assertNull(metadataTable.get(cData.blockCountKey()));
- metadataTable.delete(OzoneConsts.CONTAINER_BYTES_USED);
- assertNull(metadataTable.get(OzoneConsts.CONTAINER_BYTES_USED));
+ metadataTable.delete(cData.bytesUsedKey());
+ assertNull(metadataTable.get(cData.bytesUsedKey()));
- metadataTable.delete(OzoneConsts.PENDING_DELETE_BLOCK_COUNT);
- assertNull(metadataTable.get(OzoneConsts.PENDING_DELETE_BLOCK_COUNT));
+ metadataTable.delete(cData.pendingDeleteBlockCountKey());
+ assertNull(metadataTable.get(cData.pendingDeleteBlockCountKey()));
}
// Create a new container data object, and fill in its metadata by
@@ -278,15 +282,16 @@ public class TestSchemaOneBackwardsCompatibility {
final long expectedRegularBlocks =
TestDB.KEY_COUNT - numBlocksToDelete;
- try (DBHandle refCountedDB = BlockUtils.getDB(newKvData(), conf)) {
+ KeyValueContainerData cData = newKvData();
+ try (DBHandle refCountedDB = BlockUtils.getDB(cData, conf)) {
// Test results via block iteration.
assertEquals(expectedDeletingBlocks,
- countDeletingBlocks(refCountedDB));
+ countDeletingBlocks(refCountedDB, cData));
assertEquals(expectedDeletedBlocks,
TestDB.NUM_DELETED_BLOCKS + numberOfBlocksDeleted);
assertEquals(expectedRegularBlocks,
- countUnprefixedBlocks(refCountedDB));
+ countUnprefixedBlocks(refCountedDB, cData));
// Test table metadata.
// Because the KeyValueHandler used for the block deleting service is
@@ -295,7 +300,7 @@ public class TestSchemaOneBackwardsCompatibility {
Table<String, Long> metadataTable =
refCountedDB.getStore().getMetadataTable();
assertEquals(expectedRegularBlocks + expectedDeletingBlocks,
- (long)metadataTable.get(OzoneConsts.BLOCK_COUNT));
+ (long)metadataTable.get(cData.blockCountKey()));
}
}
@@ -320,11 +325,12 @@ public class TestSchemaOneBackwardsCompatibility {
new KeyValueHandler(conf, datanodeUuid, containerSet, volumeSet,
metrics, c -> {
});
- try (DBHandle refCountedDB = BlockUtils.getDB(newKvData(), conf)) {
+ KeyValueContainerData cData = newKvData();
+ try (DBHandle refCountedDB = BlockUtils.getDB(cData, conf)) {
// Read blocks that were already deleted before the upgrade.
List<? extends Table.KeyValue<String, ChunkInfoList>> deletedBlocks =
- refCountedDB.getStore()
- .getDeletedBlocksTable().getRangeKVs(null, 100);
+ refCountedDB.getStore().getDeletedBlocksTable()
+ .getRangeKVs(cData.startKeyEmpty(), 100);
Set<String> preUpgradeBlocks = new HashSet<>();
@@ -370,20 +376,22 @@ public class TestSchemaOneBackwardsCompatibility {
@Test
public void testReadBlockData() throws Exception {
- try (DBHandle refCountedDB = BlockUtils.getDB(newKvData(), conf)) {
+ KeyValueContainerData cData = newKvData();
+ try (DBHandle refCountedDB = BlockUtils.getDB(cData, conf)) {
Table<String, BlockData> blockDataTable =
refCountedDB.getStore().getBlockDataTable();
// Test encoding keys and decoding database values.
for (String blockID: TestDB.BLOCK_IDS) {
- BlockData blockData = blockDataTable.get(blockID);
+ String blockKey = cData.blockKey(Long.parseLong(blockID));
+ BlockData blockData = blockDataTable.get(blockKey);
Assert.assertEquals(Long.toString(blockData.getLocalID()), blockID);
}
// Test decoding keys from the database.
List<? extends Table.KeyValue<String, BlockData>> blockKeyValues =
- blockDataTable.getRangeKVs(null, 100,
- MetadataKeyFilters.getUnprefixedKeyFilter());
+ blockDataTable.getRangeKVs(cData.startKeyEmpty(), 100,
+ cData.getUnprefixedKeyFilter());
List<String> decodedKeys = new ArrayList<>();
@@ -412,20 +420,21 @@ public class TestSchemaOneBackwardsCompatibility {
@Test
public void testReadDeletingBlockData() throws Exception {
- try (DBHandle refCountedDB = BlockUtils.getDB(newKvData(), conf)) {
+ KeyValueContainerData cData = newKvData();
+ try (DBHandle refCountedDB = BlockUtils.getDB(cData, conf)) {
Table<String, BlockData> blockDataTable =
refCountedDB.getStore().getBlockDataTable();
for (String blockID: TestDB.DELETING_BLOCK_IDS) {
- BlockData blockData =
- blockDataTable.get(OzoneConsts.DELETING_KEY_PREFIX + blockID);
+ String blockKey = cData.deletingBlockKey(Long.parseLong(blockID));
+ BlockData blockData = blockDataTable.get(blockKey);
Assert.assertEquals(Long.toString(blockData.getLocalID()), blockID);
}
// Test decoding keys from the database.
List<? extends Table.KeyValue<String, BlockData>> blockKeyValues =
- blockDataTable.getRangeKVs(null, 100,
- MetadataKeyFilters.getDeletingKeyFilter());
+ blockDataTable.getRangeKVs(cData.startKeyEmpty(), 100,
+ cData.getDeletingBlockKeyFilter());
List<String> decodedKeys = new ArrayList<>();
@@ -437,14 +446,13 @@ public class TestSchemaOneBackwardsCompatibility {
// Apply the deleting prefix to the saved block IDs so we can compare
// them to the retrieved keys.
List<String> expectedKeys = TestDB.DELETING_BLOCK_IDS.stream()
- .map(key -> OzoneConsts.DELETING_KEY_PREFIX + key)
+ .map(key -> cData.deletingBlockKey(Long.parseLong(key)))
.collect(Collectors.toList());
Assert.assertEquals(expectedKeys, decodedKeys);
// Test reading deleting blocks with block iterator.
- MetadataKeyFilters.KeyPrefixFilter filter =
- MetadataKeyFilters.getDeletingKeyFilter();
+ KeyPrefixFilter filter = cData.getDeletingBlockKeyFilter();
try (BlockIterator<BlockData> iter =
refCountedDB.getStore().getBlockIterator(TestDB.CONTAINER_ID,
@@ -464,23 +472,25 @@ public class TestSchemaOneBackwardsCompatibility {
@Test
public void testReadMetadata() throws Exception {
- try (DBHandle refCountedDB = BlockUtils.getDB(newKvData(), conf)) {
+ KeyValueContainerData cData = newKvData();
+ try (DBHandle refCountedDB = BlockUtils.getDB(cData, conf)) {
Table<String, Long> metadataTable =
refCountedDB.getStore().getMetadataTable();
Assert.assertEquals(TestDB.KEY_COUNT,
- metadataTable.get(OzoneConsts.BLOCK_COUNT).longValue());
+ metadataTable.get(cData.blockCountKey()).longValue());
Assert.assertEquals(TestDB.BYTES_USED,
- metadataTable.get(OzoneConsts.CONTAINER_BYTES_USED).longValue());
+ metadataTable.get(cData.bytesUsedKey()).longValue());
Assert.assertEquals(TestDB.NUM_PENDING_DELETION_BLOCKS,
- metadataTable.get(OzoneConsts.PENDING_DELETE_BLOCK_COUNT)
+ metadataTable.get(cData.pendingDeleteBlockCountKey())
.longValue());
}
}
@Test
public void testReadDeletedBlocks() throws Exception {
- try (DBHandle refCountedDB = BlockUtils.getDB(newKvData(), conf)) {
+ KeyValueContainerData cData = newKvData();
+ try (DBHandle refCountedDB = BlockUtils.getDB(cData, conf)) {
Table<String, ChunkInfoList> deletedBlocksTable =
refCountedDB.getStore().getDeletedBlocksTable();
@@ -492,7 +502,7 @@ public class TestSchemaOneBackwardsCompatibility {
// Test decoding keys from the database.
List<? extends Table.KeyValue<String, ChunkInfoList>> chunkInfoKeyValues
=
- deletedBlocksTable.getRangeKVs(null, 100);
+ deletedBlocksTable.getRangeKVs(cData.startKeyEmpty(), 100);
List<String> decodedKeys = new ArrayList<>();
@@ -585,25 +595,28 @@ public class TestSchemaOneBackwardsCompatibility {
kvData.getNumPendingDeletionBlocks());
}
- private int countDeletedBlocks(DBHandle refCountedDB)
+ private int countDeletedBlocks(DBHandle refCountedDB,
+ KeyValueContainerData cData)
throws IOException {
return refCountedDB.getStore().getDeletedBlocksTable()
- .getRangeKVs(null, 100,
- MetadataKeyFilters.getUnprefixedKeyFilter()).size();
+ .getRangeKVs(cData.startKeyEmpty(), 100,
+ cData.getUnprefixedKeyFilter()).size();
}
- private int countDeletingBlocks(DBHandle refCountedDB)
+ private int countDeletingBlocks(DBHandle refCountedDB,
+ KeyValueContainerData cData)
throws IOException {
return refCountedDB.getStore().getBlockDataTable()
- .getRangeKVs(null, 100,
- MetadataKeyFilters.getDeletingKeyFilter()).size();
+ .getRangeKVs(cData.startKeyEmpty(), 100,
+ cData.getDeletingBlockKeyFilter()).size();
}
- private int countUnprefixedBlocks(DBHandle refCountedDB)
+ private int countUnprefixedBlocks(DBHandle refCountedDB,
+ KeyValueContainerData cData)
throws IOException {
return refCountedDB.getStore().getBlockDataTable()
- .getRangeKVs(null, 100,
- MetadataKeyFilters.getUnprefixedKeyFilter()).size();
+ .getRangeKVs(cData.startKeyEmpty(), 100,
+ cData.getUnprefixedKeyFilter()).size();
}
/**
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
index 655f400..afd5fc2 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueBlockIterator.java
@@ -219,7 +219,7 @@ public class TestKeyValueBlockIterator {
normalBlocks, deletingBlocks);
try (BlockIterator<BlockData> keyValueBlockIterator =
db.getStore().getBlockIterator(CONTAINER_ID,
- MetadataKeyFilters.getDeletingKeyFilter())) {
+ containerData.getDeletingBlockKeyFilter())) {
List<Long> deletingBlockIDs =
blockIDs.get(OzoneConsts.DELETING_KEY_PREFIX);
int counter = 0;
@@ -281,13 +281,13 @@ public class TestKeyValueBlockIterator {
Map<String, List<Long>> blockIDs = createContainerWithBlocks(CONTAINER_ID,
prefixCounts);
// Test deleting filter.
- testWithFilter(MetadataKeyFilters.getDeletingKeyFilter(),
+ testWithFilter(containerData.getDeletingBlockKeyFilter(),
blockIDs.get(OzoneConsts.DELETING_KEY_PREFIX));
// Test arbitrary filter.
MetadataKeyFilters.KeyPrefixFilter secondFilter =
new MetadataKeyFilters.KeyPrefixFilter()
- .addFilter(secondPrefix);
+ .addFilter(containerData.containerPrefix() + secondPrefix);
testWithFilter(secondFilter, blockIDs.get(secondPrefix));
}
@@ -394,8 +394,9 @@ public class TestKeyValueBlockIterator {
blockIndex++;
BlockData blockData = new BlockData(blockID);
blockData.setChunks(chunkList);
- String localID = prefix + blockID.getLocalID();
- blockDataTable.put(localID, blockData);
+ String blockKey = containerData.containerPrefix() + prefix +
+ blockID.getLocalID();
+ blockDataTable.put(blockKey, blockData);
blockIDs.get(prefix).add(blockID.getLocalID());
}
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
index ea009b7..5be7627 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainer.java
@@ -321,19 +321,20 @@ public class TestKeyValueContainer {
* Add some keys to the container.
*/
private void populate(long numberOfKeysToWrite) throws IOException {
- try (DBHandle metadataStore =
- BlockUtils.getDB(keyValueContainer.getContainerData(), CONF)) {
+ KeyValueContainerData cData = keyValueContainer.getContainerData();
+ try (DBHandle metadataStore = BlockUtils.getDB(cData, CONF)) {
Table<String, BlockData> blockDataTable =
metadataStore.getStore().getBlockDataTable();
for (long i = 0; i < numberOfKeysToWrite; i++) {
- blockDataTable.put("test" + i, new BlockData(new BlockID(i, i)));
+ blockDataTable.put(cData.blockKey(i),
+ new BlockData(new BlockID(i, i)));
}
// As now when we put blocks, we increment block count and update in DB.
// As for test, we are doing manually so adding key count to DB.
metadataStore.getStore().getMetadataTable()
- .put(OzoneConsts.BLOCK_COUNT, numberOfKeysToWrite);
+ .put(cData.blockCountKey(), numberOfKeysToWrite);
}
Map<String, String> metadata = new HashMap<>();
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java
index a4ee503..c431903 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerIntegrityChecks.java
@@ -23,7 +23,6 @@ import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.common.Checksum;
import org.apache.hadoop.ozone.common.ChecksumData;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
@@ -166,10 +165,10 @@ public class TestKeyValueContainerIntegrityChecks {
blockData.setChunks(chunkList);
// normal key
- String key = Long.toString(blockID.getLocalID());
+ String key = containerData.blockKey(blockID.getLocalID());
if (i >= normalBlocks) {
// deleted key
- key = OzoneConsts.DELETING_KEY_PREFIX + blockID.getLocalID();
+ key = containerData.deletingBlockKey(blockID.getLocalID());
}
metadataStore.getStore().getBlockDataTable().put(key, blockData);
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
index b7334ac..a2a6d6b 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/TestKeyValueContainerMetadataInspector.java
@@ -293,8 +293,8 @@ public class TestKeyValueContainerMetadataInspector
try (DBHandle db = BlockUtils.getDB(containerData, getConf())) {
Table<String, Long> metadataTable = db.getStore().getMetadataTable();
// Don't care about in memory state. Just change the DB values.
- metadataTable.put(OzoneConsts.BLOCK_COUNT, blockCount);
- metadataTable.put(OzoneConsts.CONTAINER_BYTES_USED, byteCount);
+ metadataTable.put(containerData.blockCountKey(), blockCount);
+ metadataTable.put(containerData.bytesUsedKey(), byteCount);
}
}
@@ -303,10 +303,10 @@ public class TestKeyValueContainerMetadataInspector
try (DBHandle db = BlockUtils.getDB(containerData, getConf())) {
Table<String, Long> metadataTable = db.getStore().getMetadataTable();
- long bytesUsed = metadataTable.get(OzoneConsts.CONTAINER_BYTES_USED);
+ long bytesUsed = metadataTable.get(containerData.bytesUsedKey());
Assert.assertEquals(expectedBytesUsed, bytesUsed);
- long blockCount = metadataTable.get(OzoneConsts.BLOCK_COUNT);
+ long blockCount = metadataTable.get(containerData.blockCountKey());
Assert.assertEquals(expectedBlockCount, blockCount);
}
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
index ee1ca75..0c70e79 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestContainerReader.java
@@ -124,18 +124,19 @@ public class TestContainerReader {
private void markBlocksForDelete(KeyValueContainer keyValueContainer,
boolean setMetaData, List<Long> blockNames, int count) throws Exception {
- try (DBHandle metadataStore = BlockUtils.getDB(keyValueContainer
- .getContainerData(), conf)) {
+ KeyValueContainerData cData = keyValueContainer.getContainerData();
+ try (DBHandle metadataStore = BlockUtils.getDB(cData, conf)) {
for (int i = 0; i < count; i++) {
Table<String, BlockData> blockDataTable =
metadataStore.getStore().getBlockDataTable();
- String blk = Long.toString(blockNames.get(i));
+ Long localID = blockNames.get(i);
+ String blk = cData.blockKey(localID);
BlockData blkInfo = blockDataTable.get(blk);
blockDataTable.delete(blk);
- blockDataTable.put(OzoneConsts.DELETING_KEY_PREFIX + blk, blkInfo);
+ blockDataTable.put(cData.deletingBlockKey(localID), blkInfo);
}
if (setMetaData) {
@@ -143,7 +144,7 @@ public class TestContainerReader {
// and bytes used metadata values, so those do not change.
Table<String, Long> metadataTable =
metadataStore.getStore().getMetadataTable();
- metadataTable.put(OzoneConsts.PENDING_DELETE_BLOCK_COUNT, (long)count);
+ metadataTable.put(cData.pendingDeleteBlockCountKey(), (long)count);
}
}
@@ -152,10 +153,9 @@ public class TestContainerReader {
private List<Long> addBlocks(KeyValueContainer keyValueContainer,
boolean setMetaData) throws Exception {
long containerId = keyValueContainer.getContainerData().getContainerID();
-
+ KeyValueContainerData cData = keyValueContainer.getContainerData();
List<Long> blkNames = new ArrayList<>();
- try (DBHandle metadataStore = BlockUtils.getDB(keyValueContainer
- .getContainerData(), conf)) {
+ try (DBHandle metadataStore = BlockUtils.getDB(cData, conf)) {
for (int i = 0; i < blockCount; i++) {
// Creating BlockData
@@ -172,14 +172,14 @@ public class TestContainerReader {
blockData.setChunks(chunkList);
blkNames.add(localBlockID);
metadataStore.getStore().getBlockDataTable()
- .put(Long.toString(localBlockID), blockData);
+ .put(cData.blockKey(localBlockID), blockData);
}
if (setMetaData) {
metadataStore.getStore().getMetadataTable()
- .put(OzoneConsts.BLOCK_COUNT, (long)blockCount);
+ .put(cData.blockCountKey(), (long)blockCount);
metadataStore.getStore().getMetadataTable()
- .put(OzoneConsts.CONTAINER_BYTES_USED, blockCount * blockLen);
+ .put(cData.bytesUsedKey(), blockCount * blockLen);
}
}
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
index c9d79e8..35fef4a 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainer.java
@@ -30,7 +30,6 @@ import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import
org.apache.hadoop.hdds.scm.container.common.helpers.StorageContainerException;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.hdds.utils.db.Table;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.container.common.helpers.BlockData;
import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
@@ -277,35 +276,37 @@ public class TestOzoneContainer {
long freeBytes = container.getContainerData().getMaxSize();
long containerId = container.getContainerData().getContainerID();
- DBHandle db = BlockUtils.getDB(container.getContainerData(), conf);
-
- Table<String, Long> metadataTable = db.getStore().getMetadataTable();
- Table<String, BlockData> blockDataTable =
db.getStore().getBlockDataTable();
-
- for (int bi = 0; bi < blocks; bi++) {
- // Creating BlockData
- BlockID blockID = new BlockID(containerId, bi);
- BlockData blockData = new BlockData(blockID);
- List<ContainerProtos.ChunkInfo> chunkList = new ArrayList<>();
-
- chunkList.clear();
- for (int ci = 0; ci < chunksPerBlock; ci++) {
- String chunkName = strBlock + bi + strChunk + ci;
- long offset = ci * (long) datalen;
- ChunkInfo info = new ChunkInfo(chunkName, offset, datalen);
- usedBytes += datalen;
- chunkList.add(info.getProtoBufMessage());
+ KeyValueContainerData cData = container.getContainerData();
+ try (DBHandle db = BlockUtils.getDB(cData, conf)) {
+
+ Table<String, Long> metadataTable =
+ db.getStore().getMetadataTable();
+ Table<String, BlockData> blockDataTable =
+ db.getStore().getBlockDataTable();
+
+ for (int bi = 0; bi < blocks; bi++) {
+ // Creating BlockData
+ BlockID blockID = new BlockID(containerId, bi);
+ BlockData blockData = new BlockData(blockID);
+ List<ContainerProtos.ChunkInfo> chunkList = new ArrayList<>();
+
+ chunkList.clear();
+ for (int ci = 0; ci < chunksPerBlock; ci++) {
+ String chunkName = strBlock + bi + strChunk + ci;
+ long offset = ci * (long) datalen;
+ ChunkInfo info = new ChunkInfo(chunkName, offset, datalen);
+ usedBytes += datalen;
+ chunkList.add(info.getProtoBufMessage());
+ }
+ blockData.setChunks(chunkList);
+ blockDataTable.put(cData.blockKey(blockID.getLocalID()), blockData);
}
- blockData.setChunks(chunkList);
- blockDataTable.put(Long.toString(blockID.getLocalID()), blockData);
- }
-
- // Set Block count and used bytes.
- metadataTable.put(OzoneConsts.BLOCK_COUNT, (long)blocks);
- metadataTable.put(OzoneConsts.CONTAINER_BYTES_USED, usedBytes);
+ // Set Block count and used bytes.
+ metadataTable.put(cData.blockCountKey(), (long) blocks);
+ metadataTable.put(cData.bytesUsedKey(), usedBytes);
+ }
// remaining available capacity of the container
- db.close();
return (freeBytes - usedBytes);
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataKeyFilters.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataKeyFilters.java
index e3b91ba..31660c0 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataKeyFilters.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/utils/MetadataKeyFilters.java
@@ -32,6 +32,7 @@ import com.google.common.base.Strings;
public final class MetadataKeyFilters {
private MetadataKeyFilters() { }
+ @Deprecated
public static KeyPrefixFilter getDeletingKeyFilter() {
return new MetadataKeyFilters.KeyPrefixFilter()
.addFilter(OzoneConsts.DELETING_KEY_PREFIX);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
index cdecca3..992bf40 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestStorageContainerManagerHelper.java
@@ -25,7 +25,6 @@ import java.util.Set;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.protocol.DatanodeDetails;
import
org.apache.hadoop.hdds.scm.container.common.helpers.ContainerWithPipeline;
-import org.apache.hadoop.hdds.utils.MetadataKeyFilters;
import org.apache.hadoop.hdds.utils.MetadataKeyFilters.KeyPrefixFilter;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -102,17 +101,17 @@ public class TestStorageContainerManagerHelper {
public List<String> getPendingDeletionBlocks(Long containerID)
throws IOException {
List<String> pendingDeletionBlocks = Lists.newArrayList();
- try (DBHandle meta = getContainerMetadata(containerID)) {
- KeyPrefixFilter filter =
- new KeyPrefixFilter().addFilter(OzoneConsts.DELETING_KEY_PREFIX);
+ KeyValueContainerData cData = getContainerMetadata(containerID);
+ try (DBHandle db = BlockUtils.getDB(cData, conf)) {
+ KeyPrefixFilter filter = cData.getDeletingBlockKeyFilter();
List<? extends Table.KeyValue<String, BlockData>> kvs =
- meta.getStore().getBlockDataTable()
- .getRangeKVs(null, Integer.MAX_VALUE, filter);
+ db.getStore().getBlockDataTable()
+ .getRangeKVs(cData.startKeyEmpty(), Integer.MAX_VALUE, filter);
for (Table.KeyValue<String, BlockData> entry : kvs) {
pendingDeletionBlocks
- .add(entry.getKey().replace(OzoneConsts.DELETING_KEY_PREFIX, ""));
+ .add(entry.getKey().replace(cData.deletingBlockKeyPrefix(), ""));
}
}
return pendingDeletionBlocks;
@@ -129,12 +128,13 @@ public class TestStorageContainerManagerHelper {
public List<Long> getAllBlocks(Long containeID) throws IOException {
List<Long> allBlocks = Lists.newArrayList();
- try (DBHandle meta = getContainerMetadata(containeID)) {
+ KeyValueContainerData cData = getContainerMetadata(containeID);
+ try (DBHandle db = BlockUtils.getDB(cData, conf)) {
List<? extends Table.KeyValue<String, BlockData>> kvs =
- meta.getStore().getBlockDataTable()
- .getRangeKVs(null, Integer.MAX_VALUE,
- MetadataKeyFilters.getUnprefixedKeyFilter());
+ db.getStore().getBlockDataTable()
+ .getRangeKVs(cData.startKeyEmpty(), Integer.MAX_VALUE,
+ cData.getUnprefixedKeyFilter());
for (Table.KeyValue<String, BlockData> entry : kvs) {
allBlocks.add(Long.valueOf(entry.getKey()));
@@ -146,8 +146,9 @@ public class TestStorageContainerManagerHelper {
public boolean verifyBlocksWithTxnTable(Map<Long, List<Long>>
containerBlocks)
throws IOException {
for (Map.Entry<Long, List<Long>> entry : containerBlocks.entrySet()) {
- try (DBHandle meta = getContainerMetadata(entry.getKey())) {
- DatanodeStore ds = meta.getStore();
+ KeyValueContainerData cData = getContainerMetadata(entry.getKey());
+ try (DBHandle db = BlockUtils.getDB(cData, conf)) {
+ DatanodeStore ds = db.getStore();
DatanodeStoreSchemaTwoImpl dnStoreTwoImpl =
(DatanodeStoreSchemaTwoImpl) ds;
List<? extends Table.KeyValue<Long, DeletedBlocksTransaction>>
@@ -166,7 +167,7 @@ public class TestStorageContainerManagerHelper {
return true;
}
- private DBHandle getContainerMetadata(Long containerID)
+ private KeyValueContainerData getContainerMetadata(Long containerID)
throws IOException {
ContainerWithPipeline containerWithPipeline = cluster
.getStorageContainerManager().getClientProtocolServer()
@@ -179,7 +180,7 @@ public class TestStorageContainerManagerHelper {
KeyValueContainerData containerData =
(KeyValueContainerData) containerServer.getContainerSet()
.getContainer(containerID).getContainerData();
- return BlockUtils.getDB(containerData, conf);
+ return containerData;
}
private OzoneContainer getContainerServerByDatanodeUuid(String dnUUID)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index a9f4621..3b84f9f 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -275,7 +275,8 @@ public class TestFailureHandlingByClient {
.getContainer(containerId1)).getContainerData();
try (DBHandle containerDb1 = BlockUtils.getDB(containerData1, conf)) {
BlockData blockData1 = containerDb1.getStore().getBlockDataTable().get(
- Long.toString(locationList.get(0).getBlockID().getLocalID()));
+ containerData1.blockKey(locationList.get(0).getBlockID()
+ .getLocalID()));
// The first Block could have 1 or 2 chunkSize of data
int block1NumChunks = blockData1.getChunks().size();
Assert.assertTrue(block1NumChunks >= 1);
@@ -293,7 +294,8 @@ public class TestFailureHandlingByClient {
.getContainer(containerId2)).getContainerData();
try (DBHandle containerDb2 = BlockUtils.getDB(containerData2, conf)) {
BlockData blockData2 = containerDb2.getStore().getBlockDataTable().get(
- Long.toString(locationList.get(1).getBlockID().getLocalID()));
+ containerData2.blockKey(locationList.get(1).getBlockID()
+ .getLocalID()));
// The second Block should have 0.5 chunkSize of data
Assert.assertEquals(block2ExpectedChunkCount,
blockData2.getChunks().size());
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
index fd7355b..35618d9 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneRpcClientAbstract.java
@@ -1597,11 +1597,10 @@ public abstract class TestOzoneRpcClientAbstract {
// Change first and second replica commit sequenceId
if (index < 3) {
long newBCSID = container.getBlockCommitSequenceId() - 1;
- try (DBHandle db = BlockUtils.getDB(
- (KeyValueContainerData) container.getContainerData(),
- cluster.getConf())) {
- db.getStore().getMetadataTable().put(
- OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID, newBCSID);
+ KeyValueContainerData cData =
+ (KeyValueContainerData) container.getContainerData();
+ try (DBHandle db = BlockUtils.getDB(cData, cluster.getConf())) {
+ db.getStore().getMetadataTable().put(cData.bcsIdKey(), newBCSID);
}
container.updateBlockCommitSequenceId(newBCSID);
index++;
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
index cdaf4c1..e70ffc5 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
@@ -37,7 +37,6 @@ import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
@@ -228,7 +227,7 @@ public class TestValidateBCSIDOnRestart {
// modify the bcsid for the container in the ROCKS DB thereby inducing
// corruption
db.getStore().getMetadataTable()
- .put(OzoneConsts.BLOCK_COMMIT_SEQUENCE_ID, 0L);
+ .put(keyValueContainerData.bcsIdKey(), 0L);
}
// after the restart, there will be a mismatch in BCSID of what is recorded
// in the and what is there in RockSDB and hence the container would be
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index 73f5fc0..c5dba0a 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.hdds.server.events.EventQueue;
import org.apache.hadoop.hdds.utils.db.Table;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.OzoneTestUtils;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneBucket;
@@ -439,12 +438,11 @@ public class TestBlockDeletion {
ContainerSet dnContainerSet =
datanode.getDatanodeStateMachine().getContainer().getContainerSet();
OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
- try (DBHandle db = BlockUtils.getDB(
- (KeyValueContainerData) dnContainerSet
- .getContainer(blockID.getContainerID()).getContainerData(),
- conf)) {
+ KeyValueContainerData cData = (KeyValueContainerData) dnContainerSet
+ .getContainer(blockID.getContainerID()).getContainerData();
+ try (DBHandle db = BlockUtils.getDB(cData, conf)) {
Assert.assertNotNull(db.getStore().getBlockDataTable()
- .get(Long.toString(blockID.getLocalID())));
+ .get(cData.blockKey(blockID.getLocalID())));
}
}, omKeyLocationInfoGroups);
}
@@ -456,19 +454,18 @@ public class TestBlockDeletion {
ContainerSet dnContainerSet =
datanode.getDatanodeStateMachine().getContainer().getContainerSet();
OzoneTestUtils.performOperationOnKeyContainers((blockID) -> {
- try (DBHandle db = BlockUtils.getDB(
- (KeyValueContainerData) dnContainerSet
- .getContainer(blockID.getContainerID()).getContainerData(),
- conf)) {
+ KeyValueContainerData cData = (KeyValueContainerData) dnContainerSet
+ .getContainer(blockID.getContainerID()).getContainerData();
+ try (DBHandle db = BlockUtils.getDB(cData, conf)) {
Table<String, BlockData> blockDataTable =
db.getStore().getBlockDataTable();
- String blockIDString = Long.toString(blockID.getLocalID());
+ String blockKey = cData.blockKey(blockID.getLocalID());
- BlockData blockData = blockDataTable.get(blockIDString);
+ BlockData blockData = blockDataTable.get(blockKey);
Assert.assertNull(blockData);
- String deletingKey = OzoneConsts.DELETING_KEY_PREFIX + blockIDString;
+ String deletingKey = cData.deletingBlockKey(blockID.getLocalID());
Assert.assertNull(blockDataTable.get(deletingKey));
}
containerIdsWithDeletedBlocks.add(blockID.getContainerID());
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
