This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 81c61b5b44d HDDS-16224. Wait for replica with container and block
before read in TestFailureHandlingByClient (#11065)
81c61b5b44d is described below
commit 81c61b5b44d0f5fefa10eef7cec6868e06e9d046
Author: KUAN-HAO HUANG <[email protected]>
AuthorDate: Wed Aug 26 01:54:43 2026 +0800
HDDS-16224. Wait for replica with container and block before read in
TestFailureHandlingByClient (#11065)
---
.../client/rpc/TestFailureHandlingByClient.java | 53 ++++++++++++++++------
1 file changed, 40 insertions(+), 13 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
index b672d4610fb..4d971dfeb9e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestFailureHandlingByClient.java
@@ -27,12 +27,14 @@
import static org.junit.jupiter.api.Assertions.assertInstanceOf;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import java.io.IOException;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.UUID;
import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
import org.apache.hadoop.hdds.HddsUtils;
import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.RatisReplicationConfig;
@@ -253,12 +255,7 @@ private void testBlockCountOnFailures(OmKeyInfo omKeyInfo)
throws Exception {
List<OmKeyLocationInfo> locationList =
omKeyInfo.getLatestVersionLocations()
.createLocationList();
long containerId1 = locationList.get(0).getContainerID();
- List<DatanodeDetails> block1DNs = locationList.get(0).getPipeline()
- .getNodes();
long containerId2 = locationList.get(1).getContainerID();
- List<DatanodeDetails> block2DNs = locationList.get(1).getPipeline()
- .getNodes();
-
int block2ExpectedChunkCount;
if (locationList.get(0).getLength() == 2L * chunkSize) {
@@ -275,10 +272,8 @@ private void testBlockCountOnFailures(OmKeyInfo omKeyInfo)
throws Exception {
// write or not). The 3rd chunk would not exist on the first pipeline as
// the pipeline would be closed before the last 0.5 chunk was committed
// to the block.
- KeyValueContainerData containerData1 =
- ((KeyValueContainer) cluster.getHddsDatanode(block1DNs.get(2))
- .getDatanodeStateMachine().getContainer().getContainerSet()
- .getContainer(containerId1)).getContainerData();
+ KeyValueContainerData containerData1 = waitForBlockContainer(containerId1,
+ locationList.get(0).getBlockID().getLocalID());
try (DBHandle containerDb1 = BlockUtils.getDB(containerData1, conf)) {
BlockData blockData1 = containerDb1.getStore().getBlockDataTable().get(
containerData1.getBlockKey(locationList.get(0).getBlockID()
@@ -291,10 +286,8 @@ private void testBlockCountOnFailures(OmKeyInfo omKeyInfo)
throws Exception {
}
// Verify that the second block has the remaining 0.5*chunkSize of data
- KeyValueContainerData containerData2 =
- ((KeyValueContainer) cluster.getHddsDatanode(block2DNs.get(0))
- .getDatanodeStateMachine().getContainer().getContainerSet()
- .getContainer(containerId2)).getContainerData();
+ KeyValueContainerData containerData2 = waitForBlockContainer(containerId2,
+ locationList.get(1).getBlockID().getLocalID());
try (DBHandle containerDb2 = BlockUtils.getDB(containerData2, conf)) {
BlockData blockData2 = containerDb2.getStore().getBlockDataTable().get(
containerData2.getBlockKey(locationList.get(1).getBlockID()
@@ -312,6 +305,40 @@ private void testBlockCountOnFailures(OmKeyInfo omKeyInfo)
throws Exception {
}
}
+ /**
+ * Wait for a datanode replica that has both the container and the target
+ * block applied, and return its container data. A lagging Ratis follower can
+ * hold the container without the block yet, so reading a fixed replica (or
one
+ * chosen only because it has the container) is racy; poll all replicas until
+ * one has the block.
+ */
+ private KeyValueContainerData waitForBlockContainer(long containerId,
+ long blockLocalId) throws Exception {
+ AtomicReference<KeyValueContainerData> found = new AtomicReference<>();
+ GenericTestUtils.waitFor(() -> {
+ for (HddsDatanodeService dn : cluster.getHddsDatanodes()) {
+ KeyValueContainer container = (KeyValueContainer) dn
+ .getDatanodeStateMachine().getContainer().getContainerSet()
+ .getContainer(containerId);
+ if (container == null) {
+ continue;
+ }
+ KeyValueContainerData data = container.getContainerData();
+ try (DBHandle db = BlockUtils.getDB(data, conf)) {
+ if (db.getStore().getBlockDataTable()
+ .get(data.getBlockKey(blockLocalId)) != null) {
+ found.set(data);
+ return true;
+ }
+ } catch (IOException e) {
+ // Container DB not ready on this replica yet; try the next one.
+ }
+ }
+ return false;
+ }, 500, 30000);
+ return found.get();
+ }
+
@Test
public void testWriteSmallFile() throws Exception {
String keyName = UUID.randomUUID().toString();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
