This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8a774a57df HDDS-11989. Enable SCM Ratis in tests related to
DeletedBlockLog (#7615)
8a774a57df is described below
commit 8a774a57df907c1e5c6c274054cfde21f914a33b
Author: Chung En Lee <[email protected]>
AuthorDate: Tue Jan 7 15:15:26 2025 +0800
HDDS-11989. Enable SCM Ratis in tests related to DeletedBlockLog (#7615)
---
.../hdds/scm/TestStorageContainerManager.java | 12 +-----
.../org/apache/hadoop/ozone/OzoneTestUtils.java | 33 +++++++++++++++
.../client/rpc/TestDeleteWithInAdequateDN.java | 5 +++
.../commandhandler/TestBlockDeletion.java | 48 +++++++++++-----------
.../commandhandler/TestDeleteContainerHandler.java | 8 ++++
5 files changed, 72 insertions(+), 34 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
index 94c8f91429..47f6d3823d 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestStorageContainerManager.java
@@ -321,17 +321,7 @@ public class TestStorageContainerManager {
// after sometime, all the TX should be proceed and by then
// the number of containerBlocks of all known containers will be
// empty again.
- GenericTestUtils.waitFor(() -> {
- try {
- if (SCMHAUtils.isSCMHAEnabled(cluster.getConf())) {
- cluster.getStorageContainerManager().getScmHAManager()
- .asSCMHADBTransactionBuffer().flush();
- }
- return delLog.getNumOfValidTransactions() == 0;
- } catch (IOException e) {
- return false;
- }
- }, 1000, 22000);
+ OzoneTestUtils.waitBlockDeleted(cluster.getStorageContainerManager());
assertTrue(verifyBlocksWithTxnTable(cluster, conf, containerBlocks));
// Continue the work, add some TXs that with known container names,
// but unknown block IDs.
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
index 884e435d25..0a5f7114c4 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/OzoneTestUtils.java
@@ -161,4 +161,37 @@ public final class OzoneTestUtils {
container.getState() == HddsProtos.LifeCycleState.CLOSED,
200, 30000);
}
+
+ /**
+ * Flush deleted block log & wait till something was flushed.
+ */
+ public static void flushAndWaitForDeletedBlockLog(StorageContainerManager
scm)
+ throws InterruptedException, TimeoutException {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
+ if
(scm.getScmBlockManager().getDeletedBlockLog().getNumOfValidTransactions() > 0)
{
+ return true;
+ }
+ } catch (IOException e) {
+ }
+ return false;
+ }, 100, 3000);
+ }
+
+ /**
+ * Wait till all blocks are removed.
+ */
+ public static void waitBlockDeleted(StorageContainerManager scm)
+ throws InterruptedException, TimeoutException {
+ GenericTestUtils.waitFor(() -> {
+ try {
+ if
(scm.getScmBlockManager().getDeletedBlockLog().getNumOfValidTransactions() ==
0) {
+ return true;
+ }
+ } catch (IOException e) {
+ }
+ return false;
+ }, 1000, 60000);
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithInAdequateDN.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithInAdequateDN.java
index bb42d8a0f5..2b199306b7 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithInAdequateDN.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestDeleteWithInAdequateDN.java
@@ -42,6 +42,7 @@ import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.ozone.HddsDatanodeService;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.OzoneTestUtils;
import org.apache.hadoop.ozone.RatisTestHelper;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
@@ -65,6 +66,7 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_I
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_CREATION_INTERVAL;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
@@ -73,6 +75,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
+
import org.junit.jupiter.api.AfterAll;
import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
@@ -103,6 +106,7 @@ public class TestDeleteWithInAdequateDN {
conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_SCM_HA_ENABLE_KEY, true);
conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 100,
TimeUnit.MILLISECONDS);
conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 200,
@@ -281,6 +285,7 @@ public class TestDeleteWithInAdequateDN {
//cluster.getOzoneManager().deleteKey(keyArgs);
client.getObjectStore().getVolume(volumeName).getBucket(bucketName).
deleteKey("ratis");
+
OzoneTestUtils.flushAndWaitForDeletedBlockLog(cluster.getStorageContainerManager());
// make sure the chunk was never deleted on the leader even though
// deleteBlock handler is invoked
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
index cf7d26847b..e38312e02e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestBlockDeletion.java
@@ -47,7 +47,6 @@ import
org.apache.hadoop.hdds.scm.block.SCMBlockDeletingService;
import org.apache.hadoop.hdds.scm.block.ScmBlockDeletingServiceMetrics;
import org.apache.hadoop.hdds.scm.container.ContainerID;
import org.apache.hadoop.hdds.scm.container.ContainerInfo;
-import org.apache.hadoop.hdds.scm.container.ContainerNotFoundException;
import org.apache.hadoop.hdds.scm.container.ContainerReplica;
import org.apache.hadoop.hdds.scm.container.ContainerStateManager;
import org.apache.hadoop.hdds.scm.container.replication.ReplicationManager;
@@ -95,6 +94,7 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_COMMAND_STATUS_REPORT_I
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERVAL;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_EXPIRED_CONTAINER_REPLICA_OP_SCRUB_INTERVAL;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_OWNER_CONTAINER_COUNT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
@@ -133,6 +133,7 @@ public class TestBlockDeletion {
GenericTestUtils.setLogLevel(SCMBlockDeletingService.LOG, Level.DEBUG);
GenericTestUtils.setLogLevel(ReplicationManager.LOG, Level.DEBUG);
+ conf.setBoolean(OZONE_SCM_HA_ENABLE_KEY, true);
conf.set("ozone.replication.allowed-configs",
"^(RATIS/THREE)|(EC/2-1-256k)$");
conf.setTimeDuration(OZONE_BLOCK_DELETING_SERVICE_INTERVAL, 100,
@@ -239,6 +240,7 @@ public class TestBlockDeletion {
// verify key blocks were created in DN.
GenericTestUtils.waitFor(() -> {
try {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
verifyBlocksCreated(omKeyLocationInfoGroupList);
return true;
} catch (Throwable t) {
@@ -283,6 +285,7 @@ public class TestBlockDeletion {
// The blocks should be deleted in the DN.
GenericTestUtils.waitFor(() -> {
try {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
verifyBlocksDeleted(omKeyLocationInfoGroupList);
return true;
} catch (Throwable t) {
@@ -299,6 +302,7 @@ public class TestBlockDeletion {
// Verify transactions committed
GenericTestUtils.waitFor(() -> {
try {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
verifyTransactionsCommitted();
return true;
} catch (Throwable t) {
@@ -380,10 +384,16 @@ public class TestBlockDeletion {
writeClient.deleteKey(keyArgs);
// Wait for blocks to be deleted and container reports to be processed
- GenericTestUtils.waitFor(() ->
- scm.getContainerManager().getContainers().stream()
- .allMatch(c -> c.getUsedBytes() == 0 &&
- c.getNumberOfKeys() == 0), 500, 20000);
+ GenericTestUtils.waitFor(() -> {
+ try {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ return scm.getContainerManager().getContainers().stream()
+ .allMatch(c -> c.getUsedBytes() == 0 &&
+ c.getNumberOfKeys() == 0);
+ }, 500, 20000);
Thread.sleep(5000);
// Verify that pending block delete num are as expected with resent cmds
cluster.getHddsDatanodes().forEach(dn -> {
@@ -425,6 +435,7 @@ public class TestBlockDeletion {
assertEquals(HddsProtos.LifeCycleState.DELETED,
container.getState());
try {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
assertEquals(HddsProtos.LifeCycleState.DELETED,
scm.getScmMetadataStore().getContainerTable()
.get(container.containerID()).getState());
@@ -516,14 +527,14 @@ public class TestBlockDeletion {
GenericTestUtils.waitFor(() -> {
try {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
return scm.getContainerManager().getContainerReplicas(
containerId).stream().
allMatch(replica -> replica.isEmpty());
- } catch (ContainerNotFoundException e) {
+ } catch (IOException e) {
throw new RuntimeException(e);
}
- },
- 100, 10 * 1000);
+ }, 100, 10 * 1000);
// Container state should be empty now as key got deleted
assertTrue(getContainerFromDN(
@@ -546,6 +557,7 @@ public class TestBlockDeletion {
assertEquals(HddsProtos.LifeCycleState.DELETED,
container.getState());
try {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
assertEquals(HddsProtos.LifeCycleState.DELETED,
scm.getScmMetadataStore().getContainerTable()
.get(container.containerID()).getState());
@@ -560,7 +572,6 @@ public class TestBlockDeletion {
}
return true;
}, 500, 30000);
- LOG.info(metrics.toString());
}
/**
@@ -646,14 +657,14 @@ public class TestBlockDeletion {
// Ensure isEmpty are true for all replica after delete key
GenericTestUtils.waitFor(() -> {
try {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
return scm.getContainerManager().getContainerReplicas(
containerId).stream()
.allMatch(replica -> replica.isEmpty());
- } catch (ContainerNotFoundException e) {
+ } catch (IOException e) {
throw new RuntimeException(e);
}
- },
- 500, 5 * 2000);
+ }, 500, 5 * 2000);
// Update container replica by making invalid keyCount in one replica
ContainerReplica replicaOne = ContainerReplica.newBuilder()
@@ -683,6 +694,7 @@ public class TestBlockDeletion {
assertEquals(HddsProtos.LifeCycleState.DELETED,
container.getState());
try {
+ scm.getScmHAManager().asSCMHADBTransactionBuffer().flush();
assertEquals(HddsProtos.LifeCycleState.DELETED,
scm.getScmMetadataStore().getContainerTable()
.get(container.containerID()).getState());
@@ -812,17 +824,7 @@ public class TestBlockDeletion {
}
// Wait for block delete command sent from OM
- GenericTestUtils.waitFor(() -> {
- try {
- if (scm.getScmBlockManager().getDeletedBlockLog()
- .getNumOfValidTransactions() > 0) {
- return true;
- }
- } catch (IOException e) {
- }
- return false;
- }, 100, 5000);
-
+ OzoneTestUtils.flushAndWaitForDeletedBlockLog(scm);
long start = System.currentTimeMillis();
// Wait for all blocks been deleted.
GenericTestUtils.waitFor(() -> {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteContainerHandler.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteContainerHandler.java
index 192c933f53..705ef1e0d8 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteContainerHandler.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestDeleteContainerHandler.java
@@ -75,6 +75,7 @@ import static java.nio.charset.StandardCharsets.UTF_8;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
+import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_BLOCK_DELETING_SERVICE_INTERVAL;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -97,6 +98,7 @@ public class TestDeleteContainerHandler {
@BeforeAll
public static void setup() throws Exception {
conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_SCM_HA_ENABLE_KEY, true);
conf.set(OZONE_SCM_CONTAINER_SIZE, "1GB");
conf.setStorageSize(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
0, StorageUnit.MB);
@@ -196,6 +198,8 @@ public class TestDeleteContainerHandler {
// Delete key, which will make isEmpty flag to true in containerData
objectStore.getVolume(volumeName)
.getBucket(bucketName).deleteKey(keyName);
+
OzoneTestUtils.flushAndWaitForDeletedBlockLog(cluster.getStorageContainerManager());
+ OzoneTestUtils.waitBlockDeleted(cluster.getStorageContainerManager());
// Ensure isEmpty flag is true when key is deleted and container is empty
GenericTestUtils.waitFor(() -> getContainerfromDN(
@@ -313,6 +317,8 @@ public class TestDeleteContainerHandler {
// Delete key, which will make isEmpty flag to true in containerData
objectStore.getVolume(volumeName)
.getBucket(bucketName).deleteKey(keyName);
+
OzoneTestUtils.flushAndWaitForDeletedBlockLog(cluster.getStorageContainerManager());
+ OzoneTestUtils.waitBlockDeleted(cluster.getStorageContainerManager());
// Ensure isEmpty flag is true when key is deleted and container is empty
GenericTestUtils.waitFor(() -> getContainerfromDN(
@@ -652,6 +658,8 @@ public class TestDeleteContainerHandler {
// Delete key, which will make isEmpty flag to true in containerData
objectStore.getVolume(volumeName)
.getBucket(bucketName).deleteKey(keyName);
+
OzoneTestUtils.flushAndWaitForDeletedBlockLog(cluster.getStorageContainerManager());
+ OzoneTestUtils.waitBlockDeleted(cluster.getStorageContainerManager());
// Ensure isEmpty flag is true when key is deleted
GenericTestUtils.waitFor(() -> getContainerfromDN(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
