This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 1bd721b2fd HDDS-12081. TestKeyInputStream repeats tests with default
container layout (#7704)
1bd721b2fd is described below
commit 1bd721b2fdf1811246cf460d1cd21f533b206219
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Fri Jan 24 22:01:17 2025 +0100
HDDS-12081. TestKeyInputStream repeats tests with default container layout
(#7704)
---
.../keyvalue/ContainerLayoutTestInfo.java | 2 +-
.../client/rpc/read/TestChunkInputStream.java | 18 ++---
.../ozone/client/rpc/read/TestInputStreamBase.java | 48 ++++++++++--
.../ozone/client/rpc/read/TestKeyInputStream.java | 86 ++++++++--------------
.../commandhandler/TestFinalizeBlock.java | 28 ++-----
5 files changed, 88 insertions(+), 94 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerLayoutTestInfo.java
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerLayoutTestInfo.java
index ab6e2c857c..8349005b87 100644
---
a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerLayoutTestInfo.java
+++
b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/keyvalue/ContainerLayoutTestInfo.java
@@ -122,7 +122,7 @@ private static void assertFileCount(File dir, long count) {
}
/**
- * Composite annotation for tests parameterized with {@link
ContainerLayoutTestInfo}.
+ * Composite annotation for tests parameterized with {@link
ContainerLayoutVersion}.
*/
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java
index c5301ba419..7a6873dfd0 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestChunkInputStream.java
@@ -22,12 +22,13 @@
import org.apache.hadoop.hdds.scm.storage.BlockInputStream;
import org.apache.hadoop.hdds.scm.storage.ChunkInputStream;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.io.KeyInputStream;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.keyvalue.ContainerLayoutTestInfo;
import org.apache.hadoop.ozone.om.TestBucket;
+import org.junit.jupiter.api.TestInstance;
+
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -36,6 +37,7 @@
/**
* Tests {@link ChunkInputStream}.
*/
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TestChunkInputStream extends TestInputStreamBase {
/**
@@ -44,16 +46,14 @@ class TestChunkInputStream extends TestInputStreamBase {
*/
@ContainerLayoutTestInfo.ContainerTest
void testAll(ContainerLayoutVersion layout) throws Exception {
- try (MiniOzoneCluster cluster = newCluster(layout)) {
- cluster.waitForClusterToBeReady();
+ try (OzoneClient client = getCluster().newClient()) {
+ updateConfig(layout);
- try (OzoneClient client = cluster.newClient()) {
- TestBucket bucket = TestBucket.newBuilder(client).build();
+ TestBucket bucket = TestBucket.newBuilder(client).build();
- testChunkReadBuffers(bucket);
- testBufferRelease(bucket);
- testCloseReleasesBuffers(bucket);
- }
+ testChunkReadBuffers(bucket);
+ testBufferRelease(bucket);
+ testCloseReleasesBuffers(bucket);
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java
index 256148dfb8..4835d2c3ba 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestInputStreamBase.java
@@ -25,15 +25,22 @@
import org.apache.hadoop.hdds.scm.OzoneClientConfig;
import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import
org.apache.hadoop.hdds.scm.container.replication.ReplicationManager.ReplicationManagerConfiguration;
+import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.ozone.ClientConfigForTesting;
import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConfigKeys;
+import org.apache.hadoop.ozone.container.TestHelper;
import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.TestInstance;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_DEADNODE_INTERVAL;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
-// TODO remove this class, set config as default in integration tests
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
abstract class TestInputStreamBase {
static final int CHUNK_SIZE = 1024 * 1024; // 1MB
@@ -42,8 +49,7 @@ abstract class TestInputStreamBase {
static final int BLOCK_SIZE = 2 * MAX_FLUSH_SIZE; // 8MB
static final int BYTES_PER_CHECKSUM = 256 * 1024; // 256KB
- protected static MiniOzoneCluster newCluster(
- ContainerLayoutVersion containerLayout) throws Exception {
+ protected static MiniOzoneCluster newCluster() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
OzoneClientConfig config = conf.getObject(OzoneClientConfig.class);
@@ -57,8 +63,6 @@ protected static MiniOzoneCluster newCluster(
conf.setQuietMode(false);
conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 64,
StorageUnit.MB);
- conf.set(ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY,
- containerLayout.toString());
ReplicationManagerConfiguration repConf =
conf.getObject(ReplicationManagerConfiguration.class);
@@ -81,4 +85,38 @@ static String getNewKeyName() {
return UUID.randomUUID().toString();
}
+ protected void updateConfig(ContainerLayoutVersion layout) {
+ cluster.getHddsDatanodes().forEach(dn ->
dn.getConf().setEnum(OZONE_SCM_CONTAINER_LAYOUT_KEY, layout));
+ closeContainers();
+ }
+
+ private MiniOzoneCluster cluster;
+
+ protected MiniOzoneCluster getCluster() {
+ return cluster;
+ }
+
+ @BeforeAll
+ void setup() throws Exception {
+ cluster = newCluster();
+ cluster.waitForClusterToBeReady();
+ }
+
+ @AfterAll
+ void cleanup() {
+ IOUtils.closeQuietly(cluster);
+ }
+
+ private void closeContainers() {
+ StorageContainerManager scm = cluster.getStorageContainerManager();
+ scm.getContainerManager().getContainers().forEach(container -> {
+ if (container.isOpen()) {
+ try {
+ TestHelper.waitForContainerClose(getCluster(),
container.getContainerID());
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ });
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java
index 3ab8ae3118..d9c310d8f5 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/read/TestKeyInputStream.java
@@ -34,7 +34,6 @@
import org.apache.hadoop.hdds.scm.storage.BlockExtendedInputStream;
import org.apache.hadoop.hdds.scm.storage.BlockInputStream;
import org.apache.hadoop.hdds.scm.storage.ChunkInputStream;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.io.KeyInputStream;
import org.apache.hadoop.ozone.common.utils.BufferUtils;
@@ -46,15 +45,16 @@
import org.apache.hadoop.ozone.om.helpers.OmKeyInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo;
import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.TestInstance;
+import org.junit.jupiter.api.TestMethodOrder;
import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.THREE;
import static org.apache.hadoop.ozone.container.TestHelper.countReplicas;
-import static
org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.FILE_PER_BLOCK;
-import static
org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.FILE_PER_CHUNK;
import static org.junit.jupiter.api.Assertions.assertArrayEquals;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
@@ -63,6 +63,8 @@
/**
* Tests {@link KeyInputStream}.
*/
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
class TestKeyInputStream extends TestInputStreamBase {
/**
@@ -123,20 +125,18 @@ private void validate(TestBucket bucket, KeyInputStream
keyInputStream,
*/
@ContainerLayoutTestInfo.ContainerTest
void testNonReplicationReads(ContainerLayoutVersion layout) throws Exception
{
- try (MiniOzoneCluster cluster = newCluster(layout)) {
- cluster.waitForClusterToBeReady();
-
- try (OzoneClient client = cluster.newClient()) {
- TestBucket bucket = TestBucket.newBuilder(client).build();
-
- testInputStreams(bucket);
- testSeekRandomly(bucket);
- testSeek(bucket);
- testReadChunkWithByteArray(bucket);
- testReadChunkWithByteBuffer(bucket);
- testSkip(bucket);
- testECSeek(bucket);
- }
+ try (OzoneClient client = getCluster().newClient()) {
+ updateConfig(layout);
+
+ TestBucket bucket = TestBucket.newBuilder(client).build();
+
+ testInputStreams(bucket);
+ testSeekRandomly(bucket);
+ testSeek(bucket);
+ testReadChunkWithByteArray(bucket);
+ testReadChunkWithByteBuffer(bucket);
+ testSkip(bucket);
+ testECSeek(bucket);
}
}
@@ -379,32 +379,18 @@ private void testSkip(TestBucket bucket) throws Exception
{
}
}
- private static List<Arguments> readAfterReplicationArgs() {
- return Arrays.asList(
- Arguments.arguments(FILE_PER_BLOCK, false),
- Arguments.arguments(FILE_PER_BLOCK, true),
- Arguments.arguments(FILE_PER_CHUNK, false),
- Arguments.arguments(FILE_PER_CHUNK, true)
- );
- }
-
@ParameterizedTest
- @MethodSource("readAfterReplicationArgs")
- void readAfterReplication(ContainerLayoutVersion layout,
- boolean doUnbuffer) throws Exception {
- try (MiniOzoneCluster cluster = newCluster(layout)) {
- cluster.waitForClusterToBeReady();
-
- try (OzoneClient client = cluster.newClient()) {
- TestBucket bucket = TestBucket.newBuilder(client).build();
+ @ValueSource(booleans = {false, true})
+ @Order(Integer.MAX_VALUE) // shuts down datanodes
+ void readAfterReplication(boolean doUnbuffer) throws Exception {
+ try (OzoneClient client = getCluster().newClient()) {
+ TestBucket bucket = TestBucket.newBuilder(client).build();
- testReadAfterReplication(cluster, bucket, doUnbuffer);
- }
+ testReadAfterReplication(bucket, doUnbuffer);
}
}
- private void testReadAfterReplication(MiniOzoneCluster cluster,
- TestBucket bucket, boolean doUnbuffer) throws Exception {
+ private void testReadAfterReplication(TestBucket bucket, boolean doUnbuffer)
throws Exception {
int dataLength = 2 * CHUNK_SIZE;
String keyName = getNewKeyName();
byte[] data = bucket.writeRandomBytes(keyName, dataLength);
@@ -415,7 +401,7 @@ private void testReadAfterReplication(MiniOzoneCluster
cluster,
.setKeyName(keyName)
.setReplicationConfig(RatisReplicationConfig.getInstance(THREE))
.build();
- OmKeyInfo keyInfo = cluster.getOzoneManager()
+ OmKeyInfo keyInfo = getCluster().getOzoneManager()
.getKeyInfo(keyArgs, false)
.getKeyInfo();
@@ -425,24 +411,12 @@ private void testReadAfterReplication(MiniOzoneCluster
cluster,
assertEquals(1, locationInfoList.size());
OmKeyLocationInfo loc = locationInfoList.get(0);
long containerID = loc.getContainerID();
- assertEquals(3, countReplicas(containerID, cluster));
+ assertEquals(3, countReplicas(containerID, getCluster()));
- TestHelper.waitForContainerClose(cluster, containerID);
+ TestHelper.waitForContainerClose(getCluster(), containerID);
List<DatanodeDetails> pipelineNodes = loc.getPipeline().getNodes();
- // read chunk data
- try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) {
- int b = keyInputStream.read();
- assertNotEquals(-1, b);
- if (doUnbuffer) {
- keyInputStream.unbuffer();
- }
- cluster.shutdownHddsDatanode(pipelineNodes.get(0));
- // check that we can still read it
- assertReadFully(data, keyInputStream, dataLength - 1, 1);
- }
-
// read chunk data with ByteBuffer
try (KeyInputStream keyInputStream = bucket.getKeyInputStream(keyName)) {
int b = keyInputStream.read();
@@ -450,7 +424,7 @@ private void testReadAfterReplication(MiniOzoneCluster
cluster,
if (doUnbuffer) {
keyInputStream.unbuffer();
}
- cluster.shutdownHddsDatanode(pipelineNodes.get(0));
+ getCluster().shutdownHddsDatanode(pipelineNodes.get(0));
// check that we can still read it
assertReadFullyUsingByteBuffer(data, keyInputStream, dataLength - 1, 1);
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestFinalizeBlock.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestFinalizeBlock.java
index ca3733588a..8c3f0fd025 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestFinalizeBlock.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/common/statemachine/commandhandler/TestFinalizeBlock.java
@@ -26,7 +26,6 @@
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
import org.apache.hadoop.hdds.scm.ScmConfig;
-import org.apache.hadoop.hdds.scm.ScmConfigKeys;
import org.apache.hadoop.hdds.scm.XceiverClientManager;
import org.apache.hadoop.hdds.scm.XceiverClientSpi;
import org.apache.hadoop.hdds.scm.container.ContainerID;
@@ -41,7 +40,6 @@
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.container.ContainerTestHelper;
-import org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion;
import org.apache.hadoop.ozone.container.common.interfaces.Container;
import
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration;
import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData;
@@ -51,8 +49,7 @@
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.Arguments;
-import org.junit.jupiter.params.provider.MethodSource;
+import org.junit.jupiter.params.provider.ValueSource;
import java.io.IOException;
import java.time.Duration;
@@ -61,7 +58,6 @@
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
-import java.util.stream.Stream;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.concurrent.TimeUnit.SECONDS;
@@ -74,12 +70,9 @@
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
-import static
org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.FILE_PER_BLOCK;
-import static
org.apache.hadoop.ozone.container.common.impl.ContainerLayoutVersion.FILE_PER_CHUNK;
import static
org.apache.hadoop.ozone.container.common.statemachine.DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.fail;
-import static org.junit.jupiter.params.provider.Arguments.arguments;
/**
* Tests FinalizeBlock.
@@ -94,16 +87,7 @@ public class TestFinalizeBlock {
private static String volumeName = UUID.randomUUID().toString();
private static String bucketName = UUID.randomUUID().toString();
- public static Stream<Arguments> dnLayoutParams() {
- return Stream.of(
- arguments(false, FILE_PER_CHUNK),
- arguments(true, FILE_PER_CHUNK),
- arguments(false, FILE_PER_BLOCK),
- arguments(true, FILE_PER_BLOCK)
- );
- }
-
- private void setup(boolean enableSchemaV3, ContainerLayoutVersion version)
throws Exception {
+ private void setup(boolean enableSchemaV3) throws Exception {
conf = new OzoneConfiguration();
conf.set(OZONE_SCM_CONTAINER_SIZE, "1GB");
conf.setStorageSize(OZONE_DATANODE_RATIS_VOLUME_FREE_SPACE_MIN,
@@ -116,7 +100,6 @@ private void setup(boolean enableSchemaV3,
ContainerLayoutVersion version) throw
conf.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 1, SECONDS);
conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, TimeUnit.SECONDS);
conf.setBoolean(CONTAINER_SCHEMA_V3_ENABLED, enableSchemaV3);
- conf.setEnum(ScmConfigKeys.OZONE_SCM_CONTAINER_LAYOUT_KEY, version);
DatanodeConfiguration datanodeConfiguration = conf.getObject(
DatanodeConfiguration.class);
@@ -150,10 +133,9 @@ public void shutdown() {
}
@ParameterizedTest
- @MethodSource("dnLayoutParams")
- public void testFinalizeBlock(boolean enableSchemaV3, ContainerLayoutVersion
version)
- throws Exception {
- setup(enableSchemaV3, version);
+ @ValueSource(booleans = {false, true})
+ public void testFinalizeBlock(boolean enableSchemaV3) throws Exception {
+ setup(enableSchemaV3);
String keyName = UUID.randomUUID().toString();
// create key
createKey(keyName);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
