This is an automated email from the ASF dual-hosted git repository.
ritesh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a31755a085 HDDS-12315. Speed up some Freon integration tests (#7870)
a31755a085 is described below
commit a31755a08522218a7fc27eff9aebdb3220ddcbc6
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Mon Feb 24 15:52:00 2025 +0100
HDDS-12315. Speed up some Freon integration tests (#7870)
---
.../hadoop/fs/ozone/OzoneFileSystemTests.java | 21 +-
.../ozone/freon/TestHadoopDirTreeGenerator.java | 152 ++++-----
.../ozone/freon/TestHadoopNestedDirGenerator.java | 116 ++-----
.../hadoop/ozone/freon/TestHsyncGenerator.java | 57 +---
.../ozone/freon/TestOmBucketReadWriteFileOps.java | 366 +++++++++++----------
.../ozone/freon/TestOmBucketReadWriteKeyOps.java | 282 +++++-----------
.../hadoop/ozone/freon/TestRandomKeyGenerator.java | 121 +++----
.../test/java/org/apache/ozone/test/AclTests.java | 6 +-
.../org/apache/ozone/test/ClusterForTests.java | 6 +
.../java/org/apache/ozone/test/FreonTests.java | 111 +++++++
.../test/java/org/apache/ozone/test/HATests.java | 4 +-
.../java/org/apache/ozone/test/NonHATests.java | 14 -
.../test/TestFreon.java} | 22 +-
.../hadoop/ozone/freon/BaseFreonGenerator.java | 2 +-
.../hadoop/ozone/freon/RandomKeyGenerator.java | 2 +-
15 files changed, 557 insertions(+), 725 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
index cb2c97ca6d..02e81f333e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
@@ -20,6 +20,8 @@
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -33,17 +35,30 @@
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.fs.contract.ContractTestUtils;
+import org.apache.hadoop.hdds.conf.ConfigurationTarget;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.ratis.util.Preconditions;
/**
* Common test cases for Ozone file systems.
*/
-final class OzoneFileSystemTests {
+public final class OzoneFileSystemTests {
private OzoneFileSystemTests() {
// no instances
}
+ /**
+ * Set file system listing page size. Also disable the file system cache to
+ * ensure new {@link FileSystem} instance reflects the updated page size.
+ */
+ public static void setPageSize(ConfigurationTarget conf, int pageSize) {
+ Preconditions.assertTrue(pageSize > 0, () -> "pageSize=" + pageSize + " <=
0");
+ conf.setInt(OZONE_FS_LISTING_PAGE_SIZE, pageSize);
+ conf.setBoolean(String.format("fs.%s.impl.disable.cache",
OZONE_URI_SCHEME), true);
+ conf.setBoolean(String.format("fs.%s.impl.disable.cache",
OZONE_OFS_URI_SCHEME), true);
+ }
+
/**
* Tests listStatusIterator operation on directory with different
* numbers of child directories.
@@ -60,10 +75,8 @@ public static void
listStatusIteratorOnPageSize(OzoneConfiguration conf,
pageSize + pageSize
};
OzoneConfiguration config = new OzoneConfiguration(conf);
- config.setInt(OZONE_FS_LISTING_PAGE_SIZE, pageSize);
+ setPageSize(config, pageSize);
URI uri = FileSystem.getDefaultUri(config);
- config.setBoolean(
- String.format("fs.%s.impl.disable.cache", uri.getScheme()), true);
try (FileSystem subject = FileSystem.get(uri, config)) {
Path dir = new Path(Objects.requireNonNull(rootPath),
"listStatusIterator");
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
index 937d4d4f2a..483107af46 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGenerator.java
@@ -17,148 +17,110 @@
package org.apache.hadoop.ozone.freon;
+import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_SCHEME;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
-import org.apache.commons.io.FileUtils;
+import java.util.UUID;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.ozone.OzoneFileSystemTests;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.conf.StorageSize;
import org.apache.hadoop.hdds.conf.StorageUnit;
import org.apache.hadoop.hdds.utils.IOUtils;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
-import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.raftlog.RaftLog;
+import org.apache.ozone.test.NonHATests;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.slf4j.event.Level;
/**
* Test for HadoopDirTreeGenerator.
*/
-public class TestHadoopDirTreeGenerator {
- @TempDir
- private java.nio.file.Path path;
- private OzoneConfiguration conf = null;
- private MiniOzoneCluster cluster = null;
+public abstract class TestHadoopDirTreeGenerator implements
NonHATests.TestCase {
+
+ private static final int PAGE_SIZE = 10;
+
private ObjectStore store = null;
private static final Logger LOG =
LoggerFactory.getLogger(TestHadoopDirTreeGenerator.class);
private OzoneClient client;
@BeforeEach
- public void setup() {
- GenericTestUtils.setLogLevel(RaftLog.LOG, Level.DEBUG);
- GenericTestUtils.setLogLevel(RaftServer.LOG, Level.DEBUG);
- }
-
- /**
- * Shutdown MiniDFSCluster.
- */
- private void shutdown() throws IOException {
- IOUtils.closeQuietly(client);
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- /**
- * Create a MiniDFSCluster for testing.
- *
- * @throws IOException
- */
- private void startCluster() throws Exception {
- conf = getOzoneConfiguration();
- conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
- BucketLayout.LEGACY.name());
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
- cluster.waitForClusterToBeReady();
- cluster.waitTobeOutOfSafeMode();
-
- client = OzoneClientFactory.getRpcClient(conf);
+ void setup() throws Exception {
+ client = cluster().newClient();
store = client.getObjectStore();
}
- protected OzoneConfiguration getOzoneConfiguration() {
- return new OzoneConfiguration();
+ @AfterEach
+ void cleanup() {
+ IOUtils.closeQuietly(client);
}
- @Test
- public void testNestedDirTreeGeneration() throws Exception {
- try {
- startCluster();
- FileOutputStream out = FileUtils.openOutputStream(new
File(path.toString(),
- "conf"));
- cluster.getConf().writeXml(out);
- out.getFD().sync();
- out.close();
-
- verifyDirTree("vol1", "bucket1", 1,
- 1, 1, "0");
- verifyDirTree("vol2", "bucket1", 1,
- 5, 1, "5B");
- verifyDirTree("vol3", "bucket1", 2,
- 5, 3, "1B");
- verifyDirTree("vol4", "bucket1", 3,
- 2, 4, "2B");
- verifyDirTree("vol5", "bucket1", 5,
- 4, 1, "0");
- // default page size is Constants.LISTING_PAGE_SIZE = 1024
- verifyDirTree("vol6", "bucket1", 2,
- 1, 1100, "0");
- } finally {
- shutdown();
- }
+ @ParameterizedTest
+ @EnumSource(names = {"FILE_SYSTEM_OPTIMIZED", "LEGACY"})
+ public void testNestedDirTreeGeneration(BucketLayout layout) throws
Exception {
+ String uuid = UUID.randomUUID().toString();
+ verifyDirTree("vol1-" + uuid, "bucket1", 1, 1, 1, "0", layout);
+ verifyDirTree("vol2-" + uuid, "bucket1", 1, 5, 1, "5B", layout);
+ verifyDirTree("vol3-" + uuid, "bucket1", 2, 5, 3, "1B", layout);
+ verifyDirTree("vol4-" + uuid, "bucket1", 3, 2, 4, "2B", layout);
+ verifyDirTree("vol5-" + uuid, "bucket1", 5, 4, 1, "0", layout);
+ verifyDirTree("vol6-" + uuid, "bucket1", 2, 1, PAGE_SIZE + PAGE_SIZE / 2,
"0", layout);
}
private void verifyDirTree(String volumeName, String bucketName, int depth,
- int span, int fileCount, String perFileSize)
+ int span, int fileCount, String perFileSize,
BucketLayout layout)
throws IOException {
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
- volume.createBucket(bucketName);
- String rootPath = "o3fs://" + bucketName + "." + volumeName;
- String confPath = new File(path.toString(), "conf").getAbsolutePath();
- new Freon().execute(
- new String[]{"-conf", confPath, "dtsg", "-d", depth + "", "-c",
- fileCount + "", "-s", span + "", "-n", "1", "-r", rootPath,
- "-g", perFileSize});
+ volume.createBucket(bucketName,
BucketArgs.newBuilder().setBucketLayout(layout).build());
+ String rootPath = OZONE_URI_SCHEME + "://" + bucketName + "." + volumeName;
+ String om = cluster().getConf().get(OZONE_OM_ADDRESS_KEY);
+ new Freon().getCmd().execute(
+ "-D", OZONE_OM_ADDRESS_KEY + "=" + om,
+ "dtsg",
+ "-c", String.valueOf(fileCount),
+ "-d", String.valueOf(depth),
+ "-g", perFileSize,
+ "-n", "1",
+ "-r", rootPath,
+ "-s", String.valueOf(span)
+ );
// verify the directory structure
LOG.info("Started verifying the directory structure...");
- FileSystem fileSystem = FileSystem.get(URI.create(rootPath),
- conf);
- Path rootDir = new Path(rootPath.concat("/"));
- // verify root path details
- FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
- // verify the num of peer directories, expected span count is 1
- // as it has only one dir at root.
- verifyActualSpan(1, Arrays.asList(fileStatuses));
- for (FileStatus fileStatus : fileStatuses) {
- int actualDepth =
- traverseToLeaf(fileSystem, fileStatus.getPath(), 1, depth, span,
- fileCount, StorageSize.parse(perFileSize, StorageUnit.BYTES));
- assertEquals(depth, actualDepth, "Mismatch depth in a path");
+ OzoneConfiguration conf = new OzoneConfiguration(cluster().getConf());
+ OzoneFileSystemTests.setPageSize(conf, PAGE_SIZE);
+ try (FileSystem fileSystem = FileSystem.get(URI.create(rootPath), conf)) {
+ Path rootDir = new Path(rootPath.concat("/"));
+ // verify root path details
+ FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
+ // verify the num of peer directories, expected span count is 1
+ // as it has only one dir at root.
+ verifyActualSpan(1, Arrays.asList(fileStatuses));
+ for (FileStatus fileStatus : fileStatuses) {
+ int actualDepth =
+ traverseToLeaf(fileSystem, fileStatus.getPath(), 1, depth, span,
+ fileCount, StorageSize.parse(perFileSize, StorageUnit.BYTES));
+ assertEquals(depth, actualDepth, "Mismatch depth in a path");
+ }
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java
index bd178de71b..b36397896e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopNestedDirGenerator.java
@@ -17,105 +17,55 @@
package org.apache.hadoop.ozone.freon;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.util.Collections;
import java.util.LinkedList;
-import org.apache.commons.io.FileUtils;
+import java.util.UUID;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.hdds.utils.IOUtils;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.raftlog.RaftLog;
+import org.apache.ozone.test.NonHATests;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.slf4j.event.Level;
/**
* Test for HadoopNestedDirGenerator.
*/
-
-public class TestHadoopNestedDirGenerator {
- @TempDir
- private java.nio.file.Path path;
- private OzoneConfiguration conf = null;
- private MiniOzoneCluster cluster = null;
+public abstract class TestHadoopNestedDirGenerator implements
NonHATests.TestCase {
private ObjectStore store = null;
private static final Logger LOG =
LoggerFactory.getLogger(TestHadoopNestedDirGenerator.class);
private OzoneClient client;
- @BeforeEach
- public void setup() {
- GenericTestUtils.setLogLevel(RaftLog.LOG, Level.DEBUG);
- GenericTestUtils.setLogLevel(RaftServer.LOG, Level.DEBUG);
- }
-
- /**
- * Shutdown MiniDFSCluster.
- */
-
- private void shutdown() throws IOException {
+ @AfterEach
+ void shutdown() {
IOUtils.closeQuietly(client);
- if (cluster != null) {
- cluster.shutdown();
- }
}
- /**
- * Create a MiniDFSCluster for testing.
- *
- * @throws IOException
- */
-
- private void startCluster() throws Exception {
- conf = new OzoneConfiguration();
- conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
- OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED);
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
- cluster.waitForClusterToBeReady();
- cluster.waitTobeOutOfSafeMode();
- client = OzoneClientFactory.getRpcClient(conf);
+ @BeforeEach
+ void init() throws Exception {
+ client = cluster().newClient();
store = client.getObjectStore();
}
@Test
- public void testNestedDirTreeGeneration() throws Exception {
- try {
- startCluster();
- FileOutputStream out = FileUtils.openOutputStream(new
File(path.toString(), "conf"));
- cluster.getConf().writeXml(out);
- out.getFD().sync();
- out.close();
- verifyDirTree("vol1",
- "bucket1", 1, 1);
- verifyDirTree("vol2",
- "bucket1", 1, 5);
- verifyDirTree("vol3",
- "bucket1", 2, 0);
- verifyDirTree("vol4",
- "bucket1", 3, 2);
- verifyDirTree("vol5",
- "bucket1", 5, 4);
- } finally {
- shutdown();
- }
+ public void testNestedDirTreeGeneration() throws Exception {
+ verifyDirTree("vol-" + UUID.randomUUID(), "bucket1", 1, 1);
+ verifyDirTree("vol-" + UUID.randomUUID(), "bucket1", 1, 5);
+ verifyDirTree("vol-" + UUID.randomUUID(), "bucket1", 2, 0);
+ verifyDirTree("vol-" + UUID.randomUUID(), "bucket1", 3, 2);
+ verifyDirTree("vol-" + UUID.randomUUID(), "bucket1", 5, 4);
}
private void verifyDirTree(String volumeName, String bucketName,
@@ -125,21 +75,27 @@ private void verifyDirTree(String volumeName, String
bucketName,
OzoneVolume volume = store.getVolume(volumeName);
volume.createBucket(bucketName);
String rootPath = "o3fs://" + bucketName + "." + volumeName;
- String confPath = new File(path.toString(), "conf").getAbsolutePath();
- new Freon().execute(new String[]{"-conf", confPath, "ddsg", "-d",
- actualDepth + "", "-s", span + "", "-n", "1", "-r", rootPath});
+ String om = cluster().getConf().get(OZONE_OM_ADDRESS_KEY);
+ new Freon().getCmd().execute(
+ "-D", OZONE_OM_ADDRESS_KEY + "=" + om,
+ "ddsg",
+ "-d", String.valueOf(actualDepth),
+ "-s", String.valueOf(span),
+ "-n", "1",
+ "-r", rootPath
+ );
// verify the directory structure
- FileSystem fileSystem = FileSystem.get(URI.create(rootPath),
- conf);
- Path rootDir = new Path(rootPath.concat("/"));
- // verify root path details
- FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
- Path p = null;
- for (FileStatus fileStatus : fileStatuses) {
- // verify the num of peer directories and span directories
- p = depthBFS(fileSystem, fileStatuses, span, actualDepth);
- int actualSpan = spanCheck(fileSystem, span, p);
- assertEquals(span, actualSpan, "Mismatch span in a path");
+ try (FileSystem fileSystem = FileSystem.get(URI.create(rootPath),
cluster().getConf())) {
+ Path rootDir = new Path(rootPath.concat("/"));
+ // verify root path details
+ FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
+ Path p = null;
+ for (FileStatus fileStatus : fileStatuses) {
+ // verify the num of peer directories and span directories
+ p = depthBFS(fileSystem, fileStatuses, span, actualDepth);
+ int actualSpan = spanCheck(fileSystem, span, p);
+ assertEquals(span, actualSpan, "Mismatch span in a path");
+ }
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHsyncGenerator.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHsyncGenerator.java
index 0f1836f396..bc367e19e1 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHsyncGenerator.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHsyncGenerator.java
@@ -17,83 +17,40 @@
package org.apache.hadoop.ozone.freon;
-import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import java.io.IOException;
-import java.time.Duration;
-import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
+import java.util.UUID;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
+import org.apache.ozone.test.NonHATests;
import org.junit.jupiter.api.Test;
import picocli.CommandLine;
/**
* Tests Freon HsyncGenerator with MiniOzoneCluster and validate data.
*/
-public class TestHsyncGenerator {
- private static MiniOzoneCluster cluster = null;
-
- private static void startCluster(OzoneConfiguration conf) throws Exception {
- DatanodeRatisServerConfig ratisServerConfig =
- conf.getObject(DatanodeRatisServerConfig.class);
- ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3));
- ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(10));
- conf.setFromObject(ratisServerConfig);
-
- RatisClientConfig.RaftConfig raftClientConfig =
- conf.getObject(RatisClientConfig.RaftConfig.class);
- raftClientConfig.setRpcRequestTimeout(Duration.ofSeconds(3));
- raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(10));
- conf.setFromObject(raftClientConfig);
- conf.set(OZONE_FS_HSYNC_ENABLED, "true");
-
- cluster = MiniOzoneCluster.newBuilder(conf)
- .setNumDatanodes(5).build();
- cluster.waitForClusterToBeReady();
- }
-
- static void shutdownCluster() {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
-
- @BeforeAll
- public static void init() throws Exception {
- OzoneConfiguration conf = new OzoneConfiguration();
- startCluster(conf);
- }
-
- @AfterAll
- public static void shutdown() {
- shutdownCluster();
- }
+public abstract class TestHsyncGenerator implements NonHATests.TestCase {
@Test
public void test() throws IOException {
HsyncGenerator randomKeyGenerator =
- new HsyncGenerator(cluster.getConf());
+ new HsyncGenerator(cluster().getConf());
CommandLine cmd = new CommandLine(randomKeyGenerator);
- String volumeName = "vol1";
+ String volumeName = "vol-" + UUID.randomUUID();
String bucketName = "bucket1";
- try (OzoneClient client = cluster.newClient()) {
+ try (OzoneClient client = cluster().newClient()) {
ObjectStore store = client.getObjectStore();
store.createVolume(volumeName);
OzoneVolume volume = store.getVolume(volumeName);
volume.createBucket(bucketName);
String rootPath = String.format("%s://%s/%s/%s/", OZONE_OFS_URI_SCHEME,
- cluster.getConf().get(OZONE_OM_ADDRESS_KEY), volumeName, bucketName);
+ cluster().getConf().get(OZONE_OM_ADDRESS_KEY), volumeName,
bucketName);
int exitCode = cmd.execute(
"--path", rootPath,
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
index 5d0c70d026..369f577ce4 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteFileOps.java
@@ -17,175 +17,134 @@
package org.apache.hadoop.ozone.freon;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
import java.net.URI;
-import org.apache.commons.io.FileUtils;
+import java.util.Arrays;
+import java.util.List;
+import java.util.UUID;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.utils.IOUtils;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.TestDataUtil;
+import org.apache.hadoop.ozone.client.BucketArgs;
import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
-import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.lock.OMLockMetrics;
-import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.raftlog.RaftLog;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.junit.jupiter.api.io.TempDir;
+import org.apache.ozone.test.NonHATests;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.slf4j.event.Level;
/**
* Test for OmBucketReadWriteFileOps.
*/
-public class TestOmBucketReadWriteFileOps {
- @TempDir
- private java.nio.file.Path path;
- private OzoneConfiguration conf = null;
- private MiniOzoneCluster cluster = null;
- private ObjectStore store = null;
+public abstract class TestOmBucketReadWriteFileOps implements
NonHATests.TestCase {
private static final Logger LOG =
LoggerFactory.getLogger(TestOmBucketReadWriteFileOps.class);
- private OzoneClient client;
- @BeforeEach
- public void setup() {
- GenericTestUtils.setLogLevel(RaftLog.LOG, Level.DEBUG);
- GenericTestUtils.setLogLevel(RaftServer.LOG, Level.DEBUG);
+ static List<ParameterBuilder> parameters() {
+ return Arrays.asList(
+ new ParameterBuilder()
+ .setTotalThreadCount(10)
+ .setNumOfReadOperations(10)
+ .setNumOfWriteOperations(5)
+ .setReadThreadPercentage(70)
+ .setCountForRead(10)
+ .setCountForWrite(5),
+ new ParameterBuilder()
+ .setPrefixFilePath("/dir1/dir2/dir3")
+ .setTotalThreadCount(10)
+ .setNumOfReadOperations(10)
+ .setNumOfWriteOperations(5)
+ .setReadThreadPercentage(80)
+ .setBufferSize(128)
+ .setCountForRead(10)
+ .setCountForWrite(5),
+ new ParameterBuilder()
+ .setPrefixFilePath("/")
+ .setTotalThreadCount(15)
+ .setNumOfReadOperations(5)
+ .setNumOfWriteOperations(3)
+ .setDataSize("128B")
+ .setCountForRead(5)
+ .setCountForWrite(3),
+ new ParameterBuilder()
+ .setPrefixFilePath("/dir1/")
+ .setTotalThreadCount(10)
+ .setNumOfReadOperations(5)
+ .setNumOfWriteOperations(3)
+ .setCountForRead(5)
+ .setCountForWrite(3)
+ .setDataSize("64B")
+ .setBufferSize(16),
+ new ParameterBuilder()
+ .setPrefixFilePath("/dir1/dir2/dir3")
+ .setTotalThreadCount(10)
+ .setNumOfReadOperations(5)
+ .setNumOfWriteOperations(0)
+ .setCountForRead(5),
+ new ParameterBuilder()
+ .setLength(64)
+ .setPrefixFilePath("/dir1/dir2/dir3/dir4")
+ .setTotalThreadCount(20)
+ .setNumOfReadOperations(0)
+ .setNumOfWriteOperations(5)
+ .setCountForRead(0)
+ .setCountForWrite(5)
+ );
}
- /**
- * Shutdown MiniDFSCluster.
- */
- private void shutdown() throws IOException {
- IOUtils.closeQuietly(client);
- if (cluster != null) {
- cluster.shutdown();
+ @ParameterizedTest
+ @MethodSource("parameters")
+ void testOmBucketReadWriteFileOps(ParameterBuilder parameterBuilder) throws
Exception {
+ try (OzoneClient client = cluster().newClient()) {
+ TestDataUtil.createVolumeAndBucket(client,
+ parameterBuilder.getVolumeName(),
+ parameterBuilder.getBucketName(),
+ parameterBuilder.getBucketArgs().build()
+ );
}
- }
-
- /**
- * Create a MiniDFSCluster for testing.
- *
- * @throws IOException
- */
- private void startCluster() throws Exception {
- conf = getOzoneConfiguration();
- conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
- BucketLayout.LEGACY.name());
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
- cluster.waitForClusterToBeReady();
- cluster.waitTobeOutOfSafeMode();
-
- client = OzoneClientFactory.getRpcClient(conf);
- store = client.getObjectStore();
- }
-
- protected OzoneConfiguration getOzoneConfiguration() {
- return new OzoneConfiguration();
- }
-
- @Test
- public void testOmBucketReadWriteFileOps() throws Exception {
- try {
- startCluster();
- FileOutputStream out = FileUtils.openOutputStream(new
File(path.toString(), "conf"));
- cluster.getConf().writeXml(out);
- out.getFD().sync();
- out.close();
-
- verifyFreonCommand(new ParameterBuilder().setTotalThreadCount(10)
- .setNumOfReadOperations(10).setNumOfWriteOperations(5)
- .setFileCountForRead(10).setFileCountForWrite(5));
- verifyFreonCommand(
- new ParameterBuilder().setVolumeName("vol2").setBucketName("bucket1")
- .setPrefixFilePath("/dir1/dir2/dir3").setTotalThreadCount(10)
- .setNumOfReadOperations(10).setNumOfWriteOperations(5)
- .setFileCountForRead(10).setFileCountForWrite(5));
- verifyFreonCommand(
- new ParameterBuilder().setVolumeName("vol3").setBucketName("bucket1")
- .setPrefixFilePath("/").setTotalThreadCount(15)
- .setNumOfReadOperations(5).setNumOfWriteOperations(3)
- .setFileCountForRead(5).setFileCountForWrite(3));
- verifyFreonCommand(
- new ParameterBuilder().setVolumeName("vol4").setBucketName("bucket1")
- .setPrefixFilePath("/dir1/").setTotalThreadCount(10)
- .setNumOfReadOperations(5).setNumOfWriteOperations(3)
- .setFileCountForRead(5).setFileCountForWrite(3).
- setFileSize("64B").setBufferSize(16));
- verifyFreonCommand(
- new ParameterBuilder().setVolumeName("vol5").setBucketName("bucket1")
- .setPrefixFilePath("/dir1/dir2/dir3").setTotalThreadCount(10)
- .setNumOfReadOperations(5).setNumOfWriteOperations(0)
- .setFileCountForRead(5));
- verifyFreonCommand(
- new ParameterBuilder().setVolumeName("vol6").setBucketName("bucket1")
-
.setPrefixFilePath("/dir1/dir2/dir3/dir4").setTotalThreadCount(20)
- .setNumOfReadOperations(0).setNumOfWriteOperations(5)
- .setFileCountForRead(0).setFileCountForWrite(5));
- } finally {
- shutdown();
- }
- }
-
- private void verifyFreonCommand(ParameterBuilder parameterBuilder)
- throws IOException {
- store.createVolume(parameterBuilder.volumeName);
- OzoneVolume volume = store.getVolume(parameterBuilder.volumeName);
- volume.createBucket(parameterBuilder.bucketName);
- String rootPath = "o3fs://" + parameterBuilder.bucketName + "." +
- parameterBuilder.volumeName + parameterBuilder.prefixFilePath;
- String confPath = new File(path.toString(), "conf").getAbsolutePath();
- new Freon().execute(
- new String[]{"-conf", confPath, "obrwf", "-P", rootPath,
- "-r", String.valueOf(parameterBuilder.fileCountForRead),
- "-w", String.valueOf(parameterBuilder.fileCountForWrite),
- "-g", parameterBuilder.fileSize,
- "--buffer", String.valueOf(parameterBuilder.bufferSize),
- "-l", String.valueOf(parameterBuilder.length),
- "-c", String.valueOf(parameterBuilder.totalThreadCount),
- "-T", String.valueOf(parameterBuilder.readThreadPercentage),
- "-R", String.valueOf(parameterBuilder.numOfReadOperations),
- "-W", String.valueOf(parameterBuilder.numOfWriteOperations),
- "-n", String.valueOf(1)});
+ String rootPath = "o3fs://" + parameterBuilder.getBucketName() + "." +
+ parameterBuilder.getVolumeName() +
parameterBuilder.getPrefixFilePath();
+ String om = cluster().getConf().get(OZONE_OM_ADDRESS_KEY);
+ new Freon().getCmd().execute(
+ "-D", OZONE_OM_ADDRESS_KEY + "=" + om,
+ "obrwf",
+ "-P", rootPath,
+ "-r", String.valueOf(parameterBuilder.getCountForRead()),
+ "-w", String.valueOf(parameterBuilder.getCountForWrite()),
+ "-g", parameterBuilder.getDataSize(),
+ "--buffer", String.valueOf(parameterBuilder.getBufferSize()),
+ "-l", String.valueOf(parameterBuilder.getLength()),
+ "-c", String.valueOf(parameterBuilder.getTotalThreadCount()),
+ "-T", String.valueOf(parameterBuilder.getReadThreadPercentage()),
+ "-R", String.valueOf(parameterBuilder.getNumOfReadOperations()),
+ "-W", String.valueOf(parameterBuilder.getNumOfWriteOperations()),
+ "-n", String.valueOf(1)
+ );
LOG.info("Started verifying OM bucket read/write ops file generation...");
- FileSystem fileSystem = FileSystem.get(URI.create(rootPath),
- conf);
- Path rootDir = new Path(rootPath.concat(OzoneConsts.OM_KEY_PREFIX));
- FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
- verifyFileCreation(2, fileStatuses, true);
-
- Path readDir = new Path(rootPath.concat("/readPath"));
- FileStatus[] readFileStatuses = fileSystem.listStatus(readDir);
- verifyFileCreation(parameterBuilder.fileCountForRead, readFileStatuses,
- false);
-
- int readThreadCount = (parameterBuilder.readThreadPercentage *
- parameterBuilder.totalThreadCount) / 100;
- int writeThreadCount = parameterBuilder.totalThreadCount - readThreadCount;
-
- Path writeDir = new Path(rootPath.concat("/writePath"));
- FileStatus[] writeFileStatuses = fileSystem.listStatus(writeDir);
- verifyFileCreation(writeThreadCount * parameterBuilder.fileCountForWrite *
- parameterBuilder.numOfWriteOperations, writeFileStatuses, false);
-
-
verifyOMLockMetrics(cluster.getOzoneManager().getMetadataManager().getLock()
- .getOMLockMetrics());
+ try (FileSystem fileSystem = FileSystem.get(URI.create(rootPath),
cluster().getConf())) {
+ Path rootDir = new Path(rootPath.concat(OzoneConsts.OM_KEY_PREFIX));
+ FileStatus[] fileStatuses = fileSystem.listStatus(rootDir);
+ verifyFileCreation(2, fileStatuses, true);
+
+ Path readDir = new Path(rootPath.concat("/readPath"));
+ FileStatus[] readFileStatuses = fileSystem.listStatus(readDir);
+ verifyFileCreation(parameterBuilder.getCountForRead(), readFileStatuses,
+ false);
+
+ Path writeDir = new Path(rootPath.concat("/writePath"));
+ FileStatus[] writeFileStatuses = fileSystem.listStatus(writeDir);
+ verifyFileCreation(parameterBuilder.getExpectedWriteCount(),
writeFileStatuses, false);
+
+
verifyOMLockMetrics(cluster().getOzoneManager().getMetadataManager().getLock()
+ .getOMLockMetrics());
+ }
}
private void verifyFileCreation(int expectedCount, FileStatus[] fileStatuses,
@@ -243,14 +202,16 @@ private void verifyOMLockMetrics(OMLockMetrics
omLockMetrics) {
assertThat(writeHeldSamples).isPositive();
}
- private static class ParameterBuilder {
+ static class ParameterBuilder {
- private String volumeName = "vol1";
- private String bucketName = "bucket1";
+ private static final String BUCKET_NAME = "bucket1";
+
+ private final String volumeName = "vol-" + UUID.randomUUID();
+ private final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder();
private String prefixFilePath = "/dir1/dir2";
- private int fileCountForRead = 100;
- private int fileCountForWrite = 10;
- private String fileSize = "256B";
+ private int countForRead = 100;
+ private int countForWrite = 10;
+ private String dataSize = "256B";
private int bufferSize = 64;
private int length = 10;
private int totalThreadCount = 100;
@@ -258,67 +219,112 @@ private static class ParameterBuilder {
private int numOfReadOperations = 50;
private int numOfWriteOperations = 10;
- private ParameterBuilder setVolumeName(String volumeNameParam) {
- volumeName = volumeNameParam;
- return this;
+ int getExpectedWriteCount() {
+ int readThreadCount = (getReadThreadPercentage() *
getTotalThreadCount()) / 100;
+ int writeThreadCount = getTotalThreadCount() - readThreadCount;
+ return writeThreadCount * getCountForWrite() * getNumOfWriteOperations();
}
- private ParameterBuilder setBucketName(String bucketNameParam) {
- bucketName = bucketNameParam;
+ ParameterBuilder setPrefixFilePath(String newValue) {
+ prefixFilePath = newValue;
return this;
}
- private ParameterBuilder setPrefixFilePath(String prefixFilePathParam) {
- prefixFilePath = prefixFilePathParam;
+ ParameterBuilder setCountForRead(int newValue) {
+ countForRead = newValue;
return this;
}
- private ParameterBuilder setFileCountForRead(int fileCountForReadParam) {
- fileCountForRead = fileCountForReadParam;
+ ParameterBuilder setCountForWrite(int newValue) {
+ countForWrite = newValue;
return this;
}
- private ParameterBuilder setFileCountForWrite(int fileCountForWriteParam) {
- fileCountForWrite = fileCountForWriteParam;
+ ParameterBuilder setDataSize(String newValue) {
+ dataSize = newValue;
return this;
}
- private ParameterBuilder setFileSize(String fileSizeParam) {
- fileSize = fileSizeParam;
+ ParameterBuilder setBufferSize(int newValue) {
+ bufferSize = newValue;
return this;
}
- private ParameterBuilder setBufferSize(int bufferSizeParam) {
- bufferSize = bufferSizeParam;
+ ParameterBuilder setLength(int newValue) {
+ length = newValue;
return this;
}
- private ParameterBuilder setLength(int lengthParam) {
- length = lengthParam;
+ ParameterBuilder setTotalThreadCount(int newValue) {
+ totalThreadCount = newValue;
return this;
}
- private ParameterBuilder setTotalThreadCount(int totalThreadCountParam) {
- totalThreadCount = totalThreadCountParam;
+ ParameterBuilder setReadThreadPercentage(int newValue) {
+ readThreadPercentage = newValue;
return this;
}
- private ParameterBuilder setReadThreadPercentage(
- int readThreadPercentageParam) {
- readThreadPercentage = readThreadPercentageParam;
+ ParameterBuilder setNumOfReadOperations(int newValue) {
+ numOfReadOperations = newValue;
return this;
}
- private ParameterBuilder setNumOfReadOperations(
- int numOfReadOperationsParam) {
- numOfReadOperations = numOfReadOperationsParam;
+ ParameterBuilder setNumOfWriteOperations(int newValue) {
+ numOfWriteOperations = newValue;
return this;
}
- private ParameterBuilder setNumOfWriteOperations(
- int numOfWriteOperationsParam) {
- numOfWriteOperations = numOfWriteOperationsParam;
- return this;
+ public String getVolumeName() {
+ return volumeName;
+ }
+
+ public String getBucketName() {
+ return BUCKET_NAME;
+ }
+
+ public String getPrefixFilePath() {
+ return prefixFilePath;
+ }
+
+ public BucketArgs.Builder getBucketArgs() {
+ return bucketArgs;
+ }
+
+ public int getBufferSize() {
+ return bufferSize;
+ }
+
+ public String getDataSize() {
+ return dataSize;
+ }
+
+ public int getCountForRead() {
+ return countForRead;
+ }
+
+ public int getCountForWrite() {
+ return countForWrite;
+ }
+
+ public int getLength() {
+ return length;
+ }
+
+ public int getTotalThreadCount() {
+ return totalThreadCount;
+ }
+
+ public int getReadThreadPercentage() {
+ return readThreadPercentage;
+ }
+
+ public int getNumOfReadOperations() {
+ return numOfReadOperations;
+ }
+
+ public int getNumOfWriteOperations() {
+ return numOfWriteOperations;
}
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
index d637bc1545..a3ad7c6a48 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestOmBucketReadWriteKeyOps.java
@@ -17,173 +17,129 @@
package org.apache.hadoop.ozone.freon;
+import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
-import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
-import java.nio.file.Path;
+import java.util.Arrays;
import java.util.Iterator;
-import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import java.util.List;
import org.apache.hadoop.hdds.utils.IOUtils;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.apache.hadoop.ozone.client.ObjectStore;
+import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
-import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneKey;
-import org.apache.hadoop.ozone.client.OzoneVolume;
-import org.apache.hadoop.ozone.om.OMConfigKeys;
+import
org.apache.hadoop.ozone.freon.TestOmBucketReadWriteFileOps.ParameterBuilder;
import org.apache.hadoop.ozone.om.helpers.BucketLayout;
import org.apache.hadoop.ozone.om.lock.OMLockMetrics;
-import org.apache.ozone.test.GenericTestUtils;
-import org.apache.ratis.server.RaftServer;
-import org.apache.ratis.server.raftlog.RaftLog;
+import org.apache.ozone.test.NonHATests;
+import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
-import org.junit.jupiter.params.provider.ValueSource;
+import org.junit.jupiter.params.provider.MethodSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import org.slf4j.event.Level;
/**
* Test for OmBucketReadWriteKeyOps.
*/
-public class TestOmBucketReadWriteKeyOps {
+public abstract class TestOmBucketReadWriteKeyOps implements
NonHATests.TestCase {
- // TODO: Remove code duplication of TestOmBucketReadWriteKeyOps with
- // TestOmBucketReadWriteFileOps.
- @TempDir
- private Path path;
- private OzoneConfiguration conf = null;
- private MiniOzoneCluster cluster = null;
- private ObjectStore store = null;
private static final Logger LOG =
LoggerFactory.getLogger(TestOmBucketReadWriteKeyOps.class);
private OzoneClient client;
@BeforeEach
- public void setup() {
- GenericTestUtils.setLogLevel(RaftLog.LOG, Level.DEBUG);
- GenericTestUtils.setLogLevel(RaftServer.LOG, Level.DEBUG);
+ void setup() throws Exception {
+ client = cluster().newClient();
}
- /**
- * Shutdown MiniDFSCluster.
- */
- private void shutdown() {
+ @AfterEach
+ void cleanup() {
IOUtils.closeQuietly(client);
- if (cluster != null) {
- cluster.shutdown();
- }
}
- /**
- * Create a MiniDFSCluster for testing.
- *
- * @throws IOException
- */
- private void startCluster(boolean fsPathsEnabled) throws Exception {
- conf = getOzoneConfiguration();
- conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
fsPathsEnabled);
- conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
- BucketLayout.OBJECT_STORE.name());
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
- cluster.waitForClusterToBeReady();
- cluster.waitTobeOutOfSafeMode();
-
- client = OzoneClientFactory.getRpcClient(conf);
- store = client.getObjectStore();
- }
-
- private OzoneConfiguration getOzoneConfiguration() {
- return new OzoneConfiguration();
+ static List<ParameterBuilder> parameters() {
+ return Arrays.asList(
+ new ParameterBuilder()
+ .setLength(16)
+ .setTotalThreadCount(10)
+ .setNumOfReadOperations(10)
+ .setNumOfWriteOperations(5)
+ .setReadThreadPercentage(80)
+ .setCountForRead(10)
+ .setCountForWrite(5),
+ new ParameterBuilder()
+ .setLength(32)
+ .setTotalThreadCount(10)
+ .setNumOfReadOperations(10)
+ .setNumOfWriteOperations(5)
+ .setReadThreadPercentage(70)
+ .setCountForRead(10)
+ .setCountForWrite(5),
+ new ParameterBuilder()
+ .setTotalThreadCount(15)
+ .setNumOfReadOperations(5)
+ .setNumOfWriteOperations(3)
+ .setCountForRead(5)
+ .setCountForWrite(3),
+ new ParameterBuilder()
+ .setTotalThreadCount(10)
+ .setNumOfReadOperations(5)
+ .setNumOfWriteOperations(3)
+ .setCountForRead(5)
+ .setCountForWrite(3)
+ .setDataSize("64B")
+ .setBufferSize(16),
+ new ParameterBuilder()
+ .setTotalThreadCount(10)
+ .setNumOfReadOperations(5)
+ .setNumOfWriteOperations(0)
+ .setCountForRead(5),
+ new ParameterBuilder()
+ .setTotalThreadCount(20)
+ .setNumOfReadOperations(0)
+ .setNumOfWriteOperations(5)
+ .setCountForRead(0)
+ .setCountForWrite(5)
+ );
}
@ParameterizedTest(name = "Filesystem Paths Enabled: {0}")
- @ValueSource(booleans = {false, true})
- public void testOmBucketReadWriteKeyOps(boolean fsPathsEnabled) throws
Exception {
- try {
- startCluster(fsPathsEnabled);
- FileOutputStream out = FileUtils.openOutputStream(new
File(path.toString(),
- "conf"));
- cluster.getConf().writeXml(out);
- out.getFD().sync();
- out.close();
-
- verifyFreonCommand(new ParameterBuilder().setTotalThreadCount(10)
- .setNumOfReadOperations(10).setNumOfWriteOperations(5)
- .setKeyCountForRead(10).setKeyCountForWrite(5));
- verifyFreonCommand(
- new ParameterBuilder().setVolumeName("vol2").setBucketName("bucket1")
- .setTotalThreadCount(10).setNumOfReadOperations(10)
- .setNumOfWriteOperations(5).setKeyCountForRead(10)
- .setKeyCountForWrite(5));
- verifyFreonCommand(
- new ParameterBuilder().setVolumeName("vol3").setBucketName("bucket1")
- .setTotalThreadCount(15).setNumOfReadOperations(5)
- .setNumOfWriteOperations(3).setKeyCountForRead(5)
- .setKeyCountForWrite(3));
- verifyFreonCommand(
- new ParameterBuilder().setVolumeName("vol4").setBucketName("bucket1")
- .setTotalThreadCount(10).setNumOfReadOperations(5)
- .setNumOfWriteOperations(3).setKeyCountForRead(5)
- .setKeyCountForWrite(3).setKeySize("64B")
- .setBufferSize(16));
- verifyFreonCommand(
- new ParameterBuilder().setVolumeName("vol5").setBucketName("bucket1")
- .setTotalThreadCount(10).setNumOfReadOperations(5)
- .setNumOfWriteOperations(0).setKeyCountForRead(5));
- verifyFreonCommand(
- new ParameterBuilder().setVolumeName("vol6").setBucketName("bucket1")
- .setTotalThreadCount(20).setNumOfReadOperations(0)
- .setNumOfWriteOperations(5).setKeyCountForRead(0)
- .setKeyCountForWrite(5));
-
- } finally {
- shutdown();
- }
- }
-
- private void verifyFreonCommand(ParameterBuilder parameterBuilder)
- throws IOException {
- store.createVolume(parameterBuilder.volumeName);
- OzoneVolume volume = store.getVolume(parameterBuilder.volumeName);
- volume.createBucket(parameterBuilder.bucketName);
- OzoneBucket bucket = volume.getBucket(parameterBuilder.bucketName);
- String confPath = new File(path.toString(), "conf").getAbsolutePath();
+ @MethodSource("parameters")
+ void testOmBucketReadWriteKeyOps(ParameterBuilder parameterBuilder) throws
Exception {
+ OzoneBucket bucket = TestDataUtil.createVolumeAndBucket(client,
+ parameterBuilder.getVolumeName(),
+ parameterBuilder.getBucketName(),
+
parameterBuilder.getBucketArgs().setBucketLayout(BucketLayout.OBJECT_STORE).build()
+ );
long startTime = System.currentTimeMillis();
- new Freon().execute(
- new String[]{"-conf", confPath, "obrwk",
- "-v", parameterBuilder.volumeName,
- "-b", parameterBuilder.bucketName,
- "-k", String.valueOf(parameterBuilder.keyCountForRead),
- "-w", String.valueOf(parameterBuilder.keyCountForWrite),
- "-g", parameterBuilder.keySize,
- "--buffer", String.valueOf(parameterBuilder.bufferSize),
- "-l", String.valueOf(parameterBuilder.length),
- "-c", String.valueOf(parameterBuilder.totalThreadCount),
- "-T", String.valueOf(parameterBuilder.readThreadPercentage),
- "-R", String.valueOf(parameterBuilder.numOfReadOperations),
- "-W", String.valueOf(parameterBuilder.numOfWriteOperations),
- "-n", String.valueOf(1)});
+ String om = cluster().getConf().get(OZONE_OM_ADDRESS_KEY);
+ new Freon().getCmd().execute(
+ "-D", OZONE_OM_ADDRESS_KEY + "=" + om,
+ "obrwk",
+ "-v", parameterBuilder.getVolumeName(),
+ "-b", parameterBuilder.getBucketName(),
+ "-k", String.valueOf(parameterBuilder.getCountForRead()),
+ "-w", String.valueOf(parameterBuilder.getCountForWrite()),
+ "-g", parameterBuilder.getDataSize(),
+ "--buffer", String.valueOf(parameterBuilder.getBufferSize()),
+ "-l", String.valueOf(parameterBuilder.getLength()),
+ "-c", String.valueOf(parameterBuilder.getTotalThreadCount()),
+ "-T", String.valueOf(parameterBuilder.getReadThreadPercentage()),
+ "-R", String.valueOf(parameterBuilder.getNumOfReadOperations()),
+ "-W", String.valueOf(parameterBuilder.getNumOfWriteOperations()),
+ "-n", String.valueOf(1));
long totalTime = System.currentTimeMillis() - startTime;
LOG.info("Total Execution Time: " + totalTime);
LOG.info("Started verifying OM bucket read/write ops key generation...");
- verifyKeyCreation(parameterBuilder.keyCountForRead, bucket, "/readPath/");
+ verifyKeyCreation(parameterBuilder.getCountForRead(), bucket,
"/readPath/");
+ verifyKeyCreation(parameterBuilder.getExpectedWriteCount(), bucket,
"/writePath/");
- int readThreadCount = (parameterBuilder.readThreadPercentage *
- parameterBuilder.totalThreadCount) / 100;
- int writeThreadCount = parameterBuilder.totalThreadCount - readThreadCount;
- verifyKeyCreation(writeThreadCount * parameterBuilder.keyCountForWrite *
- parameterBuilder.numOfWriteOperations, bucket, "/writePath/");
-
-
verifyOMLockMetrics(cluster.getOzoneManager().getMetadataManager().getLock()
+
verifyOMLockMetrics(cluster().getOzoneManager().getMetadataManager().getLock()
.getOMLockMetrics());
}
@@ -234,76 +190,4 @@ private void verifyOMLockMetrics(OMLockMetrics
omLockMetrics) {
assertThat(writeHeldSamples).isGreaterThan(0);
}
- private static class ParameterBuilder {
-
- private String volumeName = "vol1";
- private String bucketName = "bucket1";
- private int keyCountForRead = 100;
- private int keyCountForWrite = 10;
- private String keySize = "256B";
- private int bufferSize = 64;
- private int length = 10;
- private int totalThreadCount = 100;
- private int readThreadPercentage = 90;
- private int numOfReadOperations = 50;
- private int numOfWriteOperations = 10;
-
- private ParameterBuilder setVolumeName(String volumeNameParam) {
- volumeName = volumeNameParam;
- return this;
- }
-
- private ParameterBuilder setBucketName(String bucketNameParam) {
- bucketName = bucketNameParam;
- return this;
- }
-
- private ParameterBuilder setKeyCountForRead(int keyCountForReadParam) {
- keyCountForRead = keyCountForReadParam;
- return this;
- }
-
- private ParameterBuilder setKeyCountForWrite(int keyCountForWriteParam) {
- keyCountForWrite = keyCountForWriteParam;
- return this;
- }
-
- private ParameterBuilder setKeySize(String keySizeParam) {
- keySize = keySizeParam;
- return this;
- }
-
- private ParameterBuilder setBufferSize(int bufferSizeParam) {
- bufferSize = bufferSizeParam;
- return this;
- }
-
- private ParameterBuilder setLength(int lengthParam) {
- length = lengthParam;
- return this;
- }
-
- private ParameterBuilder setTotalThreadCount(int totalThreadCountParam) {
- totalThreadCount = totalThreadCountParam;
- return this;
- }
-
- private ParameterBuilder setReadThreadPercentage(
- int readThreadPercentageParam) {
- readThreadPercentage = readThreadPercentageParam;
- return this;
- }
-
- private ParameterBuilder setNumOfReadOperations(
- int numOfReadOperationsParam) {
- numOfReadOperations = numOfReadOperationsParam;
- return this;
- }
-
- private ParameterBuilder setNumOfWriteOperations(
- int numOfWriteOperationsParam) {
- numOfWriteOperations = numOfWriteOperationsParam;
- return this;
- }
- }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
index ed2be82dce..c7f5cd2355 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestRandomKeyGenerator.java
@@ -21,13 +21,7 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
-import java.time.Duration;
-import org.apache.hadoop.hdds.conf.DatanodeRatisServerConfig;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.hdds.ratis.conf.RatisClientConfig;
-import org.apache.hadoop.ozone.MiniOzoneCluster;
-import org.junit.jupiter.api.AfterAll;
-import org.junit.jupiter.api.BeforeAll;
+import org.apache.ozone.test.NonHATests;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.Timeout;
import picocli.CommandLine;
@@ -35,36 +29,7 @@
/**
* Tests Freon, with MiniOzoneCluster.
*/
-public class TestRandomKeyGenerator {
-
- private static MiniOzoneCluster cluster;
- private static OzoneConfiguration conf;
-
- @BeforeAll
- static void init() throws Exception {
- conf = new OzoneConfiguration();
- DatanodeRatisServerConfig ratisServerConfig =
- conf.getObject(DatanodeRatisServerConfig.class);
- ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3));
- ratisServerConfig.setWatchTimeOut(Duration.ofSeconds(3));
- conf.setFromObject(ratisServerConfig);
-
- RatisClientConfig.RaftConfig raftClientConfig =
- conf.getObject(RatisClientConfig.RaftConfig.class);
- raftClientConfig.setRpcRequestTimeout(Duration.ofSeconds(3));
- raftClientConfig.setRpcWatchRequestTimeout(Duration.ofSeconds(3));
- conf.setFromObject(raftClientConfig);
-
- cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
- cluster.waitForClusterToBeReady();
- }
-
- @AfterAll
- static void shutdown() {
- if (cluster != null) {
- cluster.shutdown();
- }
- }
+public abstract class TestRandomKeyGenerator implements NonHATests.TestCase {
@Test
@Timeout(5)
@@ -84,60 +49,50 @@ void singleFailedAttempt() {
}
@Test
- void testDefault() {
+ void testDefaultReplication() {
RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator(cluster.getConf());
+ new RandomKeyGenerator(cluster().getConf());
CommandLine cmd = new CommandLine(randomKeyGenerator);
cmd.execute("--num-of-volumes", "2",
- "--num-of-buckets", "5",
- "--num-of-keys", "10");
+ "--num-of-buckets", "3",
+ "--num-of-keys", "4",
+ "--validate-writes"
+ );
assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
- assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
- assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ assertEquals(6, randomKeyGenerator.getNumberOfBucketsCreated());
+ assertEquals(24, randomKeyGenerator.getNumberOfKeysAdded());
+ assertEquals(24, randomKeyGenerator.getTotalKeysValidated());
+ assertEquals(24, randomKeyGenerator.getSuccessfulValidationCount());
+ assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
randomKeyGenerator.printStats(System.out);
}
@Test
void testECKey() {
RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator(cluster.getConf());
+ new RandomKeyGenerator(cluster().getConf());
CommandLine cmd = new CommandLine(randomKeyGenerator);
- cmd.execute("--num-of-volumes", "2",
- "--num-of-buckets", "5",
- "--num-of-keys", "10",
+ cmd.execute("--num-of-volumes", "1",
+ "--num-of-buckets", "1",
+ "--num-of-keys", "1",
"--replication", "rs-3-2-1024k",
- "--type", "EC"
+ "--type", "EC",
+ "--validate-writes"
);
- assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
- assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
- assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
+ assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
+ assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
+ assertEquals(1, randomKeyGenerator.getTotalKeysValidated());
+ assertEquals(1, randomKeyGenerator.getSuccessfulValidationCount());
+ assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
}
@Test
void testMultiThread() {
RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator(cluster.getConf());
- CommandLine cmd = new CommandLine(randomKeyGenerator);
- cmd.execute("--num-of-volumes", "10",
- "--num-of-buckets", "1",
- "--num-of-keys", "10",
- "--num-of-threads", "10",
- "--key-size", "10KB",
- "--factor", "THREE",
- "--type", "RATIS"
- );
-
- assertEquals(10, randomKeyGenerator.getNumberOfVolumesCreated());
- assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
- assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
- }
-
- @Test
- void testRatisKey() {
- RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator(cluster.getConf());
+ new RandomKeyGenerator(cluster().getConf());
CommandLine cmd = new CommandLine(randomKeyGenerator);
cmd.execute("--num-of-volumes", "10",
"--num-of-buckets", "1",
@@ -156,7 +111,7 @@ void testRatisKey() {
@Test
void testKeyLargerThan2GB() {
RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator(cluster.getConf());
+ new RandomKeyGenerator(cluster().getConf());
CommandLine cmd = new CommandLine(randomKeyGenerator);
cmd.execute("--num-of-volumes", "1",
"--num-of-buckets", "1",
@@ -164,20 +119,18 @@ void testKeyLargerThan2GB() {
"--num-of-threads", "1",
"--key-size", "2.01GB",
"--factor", "THREE",
- "--type", "RATIS",
- "--validate-writes"
+ "--type", "RATIS"
);
assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
- assertEquals(1, randomKeyGenerator.getSuccessfulValidationCount());
}
@Test
void testZeroSizeKey() {
RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator(cluster.getConf());
+ new RandomKeyGenerator(cluster().getConf());
CommandLine cmd = new CommandLine(randomKeyGenerator);
cmd.execute("--num-of-volumes", "1",
"--num-of-buckets", "1",
@@ -192,13 +145,15 @@ void testZeroSizeKey() {
assertEquals(1, randomKeyGenerator.getNumberOfVolumesCreated());
assertEquals(1, randomKeyGenerator.getNumberOfBucketsCreated());
assertEquals(1, randomKeyGenerator.getNumberOfKeysAdded());
+ assertEquals(1, randomKeyGenerator.getTotalKeysValidated());
assertEquals(1, randomKeyGenerator.getSuccessfulValidationCount());
+ assertEquals(0, randomKeyGenerator.getUnsuccessfulValidationCount());
}
@Test
void testThreadPoolSize() {
RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator(cluster.getConf());
+ new RandomKeyGenerator(cluster().getConf());
CommandLine cmd = new CommandLine(randomKeyGenerator);
cmd.execute("--num-of-volumes", "1",
"--num-of-buckets", "1",
@@ -215,11 +170,11 @@ void testThreadPoolSize() {
@Test
void cleanObjectsTest() {
RandomKeyGenerator randomKeyGenerator =
- new RandomKeyGenerator(cluster.getConf());
+ new RandomKeyGenerator(cluster().getConf());
CommandLine cmd = new CommandLine(randomKeyGenerator);
cmd.execute("--num-of-volumes", "2",
- "--num-of-buckets", "5",
- "--num-of-keys", "10",
+ "--num-of-buckets", "3",
+ "--num-of-keys", "2",
"--num-of-threads", "10",
"--factor", "THREE",
"--type", "RATIS",
@@ -227,9 +182,9 @@ void cleanObjectsTest() {
);
assertEquals(2, randomKeyGenerator.getNumberOfVolumesCreated());
- assertEquals(10, randomKeyGenerator.getNumberOfBucketsCreated());
- assertEquals(100, randomKeyGenerator.getNumberOfKeysAdded());
+ assertEquals(6, randomKeyGenerator.getNumberOfBucketsCreated());
+ assertEquals(12, randomKeyGenerator.getNumberOfKeysAdded());
assertEquals(2, randomKeyGenerator.getNumberOfVolumesCleaned());
- assertEquals(10, randomKeyGenerator.getNumberOfBucketsCleaned());
+ assertEquals(6, randomKeyGenerator.getNumberOfBucketsCleaned());
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/AclTests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/AclTests.java
index 643b249035..9136e159c6 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/AclTests.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/AclTests.java
@@ -40,9 +40,9 @@ public abstract class AclTests extends
ClusterForTests<MiniOzoneCluster> {
public static final UserGroupInformation ADMIN_UGI =
UserGroupInformation.createUserForTesting(ADMIN_USER, new String[]
{ADMIN_GROUP});
- /** Hook method for subclasses. */
- MiniOzoneCluster.Builder newClusterBuilder() {
- return MiniOzoneCluster.newBuilder(createOzoneConfig())
+ @Override
+ protected MiniOzoneCluster.Builder newClusterBuilder() {
+ return super.newClusterBuilder()
.setNumDatanodes(3);
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java
index dba858ad36..b9264b1c8c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/ClusterForTests.java
@@ -89,6 +89,12 @@ protected C getCluster() {
return cluster;
}
+ /** Hook method for subclasses. */
+ protected MiniOzoneCluster.Builder newClusterBuilder() {
+ return MiniOzoneCluster.newBuilder(createOzoneConfig())
+ .setNumDatanodes(5);
+ }
+
@BeforeAll
void startCluster() throws Exception {
cluster = createCluster();
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/FreonTests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/FreonTests.java
new file mode 100644
index 0000000000..d0ca2a57a0
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/FreonTests.java
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ozone.test;
+
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.ozone.ClientConfigForTesting;
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.freon.TestDNRPCLoadGenerator;
+import org.apache.hadoop.ozone.freon.TestHadoopDirTreeGenerator;
+import org.apache.hadoop.ozone.freon.TestHadoopNestedDirGenerator;
+import org.apache.hadoop.ozone.freon.TestHsyncGenerator;
+import org.apache.hadoop.ozone.freon.TestOmBucketReadWriteFileOps;
+import org.apache.hadoop.ozone.freon.TestOmBucketReadWriteKeyOps;
+import org.apache.hadoop.ozone.freon.TestRandomKeyGenerator;
+import org.junit.jupiter.api.Nested;
+import org.junit.jupiter.api.TestInstance;
+
+/**
+ * Group tests that write more data than usual (e.g. Freon tests). These are
+ * separated from {@link NonHATests}, because they tend to take a bit longer.
+ * This keeps {@link NonHATests} leaner.
+ * <p/>
+ * Specific tests are implemented in separate classes, and they are subclasses
+ * here as {@link Nested} inner classes. This allows running all tests in the
+ * same cluster.
+ */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public abstract class FreonTests extends ClusterForTests<MiniOzoneCluster> {
+
+ @Override
+ protected OzoneConfiguration createOzoneConfig() {
+ OzoneConfiguration conf = super.createOzoneConfig();
+ ClientConfigForTesting.newBuilder(StorageUnit.MB)
+ .setChunkSize(4)
+ .setBlockSize(256)
+ .applyTo(conf);
+ return conf;
+ }
+
+ @Nested
+ class DNRPCLoadGenerator extends TestDNRPCLoadGenerator {
+ @Override
+ public MiniOzoneCluster cluster() {
+ return getCluster();
+ }
+ }
+
+ @Nested
+ class HadoopDirTreeGenerator extends TestHadoopDirTreeGenerator {
+ @Override
+ public MiniOzoneCluster cluster() {
+ return getCluster();
+ }
+ }
+
+ @Nested
+ class HadoopNestedDirGenerator extends TestHadoopNestedDirGenerator {
+ @Override
+ public MiniOzoneCluster cluster() {
+ return getCluster();
+ }
+ }
+
+ @Nested
+ class HsyncGenerator extends TestHsyncGenerator {
+ @Override
+ public MiniOzoneCluster cluster() {
+ return getCluster();
+ }
+ }
+
+ @Nested
+ class OmBucketReadWriteFileOps extends TestOmBucketReadWriteFileOps {
+ @Override
+ public MiniOzoneCluster cluster() {
+ return getCluster();
+ }
+ }
+
+ @Nested
+ class OmBucketReadWriteKeyOps extends TestOmBucketReadWriteKeyOps {
+ @Override
+ public MiniOzoneCluster cluster() {
+ return getCluster();
+ }
+ }
+
+ @Nested
+ class RandomKeyGenerator extends TestRandomKeyGenerator {
+ @Override
+ public MiniOzoneCluster cluster() {
+ return getCluster();
+ }
+ }
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/HATests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/HATests.java
index cf994637ab..60562aab83 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/HATests.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/HATests.java
@@ -39,8 +39,8 @@
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public abstract class HATests extends ClusterForTests<MiniOzoneHAClusterImpl> {
- /** Hook method for subclasses. */
- MiniOzoneHAClusterImpl.Builder newClusterBuilder() {
+ @Override
+ protected MiniOzoneHAClusterImpl.Builder newClusterBuilder() {
return MiniOzoneCluster.newHABuilder(createOzoneConfig())
.setOMServiceId("om-" + UUID.randomUUID())
.setNumOfOzoneManagers(3)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java
index d3340071df..b42c569257 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/NonHATests.java
@@ -32,12 +32,6 @@
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
public abstract class NonHATests extends ClusterForTests<MiniOzoneCluster> {
- /** Hook method for subclasses. */
- MiniOzoneCluster.Builder newClusterBuilder() {
- return MiniOzoneCluster.newBuilder(createOzoneConfig())
- .setNumDatanodes(5);
- }
-
/** Test cases for non-HA cluster should implement this. */
public interface TestCase {
MiniOzoneCluster cluster();
@@ -155,14 +149,6 @@ public MiniOzoneCluster cluster() {
}
}
- @Nested
- class DNRPCLoadGenerator extends
org.apache.hadoop.ozone.freon.TestDNRPCLoadGenerator {
- @Override
- public MiniOzoneCluster cluster() {
- return getCluster();
- }
- }
-
@Nested
class LeaseRecoverer extends
org.apache.hadoop.ozone.admin.om.lease.TestLeaseRecoverer {
@Override
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/TestFreon.java
similarity index 62%
rename from
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java
rename to
hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/TestFreon.java
index b13c248654..d41710b093 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/freon/TestHadoopDirTreeGeneratorWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/ozone/test/TestFreon.java
@@ -15,22 +15,18 @@
* limitations under the License.
*/
-package org.apache.hadoop.ozone.freon;
+package org.apache.ozone.test;
-import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.om.request.OMRequestTestUtils;
-
-/**
- * Test for HadoopDirTreeGenerator - prefix layout.
- */
-public class TestHadoopDirTreeGeneratorWithFSO
- extends TestHadoopDirTreeGenerator {
+import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.junit.jupiter.api.TestInstance;
+/** Test Freon with mini cluster. */
+@TestInstance(TestInstance.Lifecycle.PER_CLASS)
+public class TestFreon extends FreonTests {
@Override
- protected OzoneConfiguration getOzoneConfiguration() {
- OzoneConfiguration conf = new OzoneConfiguration();
- OMRequestTestUtils.configureFSOptimizedPaths(conf, true);
- return conf;
+ protected MiniOzoneCluster createCluster() throws Exception {
+ return newClusterBuilder()
+ .build();
}
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
index 4fd2252f85..a96e3ab95a 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/BaseFreonGenerator.java
@@ -80,7 +80,7 @@ public class BaseFreonGenerator implements FreonSubcommand {
private static final Logger LOG =
LoggerFactory.getLogger(BaseFreonGenerator.class);
- private static final int CHECK_INTERVAL_MILLIS = 1000;
+ private static final int CHECK_INTERVAL_MILLIS = 100;
private static final String DIGEST_ALGORITHM = "MD5";
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
index ae35d12e01..cbd86de81e 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/RandomKeyGenerator.java
@@ -102,7 +102,7 @@ enum FreonOps {
private static final int QUANTILES = 10;
- private static final int CHECK_INTERVAL_MILLIS = 5000;
+ private static final int CHECK_INTERVAL_MILLIS = 100;
private byte[] keyValueBuffer = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
