This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e2ebbf8 HDDS-3302. Add ability in MiniOzoneCluster to create multiple
storage directories
e2ebbf8 is described below
commit e2ebbf874d5e33565b27a24a02cfb4cee6330ea1
Author: Mukul Kumar Singh <[email protected]>
AuthorDate: Thu Apr 9 17:36:50 2020 +0200
HDDS-3302. Add ability in MiniOzoneCluster to create multiple storage
directories
Closes #763
---
.../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 12 ++++++++++++
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 20 +++++++++++++++-----
.../org/apache/hadoop/ozone/MiniOzoneCluster.java | 13 +++++++++++++
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 12 ++++++++----
.../apache/hadoop/ozone/TestMiniOzoneCluster.java | 20 ++++++++++++++++++--
5 files changed, 66 insertions(+), 11 deletions(-)
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
index 65eb86d..9357768 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/MiniOzoneChaosCluster.java
@@ -360,6 +360,18 @@ public abstract class MiniOzoneChaosCluster extends
MiniOzoneHAClusterImpl {
conf.setInt(OzoneConfigKeys.DFS_CONTAINER_RATIS_LOG_PURGE_GAP, 100);
}
+ /**
+ * Sets the number of data volumes per datanode.
+ *
+ * @param val number of volumes per datanode.
+ *
+ * @return MiniOzoneCluster.Builder
+ */
+ public Builder setNumDataVolumes(int val) {
+ numDataVolumes = val;
+ return this;
+ }
+
@Override
public MiniOzoneChaosCluster build() throws IOException {
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
index 53718e4..e1af53b 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone;
import org.apache.commons.lang3.RandomStringUtils;
+import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneVolume;
@@ -28,7 +29,8 @@ import org.junit.Ignore;
import org.junit.Test;
import picocli.CommandLine.Command;
import picocli.CommandLine.Option;
-import picocli.CommandLine;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.concurrent.TimeUnit;
@@ -38,7 +40,9 @@ import java.util.concurrent.TimeUnit;
@Ignore
@Command(description = "Starts IO with MiniOzoneChaosCluster",
name = "chaos", mixinStandardHelpOptions = true)
-public class TestMiniChaosOzoneCluster implements Runnable {
+public class TestMiniChaosOzoneCluster extends GenericCli {
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestMiniChaosOzoneCluster.class);
@Option(names = {"-d", "--numDatanodes"},
description = "num of datanodes")
@@ -69,6 +73,10 @@ public class TestMiniChaosOzoneCluster implements Runnable {
description = "no of clients writing to OM")
private static int numClients = 3;
+ @Option(names = {"-v", "--numDataVolume"},
+ description = "number of datanode volumes to create")
+ private static int numDataVolumes = 3;
+
@Option(names = {"-i", "--failureInterval"},
description = "time between failure events in seconds")
private static int failureInterval = 300; // 5 minute period between
failures.
@@ -90,6 +98,7 @@ public class TestMiniChaosOzoneCluster implements Runnable {
.setNumOzoneManagers(numOzoneManagers)
.setFailureService(failureService)
.setOMServiceID(omServiceID)
+ .setNumDataVolumes(numDataVolumes)
.build();
cluster.waitForClusterToBeReady();
@@ -117,19 +126,20 @@ public class TestMiniChaosOzoneCluster implements
Runnable {
}
}
- public void run() {
+ @Override
+ public Void call() throws Exception {
try {
init();
cluster.startChaos(failureInterval, failureInterval, TimeUnit.SECONDS);
loadGenerator.startIO(numMinutes, TimeUnit.MINUTES);
- } catch (Exception e) {
} finally {
shutdown();
}
+ return null;
}
public static void main(String... args) {
- CommandLine.run(new TestMiniChaosOzoneCluster(), System.err, args);
+ new TestMiniChaosOzoneCluster().run(args);
}
@Test
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
index 1d1ea90..4b6cf8c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java
@@ -305,6 +305,7 @@ public interface MiniOzoneCluster {
protected int numOfOmHandlers = 20;
protected int numOfScmHandlers = 20;
protected int numOfDatanodes = 3;
+ protected int numDataVolumes = 1;
protected boolean startDataNodes = true;
protected CertificateClient certClient;
protected int pipelineNumLimit = DEFAULT_PIPELIME_LIMIT;
@@ -395,6 +396,18 @@ public interface MiniOzoneCluster {
}
/**
+ * Sets the number of data volumes per datanode.
+ *
+ * @param val number of volumes per datanode.
+ *
+ * @return MiniOzoneCluster.Builder
+ */
+ public Builder setNumDataVolumes(int val) {
+ numDataVolumes = val;
+ return this;
+ }
+
+ /**
* Sets the total number of pipelines to create.
* @param val number of pipelines
* @return MiniOzoneCluster.Builder
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index 98a6eff..c3d1663 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -699,17 +699,21 @@ public class MiniOzoneClusterImpl implements
MiniOzoneCluster {
OzoneConfiguration dnConf = new OzoneConfiguration(conf);
String datanodeBaseDir = path + "/datanode-" + Integer.toString(i);
Path metaDir = Paths.get(datanodeBaseDir, "meta");
- Path dataDir = Paths.get(datanodeBaseDir, "data", "containers");
+ List<String> dataDirs = new ArrayList<>();
+ for (int j = 0; j < numDataVolumes; j++) {
+ Path dir = Paths.get(datanodeBaseDir, "data-" + j, "containers");
+ Files.createDirectories(dir);
+ dataDirs.add(dir.toString());
+ }
+ String listOfDirs = String.join(",", dataDirs);
Path ratisDir = Paths.get(datanodeBaseDir, "data", "ratis");
Path wrokDir = Paths.get(datanodeBaseDir, "data", "replication",
"work");
Files.createDirectories(metaDir);
- Files.createDirectories(dataDir);
Files.createDirectories(ratisDir);
Files.createDirectories(wrokDir);
dnConf.set(HddsConfigKeys.OZONE_METADATA_DIRS, metaDir.toString());
- dnConf.set(DFSConfigKeysLegacy.DFS_DATANODE_DATA_DIR_KEY,
- dataDir.toString());
+ dnConf.set(DFSConfigKeysLegacy.DFS_DATANODE_DATA_DIR_KEY, listOfDirs);
dnConf.set(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATANODE_STORAGE_DIR,
ratisDir.toString());
dnConf.set(OzoneConfigKeys.OZONE_CONTAINER_COPY_WORKDIR,
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
index c69aa71..386a727 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java
@@ -58,14 +58,12 @@ import org.junit.After;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
-import org.junit.Ignore;
import org.junit.Test;
import org.yaml.snakeyaml.Yaml;
/**
* Test cases for mini ozone cluster.
*/
-@Ignore
public class TestMiniOzoneCluster {
private MiniOzoneCluster cluster;
@@ -326,4 +324,22 @@ public class TestMiniOzoneCluster {
endpoint.getState());
}
}
+
+ /**
+ * Test that multiple datanode directories are created in MiniOzoneCluster.
+ * @throws Exception
+ */
+ @Test (timeout = 60000)
+ public void testMultipleDataDirs() throws Exception {
+ // Start a cluster with 3 DN
+ cluster = MiniOzoneCluster.newBuilder(conf)
+ .setNumDatanodes(1)
+ .setNumDataVolumes(3)
+ .build();
+ cluster.waitForClusterToBeReady();
+
+ Assert.assertEquals(3, cluster.getHddsDatanodes().get(0)
+ .getDatanodeStateMachine().getContainer().getVolumeSet()
+ .getVolumesList().size());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
