This is an automated email from the ASF dual-hosted git repository.
msingh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new ae8d15c HDDS-3517. Add a directory based Ozone Manager LoadGenerator.
(#893)
ae8d15c is described below
commit ae8d15cad7150f2e51df012fb4be9bbb102949ba
Author: Mukul Kumar Singh <[email protected]>
AuthorDate: Tue May 12 13:32:23 2020 +0530
HDDS-3517. Add a directory based Ozone Manager LoadGenerator. (#893)
---
.../hadoop/ozone/TestMiniChaosOzoneCluster.java | 6 +++
.../ozone/loadgenerators/AgedDirLoadGenerator.java | 50 +++++++++++++++++++
.../ozone/loadgenerators/AgedLoadGenerator.java | 1 -
.../loadgenerators/NestedDirLoadGenerator.java | 58 ++++++++++++++++++++++
.../loadgenerators/RandomDirLoadGenerator.java | 46 +++++++++++++++++
.../org/apache/hadoop/ozone/utils/LoadBucket.java | 52 +++++++++++++++++++
6 files changed, 212 insertions(+), 1 deletion(-)
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
index 5e00523..ea7fe69 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/TestMiniChaosOzoneCluster.java
@@ -27,6 +27,9 @@ import
org.apache.hadoop.ozone.loadgenerators.RandomLoadGenerator;
import org.apache.hadoop.ozone.loadgenerators.ReadOnlyLoadGenerator;
import org.apache.hadoop.ozone.loadgenerators.FilesystemLoadGenerator;
import org.apache.hadoop.ozone.loadgenerators.AgedLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.AgedDirLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.RandomDirLoadGenerator;
+import org.apache.hadoop.ozone.loadgenerators.NestedDirLoadGenerator;
import org.junit.BeforeClass;
import org.junit.AfterClass;
import org.junit.Ignore;
@@ -117,6 +120,9 @@ public class TestMiniChaosOzoneCluster extends GenericCli {
.addLoadGenerator(AgedLoadGenerator.class)
.addLoadGenerator(FilesystemLoadGenerator.class)
.addLoadGenerator(ReadOnlyLoadGenerator.class)
+ .addLoadGenerator(RandomDirLoadGenerator.class)
+ .addLoadGenerator(AgedDirLoadGenerator.class)
+ .addLoadGenerator(NestedDirLoadGenerator.class)
.build();
}
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedDirLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedDirLoadGenerator.java
new file mode 100644
index 0000000..f4ab930
--- /dev/null
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedDirLoadGenerator.java
@@ -0,0 +1,50 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.loadgenerators;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.ozone.utils.LoadBucket;
+
+/**
+ * A load generator where directories are read multiple times.
+ */
+public class AgedDirLoadGenerator extends LoadGenerator {
+ private final LoadBucket fsBucket;
+ private final int maxDirIndex;
+
+ public AgedDirLoadGenerator(DataBuffer dataBuffer, LoadBucket fsBucket) {
+ this.fsBucket = fsBucket;
+ this.maxDirIndex = 100;
+ }
+
+ @Override
+ public void generateLoad() throws Exception {
+ int index = RandomUtils.nextInt(0, maxDirIndex);
+ String keyName = getKeyName(index);
+ fsBucket.readDirectory(keyName);
+ }
+
+ @Override
+ public void initialize() throws Exception {
+ for (int i = 0; i < maxDirIndex; i++) {
+ String keyName = getKeyName(i);
+ fsBucket.createDirectory(keyName);
+ }
+ }
+}
\ No newline at end of file
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
index e122529..ecd6076 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/AgedLoadGenerator.java
@@ -34,7 +34,6 @@ import java.util.concurrent.atomic.AtomicInteger;
* The default writes to read ratio is 10:90.
*/
public class AgedLoadGenerator extends LoadGenerator {
- private static String agedSuffix = "aged";
private final AtomicInteger agedFileWrittenIndex;
private final AtomicInteger agedFileAllocationIndex;
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/NestedDirLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/NestedDirLoadGenerator.java
new file mode 100644
index 0000000..ded85a7
--- /dev/null
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/NestedDirLoadGenerator.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.loadgenerators;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.ozone.utils.LoadBucket;
+
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A Load generator where nested directories are created and read them.
+ */
+public class NestedDirLoadGenerator extends LoadGenerator {
+ private final LoadBucket fsBucket;
+ private final int maxDirDepth;
+ private final Map<Integer, String> pathMap;
+
+ public NestedDirLoadGenerator(DataBuffer dataBuffer, LoadBucket fsBucket) {
+ this.fsBucket = fsBucket;
+ this.maxDirDepth = 20;
+ this.pathMap = new ConcurrentHashMap<>();
+ }
+
+ private String createNewPath(int i, String s) {
+ String base = s != null ? s : "";
+ return base + "/" + getKeyName(i);
+ }
+
+ @Override
+ public void generateLoad() throws Exception {
+ int index = RandomUtils.nextInt(0, maxDirDepth);
+ String str = this.pathMap.compute(index, this::createNewPath);
+ fsBucket.createDirectory(str);
+ fsBucket.readDirectory(str);
+ }
+
+ @Override
+ public void initialize() throws Exception {
+ // Nothing to do here
+ }
+}
\ No newline at end of file
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomDirLoadGenerator.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomDirLoadGenerator.java
new file mode 100644
index 0000000..8eaba65
--- /dev/null
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/loadgenerators/RandomDirLoadGenerator.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.loadgenerators;
+
+import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.ozone.utils.LoadBucket;
+
+/**
+ * A simple directory based load generator.
+ */
+public class RandomDirLoadGenerator extends LoadGenerator {
+ private final LoadBucket fsBucket;
+
+ public RandomDirLoadGenerator(DataBuffer dataBuffer, LoadBucket fsBucket) {
+ this.fsBucket = fsBucket;
+ }
+
+ @Override
+ public void generateLoad() throws Exception {
+ int index = RandomUtils.nextInt();
+ String keyName = getKeyName(index);
+ fsBucket.createDirectory(keyName);
+ fsBucket.readDirectory(keyName);
+ }
+
+ @Override
+ public void initialize() {
+ // Nothing to do here
+ }
+}
\ No newline at end of file
diff --git
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/utils/LoadBucket.java
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/utils/LoadBucket.java
index 8e1ef31..51c344f 100644
---
a/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/utils/LoadBucket.java
+++
b/hadoop-ozone/fault-injection-test/mini-chaos-tests/src/test/java/org/apache/hadoop/ozone/utils/LoadBucket.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.ozone.utils;
import org.apache.commons.lang3.RandomUtils;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.ozone.OzoneFileSystem;
@@ -29,6 +30,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.client.OzoneBucket;
import java.io.InputStream;
import java.io.OutputStream;
+import org.junit.Assert;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -76,6 +78,16 @@ public class LoadBucket {
writeOp.execute();
}
+ public void createDirectory(String keyName) throws Exception {
+ Op dirOp = new DirectoryOp(keyName, false);
+ dirOp.execute();
+ }
+
+ public void readDirectory(String keyName) throws Exception {
+ Op dirOp = new DirectoryOp(keyName, true);
+ dirOp.execute();
+ }
+
// Read ops.
public void readKey(ByteBuffer buffer, String keyName) throws Exception {
readKey(isFsOp(), buffer, keyName);
@@ -148,6 +160,46 @@ public class LoadBucket {
}
/**
+ * Create and Read Directories.
+ */
+ public class DirectoryOp extends Op {
+ private final boolean readDir;
+
+ DirectoryOp(String keyName, boolean readDir) {
+ super(true, keyName);
+ this.readDir = readDir;
+ }
+
+ @Override
+ void doFsOp(Path p) throws IOException {
+ if (readDir) {
+ FileStatus status = fs.getFileStatus(p);
+ Assert.assertTrue(status.isDirectory());
+ Assert.assertEquals(p,
+ Path.getPathWithoutSchemeAndAuthority(status.getPath()));
+ } else {
+ Assert.assertTrue(fs.mkdirs(p));
+ }
+ }
+
+ @Override
+ void doBucketOp(String key) throws IOException {
+ // nothing to do here
+ }
+
+ @Override
+ void doPostOp() throws IOException {
+ // Nothing to do here
+ }
+
+ @Override
+ public String toString() {
+ return super.toString() + " "
+ + (readDir ? "readDirectory": "writeDirectory");
+ }
+ }
+
+ /**
* Write file/key to bucket.
*/
public class WriteOp extends Op {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
