This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 900682e7120 HDFS-16389.Improve NNThroughputBenchmark test mkdirs.
(#3819)
900682e7120 is described below
commit 900682e7120e7af250ebf0b4bafb40b412d3e11c
Author: jianghuazhu <[email protected]>
AuthorDate: Mon Apr 18 11:21:51 2022 +0800
HDFS-16389.Improve NNThroughputBenchmark test mkdirs. (#3819)
Reviewed-by: Viraj Jasani <[email protected]>
Reviewed-by: Akira Ajisaka <[email protected]>
Signed-off-by: Wei-Chiu Chuang <[email protected]>
---
.../hdfs/server/namenode/FileNameGenerator.java | 4 +++
.../server/namenode/NNThroughputBenchmark.java | 40 +++++++++++++++-------
2 files changed, 32 insertions(+), 12 deletions(-)
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
index 6f7087b9215..ed18696c232 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FileNameGenerator.java
@@ -77,6 +77,10 @@ public class FileNameGenerator {
return fn;
}
+ public long getFileCount() {
+ return fileCount;
+ }
+
private synchronized void reset() {
Arrays.fill(pathIndecies, -1);
fileCount = 0L;
diff --git
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
index cfbcb40ffb7..847f7dc0c12 100644
---
a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
+++
b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NNThroughputBenchmark.java
@@ -569,12 +569,20 @@ public class NNThroughputBenchmark implements Tool {
// int generatedFileIdx = 0;
LOG.info("Generate " + numOpsRequired + " intputs for " + getOpName());
fileNames = new String[numThreads][];
- for(int idx=0; idx < numThreads; idx++) {
- int threadOps = opsPerThread[idx];
- fileNames[idx] = new String[threadOps];
- for(int jdx=0; jdx < threadOps; jdx++)
- fileNames[idx][jdx] = nameGenerator.
- getNextFileName("ThroughputBench");
+ try {
+ for(int idx=0; idx < numThreads; idx++) {
+ int threadOps = opsPerThread[idx];
+ fileNames[idx] = new String[threadOps];
+ for(int jdx=0; jdx < threadOps; jdx++) {
+ fileNames[idx][jdx] = nameGenerator.
+ getNextFileName("ThroughputBench");
+ }
+ }
+ } catch (ArrayIndexOutOfBoundsException e) {
+ LOG.error("The current environment allows {} files to be created. " +
+ "If you want to test more files, please update the -filesPerDir
parameter.",
+ nameGenerator.getFileCount());
+ throw e;
}
}
@@ -669,12 +677,20 @@ public class NNThroughputBenchmark implements Tool {
false);
LOG.info("Generate " + numOpsRequired + " inputs for " + getOpName());
dirPaths = new String[numThreads][];
- for(int idx=0; idx < numThreads; idx++) {
- int threadOps = opsPerThread[idx];
- dirPaths[idx] = new String[threadOps];
- for(int jdx=0; jdx < threadOps; jdx++)
- dirPaths[idx][jdx] = nameGenerator.
- getNextFileName("ThroughputBench");
+ try {
+ for(int idx=0; idx < numThreads; idx++) {
+ int threadOps = opsPerThread[idx];
+ dirPaths[idx] = new String[threadOps];
+ for(int jdx=0; jdx < threadOps; jdx++) {
+ dirPaths[idx][jdx] = nameGenerator.
+ getNextFileName("ThroughputBench");
+ }
+ }
+ } catch (ArrayIndexOutOfBoundsException e) {
+ LOG.error("The current environment allows {} directories to be
created. " +
+ "If you want to test more directories, please update the
-dirsPerDir parameter.",
+ nameGenerator.getFileCount());
+ throw e;
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
