This is an automated email from the ASF dual-hosted git repository. shashikant pushed a commit to branch HDDS-2854 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit a2c854846041e4299ff29555de919ff946ac5006 Author: Shashikant Banerjee <[email protected]> AuthorDate: Wed Jan 8 14:51:34 2020 +0530 HDDS-2854. Add a config for the write chunk executor queue limit. --- .../src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 6 ++++++ .../src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java | 8 ++++++++ hadoop-hdds/common/src/main/resources/ozone-default.xml | 9 +++++++++ .../common/transport/server/ratis/XceiverServerRatis.java | 7 ++++++- 4 files changed, 29 insertions(+), 1 deletion(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java index 00d326e..8c11b05 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/ScmConfigKeys.java @@ -48,6 +48,12 @@ public final class ScmConfigKeys { = "dfs.container.ratis.num.write.chunk.threads"; public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT = 60; + public static final String + DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT + = "dfs.container.ratis.num.write.chunk.executor.queue.limit"; + public static final int + DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT + = 1024; public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY = "dfs.container.ratis.replication.level"; public static final ReplicationLevel diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 1700926..184124d 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -240,6 +240,14 @@ public final class OzoneConfigKeys { = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY; public static final int DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT = ScmConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT; + public static final String + DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT + = ScmConfigKeys. + DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT; + public static final int + DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT + = ScmConfigKeys. + DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT; public static final String DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY = ScmConfigKeys.DFS_CONTAINER_RATIS_REPLICATION_LEVEL_KEY; public static final ReplicationLevel diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index c83a5bf..a319f0c 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -204,6 +204,15 @@ </description> </property> <property> + <name>dfs.container.ratis.num.write.chunk.executor.queue.limit</name> + <value>60</value> + <tag>OZONE, RATIS, PERFORMANCE</tag> + <description>Maximum limit of the number of tasks that can be pending + in the thread pool that Ratis will use for writing chunks + (1024 by default). + </description> + </property> + <property> <name>dfs.container.ratis.leader.num.pending.requests</name> <value>4096</value> <tag>OZONE, RATIS, PERFORMANCE</tag> diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index dd9327a..341e2af 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -127,10 +127,15 @@ public final class XceiverServerRatis implements XceiverServerSpi { final int numWriteChunkThreads = conf.getInt( OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_KEY, OzoneConfigKeys.DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_THREADS_DEFAULT); + final int queueLimit = conf.getInt( + OzoneConfigKeys. + DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT, + OzoneConfigKeys. + DFS_CONTAINER_RATIS_NUM_WRITE_CHUNK_EXECUTOR_QUEUE_LIMIT_DEFAULT); chunkExecutor = new ThreadPoolExecutor(numWriteChunkThreads, numWriteChunkThreads, 100, TimeUnit.SECONDS, - new ArrayBlockingQueue<>(1024), + new ArrayBlockingQueue<>(queueLimit), new ThreadPoolExecutor.CallerRunsPolicy()); this.context = context; this.replicationLevel = --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
