Repository: hadoop Updated Branches: refs/heads/branch-2 6dba38f08 -> 5128e3320
MAPREDUCE-6763. Shuffle server listen queue is too small (Jason Lowe via Varun Saxena) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5128e332 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5128e332 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5128e332 Branch: refs/heads/branch-2 Commit: 5128e3320ea558e46e5f1d27cd875dec86d53796 Parents: 6dba38f Author: Varun Saxena <[email protected]> Authored: Sat Aug 20 04:05:01 2016 +0530 Committer: Varun Saxena <[email protected]> Committed: Sat Aug 20 04:05:01 2016 +0530 ---------------------------------------------------------------------- .../src/main/resources/mapred-default.xml | 6 ++++++ .../src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java | 6 ++++++ 2 files changed, 12 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5128e332/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml index 4a8d6e2..4669a2e 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml @@ -218,6 +218,12 @@ </property> <property> + <name>mapreduce.shuffle.listen.queue.size</name> + <value>128</value> + <description>The length of the shuffle server listen queue.</description> +</property> + +<property> <name>mapreduce.shuffle.connection-keep-alive.enable</name> <value>false</value> <description>set to true to support keep-alive connections.</description> http://git-wip-us.apache.org/repos/asf/hadoop/blob/5128e332/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java index ed197f2..879e6d5 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java @@ -195,6 +195,10 @@ public class ShuffleHandler extends AuxiliaryService { public static final String SHUFFLE_PORT_CONFIG_KEY = "mapreduce.shuffle.port"; public static final int DEFAULT_SHUFFLE_PORT = 13562; + public static final String SHUFFLE_LISTEN_QUEUE_SIZE = + "mapreduce.shuffle.listen.queue.size"; + public static final int DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE = 128; + public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = "mapreduce.shuffle.connection-keep-alive.enable"; public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false; @@ -504,6 +508,8 @@ public class ShuffleHandler extends AuxiliaryService { } catch (Exception ex) { throw new RuntimeException(ex); } + bootstrap.setOption("backlog", conf.getInt(SHUFFLE_LISTEN_QUEUE_SIZE, + DEFAULT_SHUFFLE_LISTEN_QUEUE_SIZE)); bootstrap.setOption("child.keepAlive", true); bootstrap.setPipelineFactory(pipelineFact); port = conf.getInt(SHUFFLE_PORT_CONFIG_KEY, DEFAULT_SHUFFLE_PORT); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
