[FLINK-8475][config][docs] Integrate netty options This closes #5386.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/7910d054 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/7910d054 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/7910d054 Branch: refs/heads/master Commit: 7910d0544b0dc74a0720055a4b38f10e63dc569c Parents: 99c07d9 Author: zentol <ches...@apache.org> Authored: Mon Jan 22 17:02:12 2018 +0100 Committer: zentol <ches...@apache.org> Committed: Mon Feb 12 17:55:13 2018 +0100 ---------------------------------------------------------------------- .../generated/netty_configuration.html | 46 ++++++++++++++++++++ docs/ops/config.md | 14 +----- flink-docs/pom.xml | 2 + .../ConfigOptionsDocGenerator.java | 2 +- .../runtime/io/network/netty/NettyConfig.java | 22 +++++++--- 5 files changed, 65 insertions(+), 21 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/7910d054/docs/_includes/generated/netty_configuration.html ---------------------------------------------------------------------- diff --git a/docs/_includes/generated/netty_configuration.html b/docs/_includes/generated/netty_configuration.html new file mode 100644 index 0000000..70a71ac --- /dev/null +++ b/docs/_includes/generated/netty_configuration.html @@ -0,0 +1,46 @@ +<table class="table table-bordered"> + <thead> + <tr> + <th class="text-left" style="width: 20%">Key</th> + <th class="text-left" style="width: 15%">Default</th> + <th class="text-left" style="width: 65%">Description</th> + </tr> + </thead> + <tbody> + <tr> + <td><h5>taskmanager.network.netty.client.connectTimeoutSec</h5></td> + <td>120</td> + <td>The Netty client connection timeout.</td> + </tr> + <tr> + <td><h5>taskmanager.network.netty.client.numThreads</h5></td> + <td>-1</td> + <td>The number of Netty client threads.</td> + </tr> + <tr> + <td><h5>taskmanager.network.netty.num-arenas</h5></td> + <td>-1</td> + <td>The number of Netty arenas.</td> + </tr> + <tr> + <td><h5>taskmanager.network.netty.sendReceiveBufferSize</h5></td> + <td>0</td> + <td>The Netty send and receive buffer size. This defaults to the system buffer size (cat /proc/sys/net/ipv4/tcp_[rw]mem) and is 4 MiB in modern Linux.</td> + </tr> + <tr> + <td><h5>taskmanager.network.netty.server.backlog</h5></td> + <td>0</td> + <td> The netty server connection backlog.</td> + </tr> + <tr> + <td><h5>taskmanager.network.netty.server.numThreads</h5></td> + <td>-1</td> + <td>The number of Netty server threads.</td> + </tr> + <tr> + <td><h5>taskmanager.network.netty.transport</h5></td> + <td>"nio"</td> + <td>The Netty transport type, either "nio" or "epoll"</td> + </tr> + </tbody> +</table> http://git-wip-us.apache.org/repos/asf/flink/blob/7910d054/docs/ops/config.md ---------------------------------------------------------------------- diff --git a/docs/ops/config.md b/docs/ops/config.md index cc0a10f..da35048 100644 --- a/docs/ops/config.md +++ b/docs/ops/config.md @@ -352,19 +352,7 @@ The following parameters configure Flink's JobManager and TaskManagers. These parameters allow for advanced tuning. The default values are sufficient when running concurrent high-throughput jobs on a large cluster. -- `taskmanager.net.num-arenas`: The number of Netty arenas (DEFAULT: **taskmanager.numberOfTaskSlots**). - -- `taskmanager.net.server.numThreads`: The number of Netty server threads (DEFAULT: **taskmanager.numberOfTaskSlots**). - -- `taskmanager.net.client.numThreads`: The number of Netty client threads (DEFAULT: **taskmanager.numberOfTaskSlots**). - -- `taskmanager.net.server.backlog`: The netty server connection backlog. - -- `taskmanager.net.client.connectTimeoutSec`: The Netty client connection timeout (DEFAULT: **120 seconds**). - -- `taskmanager.net.sendReceiveBufferSize`: The Netty send and receive buffer size. This defaults to the system buffer size (`cat /proc/sys/net/ipv4/tcp_[rw]mem`) and is 4 MiB in modern Linux. - -- `taskmanager.net.transport`: The Netty transport type, either "nio" or "epoll" (DEFAULT: **nio**). +{% include generated/netty_configuration.html %} ### Web Frontend http://git-wip-us.apache.org/repos/asf/flink/blob/7910d054/flink-docs/pom.xml ---------------------------------------------------------------------- diff --git a/flink-docs/pom.xml b/flink-docs/pom.xml index 744bf44..6abf52b 100644 --- a/flink-docs/pom.xml +++ b/flink-docs/pom.xml @@ -193,6 +193,8 @@ under the License. <!--packages with configuration classes--> <arg value="flink-core" /> <arg value="org.apache.flink.configuration" /> + <arg value="flink-runtime" /> + <arg value="org.apache.flink.runtime.io.network.netty" /> <arg value="flink-yarn" /> <arg value="org.apache.flink.yarn.configuration" /> <arg value="flink-mesos" /> http://git-wip-us.apache.org/repos/asf/flink/blob/7910d054/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java ---------------------------------------------------------------------- diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java index c67e582..52cd448 100644 --- a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java +++ b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java @@ -70,7 +70,7 @@ public class ConfigOptionsDocGenerator { private static void createTable(String rootDir, String module, String packageName, String outputDirectory) throws IOException, ClassNotFoundException { Path configDir = Paths.get(rootDir, module, "src/main/java", packageName.replaceAll("\\.", "/")); - Pattern p = Pattern.compile("(([a-zA-Z]*)(Options|Parameters))\\.java"); + Pattern p = Pattern.compile("(([a-zA-Z]*)(Options|Config|Parameters))\\.java"); try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir)) { for (Path entry : stream) { String fileName = entry.getFileName().toString(); http://git-wip-us.apache.org/repos/asf/flink/blob/7910d054/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConfig.java ---------------------------------------------------------------------- diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConfig.java b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConfig.java index e716a82..4ae77bec 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConfig.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/io/network/netty/NettyConfig.java @@ -43,37 +43,45 @@ public class NettyConfig { public static final ConfigOption<Integer> NUM_ARENAS = ConfigOptions .key("taskmanager.network.netty.num-arenas") .defaultValue(-1) - .withDeprecatedKeys("taskmanager.net.num-arenas"); + .withDeprecatedKeys("taskmanager.net.num-arenas") + .withDescription("The number of Netty arenas."); public static final ConfigOption<Integer> NUM_THREADS_SERVER = ConfigOptions .key("taskmanager.network.netty.server.numThreads") .defaultValue(-1) - .withDeprecatedKeys("taskmanager.net.server.numThreads"); + .withDeprecatedKeys("taskmanager.net.server.numThreads") + .withDescription("The number of Netty server threads."); public static final ConfigOption<Integer> NUM_THREADS_CLIENT = ConfigOptions .key("taskmanager.network.netty.client.numThreads") .defaultValue(-1) - .withDeprecatedKeys("taskmanager.net.client.numThreads"); + .withDeprecatedKeys("taskmanager.net.client.numThreads") + .withDescription("The number of Netty client threads."); public static final ConfigOption<Integer> CONNECT_BACKLOG = ConfigOptions .key("taskmanager.network.netty.server.backlog") .defaultValue(0) // default: 0 => Netty's default - .withDeprecatedKeys("taskmanager.net.server.backlog"); + .withDeprecatedKeys("taskmanager.net.server.backlog") + .withDescription(" The netty server connection backlog."); public static final ConfigOption<Integer> CLIENT_CONNECT_TIMEOUT_SECONDS = ConfigOptions .key("taskmanager.network.netty.client.connectTimeoutSec") .defaultValue(120) // default: 120s = 2min - .withDeprecatedKeys("taskmanager.net.client.connectTimeoutSec"); + .withDeprecatedKeys("taskmanager.net.client.connectTimeoutSec") + .withDescription("The Netty client connection timeout."); public static final ConfigOption<Integer> SEND_RECEIVE_BUFFER_SIZE = ConfigOptions .key("taskmanager.network.netty.sendReceiveBufferSize") .defaultValue(0) // default: 0 => Netty's default - .withDeprecatedKeys("taskmanager.net.sendReceiveBufferSize"); + .withDeprecatedKeys("taskmanager.net.sendReceiveBufferSize") + .withDescription("The Netty send and receive buffer size. This defaults to the system buffer size" + + " (cat /proc/sys/net/ipv4/tcp_[rw]mem) and is 4 MiB in modern Linux."); public static final ConfigOption<String> TRANSPORT_TYPE = ConfigOptions .key("taskmanager.network.netty.transport") .defaultValue("nio") - .withDeprecatedKeys("taskmanager.net.transport"); + .withDeprecatedKeys("taskmanager.net.transport") + .withDescription("The Netty transport type, either \"nio\" or \"epoll\""); // ------------------------------------------------------------------------
