[FLINK-8475][config][docs] Integrate Mesos options This closes #5461.
Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/99c07d96 Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/99c07d96 Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/99c07d96 Branch: refs/heads/master Commit: 99c07d967a7a27ce06ded601cbc4c36d1b6702e2 Parents: 023ab74 Author: zentol <ches...@apache.org> Authored: Tue Jan 23 14:20:12 2018 +0100 Committer: zentol <ches...@apache.org> Committed: Mon Feb 12 17:04:14 2018 +0100 ---------------------------------------------------------------------- .../generated/mesos_configuration.html | 66 ++++++++++++++++++ .../mesos_task_manager_configuration.html | 71 ++++++++++++++++++++ docs/ops/config.md | 47 +------------ flink-docs/pom.xml | 9 +++ .../ConfigOptionsDocGenerator.java | 4 +- .../flink/mesos/configuration/MesosOptions.java | 39 ++++++++--- .../MesosTaskManagerParameters.java | 23 +++++-- 7 files changed, 195 insertions(+), 64 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/99c07d96/docs/_includes/generated/mesos_configuration.html ---------------------------------------------------------------------- diff --git a/docs/_includes/generated/mesos_configuration.html b/docs/_includes/generated/mesos_configuration.html new file mode 100644 index 0000000..a3fd2cd --- /dev/null +++ b/docs/_includes/generated/mesos_configuration.html @@ -0,0 +1,66 @@ +<table class="table table-bordered"> + <thead> + <tr> + <th class="text-left" style="width: 20%">Key</th> + <th class="text-left" style="width: 15%">Default</th> + <th class="text-left" style="width: 65%">Description</th> + </tr> + </thead> + <tbody> + <tr> + <td><h5>mesos.failover-timeout</h5></td> + <td>600</td> + <td>The failover timeout in seconds for the Mesos scheduler, after which running tasks are automatically shut down.</td> + </tr> + <tr> + <td><h5>mesos.initial-tasks</h5></td> + <td>0</td> + <td>The initial workers to bring up when the master starts</td> + </tr> + <tr> + <td><h5>mesos.master</h5></td> + <td>(none)</td> + <td>The Mesos master URL. The value should be in one of the following forms: "host:port", "zk://host1:port1,host2:port2,.../path", "zk://username:password@host1:port1,host2:port2,.../path" or "file:///path/to/file"</td> + </tr> + <tr> + <td><h5>mesos.maximum-failed-tasks</h5></td> + <td>-1</td> + <td>The maximum number of failed workers before the cluster fails. May be set to -1 to disable this feature</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.artifactserver.port</h5></td> + <td>0</td> + <td>The config parameter defining the Mesos artifact server port to use. Setting the port to 0 will let the OS choose an available port.</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.artifactserver.ssl.enabled</h5></td> + <td>true</td> + <td>Enables SSL for the Flink artifact server. Note that security.ssl.enabled also needs to be set to true encryption to enable encryption.</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.framework.name</h5></td> + <td>"Flink"</td> + <td>Mesos framework name</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.framework.principal</h5></td> + <td>(none)</td> + <td>Mesos framework principal</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.framework.role</h5></td> + <td>"*"</td> + <td>Mesos framework role definition</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.framework.secret</h5></td> + <td>(none)</td> + <td>Mesos framework secret</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.framework.user</h5></td> + <td>(none)</td> + <td>Mesos framework user</td> + </tr> + </tbody> +</table> http://git-wip-us.apache.org/repos/asf/flink/blob/99c07d96/docs/_includes/generated/mesos_task_manager_configuration.html ---------------------------------------------------------------------- diff --git a/docs/_includes/generated/mesos_task_manager_configuration.html b/docs/_includes/generated/mesos_task_manager_configuration.html new file mode 100644 index 0000000..d05f22d --- /dev/null +++ b/docs/_includes/generated/mesos_task_manager_configuration.html @@ -0,0 +1,71 @@ +<table class="table table-bordered"> + <thead> + <tr> + <th class="text-left" style="width: 20%">Key</th> + <th class="text-left" style="width: 15%">Default</th> + <th class="text-left" style="width: 65%">Description</th> + </tr> + </thead> + <tbody> + <tr> + <td><h5>mesos.constraints.hard.hostattribute</h5></td> + <td>(none)</td> + <td>Constraints for task placement on mesos.</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.tasks.bootstrap-cmd</h5></td> + <td>(none)</td> + <td></td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.tasks.container.docker.parameters</h5></td> + <td>(none)</td> + <td>Custom parameters to be passed into docker run command when using the docker containerizer. Comma separated list of "key=value" pairs. The "value" may contain '='.</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.tasks.container.image.name</h5></td> + <td>(none)</td> + <td>Image name to use for the container.</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.tasks.container.type</h5></td> + <td>"mesos"</td> + <td>Type of the containerization used: âmesosâ or âdockerâ.</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.tasks.container.volumes</h5></td> + <td>(none)</td> + <td>A comma separated list of [host_path:]container_path[:RO|RW]. This allows for mounting additional volumes into your container.</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.tasks.cpus</h5></td> + <td>0.0</td> + <td>CPUs to assign to the Mesos workers.</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.tasks.gpus</h5></td> + <td>0</td> + <td></td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.tasks.hostname</h5></td> + <td>(none)</td> + <td></td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.tasks.mem</h5></td> + <td>1024</td> + <td>Memory to assign to the Mesos workers in MB.</td> + </tr> + <tr> + <td><h5>mesos.resourcemanager.tasks.taskmanager-cmd</h5></td> + <td>"$FLINK_HOME/bin/mesos-taskmanager.sh"</td> + <td></td> + </tr> + <tr> + <td><h5>taskmanager.numberOfTaskSlots</h5></td> + <td>1</td> + <td></td> + </tr> + </tbody> +</table> http://git-wip-us.apache.org/repos/asf/flink/blob/99c07d96/docs/ops/config.md ---------------------------------------------------------------------- diff --git a/docs/ops/config.md b/docs/ops/config.md index 3236199..cc0a10f 100644 --- a/docs/ops/config.md +++ b/docs/ops/config.md @@ -439,52 +439,11 @@ use the `env.java.opts` setting, which is the `%jvmopts%` variable in the String ### Mesos +{% include generated/mesos_configuration.html %} -- `mesos.initial-tasks`: The initial workers to bring up when the master starts (**DEFAULT**: The number of workers specified at cluster startup). +#### Mesos TaskManager -- `mesos.constraints.hard.hostattribute`: Constraints for task placement on mesos (**DEFAULT**: None). - -- `mesos.maximum-failed-tasks`: The maximum number of failed workers before the cluster fails (**DEFAULT**: Number of initial workers). -May be set to -1 to disable this feature. - -- `mesos.master`: The Mesos master URL. The value should be in one of the following forms: - * `host:port` - * `zk://host1:port1,host2:port2,.../path` - * `zk://username:password@host1:port1,host2:port2,.../path` - * `file:///path/to/file` - - -- `mesos.failover-timeout`: The failover timeout in seconds for the Mesos scheduler, after which running tasks are automatically shut down (**DEFAULT:** 600). - -- `mesos.resourcemanager.artifactserver.port`:The config parameter defining the Mesos artifact server port to use. Setting the port to 0 will let the OS choose an available port. - -- `mesos.resourcemanager.framework.name`: Mesos framework name (**DEFAULT:** Flink) - -- `mesos.resourcemanager.framework.role`: Mesos framework role definition (**DEFAULT:** *) - -- `mesos.resourcemanager.framework.principal`: Mesos framework principal (**NO DEFAULT**) - -- `mesos.resourcemanager.framework.secret`: Mesos framework secret (**NO DEFAULT**) - -- `mesos.resourcemanager.framework.user`: Mesos framework user (**DEFAULT:**"") - -- `mesos.resourcemanager.artifactserver.ssl.enabled`: Enables SSL for the Flink artifact server (**DEFAULT**: true). Note that `security.ssl.enabled` also needs to be set to `true` encryption to enable encryption. - -- `mesos.resourcemanager.tasks.mem`: Memory to assign to the Mesos workers in MB (**DEFAULT**: 1024) - -- `mesos.resourcemanager.tasks.cpus`: CPUs to assign to the Mesos workers (**DEFAULT**: 0.0) - -- `mesos.resourcemanager.tasks.gpus`: GPUs to assign to the Mesos workers (**DEFAULT**: 0.0) - -- `mesos.resourcemanager.tasks.container.type`: Type of the containerization used: "mesos" or "docker" (DEFAULT: mesos); - -- `mesos.resourcemanager.tasks.container.image.name`: Image name to use for the container (**NO DEFAULT**) - -- `mesos.resourcemanager.tasks.container.volumes`: A comma separated list of `[host_path:]`container_path`[:RO|RW]`. This allows for mounting additional volumes into your container. (**NO DEFAULT**) - -- `mesos.resourcemanager.tasks.container.docker.parameters`: Custom parameters to be passed into docker run command when using the docker containerizer. Comma separated list of `key=value` pairs. `value` may contain '=' (**NO DEFAULT**) - -- `high-availability.zookeeper.path.mesos-workers`: The ZooKeeper root path for persisting the Mesos worker information. +{% include generated/mesos_task_manager_configuration.html %} ### High Availability (HA) http://git-wip-us.apache.org/repos/asf/flink/blob/99c07d96/flink-docs/pom.xml ---------------------------------------------------------------------- diff --git a/flink-docs/pom.xml b/flink-docs/pom.xml index 68a442d..744bf44 100644 --- a/flink-docs/pom.xml +++ b/flink-docs/pom.xml @@ -53,6 +53,11 @@ under the License. <artifactId>flink-yarn_${scala.binary.version}</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.flink</groupId> + <artifactId>flink-mesos_${scala.binary.version}</artifactId> + <version>${project.version}</version> + </dependency> <dependency> <groupId>org.apache.flink</groupId> @@ -190,6 +195,10 @@ under the License. <arg value="org.apache.flink.configuration" /> <arg value="flink-yarn" /> <arg value="org.apache.flink.yarn.configuration" /> + <arg value="flink-mesos" /> + <arg value="org.apache.flink.mesos.configuration" /> + <arg value="flink-mesos" /> + <arg value="org.apache.flink.mesos.runtime.clusterframework" /> </java> </target> </configuration> http://git-wip-us.apache.org/repos/asf/flink/blob/99c07d96/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java ---------------------------------------------------------------------- diff --git a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java index 1c60a22..c67e582 100644 --- a/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java +++ b/flink-docs/src/main/java/org/apache/flink/docs/configuration/ConfigOptionsDocGenerator.java @@ -70,8 +70,8 @@ public class ConfigOptionsDocGenerator { private static void createTable(String rootDir, String module, String packageName, String outputDirectory) throws IOException, ClassNotFoundException { Path configDir = Paths.get(rootDir, module, "src/main/java", packageName.replaceAll("\\.", "/")); - Pattern p = Pattern.compile("(([a-zA-Z]*)(Options))\\.java"); - try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir, "*Options.java")) { + Pattern p = Pattern.compile("(([a-zA-Z]*)(Options|Parameters))\\.java"); + try (DirectoryStream<Path> stream = Files.newDirectoryStream(configDir)) { for (Path entry : stream) { String fileName = entry.getFileName().toString(); Matcher matcher = p.matcher(fileName); http://git-wip-us.apache.org/repos/asf/flink/blob/99c07d96/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java ---------------------------------------------------------------------- diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java b/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java index 8616cad..6c802fa 100644 --- a/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java +++ b/flink-mesos/src/main/java/org/apache/flink/mesos/configuration/MesosOptions.java @@ -32,7 +32,8 @@ public class MesosOptions { */ public static final ConfigOption<Integer> INITIAL_TASKS = key("mesos.initial-tasks") - .defaultValue(0); + .defaultValue(0) + .withDescription("The initial workers to bring up when the master starts"); /** * The maximum number of failed Mesos tasks before entirely stopping @@ -42,7 +43,9 @@ public class MesosOptions { */ public static final ConfigOption<Integer> MAX_FAILED_TASKS = key("mesos.maximum-failed-tasks") - .defaultValue(-1); + .defaultValue(-1) + .withDescription("The maximum number of failed workers before the cluster fails. May be set to -1 to disable" + + " this feature"); /** * The Mesos master URL. @@ -59,14 +62,19 @@ public class MesosOptions { */ public static final ConfigOption<String> MASTER_URL = key("mesos.master") - .noDefaultValue(); + .noDefaultValue() + .withDescription("The Mesos master URL. The value should be in one of the following forms:" + + " \"host:port\", \"zk://host1:port1,host2:port2,.../path\"," + + " \"zk://username:password@host1:port1,host2:port2,.../path\" or \"file:///path/to/file\""); /** * The failover timeout for the Mesos scheduler, after which running tasks are automatically shut down. */ public static final ConfigOption<Integer> FAILOVER_TIMEOUT_SECONDS = key("mesos.failover-timeout") - .defaultValue(600); + .defaultValue(600) + .withDescription("The failover timeout in seconds for the Mesos scheduler, after which running tasks are" + + " automatically shut down."); /** * The config parameter defining the Mesos artifact server port to use. @@ -74,33 +82,42 @@ public class MesosOptions { */ public static final ConfigOption<Integer> ARTIFACT_SERVER_PORT = key("mesos.resourcemanager.artifactserver.port") - .defaultValue(0); + .defaultValue(0) + .withDescription("The config parameter defining the Mesos artifact server port to use. Setting the port to" + + " 0 will let the OS choose an available port."); public static final ConfigOption<String> RESOURCEMANAGER_FRAMEWORK_NAME = key("mesos.resourcemanager.framework.name") - .defaultValue("Flink"); + .defaultValue("Flink") + .withDescription("Mesos framework name"); public static final ConfigOption<String> RESOURCEMANAGER_FRAMEWORK_ROLE = key("mesos.resourcemanager.framework.role") - .defaultValue("*"); + .defaultValue("*") + .withDescription("Mesos framework role definition"); public static final ConfigOption<String> RESOURCEMANAGER_FRAMEWORK_PRINCIPAL = key("mesos.resourcemanager.framework.principal") - .noDefaultValue(); + .noDefaultValue() + .withDescription("Mesos framework principal"); public static final ConfigOption<String> RESOURCEMANAGER_FRAMEWORK_SECRET = key("mesos.resourcemanager.framework.secret") - .noDefaultValue(); + .noDefaultValue() + .withDescription("Mesos framework secret"); public static final ConfigOption<String> RESOURCEMANAGER_FRAMEWORK_USER = key("mesos.resourcemanager.framework.user") - .defaultValue(""); + .defaultValue("") + .withDescription("Mesos framework user"); /** * Config parameter to override SSL support for the Artifact Server. */ public static final ConfigOption<Boolean> ARTIFACT_SERVER_SSL_ENABLED = key("mesos.resourcemanager.artifactserver.ssl.enabled") - .defaultValue(true); + .defaultValue(true) + .withDescription("Enables SSL for the Flink artifact server. Note that security.ssl.enabled also needs to" + + " be set to true encryption to enable encryption."); } http://git-wip-us.apache.org/repos/asf/flink/blob/99c07d96/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java ---------------------------------------------------------------------- diff --git a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java index 8e52a80..68061d8 100644 --- a/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java +++ b/flink-mesos/src/main/java/org/apache/flink/mesos/runtime/clusterframework/MesosTaskManagerParameters.java @@ -56,11 +56,13 @@ public class MesosTaskManagerParameters { public static final ConfigOption<Integer> MESOS_RM_TASKS_MEMORY_MB = key("mesos.resourcemanager.tasks.mem") - .defaultValue(1024); + .defaultValue(1024) + .withDescription("Memory to assign to the Mesos workers in MB."); public static final ConfigOption<Double> MESOS_RM_TASKS_CPUS = key("mesos.resourcemanager.tasks.cpus") - .defaultValue(0.0); + .defaultValue(0.0) + .withDescription("CPUs to assign to the Mesos workers."); public static final ConfigOption<Integer> MESOS_RM_TASKS_GPUS = key("mesos.resourcemanager.tasks.gpus") @@ -68,11 +70,13 @@ public class MesosTaskManagerParameters { public static final ConfigOption<String> MESOS_RM_CONTAINER_TYPE = key("mesos.resourcemanager.tasks.container.type") - .defaultValue("mesos"); + .defaultValue("mesos") + .withDescription("Type of the containerization used: âmesosâ or âdockerâ."); public static final ConfigOption<String> MESOS_RM_CONTAINER_IMAGE_NAME = key("mesos.resourcemanager.tasks.container.image.name") - .noDefaultValue(); + .noDefaultValue() + .withDescription("Image name to use for the container."); public static final ConfigOption<String> MESOS_TM_HOSTNAME = key("mesos.resourcemanager.tasks.hostname") @@ -88,15 +92,20 @@ public class MesosTaskManagerParameters { public static final ConfigOption<String> MESOS_RM_CONTAINER_VOLUMES = key("mesos.resourcemanager.tasks.container.volumes") - .noDefaultValue(); + .noDefaultValue() + .withDescription("A comma separated list of [host_path:]container_path[:RO|RW]. This allows for mounting" + + " additional volumes into your container."); public static final ConfigOption<String> MESOS_RM_CONTAINER_DOCKER_PARAMETERS = key("mesos.resourcemanager.tasks.container.docker.parameters") - .noDefaultValue(); + .noDefaultValue() + .withDescription("Custom parameters to be passed into docker run command when using the docker containerizer." + + " Comma separated list of \"key=value\" pairs. The \"value\" may contain '='."); public static final ConfigOption<String> MESOS_CONSTRAINTS_HARD_HOSTATTR = key("mesos.constraints.hard.hostattribute") - .noDefaultValue(); + .noDefaultValue() + .withDescription("Constraints for task placement on mesos."); /** * Value for {@code MESOS_RESOURCEMANAGER_TASKS_CONTAINER_TYPE} setting. Tells to use the Mesos containerizer.
