dawidwys closed pull request #6294: [FLINK-9013][docs] Document
yarn.containers.vcores only being effective whe…
URL: https://github.com/apache/flink/pull/6294
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/docs/_includes/generated/yarn_config_configuration.html
b/docs/_includes/generated/yarn_config_configuration.html
index fb16e5340c8..bbe25499f17 100644
--- a/docs/_includes/generated/yarn_config_configuration.html
+++ b/docs/_includes/generated/yarn_config_configuration.html
@@ -30,7 +30,7 @@
<tr>
<td><h5>yarn.containers.vcores</h5></td>
<td style="word-wrap: break-word;">-1</td>
- <td>The number of virtual cores (vcores) per YARN container. By
default, the number of vcores is set to the number of slots per TaskManager, if
set, or to 1, otherwise.</td>
+ <td>The number of virtual cores (vcores) per YARN container. By
default, the number of vcores is set to the number of slots per TaskManager, if
set, or to 1, otherwise. In order for this parameter to be used your cluster
must have CPU scheduling enabled. You can do this by setting the <span
markdown="span">`org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler`</span>.</td>
</tr>
<tr>
<td><h5>yarn.heartbeat-delay</h5></td>
diff --git a/docs/ops/deployment/yarn_setup.md
b/docs/ops/deployment/yarn_setup.md
index 9c255d27b92..18e693e1c7d 100644
--- a/docs/ops/deployment/yarn_setup.md
+++ b/docs/ops/deployment/yarn_setup.md
@@ -131,8 +131,8 @@ Once Flink is deployed in your YARN cluster, it will show
you the connection det
Stop the YARN session by stopping the unix process (using CTRL+C) or by
entering 'stop' into the client.
Flink on YARN will only start all requested containers if enough resources are
available on the cluster. Most YARN schedulers account for the requested memory
of the containers,
-some account also for the number of vcores. By default, the number of vcores
is equal to the processing slots (`-s`) argument. The `yarn.containers.vcores`
allows overwriting the
-number of vcores with a custom value.
+some account also for the number of vcores. By default, the number of vcores
is equal to the processing slots (`-s`) argument. The
[`yarn.containers.vcores`]({{ site.baseurl
}}/ops/config.html#yarn-containers-vcores) allows overwriting the
+number of vcores with a custom value. In order for this parameter to work you
should enable CPU scheduling in your cluster.
#### Detached YARN Session
diff --git
a/flink-core/src/main/java/org/apache/flink/configuration/description/Description.java
b/flink-core/src/main/java/org/apache/flink/configuration/description/Description.java
index 25a6a640b58..c00890d7573 100644
---
a/flink-core/src/main/java/org/apache/flink/configuration/description/Description.java
+++
b/flink-core/src/main/java/org/apache/flink/configuration/description/Description.java
@@ -78,6 +78,17 @@ public DescriptionBuilder text(String text) {
return this;
}
+ /**
+ * Block of description add.
+ *
+ * @param block block of description to add
+ * @return block of description
+ */
+ public DescriptionBuilder add(BlockElement block) {
+ blocks.add(block);
+ return this;
+ }
+
/**
* Creates a line break in the description.
*/
diff --git
a/flink-core/src/main/java/org/apache/flink/configuration/description/Formatter.java
b/flink-core/src/main/java/org/apache/flink/configuration/description/Formatter.java
index d3fcf403d5a..fdf7db67ea5 100644
---
a/flink-core/src/main/java/org/apache/flink/configuration/description/Formatter.java
+++
b/flink-core/src/main/java/org/apache/flink/configuration/description/Formatter.java
@@ -18,6 +18,8 @@
package org.apache.flink.configuration.description;
+import java.util.EnumSet;
+
/**
* Allows providing multiple formatters for the description. E.g. Html
formatter, Markdown formatter etc.
*/
@@ -49,7 +51,7 @@ public void format(TextElement element) {
return formatter.finalizeFormatting();
}
).toArray(String[]::new);
- formatText(state, escapeFormatPlaceholder(element.getFormat()),
inlineElements);
+ formatText(state, escapeFormatPlaceholder(element.getFormat()),
inlineElements, element.getStyles());
}
public void format(LineBreakElement element) {
@@ -76,7 +78,11 @@ private String finalizeFormatting() {
protected abstract void formatLineBreak(StringBuilder state);
- protected abstract void formatText(StringBuilder state, String format,
String[] elements);
+ protected abstract void formatText(
+ StringBuilder state,
+ String format,
+ String[] elements,
+ EnumSet<TextElement.TextStyle> styles);
protected abstract void formatList(StringBuilder state, String[]
entries);
diff --git
a/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java
b/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java
index 72531abc5f4..a47530315c5 100644
---
a/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java
+++
b/flink-core/src/main/java/org/apache/flink/configuration/description/HtmlFormatter.java
@@ -18,6 +18,8 @@
package org.apache.flink.configuration.description;
+import java.util.EnumSet;
+
/**
* Formatter that transforms {@link Description} into Html representation.
*/
@@ -34,9 +36,22 @@ protected void formatLineBreak(StringBuilder state) {
}
@Override
- protected void formatText(StringBuilder state, String format, String[]
elements) {
+ protected void formatText(
+ StringBuilder state,
+ String format,
+ String[] elements,
+ EnumSet<TextElement.TextStyle> styles) {
String escapedFormat = escapeCharacters(format);
+
+ String prefix = "";
+ String suffix = "";
+ if (styles.contains(TextElement.TextStyle.CODE)) {
+ prefix = "<span markdown=\"span\">`";
+ suffix = "`</span>";
+ }
+ state.append(prefix);
state.append(String.format(escapedFormat, elements));
+ state.append(suffix);
}
@Override
diff --git
a/flink-core/src/main/java/org/apache/flink/configuration/description/TextElement.java
b/flink-core/src/main/java/org/apache/flink/configuration/description/TextElement.java
index aeb1d7aeaee..80bdfcf6cc7 100644
---
a/flink-core/src/main/java/org/apache/flink/configuration/description/TextElement.java
+++
b/flink-core/src/main/java/org/apache/flink/configuration/description/TextElement.java
@@ -20,6 +20,7 @@
import java.util.Arrays;
import java.util.Collections;
+import java.util.EnumSet;
import java.util.List;
/**
@@ -28,6 +29,7 @@
public class TextElement implements BlockElement, InlineElement {
private final String format;
private final List<InlineElement> elements;
+ private final EnumSet<TextStyle> textStyles =
EnumSet.noneOf(TextStyle.class);
/**
* Creates a block of text with placeholders ("%s") that will be
replaced with proper string representation of
@@ -35,7 +37,7 @@
*
* <p>{@code text("This is a text with a link %s",
link("https://somepage";, "to here"))}
*
- * @param format text with placeholders for elements
+ * @param format text with placeholders for elements
* @param elements elements to be put in the text
* @return block of text
*/
@@ -53,6 +55,18 @@ public static TextElement text(String text) {
return new TextElement(text, Collections.emptyList());
}
+ /**
+ * Creates a block of text formatted as code.
+ *
+ * @param text a block of text that will be formatted as code
+ * @return block of text formatted as code
+ */
+ public static TextElement code(String text) {
+ TextElement element = text(text);
+ element.textStyles.add(TextStyle.CODE);
+ return element;
+ }
+
public String getFormat() {
return format;
}
@@ -61,6 +75,10 @@ public String getFormat() {
return elements;
}
+ public EnumSet<TextStyle> getStyles() {
+ return textStyles;
+ }
+
private TextElement(String format, List<InlineElement> elements) {
this.format = format;
this.elements = elements;
@@ -70,4 +88,11 @@ private TextElement(String format, List<InlineElement>
elements) {
public void format(Formatter formatter) {
formatter.format(this);
}
+
+ /**
+ * Styles that can be applied to {@link TextElement} e.g. code, bold
etc.
+ */
+ public enum TextStyle {
+ CODE
+ }
}
diff --git
a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
index 255a6c71f8a..b0594756e9e 100644
---
a/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
+++
b/flink-yarn/src/main/java/org/apache/flink/yarn/configuration/YarnConfigOptions.java
@@ -19,8 +19,10 @@
package org.apache.flink.yarn.configuration;
import org.apache.flink.configuration.ConfigOption;
+import org.apache.flink.configuration.description.Description;
import static org.apache.flink.configuration.ConfigOptions.key;
+import static org.apache.flink.configuration.description.TextElement.code;
/**
* This class holds configuration constants used by Flink's YARN runners.
@@ -63,9 +65,14 @@
*/
public static final ConfigOption<Integer> VCORES =
key("yarn.containers.vcores")
- .defaultValue(-1)
- .withDescription("The number of virtual cores (vcores) per YARN
container. By default, the number of vcores" +
- " is set to the number of slots per TaskManager, if
set, or to 1, otherwise.");
+ .defaultValue(-1)
+ .withDescription(Description.builder().text(
+ "The number of virtual cores (vcores)
per YARN container. By default, the number of vcores" +
+ " is set to the number of slots per
TaskManager, if set, or to 1, otherwise. In order for this" +
+ " parameter to be used your cluster
must have CPU scheduling enabled. You can do this by setting" +
+ " the %s.",
+
code("org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"))
+ .build());
/**
* The maximum number of failed YARN containers before entirely stopping
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
With regards,
Apache Git Services
