xintongsong commented on a change in pull request #11445: [FLINK-16615]
Introduce data structures and utilities to calculate Job Manager memory
components
URL: https://github.com/apache/flink/pull/11445#discussion_r396942397
##########
File path:
flink-core/src/main/java/org/apache/flink/configuration/JobManagerOptions.java
##########
@@ -94,9 +97,116 @@
@Deprecated
public static final ConfigOption<Integer> JOB_MANAGER_HEAP_MEMORY_MB =
key("jobmanager.heap.mb")
+ .intType()
.defaultValue(1024)
.withDescription("JVM heap size (in megabytes) for the
JobManager.");
+ /**
+ * Total Process Memory size for the JobManager.
+ */
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
+ @Documentation.ExcludeFromDocumentation("New JM memory model is still
in development.")
+ public static final ConfigOption<MemorySize> TOTAL_PROCESS_MEMORY =
+ key("jobmanager.memory.process.size")
+ .memoryType()
+ .noDefaultValue()
+ .withDescription(String.format(
+ "Total Process Memory size for the JobManager.
This includes all the memory that a " +
+ "JobManager JVM process consumes,
consisting of Total Flink Memory, JVM Metaspace, and JVM Overhead. " +
+ "In containerized setups, this should
be set to the container memory. See also " +
+ "'jobmanager.memory.flink.size' for
Total Flink Memory size configuration."));
+
+ /**
+ * Total Flink Memory size for the JobManager.
+ */
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
+ @Documentation.ExcludeFromDocumentation("New JM memory model is still
in development.")
+ public static final ConfigOption<MemorySize> TOTAL_FLINK_MEMORY =
+ key("jobmanager.memory.flink.size")
+ .memoryType()
+ .noDefaultValue()
+ .withDescription(String.format(
+ "Total Flink Memory size for the JobManager.
This includes all the " +
+ "memory that a JobManager consumes,
except for JVM Metaspace and JVM Overhead. It consists of " +
+ "JVM Heap Memory and Off-heap Memory.
See also '%s' for total process memory size configuration.",
+ TOTAL_PROCESS_MEMORY.key()));
+
+ /**
+ * JVM Heap Memory size for the JobManager.
+ */
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
+ @Documentation.ExcludeFromDocumentation("New JM memory model is still
in development.")
+ public static final ConfigOption<MemorySize> JVM_HEAP_MEMORY =
+ key("jobmanager.memory.heap.size")
+ .memoryType()
+ .noDefaultValue()
+ .withDescription("JVM Heap Memory size for JobManager.
The minimum recommended JVM Heap size is " +
+ MIN_JVM_HEAP_SIZE.toHumanReadableString() +
'.');
+
+ /**
+ * Off-heap Memory size for the JobManager.
+ */
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
+ @Documentation.ExcludeFromDocumentation("New JM memory model is still
in development.")
+ public static final ConfigOption<MemorySize> OFF_HEAP_MEMORY =
+ key("jobmanager.memory.off-heap.size")
+ .memoryType()
+ .defaultValue(MemorySize.ofMebiBytes(128))
+ .withDescription("Off-heap Memory size for JobManager.
The JVM direct memory limit of the Job Manager " +
+ "process (-XX:MaxDirectMemorySize) will be set
to this value. This option covers all off-heap memory " +
+ "usage including direct and native memory
allocation.");
+
+ /**
+ * JVM Metaspace Size for the JobManager.
+ */
+ @Documentation.Section(Documentation.Sections.COMMON_MEMORY)
+ @Documentation.ExcludeFromDocumentation("New JM memory model is still
in development.")
+ public static final ConfigOption<MemorySize> JVM_METASPACE =
+ key("jobmanager.memory.jvm-metaspace.size")
+ .memoryType()
+ .defaultValue(MemorySize.ofMebiBytes(256))
+ .withDescription("JVM Metaspace Size for the
JobManager.");
+
+ private static final String JVM_OVERHEAD_DESCRIPTION = "This is
off-heap memory reserved for JVM " +
+ "overhead, such as thread stack space, compile cache, etc. This
includes native memory but not direct " +
+ "memory, and will not be counted when Flink calculates JVM max
direct memory size parameter. The size " +
+ "of JVM Overhead is derived to make up the configured fraction
of the Total Process Memory. If the " +
+ "derived size is less or greater than the configured min or max
size, the min or max size will be used. The " +
+ "exact size of JVM Overhead can be explicitly specified by
setting the min or max size to the same value.";
Review comment:
```suggestion
"exact size of JVM Overhead can be explicitly specified by
setting the min and max size to the same value.";
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
