azagrebin commented on a change in pull request #12533:
URL: https://github.com/apache/flink/pull/12533#discussion_r437529637
##########
File path:
flink-runtime/src/main/java/org/apache/flink/runtime/util/bash/BashJavaUtils.java
##########
@@ -39,6 +45,7 @@
* Utility class for using java utilities in bash scripts.
*/
public class BashJavaUtils {
+ private static final Logger LOG =
LoggerFactory.getLogger(BashJavaUtils.class);
Review comment:
We cannot rely on external logging in `BashJavaUtils`.
The logs have to be forced into the stdout independently from logging config
to pass them to `FLINK_INHERITED_LOGS` env var which is used by the main Flink
process to log on its startup.
##########
File path:
flink-runtime/src/main/java/org/apache/flink/runtime/util/bash/BashJavaUtils.java
##########
@@ -92,9 +102,42 @@ public static void main(String[] args) throws
FlinkException {
JobManagerProcessSpec jobManagerProcessSpec =
JobManagerProcessUtils.processSpecFromConfigWithNewOptionToInterpretLegacyHeap(
configuration,
JobManagerOptions.JVM_HEAP_MEMORY);
+
+ logMasterConfiguration(jobManagerProcessSpec);
+
return
Collections.singletonList(ProcessMemoryUtils.generateJvmParametersStr(jobManagerProcessSpec));
}
+ private static void logMasterConfiguration(JobManagerProcessSpec spec) {
+ JobManagerFlinkMemory flinkMemory = spec.getFlinkMemory();
+ LOG.info("Final Master Memory configuration:");
+ LOG.info(" Total Process Memory: {}",
spec.getTotalProcessMemorySize().toHumanReadableString());
+ LOG.info(" Total Flink Memory: {}",
flinkMemory.getTotalFlinkMemorySize().toHumanReadableString());
+ LOG.info(" Heap: {}",
flinkMemory.getJvmHeapMemorySize().toHumanReadableString());
+ LOG.info(" Off-heap: {}",
flinkMemory.getJvmDirectMemorySize().toHumanReadableString());
+ LOG.info(" JVM Metaspace: {}",
spec.getJvmMetaspaceSize().toHumanReadableString());
+ LOG.info(" JVM Overhead: {}",
spec.getJvmOverheadSize().toHumanReadableString());
+ }
+
+ private static void
logTaskExecutorConfiguration(TaskExecutorProcessSpec spec) {
+ TaskExecutorFlinkMemory flinkMemory = spec.getFlinkMemory();
+ MemorySize totalOffHeapMemory =
flinkMemory.getManaged().add(flinkMemory.getJvmDirectMemorySize());
+ LOG.info("Final TaskExecutor Memory configuration:");
Review comment:
This might be a bit tedious but I think it would be really cool to
output this info in the format we use in JavaDocs for `TaskExecutorProcessSpec`
if it works. Same for JM.
##########
File path:
flink-runtime/src/main/java/org/apache/flink/runtime/util/bash/BashJavaUtils.java
##########
@@ -92,9 +102,42 @@ public static void main(String[] args) throws
FlinkException {
JobManagerProcessSpec jobManagerProcessSpec =
JobManagerProcessUtils.processSpecFromConfigWithNewOptionToInterpretLegacyHeap(
configuration,
JobManagerOptions.JVM_HEAP_MEMORY);
+
+ logMasterConfiguration(jobManagerProcessSpec);
+
return
Collections.singletonList(ProcessMemoryUtils.generateJvmParametersStr(jobManagerProcessSpec));
}
+ private static void logMasterConfiguration(JobManagerProcessSpec spec) {
+ JobManagerFlinkMemory flinkMemory = spec.getFlinkMemory();
+ LOG.info("Final Master Memory configuration:");
+ LOG.info(" Total Process Memory: {}",
spec.getTotalProcessMemorySize().toHumanReadableString());
+ LOG.info(" Total Flink Memory: {}",
flinkMemory.getTotalFlinkMemorySize().toHumanReadableString());
+ LOG.info(" Heap: {}",
flinkMemory.getJvmHeapMemorySize().toHumanReadableString());
Review comment:
```suggestion
LOG.info(" JVM Heap: {}",
flinkMemory.getJvmHeapMemorySize().toHumanReadableString());
```
##########
File path:
flink-runtime/src/main/java/org/apache/flink/runtime/util/bash/BashJavaUtils.java
##########
@@ -92,9 +102,42 @@ public static void main(String[] args) throws
FlinkException {
JobManagerProcessSpec jobManagerProcessSpec =
JobManagerProcessUtils.processSpecFromConfigWithNewOptionToInterpretLegacyHeap(
configuration,
JobManagerOptions.JVM_HEAP_MEMORY);
+
+ logMasterConfiguration(jobManagerProcessSpec);
+
return
Collections.singletonList(ProcessMemoryUtils.generateJvmParametersStr(jobManagerProcessSpec));
}
+ private static void logMasterConfiguration(JobManagerProcessSpec spec) {
+ JobManagerFlinkMemory flinkMemory = spec.getFlinkMemory();
+ LOG.info("Final Master Memory configuration:");
+ LOG.info(" Total Process Memory: {}",
spec.getTotalProcessMemorySize().toHumanReadableString());
+ LOG.info(" Total Flink Memory: {}",
flinkMemory.getTotalFlinkMemorySize().toHumanReadableString());
+ LOG.info(" Heap: {}",
flinkMemory.getJvmHeapMemorySize().toHumanReadableString());
+ LOG.info(" Off-heap: {}",
flinkMemory.getJvmDirectMemorySize().toHumanReadableString());
+ LOG.info(" JVM Metaspace: {}",
spec.getJvmMetaspaceSize().toHumanReadableString());
+ LOG.info(" JVM Overhead: {}",
spec.getJvmOverheadSize().toHumanReadableString());
+ }
+
+ private static void
logTaskExecutorConfiguration(TaskExecutorProcessSpec spec) {
+ TaskExecutorFlinkMemory flinkMemory = spec.getFlinkMemory();
+ MemorySize totalOffHeapMemory =
flinkMemory.getManaged().add(flinkMemory.getJvmDirectMemorySize());
+ LOG.info("Final TaskExecutor Memory configuration:");
+ LOG.info(" Total Process Memory: {}",
spec.getTotalProcessMemorySize().toHumanReadableString());
+ LOG.info(" Total Flink Memory: {}",
flinkMemory.getTotalFlinkMemorySize().toHumanReadableString());
+ LOG.info(" Total Heap Memory: {}",
flinkMemory.getJvmHeapMemorySize().toHumanReadableString());
+ LOG.info(" Framework: {}",
flinkMemory.getFrameworkHeap().toHumanReadableString());
+ LOG.info(" Task: {}",
flinkMemory.getTaskHeap().toHumanReadableString());
+ LOG.info(" Total Off-heap Memory: {}",
totalOffHeapMemory.toHumanReadableString());
+ LOG.info(" Managed: {}",
flinkMemory.getManaged().toHumanReadableString());
+ LOG.info(" Total Direct Memory: {}",
flinkMemory.getJvmDirectMemorySize().toHumanReadableString());
Review comment:
```suggestion
LOG.info(" Total JVM Direct Memory: {}",
flinkMemory.getJvmDirectMemorySize().toHumanReadableString());
```
##########
File path:
flink-runtime/src/main/java/org/apache/flink/runtime/util/bash/BashJavaUtils.java
##########
@@ -92,9 +102,42 @@ public static void main(String[] args) throws
FlinkException {
JobManagerProcessSpec jobManagerProcessSpec =
JobManagerProcessUtils.processSpecFromConfigWithNewOptionToInterpretLegacyHeap(
configuration,
JobManagerOptions.JVM_HEAP_MEMORY);
+
+ logMasterConfiguration(jobManagerProcessSpec);
+
return
Collections.singletonList(ProcessMemoryUtils.generateJvmParametersStr(jobManagerProcessSpec));
}
+ private static void logMasterConfiguration(JobManagerProcessSpec spec) {
+ JobManagerFlinkMemory flinkMemory = spec.getFlinkMemory();
+ LOG.info("Final Master Memory configuration:");
+ LOG.info(" Total Process Memory: {}",
spec.getTotalProcessMemorySize().toHumanReadableString());
+ LOG.info(" Total Flink Memory: {}",
flinkMemory.getTotalFlinkMemorySize().toHumanReadableString());
+ LOG.info(" Heap: {}",
flinkMemory.getJvmHeapMemorySize().toHumanReadableString());
+ LOG.info(" Off-heap: {}",
flinkMemory.getJvmDirectMemorySize().toHumanReadableString());
+ LOG.info(" JVM Metaspace: {}",
spec.getJvmMetaspaceSize().toHumanReadableString());
+ LOG.info(" JVM Overhead: {}",
spec.getJvmOverheadSize().toHumanReadableString());
+ }
+
+ private static void
logTaskExecutorConfiguration(TaskExecutorProcessSpec spec) {
+ TaskExecutorFlinkMemory flinkMemory = spec.getFlinkMemory();
+ MemorySize totalOffHeapMemory =
flinkMemory.getManaged().add(flinkMemory.getJvmDirectMemorySize());
+ LOG.info("Final TaskExecutor Memory configuration:");
+ LOG.info(" Total Process Memory: {}",
spec.getTotalProcessMemorySize().toHumanReadableString());
+ LOG.info(" Total Flink Memory: {}",
flinkMemory.getTotalFlinkMemorySize().toHumanReadableString());
+ LOG.info(" Total Heap Memory: {}",
flinkMemory.getJvmHeapMemorySize().toHumanReadableString());
Review comment:
```suggestion
LOG.info(" Total JVM Heap Memory: {}",
flinkMemory.getJvmHeapMemorySize().toHumanReadableString());
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
