This is an automated email from the ASF dual-hosted git repository. pnowojski pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
commit fd66a8cdde8ed4901c4b8d93ae3d22712534c78c Author: Anton Kalashnikov <[email protected]> AuthorDate: Wed Jul 28 17:47:55 2021 +0200 [FLINK-23453][core] Added the rest configuration for buffer-debloater --- .../generated/all_taskmanager_network_section.html | 18 ++++++++++ .../generated/all_taskmanager_section.html | 6 ++++ .../generated/task_manager_configuration.html | 18 ++++++++++ .../task_manager_memory_configuration.html | 6 ++++ .../flink/configuration/TaskManagerOptions.java | 41 ++++++++++++++++++++++ 5 files changed, 89 insertions(+) diff --git a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html index 8a9264b..135cbfa 100644 --- a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html +++ b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html @@ -39,6 +39,24 @@ <td>The number of the last buffer size values that will be taken for the correct calculation of the new one.</td> </tr> <tr> + <td><h5>taskmanager.network.memory.buffer-debloat.enabled</h5></td> + <td style="word-wrap: break-word;">false</td> + <td>Boolean</td> + <td>The switch of the automatic buffered debloating feature. If enabled the amount of in-flight data will be adjusted automatically accordingly to the measured throughput.</td> + </tr> + <tr> + <td><h5>taskmanager.network.memory.buffer-debloat.target</h5></td> + <td style="word-wrap: break-word;">1 s</td> + <td>Duration</td> + <td>The target total time after which buffered in-flight data should be fully consumed. This configuration option will be used, in combination with the measured throughput, to adjust the amount of in-flight data.</td> + </tr> + <tr> + <td><h5>taskmanager.network.memory.buffer-debloat.threshold-percentages</h5></td> + <td style="word-wrap: break-word;">50</td> + <td>Integer</td> + <td>The minimum difference in percentage between the newly calculated buffer size and the old one to announce the new value. Can be used to avoid constant back and forth small adjustments.</td> + </tr> + <tr> <td><h5>taskmanager.network.memory.buffers-per-channel</h5></td> <td style="word-wrap: break-word;">2</td> <td>Integer</td> diff --git a/docs/layouts/shortcodes/generated/all_taskmanager_section.html b/docs/layouts/shortcodes/generated/all_taskmanager_section.html index 4a4ed48..bc91856 100644 --- a/docs/layouts/shortcodes/generated/all_taskmanager_section.html +++ b/docs/layouts/shortcodes/generated/all_taskmanager_section.html @@ -63,6 +63,12 @@ <td>Whether to kill the TaskManager when the task thread throws an OutOfMemoryError.</td> </tr> <tr> + <td><h5>taskmanager.memory.min-segment-size</h5></td> + <td style="word-wrap: break-word;">1 kb</td> + <td>MemorySize</td> + <td>Minimum possible size of memory buffers used by the network stack and the memory manager. ex. can be used for automatic buffer size adjustment.</td> + </tr> + <tr> <td><h5>taskmanager.memory.segment-size</h5></td> <td style="word-wrap: break-word;">32 kb</td> <td>MemorySize</td> diff --git a/docs/layouts/shortcodes/generated/task_manager_configuration.html b/docs/layouts/shortcodes/generated/task_manager_configuration.html index 8919882..b6bc41c 100644 --- a/docs/layouts/shortcodes/generated/task_manager_configuration.html +++ b/docs/layouts/shortcodes/generated/task_manager_configuration.html @@ -76,6 +76,24 @@ <td>The number of the last buffer size values that will be taken for the correct calculation of the new one.</td> </tr> <tr> + <td><h5>taskmanager.network.memory.buffer-debloat.enabled</h5></td> + <td style="word-wrap: break-word;">false</td> + <td>Boolean</td> + <td>The switch of the automatic buffered debloating feature. If enabled the amount of in-flight data will be adjusted automatically accordingly to the measured throughput.</td> + </tr> + <tr> + <td><h5>taskmanager.network.memory.buffer-debloat.target</h5></td> + <td style="word-wrap: break-word;">1 s</td> + <td>Duration</td> + <td>The target total time after which buffered in-flight data should be fully consumed. This configuration option will be used, in combination with the measured throughput, to adjust the amount of in-flight data.</td> + </tr> + <tr> + <td><h5>taskmanager.network.memory.buffer-debloat.threshold-percentages</h5></td> + <td style="word-wrap: break-word;">50</td> + <td>Integer</td> + <td>The minimum difference in percentage between the newly calculated buffer size and the old one to announce the new value. Can be used to avoid constant back and forth small adjustments.</td> + </tr> + <tr> <td><h5>taskmanager.numberOfTaskSlots</h5></td> <td style="word-wrap: break-word;">1</td> <td>Integer</td> diff --git a/docs/layouts/shortcodes/generated/task_manager_memory_configuration.html b/docs/layouts/shortcodes/generated/task_manager_memory_configuration.html index 589a61c..7e4ccea 100644 --- a/docs/layouts/shortcodes/generated/task_manager_memory_configuration.html +++ b/docs/layouts/shortcodes/generated/task_manager_memory_configuration.html @@ -75,6 +75,12 @@ <td>Managed Memory size for TaskExecutors. This is the size of off-heap memory managed by the memory manager, reserved for sorting, hash tables, caching of intermediate results and RocksDB state backend. Memory consumers can either allocate memory from the memory manager in the form of MemorySegments, or reserve bytes from the memory manager and keep their memory usage within that boundary. If unspecified, it will be derived to make up the configured fraction of the Total Fli [...] </tr> <tr> + <td><h5>taskmanager.memory.min-segment-size</h5></td> + <td style="word-wrap: break-word;">1 kb</td> + <td>MemorySize</td> + <td>Minimum possible size of memory buffers used by the network stack and the memory manager. ex. can be used for automatic buffer size adjustment.</td> + </tr> + <tr> <td><h5>taskmanager.memory.network.fraction</h5></td> <td style="word-wrap: break-word;">0.1</td> <td>Float</td> diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java index 2d1e5cb..b3a0bcd 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java @@ -277,6 +277,16 @@ public class TaskManagerOptions { .withDescription( "Size of memory buffers used by the network stack and the memory manager."); + /** Minimum possible size of memory buffers used by the network stack and the memory manager. */ + @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER) + public static final ConfigOption<MemorySize> MIN_MEMORY_SEGMENT_SIZE = + key("taskmanager.memory.min-segment-size") + .memoryType() + .defaultValue(MemorySize.parse("1kb")) + .withDescription( + "Minimum possible size of memory buffers used by the network stack and the memory manager. " + + "ex. can be used for automatic buffer size adjustment."); + /** * The config parameter for automatically defining the TaskManager's binding address, if {@link * #HOST} configuration option is not set. @@ -531,6 +541,37 @@ public class TaskManagerOptions { .withDescription( "The number of the last buffer size values that will be taken for the correct calculation of the new one."); + /** The total time for which automated adjusted buffers should be fully consumed. */ + @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK) + public static final ConfigOption<Duration> BUFFER_DEBLOAT_TARGET = + ConfigOptions.key("taskmanager.network.memory.buffer-debloat.target") + .durationType() + .defaultValue(Duration.ofSeconds(1)) + .withDescription( + "The target total time after which buffered in-flight data should be fully consumed. " + + "This configuration option will be used, in combination with the measured throughput, to adjust the amount of in-flight data."); + + @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK) + public static final ConfigOption<Boolean> BUFFER_DEBLOAT_ENABLED = + ConfigOptions.key("taskmanager.network.memory.buffer-debloat.enabled") + .booleanType() + .defaultValue(false) + .withDescription( + "The switch of the automatic buffered debloating feature. " + + "If enabled the amount of in-flight data will be adjusted automatically accordingly to the measured throughput."); + + /** + * Difference between the new and the old buffer size for applying the new value(in percent). + */ + @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK) + public static final ConfigOption<Integer> BUFFER_DEBLOAT_THRESHOLD_PERCENTAGES = + ConfigOptions.key("taskmanager.network.memory.buffer-debloat.threshold-percentages") + .intType() + .defaultValue(50) + .withDescription( + "The minimum difference in percentage between the newly calculated buffer size and the old one to announce the new value. " + + "Can be used to avoid constant back and forth small adjustments."); + /** * Size of direct memory used by blocking shuffle for shuffle data read (currently only used by * sort-merge shuffle).
