This is an automated email from the ASF dual-hosted git repository. xtsong pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
commit 501370dda7a3be20fb217bf16d91240b3881ce81 Author: sxnan <[email protected]> AuthorDate: Mon May 27 15:29:50 2024 +0800 [FLINK-35461][config] Deprecate options related to hash-based blocking shuffle --- .../generated/all_taskmanager_network_section.html | 12 ------------ .../generated/netty_shuffle_environment_configuration.html | 12 ------------ .../flink/configuration/NettyShuffleEnvironmentOptions.java | 11 +++++++++-- 3 files changed, 9 insertions(+), 26 deletions(-) diff --git a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html index 0721df06f92..fae3bc786da 100644 --- a/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html +++ b/docs/layouts/shortcodes/generated/all_taskmanager_network_section.html @@ -14,12 +14,6 @@ <td>Boolean</td> <td>Boolean flag indicating whether the shuffle data will be compressed for batch shuffle mode. Note that data is compressed per buffer and compression can incur extra CPU overhead, so it is more effective for IO bounded scenario when compression ratio is high.</td> </tr> - <tr> - <td><h5>taskmanager.network.blocking-shuffle.type</h5></td> - <td style="word-wrap: break-word;">"file"</td> - <td>String</td> - <td>The blocking shuffle type, either "mmap" or "file". The "auto" means selecting the property type automatically based on system memory architecture (64 bit for mmap and 32 bit for file). Note that the memory usage of mmap is not accounted by configured memory limits, but some resource frameworks like yarn would track this memory usage and kill the container once memory exceeding some threshold. Also note that this option is experimental and might be changed future.</td> - </tr> <tr> <td><h5>taskmanager.network.compression.codec</h5></td> <td style="word-wrap: break-word;">LZ4</td> @@ -218,12 +212,6 @@ <td>Integer</td> <td>Minimum number of network buffers required per blocking result partition for sort-shuffle. For production usage, it is suggested to increase this config value to at least 2048 (64M memory if the default 32K memory segment size is used) to improve the data compression ratio and reduce the small network packets. Usually, several hundreds of megabytes memory is enough for large scale batch jobs. Note: you may also need to increase the size of total network memory to avoid th [...] </tr> - <tr> - <td><h5>taskmanager.network.sort-shuffle.min-parallelism</h5></td> - <td style="word-wrap: break-word;">1</td> - <td>Integer</td> - <td>Parallelism threshold to switch between sort-based blocking shuffle and hash-based blocking shuffle, which means for batch jobs of smaller parallelism, hash-shuffle will be used and for batch jobs of larger or equal parallelism, sort-shuffle will be used. The value 1 means that sort-shuffle is the default option. Note: For production usage, you may also need to tune 'taskmanager.network.sort-shuffle.min-buffers' and 'taskmanager.memory.framework.off-heap.batch-shuffle.siz [...] - </tr> <tr> <td><h5>taskmanager.network.tcp-connection.enable-reuse-across-jobs</h5></td> <td style="word-wrap: break-word;">true</td> diff --git a/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html b/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html index 22b71b6139c..713bc705663 100644 --- a/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html +++ b/docs/layouts/shortcodes/generated/netty_shuffle_environment_configuration.html @@ -32,12 +32,6 @@ <td>Boolean</td> <td>Boolean flag indicating whether the shuffle data will be compressed for batch shuffle mode. Note that data is compressed per buffer and compression can incur extra CPU overhead, so it is more effective for IO bounded scenario when compression ratio is high.</td> </tr> - <tr> - <td><h5>taskmanager.network.blocking-shuffle.type</h5></td> - <td style="word-wrap: break-word;">"file"</td> - <td>String</td> - <td>The blocking shuffle type, either "mmap" or "file". The "auto" means selecting the property type automatically based on system memory architecture (64 bit for mmap and 32 bit for file). Note that the memory usage of mmap is not accounted by configured memory limits, but some resource frameworks like yarn would track this memory usage and kill the container once memory exceeding some threshold. Also note that this option is experimental and might be changed future.</td> - </tr> <tr> <td><h5>taskmanager.network.compression.codec</h5></td> <td style="word-wrap: break-word;">LZ4</td> @@ -206,12 +200,6 @@ <td>Integer</td> <td>Minimum number of network buffers required per blocking result partition for sort-shuffle. For production usage, it is suggested to increase this config value to at least 2048 (64M memory if the default 32K memory segment size is used) to improve the data compression ratio and reduce the small network packets. Usually, several hundreds of megabytes memory is enough for large scale batch jobs. Note: you may also need to increase the size of total network memory to avoid th [...] </tr> - <tr> - <td><h5>taskmanager.network.sort-shuffle.min-parallelism</h5></td> - <td style="word-wrap: break-word;">1</td> - <td>Integer</td> - <td>Parallelism threshold to switch between sort-based blocking shuffle and hash-based blocking shuffle, which means for batch jobs of smaller parallelism, hash-shuffle will be used and for batch jobs of larger or equal parallelism, sort-shuffle will be used. The value 1 means that sort-shuffle is the default option. Note: For production usage, you may also need to tune 'taskmanager.network.sort-shuffle.min-buffers' and 'taskmanager.memory.framework.off-heap.batch-shuffle.siz [...] - </tr> <tr> <td><h5>taskmanager.network.tcp-connection.enable-reuse-across-jobs</h5></td> <td style="word-wrap: break-word;">true</td> diff --git a/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java index c69f052aa13..02c048f1445 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/NettyShuffleEnvironmentOptions.java @@ -316,8 +316,11 @@ public class NettyShuffleEnvironmentOptions { /** * Parallelism threshold to switch between sort-based blocking shuffle and hash-based blocking * shuffle. + * + * @deprecated The hash-based blocking shuffle is deprecated in 1.20 and will be totally removed + * in 2.0. */ - @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK) + @Deprecated public static final ConfigOption<Integer> NETWORK_SORT_SHUFFLE_MIN_PARALLELISM = key("taskmanager.network.sort-shuffle.min-parallelism") .intType() @@ -455,7 +458,11 @@ public class NettyShuffleEnvironmentOptions { + "the \"Insufficient number of network buffers\" exception, while the workloads may suffer performance " + "reduction silently."); - @Documentation.Section(Documentation.Sections.ALL_TASK_MANAGER_NETWORK) + /** + * @deprecated The hash-based blocking shuffle is deprecated in 1.20 and will be totally removed + * in 2.0. + */ + @Deprecated public static final ConfigOption<String> NETWORK_BLOCKING_SHUFFLE_TYPE = key("taskmanager.network.blocking-shuffle.type") .stringType()
