This is an automated email from the ASF dual-hosted git repository. yingjie pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/flink.git
commit e1878fb899d4598c36fbb0740cefebf31de8c3d9 Author: kevin.cyj <[email protected]> AuthorDate: Thu Jan 13 17:24:04 2022 +0800 [FLINK-25639][network] Increase the default read buffer size of sort-shuffle to 64M This closes #18350. --- docs/layouts/shortcodes/generated/common_memory_section.html | 4 ++-- .../shortcodes/generated/task_manager_memory_configuration.html | 4 ++-- .../java/org/apache/flink/configuration/TaskManagerOptions.java | 6 +++--- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/layouts/shortcodes/generated/common_memory_section.html b/docs/layouts/shortcodes/generated/common_memory_section.html index 1d224e8..6e520e2 100644 --- a/docs/layouts/shortcodes/generated/common_memory_section.html +++ b/docs/layouts/shortcodes/generated/common_memory_section.html @@ -76,9 +76,9 @@ </tr> <tr> <td><h5>taskmanager.memory.framework.off-heap.batch-shuffle.size</h5></td> - <td style="word-wrap: break-word;">32 mb</td> + <td style="word-wrap: break-word;">64 mb</td> <td>MemorySize</td> - <td>Size of memory used by blocking shuffle for shuffle data read (currently only used by sort-merge shuffle). Notes: 1) The memory is cut from 'taskmanager.memory.framework.off-heap.size' so must be smaller than that, which means you may also need to increase 'taskmanager.memory.framework.off-heap.size' after you increase this config value; 2) This memory size can influence the shuffle performance and you can increase this config value for large-scale batch jobs (for example [...] + <td>Size of memory used by blocking shuffle for shuffle data read (currently only used by sort-shuffle). Notes: 1) The memory is cut from 'taskmanager.memory.framework.off-heap.size' so must be smaller than that, which means you may also need to increase 'taskmanager.memory.framework.off-heap.size' after you increase this config value; 2) This memory size can influence the shuffle performance and you can increase this config value for large-scale batch jobs (for example, to 1 [...] </tr> <tr> <td><h5>taskmanager.memory.framework.off-heap.size</h5></td> diff --git a/docs/layouts/shortcodes/generated/task_manager_memory_configuration.html b/docs/layouts/shortcodes/generated/task_manager_memory_configuration.html index bd9248e..d572e80 100644 --- a/docs/layouts/shortcodes/generated/task_manager_memory_configuration.html +++ b/docs/layouts/shortcodes/generated/task_manager_memory_configuration.html @@ -22,9 +22,9 @@ </tr> <tr> <td><h5>taskmanager.memory.framework.off-heap.batch-shuffle.size</h5></td> - <td style="word-wrap: break-word;">32 mb</td> + <td style="word-wrap: break-word;">64 mb</td> <td>MemorySize</td> - <td>Size of memory used by blocking shuffle for shuffle data read (currently only used by sort-merge shuffle). Notes: 1) The memory is cut from 'taskmanager.memory.framework.off-heap.size' so must be smaller than that, which means you may also need to increase 'taskmanager.memory.framework.off-heap.size' after you increase this config value; 2) This memory size can influence the shuffle performance and you can increase this config value for large-scale batch jobs (for example [...] + <td>Size of memory used by blocking shuffle for shuffle data read (currently only used by sort-shuffle). Notes: 1) The memory is cut from 'taskmanager.memory.framework.off-heap.size' so must be smaller than that, which means you may also need to increase 'taskmanager.memory.framework.off-heap.size' after you increase this config value; 2) This memory size can influence the shuffle performance and you can increase this config value for large-scale batch jobs (for example, to 1 [...] </tr> <tr> <td><h5>taskmanager.memory.framework.off-heap.size</h5></td> diff --git a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java index 7fecad5..2ad7333 100644 --- a/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java +++ b/flink-core/src/main/java/org/apache/flink/configuration/TaskManagerOptions.java @@ -581,17 +581,17 @@ public class TaskManagerOptions { /** * Size of direct memory used by blocking shuffle for shuffle data read (currently only used by - * sort-merge shuffle). + * sort-shuffle). */ @Documentation.Section(Documentation.Sections.COMMON_MEMORY) public static final ConfigOption<MemorySize> NETWORK_BATCH_SHUFFLE_READ_MEMORY = key("taskmanager.memory.framework.off-heap.batch-shuffle.size") .memoryType() - .defaultValue(MemorySize.parse("32m")) + .defaultValue(MemorySize.parse("64m")) .withDescription( String.format( "Size of memory used by blocking shuffle for shuffle data read " - + "(currently only used by sort-merge shuffle). Notes: " + + "(currently only used by sort-shuffle). Notes: " + "1) The memory is cut from '%s' so must be smaller than" + " that, which means you may also need to increase '%s' " + "after you increase this config value; 2) This memory"
