This is an automated email from the ASF dual-hosted git repository. azagrebin pushed a commit to branch release-1.10 in repository https://gitbox.apache.org/repos/asf/flink.git
commit ff7e25cad70a74da1795c69fd651d788d5972f1c Author: Andrey Zagrebin <azagre...@apache.org> AuthorDate: Fri Jan 31 12:51:30 2020 +0100 [FLINK-15143][docs] Add new memory configuration guide for FLIP-49 --- docs/fig/detailed-mem-model.svg | 21 +++++ docs/fig/simple_mem_model.svg | 21 +++++ docs/ops/cli.md | 2 +- docs/ops/cli.zh.md | 2 +- docs/ops/config.md | 2 +- docs/ops/config.zh.md | 2 +- docs/ops/memory/index.md | 24 ++++++ docs/ops/memory/index.zh.md | 24 ++++++ docs/ops/memory/mem_detail.md | 149 ++++++++++++++++++++++++++++++++++++ docs/ops/memory/mem_detail.zh.md | 149 ++++++++++++++++++++++++++++++++++++ docs/ops/memory/mem_setup.md | 132 ++++++++++++++++++++++++++++++++ docs/ops/memory/mem_setup.zh.md | 132 ++++++++++++++++++++++++++++++++ docs/ops/plugins.md | 2 +- docs/ops/plugins.zh.md | 2 +- docs/ops/production_ready.md | 2 +- docs/ops/production_ready.zh.md | 2 +- docs/ops/python_shell.md | 2 +- docs/ops/python_shell.zh.md | 2 +- docs/ops/scala_shell.md | 2 +- docs/ops/scala_shell.zh.md | 2 +- docs/ops/security-ssl.md | 2 +- docs/ops/security-ssl.zh.md | 2 +- docs/ops/upgrading.md | 2 +- docs/ops/upgrading.zh.md | 2 +- docs/release-notes/flink-1.10.md | 2 + docs/release-notes/flink-1.10.zh.md | 2 + 26 files changed, 672 insertions(+), 16 deletions(-) diff --git a/docs/fig/detailed-mem-model.svg b/docs/fig/detailed-mem-model.svg new file mode 100644 index 0000000..215d99a --- /dev/null +++ b/docs/fig/detailed-mem-model.svg @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="UTF-8" standalone="no"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<svg version="1.1" viewBox="0.0 0.0 351.0 790.7060367454068" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l351.0 0l0 790.70605l-351.0 0l0 -790.70605z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l351.0 0l0 790.70605l-351.0 0z" fill-rule="evenodd"/><path fill="#ffffff" d="m0 0l350.2047 0l0 701. [...] diff --git a/docs/fig/simple_mem_model.svg b/docs/fig/simple_mem_model.svg new file mode 100644 index 0000000..415b032 --- /dev/null +++ b/docs/fig/simple_mem_model.svg @@ -0,0 +1,21 @@ +<?xml version="1.0" standalone="yes"?> +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +<svg version="1.1" viewBox="0.0 0.0 350.04199475065616 393.7979002624672" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l350.042 0l0 393.7979l-350.042 0l0 -393.7979z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l350.042 0l0 393.7979l-350.042 0z" fill-rule="evenodd"/><path fill="#ffffff" d="m0 0 [...] diff --git a/docs/ops/cli.md b/docs/ops/cli.md index df75f4e..4018dd1 100644 --- a/docs/ops/cli.md +++ b/docs/ops/cli.md @@ -2,7 +2,7 @@ title: "Command-Line Interface" nav-title: CLI nav-parent_id: ops -nav-pos: 6 +nav-pos: 7 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/cli.zh.md b/docs/ops/cli.zh.md index d86ab40..649800d 100644 --- a/docs/ops/cli.zh.md +++ b/docs/ops/cli.zh.md @@ -2,7 +2,7 @@ title: "命令行界面" nav-title: CLI nav-parent_id: ops -nav-pos: 6 +nav-pos: 7 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/config.md b/docs/ops/config.md index ff07d75..992a838 100644 --- a/docs/ops/config.md +++ b/docs/ops/config.md @@ -154,7 +154,7 @@ These configuration values control the way that TaskManagers and JobManagers use Flink tries to shield users as much as possible from the complexity of configuring the JVM for data-intensive processing. In most cases, users should only need to set the values `taskmanager.memory.process.size` or `taskmanager.memory.flink.size` (depending on how the setup), and possibly adjusting the ratio of JVM heap and Managed Memory via `taskmanager.memory.managed.fraction`. The other options below can be used for performane tuning and fixing memory related errors. -For a detailed explanation of how these options interact, see the [documentation on TaskManager memory configuration]({{site.baseurl}}/ops/memory_configuration.html). +For a detailed explanation of how these options interact, see the [documentation on TaskManager memory configuration]({{site.baseurl}}/ops/memory/mem_setup.html). {% include generated/common_memory_section.html %} diff --git a/docs/ops/config.zh.md b/docs/ops/config.zh.md index 99d7092..23b7ce1 100644 --- a/docs/ops/config.zh.md +++ b/docs/ops/config.zh.md @@ -154,7 +154,7 @@ These configuration values control the way that TaskManagers and JobManagers use Flink tries to shield users as much as possible from the complexity of configuring the JVM for data-intensive processing. In most cases, users should only need to set the values `taskmanager.memory.process.size` or `taskmanager.memory.flink.size` (depending on how the setup), and possibly adjusting the ratio of JVM heap and Managed Memory via `taskmanager.memory.managed.fraction`. The other options below can be used for performane tuning and fixing memory related errors. -For a detailed explanation of how these options interact, see the [documentation on TaskManager memory configuration]({{site.baseurl}}/ops/memory_configuration.html). +For a detailed explanation of how these options interact, see the [documentation on TaskManager memory configuration]({{site.baseurl}}/ops/memory/mem_setup.html). {% include generated/common_memory_section.html %} diff --git a/docs/ops/memory/index.md b/docs/ops/memory/index.md new file mode 100644 index 0000000..e2190e1 --- /dev/null +++ b/docs/ops/memory/index.md @@ -0,0 +1,24 @@ +--- +nav-title: 'Memory Configuration' +nav-id: ops_mem +nav-parent_id: ops +nav-pos: 5 +--- +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> diff --git a/docs/ops/memory/index.zh.md b/docs/ops/memory/index.zh.md new file mode 100644 index 0000000..e2190e1 --- /dev/null +++ b/docs/ops/memory/index.zh.md @@ -0,0 +1,24 @@ +--- +nav-title: 'Memory Configuration' +nav-id: ops_mem +nav-parent_id: ops +nav-pos: 5 +--- +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> diff --git a/docs/ops/memory/mem_detail.md b/docs/ops/memory/mem_detail.md new file mode 100644 index 0000000..a5b476f --- /dev/null +++ b/docs/ops/memory/mem_detail.md @@ -0,0 +1,149 @@ +--- +title: "Detailed Memory Model" +nav-parent_id: ops_mem +nav-pos: 2 +--- +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +This section gives a detailed description of all components in Flink’s memory model of task executor. +Check [memory configuration guide](mem_setup.html) for the basic memory setup. + +* toc +{:toc} + +## Overview + +<br /> +<center> + <img src="{{ site.baseurl }}/fig/detailed-mem-model.svg" width="300px" alt="Simple memory model" usemap="#simple-mem-model"> +</center> +<br /> + +The following table lists all memory components, depicted above, and references Flink configuration options +which affect the size of the respective components: + +| **Component** | **Configuration options** | **Description** [...] +| :-------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------------- [...] +| [Framework Heap Memory](#framework-memory) | [`taskmanager.memory.framework.heap.size`](../config.html#taskmanager-memory-framework-heap-size) | JVM heap memory dedicated to Flink framework (advanced option) [...] +| [Task Heap Memory](mem_setup.html#task-operator-heap-memory) | [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) | JVM heap memory dedicated to Flink application to run operators and user code [...] +| [Managed memory](mem_setup.html#managed-memory) | [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) <br/> [`taskmanager.memory.managed.fraction`](../config.html#taskmanager-memory-managed-fraction) | Native memory managed by Flink, reserved for sorting, hash tables, caching of intermediate results and RocksDB state backe [...] +| [Framework Off-heap Memory](#framework-memory) | [`taskmanager.memory.framework.off-heap.size`](../config.html#taskmanager-memory-framework-off-heap-size) | [Off-heap direct (or native) memory](mem_setup.html#configure-off-heap-memory-direct-or-native) dedicated to Flink framework [...] +| [Task Off-heap Memory](mem_setup.html#configure-off-heap-memory-direct-or-native) | [`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size) | [Off-heap direct (or native) memory](mem_setup.html#configure-off-heap-memory-direct-or-native) dedicated to Flink applicati [...] +| Network Memory | [`taskmanager.memory.network.min`](../config.html#taskmanager-memory-network-min) <br/> [`taskmanager.memory.network.max`](../config.html#taskmanager-memory-network-max) <br/> [`taskmanager.memory.network.fraction`](../config.html#taskmanager-memory-network-fraction) | Direct memory reserved for data record exchange between tasks (e.g. buffering for the transfer over the network), it is [...] +| [JVM metaspace](#jvm-parameters) | [`taskmanager.memory.jvm-metaspace.size`](../config.html#taskmanager-memory-jvm-metaspace-size) | Metaspace size of the Flink JVM process [...] +| JVM Overhead | [`taskmanager.memory.jvm-overhead.min`](../config.html#taskmanager-memory-jvm-overhead-min) <br/> [`taskmanager.memory.jvm-overhead.max`](../config.html#taskmanager-memory-jvm-overhead-max) <br/> [`taskmanager.memory.jvm-overhead.fraction`](../config.html#taskmanager-memory-jvm-overhead-fraction) | Native memory reserved for other JVM overhead: e.g. thread stacks, code cache, garbage collection space et [...] +{:.table-bordered} +<br/> + +As you can see, the size of some memory components can be simply set by the respective option. +Other components can be tuned using multiple options. + +## Framework Memory + +The *framework heap memory* and *framework off-heap memory* options are not supposed to be changed without a good reason. +Adjust them only if you are sure that Flink needs more memory for some internal data structures or operations. +It can be related to a particular deployment environment or job structure, like high parallelism. +In addition, Flink dependencies, such as Hadoop may consume more direct or native memory in certain setups. + +<span class="label label-info">Note</span> Neither heap nor off-heap versions of framework and task memory are currently isolated within Flink. +The separation of framework and task memory can be used in future releases for further optimizations. + +## Capped Fractionated Components + +This section describes the configuration details of the following options which can be a fraction of a certain +[total memory](mem_setup.html#configure-total-memory): + +* *Network memory* can be a fraction of the *total Flink memory* +* *JVM overhead* can be a fraction of the *total process memory* + +See also [detailed memory model](#overview). + +The size of those components always has to be between its maximum and minimum value, otherwise Flink startup will fail. +The maximum and minimum values have defaults or can be explicitly set by corresponding configuration options. +For example, if only the following memory options are set: +- total Flink memory = 1000Mb, +- network min = 64Mb, +- network max = 128Mb, +- network fraction = 0.1 + +then the network memory will be 1000Mb x 0.1 = 100Mb which is within the range 64-128Mb. + +Notice if you configure the same maximum and minimum value it effectively means that its size is fixed to that value. + +If the component memory is not explicitly configured, then Flink will use the fraction to calculate the memory size +based on the total memory. The calculated value is capped by its corresponding min/max options. +For example, if only the following memory options are set: +- total Flink memory = 1000Mb, +- network min = 128Mb, +- network max = 256Mb, +- network fraction = 0.1 + +then the network memory will be 128Mb because the size derived from fraction is 100Mb and it is less than the minimum. + +It can also happen that the fraction is ignored if the sizes of the total memory and its other components are defined. +In this case, the network memory is the rest of the total memory. The derived value still has to be within its min/max +range otherwise the configuration fails. For example, suppose only the following memory options are set: +- total Flink memory = 1000Mb, +- task heap = 100Mb, +- network min = 64Mb, +- network max = 256Mb, +- network fraction = 0.1 + +All other components of the total Flink memory have default values, including the default managed memory fraction. +Then the network memory is not the fraction (1000Mb x 0.1 = 100Mb) but the rest of the total Flink memory +which will either be within the range 64-256Mb or fail. + +## JVM Parameters + +Flink explicitly adds the following memory related JVM arguments while starting the task executor process, +based on the configured or derived memory component sizes: + +| **JVM Arguments** | **Value** | +| :---------------------------------------- | :----------------------------------------- | +| *-Xmx* and *-Xms* | Framework + Task Heap Memory | +| *-XX:MaxDirectMemorySize* | Framework + Task Off-Heap + Network Memory | +| *-XX:MaxMetaspaceSize* | JVM Metaspace | +{:.table-bordered} +<br/> + +See also [detailed memory model](#overview). + +## Local Execution +If you start Flink locally on your machine as a single java program without creating a cluster (e.g. from your IDE) +then all components are ignored except for the following: + +| **Memory component** | **Relevant options** | **Default value for the local execution** | +| :------------------------------------------- | :-------------------------------------------------------------------------------------------- | :---------------------------------------------------------------------------- | +| Task heap | [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) | infinite | +| Task off-heap | [`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size) | infinite | +| Managed memory | [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) | 128Mb | +| Network memory | [`taskmanager.memory.network.min`](../config.html#taskmanager-memory-network-min) <br /> [`taskmanager.memory.network.max`](../config.html#taskmanager-memory-network-max) | 64Mb | +{:.table-bordered} +<br/> + +All of the components listed above can be but do not have to be explicitly configured for the local execution. +If they are not configured they are set to their default values. [Task heap memory](mem_setup.html#task-operator-heap-memory) and +*task off-heap memory* are considered to be infinite (*Long.MAX_VALUE* bytes) and [managed memory](mem_setup.html#managed-memory) +has a default value of 128Mb only for the local execution mode. + +<span class="label label-info">Note</span> The task heap size is not related in any way to the real heap size in this case. +It can become relevant for future optimizations coming with next releases. The actual JVM heap size of the started +local process is not controlled by Flink and depends on how you start the process. +If you want to control the JVM heap size you have to explicitly pass the corresponding JVM arguments, e.g. *-Xmx*, *-Xms*. diff --git a/docs/ops/memory/mem_detail.zh.md b/docs/ops/memory/mem_detail.zh.md new file mode 100644 index 0000000..a5b476f --- /dev/null +++ b/docs/ops/memory/mem_detail.zh.md @@ -0,0 +1,149 @@ +--- +title: "Detailed Memory Model" +nav-parent_id: ops_mem +nav-pos: 2 +--- +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +This section gives a detailed description of all components in Flink’s memory model of task executor. +Check [memory configuration guide](mem_setup.html) for the basic memory setup. + +* toc +{:toc} + +## Overview + +<br /> +<center> + <img src="{{ site.baseurl }}/fig/detailed-mem-model.svg" width="300px" alt="Simple memory model" usemap="#simple-mem-model"> +</center> +<br /> + +The following table lists all memory components, depicted above, and references Flink configuration options +which affect the size of the respective components: + +| **Component** | **Configuration options** | **Description** [...] +| :-------------------------------------------------------------------------------- | :------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | :----------------------------------------------------------------------------------------------------------------------------- [...] +| [Framework Heap Memory](#framework-memory) | [`taskmanager.memory.framework.heap.size`](../config.html#taskmanager-memory-framework-heap-size) | JVM heap memory dedicated to Flink framework (advanced option) [...] +| [Task Heap Memory](mem_setup.html#task-operator-heap-memory) | [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) | JVM heap memory dedicated to Flink application to run operators and user code [...] +| [Managed memory](mem_setup.html#managed-memory) | [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) <br/> [`taskmanager.memory.managed.fraction`](../config.html#taskmanager-memory-managed-fraction) | Native memory managed by Flink, reserved for sorting, hash tables, caching of intermediate results and RocksDB state backe [...] +| [Framework Off-heap Memory](#framework-memory) | [`taskmanager.memory.framework.off-heap.size`](../config.html#taskmanager-memory-framework-off-heap-size) | [Off-heap direct (or native) memory](mem_setup.html#configure-off-heap-memory-direct-or-native) dedicated to Flink framework [...] +| [Task Off-heap Memory](mem_setup.html#configure-off-heap-memory-direct-or-native) | [`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size) | [Off-heap direct (or native) memory](mem_setup.html#configure-off-heap-memory-direct-or-native) dedicated to Flink applicati [...] +| Network Memory | [`taskmanager.memory.network.min`](../config.html#taskmanager-memory-network-min) <br/> [`taskmanager.memory.network.max`](../config.html#taskmanager-memory-network-max) <br/> [`taskmanager.memory.network.fraction`](../config.html#taskmanager-memory-network-fraction) | Direct memory reserved for data record exchange between tasks (e.g. buffering for the transfer over the network), it is [...] +| [JVM metaspace](#jvm-parameters) | [`taskmanager.memory.jvm-metaspace.size`](../config.html#taskmanager-memory-jvm-metaspace-size) | Metaspace size of the Flink JVM process [...] +| JVM Overhead | [`taskmanager.memory.jvm-overhead.min`](../config.html#taskmanager-memory-jvm-overhead-min) <br/> [`taskmanager.memory.jvm-overhead.max`](../config.html#taskmanager-memory-jvm-overhead-max) <br/> [`taskmanager.memory.jvm-overhead.fraction`](../config.html#taskmanager-memory-jvm-overhead-fraction) | Native memory reserved for other JVM overhead: e.g. thread stacks, code cache, garbage collection space et [...] +{:.table-bordered} +<br/> + +As you can see, the size of some memory components can be simply set by the respective option. +Other components can be tuned using multiple options. + +## Framework Memory + +The *framework heap memory* and *framework off-heap memory* options are not supposed to be changed without a good reason. +Adjust them only if you are sure that Flink needs more memory for some internal data structures or operations. +It can be related to a particular deployment environment or job structure, like high parallelism. +In addition, Flink dependencies, such as Hadoop may consume more direct or native memory in certain setups. + +<span class="label label-info">Note</span> Neither heap nor off-heap versions of framework and task memory are currently isolated within Flink. +The separation of framework and task memory can be used in future releases for further optimizations. + +## Capped Fractionated Components + +This section describes the configuration details of the following options which can be a fraction of a certain +[total memory](mem_setup.html#configure-total-memory): + +* *Network memory* can be a fraction of the *total Flink memory* +* *JVM overhead* can be a fraction of the *total process memory* + +See also [detailed memory model](#overview). + +The size of those components always has to be between its maximum and minimum value, otherwise Flink startup will fail. +The maximum and minimum values have defaults or can be explicitly set by corresponding configuration options. +For example, if only the following memory options are set: +- total Flink memory = 1000Mb, +- network min = 64Mb, +- network max = 128Mb, +- network fraction = 0.1 + +then the network memory will be 1000Mb x 0.1 = 100Mb which is within the range 64-128Mb. + +Notice if you configure the same maximum and minimum value it effectively means that its size is fixed to that value. + +If the component memory is not explicitly configured, then Flink will use the fraction to calculate the memory size +based on the total memory. The calculated value is capped by its corresponding min/max options. +For example, if only the following memory options are set: +- total Flink memory = 1000Mb, +- network min = 128Mb, +- network max = 256Mb, +- network fraction = 0.1 + +then the network memory will be 128Mb because the size derived from fraction is 100Mb and it is less than the minimum. + +It can also happen that the fraction is ignored if the sizes of the total memory and its other components are defined. +In this case, the network memory is the rest of the total memory. The derived value still has to be within its min/max +range otherwise the configuration fails. For example, suppose only the following memory options are set: +- total Flink memory = 1000Mb, +- task heap = 100Mb, +- network min = 64Mb, +- network max = 256Mb, +- network fraction = 0.1 + +All other components of the total Flink memory have default values, including the default managed memory fraction. +Then the network memory is not the fraction (1000Mb x 0.1 = 100Mb) but the rest of the total Flink memory +which will either be within the range 64-256Mb or fail. + +## JVM Parameters + +Flink explicitly adds the following memory related JVM arguments while starting the task executor process, +based on the configured or derived memory component sizes: + +| **JVM Arguments** | **Value** | +| :---------------------------------------- | :----------------------------------------- | +| *-Xmx* and *-Xms* | Framework + Task Heap Memory | +| *-XX:MaxDirectMemorySize* | Framework + Task Off-Heap + Network Memory | +| *-XX:MaxMetaspaceSize* | JVM Metaspace | +{:.table-bordered} +<br/> + +See also [detailed memory model](#overview). + +## Local Execution +If you start Flink locally on your machine as a single java program without creating a cluster (e.g. from your IDE) +then all components are ignored except for the following: + +| **Memory component** | **Relevant options** | **Default value for the local execution** | +| :------------------------------------------- | :-------------------------------------------------------------------------------------------- | :---------------------------------------------------------------------------- | +| Task heap | [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) | infinite | +| Task off-heap | [`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size) | infinite | +| Managed memory | [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) | 128Mb | +| Network memory | [`taskmanager.memory.network.min`](../config.html#taskmanager-memory-network-min) <br /> [`taskmanager.memory.network.max`](../config.html#taskmanager-memory-network-max) | 64Mb | +{:.table-bordered} +<br/> + +All of the components listed above can be but do not have to be explicitly configured for the local execution. +If they are not configured they are set to their default values. [Task heap memory](mem_setup.html#task-operator-heap-memory) and +*task off-heap memory* are considered to be infinite (*Long.MAX_VALUE* bytes) and [managed memory](mem_setup.html#managed-memory) +has a default value of 128Mb only for the local execution mode. + +<span class="label label-info">Note</span> The task heap size is not related in any way to the real heap size in this case. +It can become relevant for future optimizations coming with next releases. The actual JVM heap size of the started +local process is not controlled by Flink and depends on how you start the process. +If you want to control the JVM heap size you have to explicitly pass the corresponding JVM arguments, e.g. *-Xmx*, *-Xms*. diff --git a/docs/ops/memory/mem_setup.md b/docs/ops/memory/mem_setup.md new file mode 100644 index 0000000..4b2786f --- /dev/null +++ b/docs/ops/memory/mem_setup.md @@ -0,0 +1,132 @@ +--- +title: "Set up Task Executor Memory" +nav-parent_id: ops_mem +nav-pos: 1 +--- +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +Apache Flink provides efficient workloads on top of the JVM by tightly controlling the memory usage of its various components. +While the community strives to offer sensible defaults to all configurations, the full breadth of applications +that users deploy on Flink means this isn't always possible. To provide the most production value to our users, +Flink allows both high level and fine-grained tuning of memory allocation within clusters. + +* toc +{:toc} + +The further described memory configuration is applicable starting with the release version *1.10*. + +<span class="label label-info">Note</span> This memory setup guide is relevant <strong>only for task executors</strong>! +Check [job manager related configuration options](../config.html#jobmanager-heap-size) for the memory setup of job manager. + +## Configure Total Memory + +The *total process memory* of Flink JVM processes consists of memory consumed by Flink application (*total Flink memory*) +and by the JVM to run the process. The *total Flink memory* consumption includes usage of JVM heap, +*managed memory* (managed by Flink) and other direct (or native) memory. + +<center> + <img src="{{ site.baseurl }}/fig/simple_mem_model.svg" width="300px" alt="Simple memory model" usemap="#simple-mem-model"> +</center> +<br /> + +If you run FIink locally (e.g. from your IDE) without creating a cluster, then only a subset of the memory configuration +options are relevant, see also [local execution](mem_detail.html#local-execution) for more details. + +Otherwise, the simplest way to setup memory in Flink is to configure either of the two following options: +* Total Flink memory ([`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)) +* Total process memory ([`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)) + +The rest of the memory components will be adjusted automatically, based on default values or additionally configured options. +[Here](mem_detail.html#detailed-memory-model) are more details about the other memory components. + +Configuring *total Flink memory* is better suited for standalone deployments where you want to declare how much memory +is given to Flink itself. The *total Flink memory* splits up into JVM heap, [managed memory size](#managed-memory) +and *direct memory*. + +If you configure *total process memory* you declare how much memory in total should be assigned to the Flink *JVM process*. +For the containerized deployments it corresponds to the size of the requested container, see also +[how to configure memory for containers](#heading=h.q0nx4u2c3pzx) +([Kubernetes](../deployment/kubernetes.html), [Yarn](../deployment/yarn_setup.html) or [Mesos](../deployment/mesos.html)). + +Another way to setup the memory is to set [task heap](#task-operator-heap-memory) and [managed memory](#managed-memory) +([`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) and [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size)). +This more fine-grained approach is described in more detail [here](#configure-heap-and-managed-memory). + +<span class="label label-info">Note</span> One of the three mentioned ways has to be used to configure Flink’s memory (except for local execution), or the Flink startup will fail. +This means that one of the following option subsets, which do not have default values, have to be configured explicitly: +* [`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size) +* [`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size) +* [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) and [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) + +<span class="label label-info">Note</span> Explicitly configuring both *total process memory* and *total Flink memory* is not recommended. +It may lead to deployment failures due to potential memory configuration conflicts. Additional configuration +of other memory components also requires caution as it can produce further configuration conflicts. + +## Configure Heap and Managed Memory + +As mentioned before in [total memory description](#configure-total-memory), another way to setup memory in Flink is +to specify explicitly both [task heap](#task-operator-heap-memory) and [managed memory](#managed-memory). +It gives more control over the available JVM heap to Flink’s tasks and its [managed memory](#managed-memory). + +The rest of the memory components will be adjusted automatically, based on default values or additionally configured options. +[Here](mem_detail.html#detailed-memory-model) are more details about the other memory components. + +<span class="label label-info">Note</span> If you have configured the task heap and managed memory explicitly, it is recommended to set neither +*total process memory* nor *total Flink memory*. Otherwise, it may easily lead to memory configuration conflicts. + +### Task (Operator) Heap Memory + +If you want to guarantee that a certain amount of JVM heap is available for your user code, you can set the *task heap memory* +explicitly ([`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size)). +It will be added to the JVM heap size and will be dedicated to Flink’s operators running the user code. + +### Managed Memory + +*Managed memory* is managed by Flink and is allocated as native memory (off-heap). The following workloads use *managed memory*: +* Streaming jobs can use it for [RocksDB state backend](../state/state_backends.html#the-rocksdbstatebackend). +* [Batch jobs](../../dev/batch) can use it for sorting, hash tables, caching of intermediate results. + +The size of *managed memory* can be +* either configured explicitly via [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) +* or computed as a fraction of *total Flink memory* via [`taskmanager.memory.managed.fraction`](../config.html#taskmanager-memory-managed-fraction). + +*Size* will override *fraction*, if both are set. +If neither *size* nor *fraction* is explicitly configured, the [default fraction](../config.html#taskmanager-memory-managed-fraction) will be used. + +See also [how to configure memory for state backends](#heading=h.srelwz7nbzwa) and [batch jobs](#heading=h.d6mjc9yd85c0). + +## Configure Off-Heap Memory (direct or native) + +The off-heap memory which is allocated by user code should be accounted for in *task off-heap memory* +([`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size)). + +<span class="label label-info">Note</span> You can also adjust the [framework off-heap memory](mem_detail.html#framework-memory). This option is advanced +and only recommended to be changed if you are sure that the Flink framework needs more memory. + +Flink includes the *framework off-heap memory* and *task off-heap memory* into the *direct memory* limit of the JVM, +see also [JVM parameters](mem_detail.html#jvm-parameters). + +<span class="label label-info">Note</span> Although, native non-direct memory usage can be accounted for as a part of the +*framework off-heap memory* or *task off-heap memory*, it will result in a higher JVM's *direct memory* limit in this case. + +<span class="label label-info">Note</span> The *network memory* is also part of JVM *direct memory* but it is managed by Flink and guaranteed +to never exceed its configured size. Therefore, resizing the *network memory* will not help in this situation. + +See also [the detailed memory model](mem_detail.html#detailed-memory-model). diff --git a/docs/ops/memory/mem_setup.zh.md b/docs/ops/memory/mem_setup.zh.md new file mode 100644 index 0000000..f2a2b06 --- /dev/null +++ b/docs/ops/memory/mem_setup.zh.md @@ -0,0 +1,132 @@ +--- +title: "Set up Task Executor Memory" +nav-parent_id: ops_mem +nav-pos: 1 +--- +<!-- +Licensed to the Apache Software Foundation (ASF) under one +or more contributor license agreements. See the NOTICE file +distributed with this work for additional information +regarding copyright ownership. The ASF licenses this file +to you under the Apache License, Version 2.0 (the +"License"); you may not use this file except in compliance +with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +--> + +Apache Flink provides efficient workloads on top of the JVM by tightly controlling the memory usage of its various components. +While the community strives to offer sensible defaults to all configurations, the full breadth of applications +that users deploy on Flink means this isn't always possible. To provide the most production value to our users, +Flink allows both high level and fine-grained tuning of memory allocation within clusters. + +* toc +{:toc} + +The further described memory configuration is applicable starting with the release version *1.10*. + +<span class="label label-info">Note</span> This memory setup guide is relevant <strong>only for task executors</strong>! +Check [job manager related configuration options](../config.html#jobmanager-heap-size) for the memory setup of job manager. + +## Configure Total Memory + +The *total process memory* of Flink JVM processes consists of memory consumed by Flink application (*total Flink memory*) +and by the JVM to run the process. The *total Flink memory* consumption includes usage of JVM heap, +*managed memory* (managed by Flink) and other direct (or native) memory. + +<center> + <img src="{{ site.baseurl }}/fig/simple_mem_model.svg" width="300px" alt="Simple memory model" usemap="#simple-mem-model"> +</center> +<br /> + +If you run FIink locally (e.g. from your IDE) without creating a cluster, then only a subset of the memory configuration +options are relevant, see also [local execution](mem_detail.html#local-execution) for more details. + +Otherwise, the simplest way to setup memory in Flink is to configure either of the two following options: +* Total Flink memory ([`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size)) +* Total process memory ([`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size)) + +The rest of the memory components will be adjusted automatically, based on default values or additionally configured options. +[Here](mem_detail.html#detailed-memory-model) are more details about the other memory components. + +Configuring *total Flink memory* is better suited for standalone deployments where you want to declare how much memory +is given to Flink itself. The *total Flink memory* splits up into JVM heap, [managed memory size](#managed-memory) +and *direct memory*. + +If you configure *total process memory* you declare how much memory in total should be assigned to the Flink *JVM process*. +For the containerized deployments it corresponds to the size of the requested container, see also +[how to configure memory for containers](mem_tuning.html#configure-memory-for-containers) +([Kubernetes](../deployment/kubernetes.html), [Yarn](../deployment/yarn_setup.html) or [Mesos](../deployment/mesos.html)). + +Another way to setup the memory is to set [task heap](#task-operator-heap-memory) and [managed memory](#managed-memory) +([`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) and [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size)). +This more fine-grained approach is described in more detail [here](#configure-heap-and-managed-memory). + +<span class="label label-info">Note</span> One of the three mentioned ways has to be used to configure Flink’s memory (except for local execution), or the Flink startup will fail. +This means that one of the following option subsets, which do not have default values, have to be configured explicitly: +* [`taskmanager.memory.flink.size`](../config.html#taskmanager-memory-flink-size) +* [`taskmanager.memory.process.size`](../config.html#taskmanager-memory-process-size) +* [`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size) and [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) + +<span class="label label-info">Note</span> Explicitly configuring both *total process memory* and *total Flink memory* is not recommended. +It may lead to deployment failures due to potential memory configuration conflicts. Additional configuration +of other memory components also requires caution as it can produce further configuration conflicts. + +## Configure Heap and Managed Memory + +As mentioned before in [total memory description](#configure-total-memory), another way to setup memory in Flink is +to specify explicitly both [task heap](#task-operator-heap-memory) and [managed memory](#managed-memory). +It gives more control over the available JVM heap to Flink’s tasks and its [managed memory](#managed-memory). + +The rest of the memory components will be adjusted automatically, based on default values or additionally configured options. +[Here](mem_detail.html#detailed-memory-model) are more details about the other memory components. + +<span class="label label-info">Note</span> If you have configured the task heap and managed memory explicitly, it is recommended to set neither +*total process memory* nor *total Flink memory*. Otherwise, it may easily lead to memory configuration conflicts. + +### Task (Operator) Heap Memory + +If you want to guarantee that a certain amount of JVM heap is available for your user code, you can set the *task heap memory* +explicitly ([`taskmanager.memory.task.heap.size`](../config.html#taskmanager-memory-task-heap-size)). +It will be added to the JVM heap size and will be dedicated to Flink’s operators running the user code. + +### Managed Memory + +*Managed memory* is managed by Flink and is allocated as native memory (off-heap). The following workloads use *managed memory*: +* Streaming jobs can use it for [RocksDB state backend](../state/state_backends.html#the-rocksdbstatebackend). +* [Batch jobs](../../dev/batch) can use it for sorting, hash tables, caching of intermediate results. + +The size of *managed memory* can be +* either configured explicitly via [`taskmanager.memory.managed.size`](../config.html#taskmanager-memory-managed-size) +* or computed as a fraction of *total Flink memory* via [`taskmanager.memory.managed.fraction`](../config.html#taskmanager-memory-managed-fraction). + +*Size* will override *fraction*, if both are set. +If neither *size* nor *fraction* is explicitly configured, the [default fraction](../config.html#taskmanager-memory-managed-fraction) will be used. + +See also [how to configure memory for state backends](mem_tuning.html#configure-memory-for-state-backends) and [batch jobs](mem_tuning.html#configure-memory-for-batch-jobs). + +## Configure Off-Heap Memory (direct or native) + +The off-heap memory which is allocated by user code should be accounted for in *task off-heap memory* +([`taskmanager.memory.task.off-heap.size`](../config.html#taskmanager-memory-task-off-heap-size)). + +<span class="label label-info">Note</span> You can also adjust the [framework off-heap memory](mem_detail.html#framework-memory). This option is advanced +and only recommended to be changed if you are sure that the Flink framework needs more memory. + +Flink includes the *framework off-heap memory* and *task off-heap memory* into the *direct memory* limit of the JVM, +see also [JVM parameters](mem_detail.html#jvm-parameters). + +<span class="label label-info">Note</span> Although, native non-direct memory usage can be accounted for as a part of the +*framework off-heap memory* or *task off-heap memory*, it will result in a higher JVM's *direct memory* limit in this case. + +<span class="label label-info">Note</span> The *network memory* is also part of JVM *direct memory* but it is managed by Flink and guaranteed +to never exceed its configured size. Therefore, resizing the *network memory* will not help in this situation. + +See also [the detailed memory model](mem_detail.html#detailed-memory-model). diff --git a/docs/ops/plugins.md b/docs/ops/plugins.md index 1be3834..759d5dc 100644 --- a/docs/ops/plugins.md +++ b/docs/ops/plugins.md @@ -2,7 +2,7 @@ title: "Plugins" nav-id: plugins nav-parent_id: ops -nav-pos: 16 +nav-pos: 14 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/plugins.zh.md b/docs/ops/plugins.zh.md index 1be3834..759d5dc 100644 --- a/docs/ops/plugins.zh.md +++ b/docs/ops/plugins.zh.md @@ -2,7 +2,7 @@ title: "Plugins" nav-id: plugins nav-parent_id: ops -nav-pos: 16 +nav-pos: 14 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/production_ready.md b/docs/ops/production_ready.md index ef97173..dcd6913 100644 --- a/docs/ops/production_ready.md +++ b/docs/ops/production_ready.md @@ -1,7 +1,7 @@ --- title: "Production Readiness Checklist" nav-parent_id: ops -nav-pos: 5 +nav-pos: 6 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/production_ready.zh.md b/docs/ops/production_ready.zh.md index e5cff46..dbca6dc 100644 --- a/docs/ops/production_ready.zh.md +++ b/docs/ops/production_ready.zh.md @@ -1,7 +1,7 @@ --- title: "生产就绪情况核对清单" nav-parent_id: ops -nav-pos: 5 +nav-pos: 6 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/python_shell.md b/docs/ops/python_shell.md index 91ddd19..64f84d6 100644 --- a/docs/ops/python_shell.md +++ b/docs/ops/python_shell.md @@ -1,7 +1,7 @@ --- title: "Python REPL" nav-parent_id: ops -nav-pos: 7 +nav-pos: 8 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/python_shell.zh.md b/docs/ops/python_shell.zh.md index 32ac419..f586a88 100644 --- a/docs/ops/python_shell.zh.md +++ b/docs/ops/python_shell.zh.md @@ -1,7 +1,7 @@ --- title: "Python REPL" nav-parent_id: ops -nav-pos: 7 +nav-pos: 8 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/scala_shell.md b/docs/ops/scala_shell.md index f233d9d..a2290b0 100644 --- a/docs/ops/scala_shell.md +++ b/docs/ops/scala_shell.md @@ -1,7 +1,7 @@ --- title: "Scala REPL" nav-parent_id: ops -nav-pos: 7 +nav-pos: 9 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/scala_shell.zh.md b/docs/ops/scala_shell.zh.md index f233d9d..a2290b0 100644 --- a/docs/ops/scala_shell.zh.md +++ b/docs/ops/scala_shell.zh.md @@ -1,7 +1,7 @@ --- title: "Scala REPL" nav-parent_id: ops -nav-pos: 7 +nav-pos: 9 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/security-ssl.md b/docs/ops/security-ssl.md index fd26599..5624174 100644 --- a/docs/ops/security-ssl.md +++ b/docs/ops/security-ssl.md @@ -1,7 +1,7 @@ --- title: "SSL Setup" nav-parent_id: ops -nav-pos: 10 +nav-pos: 11 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/security-ssl.zh.md b/docs/ops/security-ssl.zh.md index 8ffccba..a00c4b8 100644 --- a/docs/ops/security-ssl.zh.md +++ b/docs/ops/security-ssl.zh.md @@ -1,7 +1,7 @@ --- title: "SSL 配置" nav-parent_id: ops -nav-pos: 10 +nav-pos: 11 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/upgrading.md b/docs/ops/upgrading.md index 529d45d..aff4a4f 100644 --- a/docs/ops/upgrading.md +++ b/docs/ops/upgrading.md @@ -1,7 +1,7 @@ --- title: "Upgrading Applications and Flink Versions" nav-parent_id: ops -nav-pos: 15 +nav-pos: 13 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/ops/upgrading.zh.md b/docs/ops/upgrading.zh.md index 2704588..4aee9bc 100644 --- a/docs/ops/upgrading.zh.md +++ b/docs/ops/upgrading.zh.md @@ -1,7 +1,7 @@ --- title: "升级应用程序和 Flink 版本" nav-parent_id: ops -nav-pos: 15 +nav-pos: 13 --- <!-- Licensed to the Apache Software Foundation (ASF) under one diff --git a/docs/release-notes/flink-1.10.md b/docs/release-notes/flink-1.10.md index cec6353..ab245b7 100644 --- a/docs/release-notes/flink-1.10.md +++ b/docs/release-notes/flink-1.10.md @@ -100,6 +100,8 @@ If you try to reuse your previous Flink configuration without any adjustments, the new memory model can result in differently computed memory parameters for the JVM and, thus, performance changes. +Please, check [the user documentation](../ops/memory/mem_setup.html) for more details. + ##### Deprecation and breaking changes The following options have been removed and have no effect anymore: diff --git a/docs/release-notes/flink-1.10.zh.md b/docs/release-notes/flink-1.10.zh.md index cec6353..ab245b7 100644 --- a/docs/release-notes/flink-1.10.zh.md +++ b/docs/release-notes/flink-1.10.zh.md @@ -100,6 +100,8 @@ If you try to reuse your previous Flink configuration without any adjustments, the new memory model can result in differently computed memory parameters for the JVM and, thus, performance changes. +Please, check [the user documentation](../ops/memory/mem_setup.html) for more details. + ##### Deprecation and breaking changes The following options have been removed and have no effect anymore: