This is an automated email from the ASF dual-hosted git repository.
haonan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new e434a120246 feat(env):add MEMORY_SIZE parameters into env.sh/env.bat
(#11380)
e434a120246 is described below
commit e434a12024645a04f466dcbae5bcd31296b963a0
Author: CritasWang <[email protected]>
AuthorDate: Thu Oct 26 09:26:41 2023 +0800
feat(env):add MEMORY_SIZE parameters into env.sh/env.bat (#11380)
---
.../src/assembly/resources/conf/confignode-env.bat | 72 ++++++++-----
.../src/assembly/resources/conf/confignode-env.sh | 113 +++++++++++----------
.../src/assembly/resources/conf/datanode-env.bat | 69 ++++++++-----
.../src/assembly/resources/conf/datanode-env.sh | 105 +++++++++----------
4 files changed, 201 insertions(+), 158 deletions(-)
diff --git
a/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.bat
b/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.bat
index a165d5d19df..f6de7c93fd9 100644
--- a/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.bat
+++ b/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.bat
@@ -18,6 +18,10 @@
@REM
@echo off
+
+@REM You can set datanode memory size, example '2G' or '2048M'
+set MEMORY_SIZE=
+
@REM true or false
@REM DO NOT FORGET TO MODIFY THE PASSWORD FOR SECURITY
(%CONFIGNODE_CONF%\jmx.password and %{CONFIGNODE_CONF%\jmx.access)
set JMX_LOCAL="true"
@@ -60,47 +64,61 @@ for /f "tokens=*" %%a in ('cscript //nologo
%CONFIGNODE_HOME%\sbin\tmp.vbs') do
del %CONFIGNODE_HOME%\sbin\tmp.vbs
set system_memory_in_mb=%system_memory_in_mb:,=%
-set /a half_=%system_memory_in_mb%/2
-set /a quarter_=%half_%/2
+@REM suggest using memory, system memory 3 / 10
+set /a suggest_=%system_memory_in_mb%/10*3
-if %half_% GTR 1024 set half_=1024
-if %quarter_% GTR 65536 set quarter_=65536
-
-if %half_% GTR %quarter_% (
- set max_heap_size_in_mb=%half_%
-) else set max_heap_size_in_mb=%quarter_%
+if "%MEMORY_SIZE%"=="" (
+ set /a memory_size_in_mb=%suggest_%
+) else (
+ if "%MEMORY_SIZE:~-1%"=="M" (
+ set /a memory_size_in_mb=%MEMORY_SIZE:~0,-1%
+ ) else if "%MEMORY_SIZE:~-1%"=="G" (
+ set /a memory_size_in_mb=%MEMORY_SIZE:~0,-1%*1024
+ ) else (
+ echo "Invalid format of MEMORY_SIZE, please use the format like 2048M or
2G."
+ exit /b 1
+ )
+)
-set MAX_HEAP_SIZE=%max_heap_size_in_mb%M
-set max_sensible_yg_per_core_in_mb=100
-set /a
max_sensible_yg_in_mb=%max_sensible_yg_per_core_in_mb%*%system_cpu_cores%
-set /a desired_yg_in_mb=%max_heap_size_in_mb%/4
+@REM set on heap memory size
+@REM when memory_size_in_mb is less than 4 * 1024, we will set on heap memory
size to memory_size_in_mb / 4 * 3
+@REM when memory_size_in_mb is greater than 4 * 1024 and less than 16 * 1024,
we will set on heap memory size to memory_size_in_mb / 5 * 4
+@REM when memory_size_in_mb is greater than 16 * 1024 and less than 128 *
1024, we will set on heap memory size to memory_size_in_mb / 8 * 7
+@REM when memory_size_in_mb is greater than 128 * 1024, we will set on heap
memory size to memory_size_in_mb - 16 * 1024
+if %memory_size_in_mb% LSS 4096 (
+ set /a on_heap_memory_size_in_mb=%memory_size_in_mb%/4*3
+) else if %memory_size_in_mb% LSS 16384 (
+ set /a on_heap_memory_size_in_mb=%memory_size_in_mb%/5*4
+) else if %memory_size_in_mb% LSS 131072 (
+ set /a on_heap_memory_size_in_mb=%memory_size_in_mb%/8*7
+) else (
+ set /a on_heap_memory_size_in_mb=%memory_size_in_mb%-16384
+)
+set /a
off_heap_memory_size_in_mb=%memory_size_in_mb%-%on_heap_memory_size_in_mb%
-if %desired_yg_in_mb% GTR %max_sensible_yg_in_mb% (
- set HEAP_NEWSIZE=%max_sensible_yg_in_mb%M
-) else set HEAP_NEWSIZE=%desired_yg_in_mb%M
+set ON_HEAP_MEMORY=%on_heap_memory_size_in_mb%M
+set OFF_HEAP_MEMORY=%off_heap_memory_size_in_mb%M
@REM if the heap size is larger than 16GB, we will forbid writing the heap
dump file
-if %desired_yg_in_mb% GTR 16384 (
+if %off_heap_memory_size_in_mb% GTR 16384 (
set IOTDB_ALLOW_HEAP_DUMP="false"
) else set IOTDB_ALLOW_HEAP_DUMP="true"
-@REM Maximum heap size
-@REM set MAX_HEAP_SIZE="2G"
-@REM Minimum heap size
-@REM set HEAP_NEWSIZE="2G"
+@REM on heap memory size
+@REM set ON_HEAP_MEMORY=2G
+@REM off heap memory size
+@REM set OFF_HEAP_MEMORY=512M
-@REM maximum direct memory size
-set MAX_DIRECT_MEMORY_SIZE=%MAX_HEAP_SIZE%
@REM threads number that may use direct memory, including query threads(8) +
merge threads(4) + space left for system(4)
set threads_number=16
@REM the size of buffer cache pool(IOV_MAX) depends on operating system
set temp_buffer_pool_size=1024
@REM Max cached buffer size, Note: unit can only be B!
@REM which equals DIRECT_MEMORY_SIZE / threads_number / temp_buffer_pool_size
-set
MAX_CACHED_BUFFER_SIZE=%max_heap_size_in_mb%*1024*1024/%threads_number%/%temp_buffer_pool_size%
+set
MAX_CACHED_BUFFER_SIZE=%on_heap_memory_size_in_mb%*1024*1024/%threads_number%/%temp_buffer_pool_size%
-set CONFIGNODE_HEAP_OPTS=-Xmx%MAX_HEAP_SIZE% -Xms%HEAP_NEWSIZE%
-set CONFIGNODE_HEAP_OPTS=%CONFIGNODE_HEAP_OPTS%
-XX:MaxDirectMemorySize=%MAX_DIRECT_MEMORY_SIZE%
+set CONFIGNODE_HEAP_OPTS=-Xmx%ON_HEAP_MEMORY% -Xms%ON_HEAP_MEMORY%
+set CONFIGNODE_HEAP_OPTS=%CONFIGNODE_HEAP_OPTS%
-XX:MaxDirectMemorySize=%OFF_HEAP_MEMORY%
set CONFIGNODE_HEAP_OPTS=%CONFIGNODE_HEAP_OPTS%
-Djdk.nio.maxCachedBufferSize=%MAX_CACHED_BUFFER_SIZE%
@REM You can put your env variable here
@@ -129,5 +147,5 @@ IF "%JAVA_VERSION%" == "8" (
--add-opens=java.base/java.net=ALL-UNNAMED
)
-echo Maximum memory allocation pool = %MAX_HEAP_SIZE%, initial memory
allocation pool = %HEAP_NEWSIZE%
-echo If you want to change this configuration, please check
conf/confignode-env.sh(Unix or OS X, if you use Windows, check
conf/confignode-env.bat).
+echo ConfigNode on heap memory size = %ON_HEAP_MEMORY%B, off heap memory size
= %OFF_HEAP_MEMORY%B
+echo If you want to change this configuration, please check
conf/confignode-env.bat.
diff --git
a/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
b/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
index 08ed71cddb9..51dea4bc6c5 100644
--- a/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
+++ b/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
@@ -18,6 +18,9 @@
# under the License.
#
+# You can set ConfigNode memory size, example '2G' or '2048M'
+MEMORY_SIZE=
+
# You can put your env variable here
# export JAVA_HOME=$JAVA_HOME
@@ -60,7 +63,7 @@ esac
# whether we allow enable heap dump files
IOTDB_ALLOW_HEAP_DUMP="true"
-calculate_heap_sizes()
+calculate_memory_sizes()
{
case "`uname`" in
Linux)
@@ -95,48 +98,61 @@ calculate_heap_sizes()
system_cpu_cores="1"
fi
- # set max heap size based on the following
- # max(min(1/2 ram, 1024MB), min(1/4 ram, 64GB))
- # calculate 1/2 ram and cap to 1024MB
- # calculate 1/4 ram and cap to 65536MB
- # pick the max
- half_system_memory_in_mb=`expr $system_memory_in_mb / 2`
- quarter_system_memory_in_mb=`expr $half_system_memory_in_mb / 2`
- if [ "$half_system_memory_in_mb" -gt "1024" ]
+ # suggest using memory, system memory 3 / 10
+ suggest_using_memory_in_mb=`expr $system_memory_in_mb / 10 \* 3`
+
+ if [ -n "$MEMORY_SIZE" ]
then
- half_system_memory_in_mb="1024"
+ if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ] || [
"${MEMORY_SIZE%"M"}" != "$MEMORY_SIZE" ]
+ then
+ if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ]
+ then
+ memory_size_in_mb=`expr ${MEMORY_SIZE%"G"} "*" 1024`
+ else
+ memory_size_in_mb=`expr ${MEMORY_SIZE%"M"}`
+ fi
+ else
+ echo "Invalid format of MEMORY_SIZE, please use the format like
2048M or 2G"
+ exit 1
+ fi
+ else
+ # set memory size to suggest using memory, if suggest using memory is
greater than 8GB, set memory size to 8GB
+ if [ "$suggest_using_memory_in_mb" -gt "8192" ]
+ then
+ memory_size_in_mb="8192"
+ else
+ memory_size_in_mb=$suggest_using_memory_in_mb
+ fi
fi
- if [ "$quarter_system_memory_in_mb" -gt "65536" ]
+
+ # set on heap memory size
+ # when memory_size_in_mb is less than 4 * 1024, we will set on heap memory
size to memory_size_in_mb / 4 * 3
+ # when memory_size_in_mb is greater than 4 * 1024 and less than 16 * 1024,
we will set on heap memory size to memory_size_in_mb / 5 * 4
+ # when memory_size_in_mb is greater than 16 * 1024 and less than 128 *
1024, we will set on heap memory size to memory_size_in_mb / 8 * 7
+ # when memory_size_in_mb is greater than 128 * 1024, we will set on heap
memory size to memory_size_in_mb - 16 * 1024
+ if [ "$memory_size_in_mb" -lt "4096" ]
then
- quarter_system_memory_in_mb="65536"
- fi
- if [ "$half_system_memory_in_mb" -gt "$quarter_system_memory_in_mb" ]
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 4 \* 3`
+ elif [ "$memory_size_in_mb" -lt "16384" ]
then
- max_heap_size_in_mb="$half_system_memory_in_mb"
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 5 \* 4`
+ elif [ "$memory_size_in_mb" -lt "131072" ]
+ then
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 8 \* 7`
else
- max_heap_size_in_mb="$quarter_system_memory_in_mb"
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb - 16384`
fi
- MAX_HEAP_SIZE="${max_heap_size_in_mb}M"
+ off_heap_memory_size_in_mb=`expr $memory_size_in_mb -
$on_heap_memory_size_in_mb`
+
+ ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M"
+ OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M"
# if the heap size is larger than 16GB, we will forbid writing the heap
dump file
- if [ "$max_heap_size_in_mb" -gt "16384" ]
+ if [ "$on_heap_memory_size_in_mb" -gt "16384" ]
then
- echo "IoTDB memory is too large ($max_heap_size_in_mb MB), will forbid
writing heap dump file"
+ echo "IoTDB memory is too large ($on_heap_memory_size_in_mb MB), will
forbid writing heap dump file"
IOTDB_ALLOW_HEAP_DUMP="false"
fi
-
- # Young gen: min(max_sensible_per_modern_cpu_core * num_cores, 1/4 * heap
size)
- max_sensible_yg_per_core_in_mb="100"
- max_sensible_yg_in_mb=`expr $max_sensible_yg_per_core_in_mb "*"
$system_cpu_cores`
-
- desired_yg_in_mb=`expr $max_heap_size_in_mb / 4`
-
- if [ "$desired_yg_in_mb" -gt "$max_sensible_yg_in_mb" ]
- then
- HEAP_NEWSIZE="${max_sensible_yg_in_mb}M"
- else
- HEAP_NEWSIZE="${desired_yg_in_mb}M"
- fi
}
@@ -210,29 +226,20 @@ else
fi
+calculate_memory_sizes
-calculate_heap_sizes
-
-## Set heap size by percentage of total memory
-#max_percentage=90
-#min_percentage=50
-#MAX_HEAP_SIZE="`expr $system_memory_in_mb \* $max_percentage / 100`M"
-#HEAP_NEWSIZE="`expr $system_memory_in_mb \* $min_percentage / 100`M"
-
-# Maximum heap size
-#MAX_HEAP_SIZE="2G"
-# Minimum heap size
-#HEAP_NEWSIZE="2G"
-# Maximum direct memory size
-MAX_DIRECT_MEMORY_SIZE=${MAX_HEAP_SIZE}
+# on heap memory size
+#ON_HEAP_MEMORY="2G"
+# off heap memory size
+#OFF_HEAP_MEMORY="512M"
# threads number that may use direct memory, including read threads(8) + merge
threads(4) + space left for system(4)
threads_number="16"
# the size of buffer cache pool(IOV_MAX) depends on operating system
temp_buffer_pool_size="1024"
# Max cached buffer size, Note: unit can only be B!
-# which equals DIRECT_MEMORY_SIZE / threads_number / temp_buffer_pool_size
-MAX_CACHED_BUFFER_SIZE=`expr $max_heap_size_in_mb \* 1024 \* 1024 /
$threads_number / $temp_buffer_pool_size`
+# which equals OFF_HEAP_MEMORY / threads_number / temp_buffer_pool_size
+MAX_CACHED_BUFFER_SIZE=`expr $on_heap_memory_size_in_mb \* 1024 \* 1024 /
$threads_number / $temp_buffer_pool_size`
#true or false
#DO NOT FORGET TO MODIFY THE PASSWORD FOR SECURITY
(${CONFIGNODE_CONF}/jmx.password and ${CONFIGNODE_CONF}/jmx.access)
@@ -264,11 +271,11 @@ else
fi
CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS -Diotdb.jmx.local=$JMX_LOCAL"
-CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS -Xms${HEAP_NEWSIZE}"
-CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS -Xmx${MAX_HEAP_SIZE}"
-CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS
-XX:MaxDirectMemorySize=${MAX_DIRECT_MEMORY_SIZE}"
+CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS -Xms${ON_HEAP_MEMORY}"
+CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS -Xmx${ON_HEAP_MEMORY}"
+CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS
-XX:MaxDirectMemorySize=${OFF_HEAP_MEMORY}"
CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS
-Djdk.nio.maxCachedBufferSize=${MAX_CACHED_BUFFER_SIZE}"
-echo "Maximum memory allocation pool = ${MAX_HEAP_SIZE}B, initial memory
allocation pool = ${HEAP_NEWSIZE}B"
-echo "If you want to change this configuration, please check
conf/confignode-env.sh(Unix or OS X, if you use Windows, check
conf/confignode-env.bat)."
+echo "ConfigNode on heap memory size = ${ON_HEAP_MEMORY}B, off heap memory
size = ${OFF_HEAP_MEMORY}B"
+echo "If you want to change this configuration, please check
conf/confignode-env.sh."
diff --git a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
index d29a18116ed..3c95bdf6993 100644
--- a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
+++ b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
@@ -18,6 +18,10 @@
@REM
@echo off
+
+@REM You can set datanode memory size, example '2G' or '2048M'
+set MEMORY_SIZE=
+
@REM true or false
@REM DO NOT FORGET TO MODIFY THE PASSWORD FOR SECURITY
(%IOTDB_CONF%\jmx.password and %{IOTDB_CONF%\jmx.access)
set JMX_LOCAL="true"
@@ -60,47 +64,60 @@ for /f "tokens=*" %%a in ('cscript //nologo
%IOTDB_HOME%\sbin\tmp.vbs') do set s
del %IOTDB_HOME%\sbin\tmp.vbs
set system_memory_in_mb=%system_memory_in_mb:,=%
-set /a half_=%system_memory_in_mb%/2
-set /a quarter_=%half_%/2
+set /a suggest_=%system_memory_in_mb%/2
-if %half_% GTR 1024 set half_=1024
-if %quarter_% GTR 65536 set quarter_=65536
-
-if %half_% GTR %quarter_% (
- set max_heap_size_in_mb=%half_%
-) else set max_heap_size_in_mb=%quarter_%
+if "%MEMORY_SIZE%"=="" (
+ set /a memory_size_in_mb=%suggest_%
+) else (
+ if "%MEMORY_SIZE:~-1%"=="M" (
+ set /a memory_size_in_mb=%MEMORY_SIZE:~0,-1%
+ ) else if "%MEMORY_SIZE:~-1%"=="G" (
+ set /a memory_size_in_mb=%MEMORY_SIZE:~0,-1%*1024
+ ) else (
+ echo "Invalid format of MEMORY_SIZE, please use the format like 2048M or
2G."
+ exit /b 1
+ )
+)
-set MAX_HEAP_SIZE=%max_heap_size_in_mb%M
-set max_sensible_yg_per_core_in_mb=100
-set /a
max_sensible_yg_in_mb=%max_sensible_yg_per_core_in_mb%*%system_cpu_cores%
-set /a desired_yg_in_mb=%max_heap_size_in_mb%/4
+@REM set on heap memory size
+@REM when memory_size_in_mb is less than 4 * 1024, we will set on heap memory
size to memory_size_in_mb / 4 * 3
+@REM when memory_size_in_mb is greater than 4 * 1024 and less than 16 * 1024,
we will set on heap memory size to memory_size_in_mb / 5 * 4
+@REM when memory_size_in_mb is greater than 16 * 1024 and less than 128 *
1024, we will set on heap memory size to memory_size_in_mb / 8 * 7
+@REM when memory_size_in_mb is greater than 128 * 1024, we will set on heap
memory size to memory_size_in_mb - 16 * 1024
+if %memory_size_in_mb% LSS 4096 (
+ set /a on_heap_memory_size_in_mb=%memory_size_in_mb%/4*3
+) else if %memory_size_in_mb% LSS 16384 (
+ set /a on_heap_memory_size_in_mb=%memory_size_in_mb%/5*4
+) else if %memory_size_in_mb% LSS 131072 (
+ set /a on_heap_memory_size_in_mb=%memory_size_in_mb%/8*7
+) else (
+ set /a on_heap_memory_size_in_mb=%memory_size_in_mb%-16384
+)
+set /a
off_heap_memory_size_in_mb=%memory_size_in_mb%-%on_heap_memory_size_in_mb%
-if %desired_yg_in_mb% GTR %max_sensible_yg_in_mb% (
- set HEAP_NEWSIZE=%max_sensible_yg_in_mb%M
-) else set HEAP_NEWSIZE=%desired_yg_in_mb%M
+set ON_HEAP_MEMORY=%on_heap_memory_size_in_mb%M
+set OFF_HEAP_MEMORY=%off_heap_memory_size_in_mb%M
@REM if the heap size is larger than 16GB, we will forbid writing the heap
dump file
-if %desired_yg_in_mb% GTR 16384 (
+if %off_heap_memory_size_in_mb% GTR 16384 (
set IOTDB_ALLOW_HEAP_DUMP="false"
) else set IOTDB_ALLOW_HEAP_DUMP="true"
-@REM Maximum heap size
-@REM set MAX_HEAP_SIZE="2G"
-@REM Minimum heap size
-@REM set HEAP_NEWSIZE="2G"
+@REM on heap memory size
+@REM set ON_HEAP_MEMORY=2G
+@REM off heap memory size
+@REM set OFF_HEAP_MEMORY=512M
-@REM maximum direct memory size
-set MAX_DIRECT_MEMORY_SIZE=%MAX_HEAP_SIZE%
@REM threads number that may use direct memory, including query threads(8) +
merge threads(4) + space left for system(4)
set threads_number=16
@REM the size of buffer cache pool(IOV_MAX) depends on operating system
set temp_buffer_pool_size=1024
@REM Max cached buffer size, Note: unit can only be B!
@REM which equals DIRECT_MEMORY_SIZE / threads_number / temp_buffer_pool_size
-set
MAX_CACHED_BUFFER_SIZE=%max_heap_size_in_mb%*1024*1024/%threads_number%/%temp_buffer_pool_size%
+set
MAX_CACHED_BUFFER_SIZE=%on_heap_memory_size_in_mb%*1024*1024/%threads_number%/%temp_buffer_pool_size%
-set IOTDB_HEAP_OPTS=-Xmx%MAX_HEAP_SIZE% -Xms%HEAP_NEWSIZE%
-set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS%
-XX:MaxDirectMemorySize=%MAX_DIRECT_MEMORY_SIZE%
+set IOTDB_HEAP_OPTS=-Xmx%ON_HEAP_MEMORY% -Xms%ON_HEAP_MEMORY%
+set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:MaxDirectMemorySize=%OFF_HEAP_MEMORY%
set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS%
-Djdk.nio.maxCachedBufferSize=%MAX_CACHED_BUFFER_SIZE%
@REM You can put your env variable here
@@ -129,5 +146,5 @@ IF "%JAVA_VERSION%" == "8" (
--add-opens=java.base/java.net=ALL-UNNAMED
)
-echo Maximum memory allocation pool = %MAX_HEAP_SIZE%, initial memory
allocation pool = %HEAP_NEWSIZE%
+echo DataNode on heap memory size = %ON_HEAP_MEMORY%B, off heap memory size =
%OFF_HEAP_MEMORY%B
echo If you want to change this configuration, please check
conf\datanode-env.bat.
diff --git a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
index a79086612c4..905ce9a6424 100755
--- a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
+++ b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
@@ -18,6 +18,9 @@
# under the License.
#
+# You can set DataNode memory size, example '2G' or '2048M'
+MEMORY_SIZE=
+
# You can put your env variable here
# export JAVA_HOME=$JAVA_HOME
@@ -60,7 +63,7 @@ esac
# whether we allow enable heap dump files
IOTDB_ALLOW_HEAP_DUMP="true"
-calculate_heap_sizes()
+calculate_memory_sizes()
{
case "`uname`" in
Linux)
@@ -95,48 +98,55 @@ calculate_heap_sizes()
system_cpu_cores="1"
fi
- # set max heap size based on the following
- # max(min(1/2 ram, 1024MB), min(1/4 ram, 64GB))
- # calculate 1/2 ram and cap to 1024MB
- # calculate 1/4 ram and cap to 65536MB
- # pick the max
- half_system_memory_in_mb=`expr $system_memory_in_mb / 2`
- quarter_system_memory_in_mb=`expr $half_system_memory_in_mb / 2`
- if [ "$half_system_memory_in_mb" -gt "1024" ]
+ # suggest using memory, system memory 1 / 2
+ suggest_using_memory_in_mb=`expr $system_memory_in_mb / 2`
+
+ if [ -n "$MEMORY_SIZE" ]
then
- half_system_memory_in_mb="1024"
+ if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ] || [
"${MEMORY_SIZE%"M"}" != "$MEMORY_SIZE" ]
+ then
+ if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ]
+ then
+ memory_size_in_mb=`expr ${MEMORY_SIZE%"G"} "*" 1024`
+ else
+ memory_size_in_mb=`expr ${MEMORY_SIZE%"M"}`
+ fi
+ else
+ echo "Invalid format of MEMORY_SIZE, please use the format like
2048M or 2G"
+ exit 1
+ fi
+ else
+ memory_size_in_mb=$suggest_using_memory_in_mb
fi
- if [ "$quarter_system_memory_in_mb" -gt "65536" ]
+
+ # set on heap memory size
+ # when memory_size_in_mb is less than 4 * 1024, we will set on heap memory
size to memory_size_in_mb / 4 * 3
+ # when memory_size_in_mb is greater than 4 * 1024 and less than 16 * 1024,
we will set on heap memory size to memory_size_in_mb / 5 * 4
+ # when memory_size_in_mb is greater than 16 * 1024 and less than 128 *
1024, we will set on heap memory size to memory_size_in_mb / 8 * 7
+ # when memory_size_in_mb is greater than 128 * 1024, we will set on heap
memory size to memory_size_in_mb - 16 * 1024
+ if [ "$memory_size_in_mb" -lt "4096" ]
then
- quarter_system_memory_in_mb="65536"
- fi
- if [ "$half_system_memory_in_mb" -gt "$quarter_system_memory_in_mb" ]
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 4 \* 3`
+ elif [ "$memory_size_in_mb" -lt "16384" ]
then
- max_heap_size_in_mb="$half_system_memory_in_mb"
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 5 \* 4`
+ elif [ "$memory_size_in_mb" -lt "131072" ]
+ then
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 8 \* 7`
else
- max_heap_size_in_mb="$quarter_system_memory_in_mb"
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb - 16384`
fi
- MAX_HEAP_SIZE="${max_heap_size_in_mb}M"
+ off_heap_memory_size_in_mb=`expr $memory_size_in_mb -
$on_heap_memory_size_in_mb`
+
+ ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M"
+ OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M"
# if the heap size is larger than 16GB, we will forbid writing the heap
dump file
- if [ "$max_heap_size_in_mb" -gt "16384" ]
+ if [ "$on_heap_memory_size_in_mb" -gt "16384" ]
then
- echo "IoTDB memory is too large ($max_heap_size_in_mb MB), will forbid
writing heap dump file"
+ echo "IoTDB memory is too large ($on_heap_memory_size_in_mb MB), will
forbid writing heap dump file"
IOTDB_ALLOW_HEAP_DUMP="false"
fi
-
- # Young gen: min(max_sensible_per_modern_cpu_core * num_cores, 1/4 * heap
size)
- max_sensible_yg_per_core_in_mb="100"
- max_sensible_yg_in_mb=`expr $max_sensible_yg_per_core_in_mb "*"
$system_cpu_cores`
-
- desired_yg_in_mb=`expr $max_heap_size_in_mb / 4`
-
- if [ "$desired_yg_in_mb" -gt "$max_sensible_yg_in_mb" ]
- then
- HEAP_NEWSIZE="${max_sensible_yg_in_mb}M"
- else
- HEAP_NEWSIZE="${desired_yg_in_mb}M"
- fi
}
@@ -210,29 +220,20 @@ else
fi
+calculate_memory_sizes
-calculate_heap_sizes
-
-## Set heap size by percentage of total memory
-#max_percentage=90
-#min_percentage=50
-#MAX_HEAP_SIZE="`expr $system_memory_in_mb \* $max_percentage / 100`M"
-#HEAP_NEWSIZE="`expr $system_memory_in_mb \* $min_percentage / 100`M"
-
-# Maximum heap size
-#MAX_HEAP_SIZE="2G"
-# Minimum heap size
-#HEAP_NEWSIZE="2G"
-# Maximum direct memory size
-MAX_DIRECT_MEMORY_SIZE=${MAX_HEAP_SIZE}
+# on heap memory size
+#ON_HEAP_MEMORY="2G"
+# off heap memory size
+#OFF_HEAP_MEMORY="512M"
# threads number that may use direct memory, including query threads(8) +
merge threads(4) + space left for system(4)
threads_number="16"
# the size of buffer cache pool(IOV_MAX) depends on operating system
temp_buffer_pool_size="1024"
# Max cached buffer size, Note: unit can only be B!
-# which equals DIRECT_MEMORY_SIZE / threads_number / temp_buffer_pool_size
-MAX_CACHED_BUFFER_SIZE=`expr $max_heap_size_in_mb \* 1024 \* 1024 /
$threads_number / $temp_buffer_pool_size`
+# which equals OFF_HEAP_MEMORY / threads_number / temp_buffer_pool_size
+MAX_CACHED_BUFFER_SIZE=`expr $on_heap_memory_size_in_mb \* 1024 \* 1024 /
$threads_number / $temp_buffer_pool_size`
#true or false
#DO NOT FORGET TO MODIFY THE PASSWORD FOR SECURITY (${IOTDB_CONF}/jmx.password
and ${IOTDB_CONF}/jmx.access)
@@ -264,11 +265,11 @@ else
fi
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Diotdb.jmx.local=$JMX_LOCAL"
-IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xms${HEAP_NEWSIZE}"
-IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xmx${MAX_HEAP_SIZE}"
-IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS
-XX:MaxDirectMemorySize=${MAX_DIRECT_MEMORY_SIZE}"
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xms${ON_HEAP_MEMORY}"
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xmx${ON_HEAP_MEMORY}"
+IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:MaxDirectMemorySize=${OFF_HEAP_MEMORY}"
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS
-Djdk.nio.maxCachedBufferSize=${MAX_CACHED_BUFFER_SIZE}"
-echo "Maximum memory allocation pool = ${MAX_HEAP_SIZE}B, initial memory
allocation pool = ${HEAP_NEWSIZE}B"
+echo "DataNode on heap memory size = ${ON_HEAP_MEMORY}B, off heap memory size
= ${OFF_HEAP_MEMORY}B"
echo "If you want to change this configuration, please check
conf/datanode-env.sh."
