This is an automated email from the ASF dual-hosted git repository.
jackietien pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 9c05a54f238 feat(env): set MAX_CACHED_BUFFER_SIZE equals
OFF_HEAP_MEMORY / IO_THREADS_NUMBER
9c05a54f238 is described below
commit 9c05a54f238c271c946fd8fb73d396887586285a
Author: CritasWang <[email protected]>
AuthorDate: Tue Oct 31 23:24:27 2023 +0800
feat(env): set MAX_CACHED_BUFFER_SIZE equals OFF_HEAP_MEMORY /
IO_THREADS_NUMBER
---
.../src/assembly/resources/conf/confignode-env.bat | 21 +++++++++---------
.../src/assembly/resources/conf/confignode-env.sh | 24 ++++++++++-----------
.../src/assembly/resources/conf/datanode-env.bat | 21 +++++++++---------
.../src/assembly/resources/conf/datanode-env.sh | 25 +++++++++++-----------
4 files changed, 45 insertions(+), 46 deletions(-)
diff --git
a/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.bat
b/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.bat
index f6de7c93fd9..263fa0d328e 100644
--- a/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.bat
+++ b/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.bat
@@ -99,23 +99,24 @@ set /a
off_heap_memory_size_in_mb=%memory_size_in_mb%-%on_heap_memory_size_in_mb
set ON_HEAP_MEMORY=%on_heap_memory_size_in_mb%M
set OFF_HEAP_MEMORY=%off_heap_memory_size_in_mb%M
-@REM if the heap size is larger than 16GB, we will forbid writing the heap
dump file
-if %off_heap_memory_size_in_mb% GTR 16384 (
- set IOTDB_ALLOW_HEAP_DUMP="false"
-) else set IOTDB_ALLOW_HEAP_DUMP="true"
+set IOTDB_ALLOW_HEAP_DUMP="true"
@REM on heap memory size
@REM set ON_HEAP_MEMORY=2G
@REM off heap memory size
@REM set OFF_HEAP_MEMORY=512M
-@REM threads number that may use direct memory, including query threads(8) +
merge threads(4) + space left for system(4)
-set threads_number=16
-@REM the size of buffer cache pool(IOV_MAX) depends on operating system
-set temp_buffer_pool_size=1024
+if "%OFF_HEAP_MEMORY:~-1%"=="M" (
+ set /a off_heap_memory_size_in_mb=%OFF_HEAP_MEMORY:~0,-1%
+ ) else if "%OFF_HEAP_MEMORY:~-1%"=="G" (
+ set /a off_heap_memory_size_in_mb=%OFF_HEAP_MEMORY:~0,-1%*1024
+ )
+
+@REM threads number of io
+set IO_THREADS_NUMBER=100
@REM Max cached buffer size, Note: unit can only be B!
-@REM which equals DIRECT_MEMORY_SIZE / threads_number / temp_buffer_pool_size
-set
MAX_CACHED_BUFFER_SIZE=%on_heap_memory_size_in_mb%*1024*1024/%threads_number%/%temp_buffer_pool_size%
+@REM which equals OFF_HEAP_MEMORY / IO_THREADS_NUMBER
+set /a
MAX_CACHED_BUFFER_SIZE=%off_heap_memory_size_in_mb%/%IO_THREADS_NUMBER%*1024*1024
set CONFIGNODE_HEAP_OPTS=-Xmx%ON_HEAP_MEMORY% -Xms%ON_HEAP_MEMORY%
set CONFIGNODE_HEAP_OPTS=%CONFIGNODE_HEAP_OPTS%
-XX:MaxDirectMemorySize=%OFF_HEAP_MEMORY%
diff --git
a/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
b/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
index 51dea4bc6c5..c77d787b628 100644
--- a/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
+++ b/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
@@ -146,13 +146,6 @@ calculate_memory_sizes()
ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M"
OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M"
-
- # if the heap size is larger than 16GB, we will forbid writing the heap
dump file
- if [ "$on_heap_memory_size_in_mb" -gt "16384" ]
- then
- echo "IoTDB memory is too large ($on_heap_memory_size_in_mb MB), will
forbid writing heap dump file"
- IOTDB_ALLOW_HEAP_DUMP="false"
- fi
}
@@ -233,13 +226,18 @@ calculate_memory_sizes
# off heap memory size
#OFF_HEAP_MEMORY="512M"
-# threads number that may use direct memory, including read threads(8) + merge
threads(4) + space left for system(4)
-threads_number="16"
-# the size of buffer cache pool(IOV_MAX) depends on operating system
-temp_buffer_pool_size="1024"
+if [ "${OFF_HEAP_MEMORY%"G"}" != "$OFF_HEAP_MEMORY" ]
+then
+ off_heap_memory_size_in_mb=`expr ${OFF_HEAP_MEMORY%"G"} "*" 1024`
+else
+ off_heap_memory_size_in_mb=`expr ${OFF_HEAP_MEMORY%"M"}`
+fi
+
+# threads number of io
+IO_THREADS_NUMBER="100"
# Max cached buffer size, Note: unit can only be B!
-# which equals OFF_HEAP_MEMORY / threads_number / temp_buffer_pool_size
-MAX_CACHED_BUFFER_SIZE=`expr $on_heap_memory_size_in_mb \* 1024 \* 1024 /
$threads_number / $temp_buffer_pool_size`
+# which equals OFF_HEAP_MEMORY / IO_THREADS_NUMBER
+MAX_CACHED_BUFFER_SIZE=`expr $off_heap_memory_size_in_mb \* 1024 \* 1024 /
$IO_THREADS_NUMBER`
#true or false
#DO NOT FORGET TO MODIFY THE PASSWORD FOR SECURITY
(${CONFIGNODE_CONF}/jmx.password and ${CONFIGNODE_CONF}/jmx.access)
diff --git a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
index 6d2ce6d0f80..16ad87390f0 100644
--- a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
+++ b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.bat
@@ -98,23 +98,24 @@ set /a
off_heap_memory_size_in_mb=%memory_size_in_mb%-%on_heap_memory_size_in_mb
set ON_HEAP_MEMORY=%on_heap_memory_size_in_mb%M
set OFF_HEAP_MEMORY=%off_heap_memory_size_in_mb%M
-@REM if the heap size is larger than 16GB, we will forbid writing the heap
dump file
-if %off_heap_memory_size_in_mb% GTR 16384 (
- set IOTDB_ALLOW_HEAP_DUMP="false"
-) else set IOTDB_ALLOW_HEAP_DUMP="true"
+set IOTDB_ALLOW_HEAP_DUMP="true"
@REM on heap memory size
@REM set ON_HEAP_MEMORY=2G
@REM off heap memory size
@REM set OFF_HEAP_MEMORY=512M
-@REM threads number that may use direct memory, including query threads(8) +
merge threads(4) + space left for system(4)
-set threads_number=16
-@REM the size of buffer cache pool(IOV_MAX) depends on operating system
-set temp_buffer_pool_size=1024
+if "%OFF_HEAP_MEMORY:~-1%"=="M" (
+ set /a off_heap_memory_size_in_mb=%OFF_HEAP_MEMORY:~0,-1%
+ ) else if "%OFF_HEAP_MEMORY:~-1%"=="G" (
+ set /a off_heap_memory_size_in_mb=%OFF_HEAP_MEMORY:~0,-1%*1024
+ )
+
+@REM threads number of io
+set IO_THREADS_NUMBER=1000
@REM Max cached buffer size, Note: unit can only be B!
-@REM which equals DIRECT_MEMORY_SIZE / threads_number / temp_buffer_pool_size
-set
MAX_CACHED_BUFFER_SIZE=%on_heap_memory_size_in_mb%*1024*1024/%threads_number%/%temp_buffer_pool_size%
+@REM which equals OFF_HEAP_MEMORY / IO_THREADS_NUMBER
+set /a
MAX_CACHED_BUFFER_SIZE=%off_heap_memory_size_in_mb%/%IO_THREADS_NUMBER%*1024*1024
set IOTDB_HEAP_OPTS=-Xmx%ON_HEAP_MEMORY% -Xms%ON_HEAP_MEMORY%
set IOTDB_HEAP_OPTS=%IOTDB_HEAP_OPTS% -XX:MaxDirectMemorySize=%OFF_HEAP_MEMORY%
diff --git a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
index f15bd84f207..8dcf93cb149 100755
--- a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
+++ b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
@@ -140,13 +140,6 @@ calculate_memory_sizes()
ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M"
OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M"
-
- # if the heap size is larger than 16GB, we will forbid writing the heap
dump file
- if [ "$on_heap_memory_size_in_mb" -gt "16384" ]
- then
- echo "IoTDB memory is too large ($on_heap_memory_size_in_mb MB), will
forbid writing heap dump file"
- IOTDB_ALLOW_HEAP_DUMP="false"
- fi
}
@@ -227,13 +220,19 @@ calculate_memory_sizes
# off heap memory size
#OFF_HEAP_MEMORY="512M"
-# threads number that may use direct memory, including query threads(8) +
merge threads(4) + space left for system(4)
-threads_number="16"
-# the size of buffer cache pool(IOV_MAX) depends on operating system
-temp_buffer_pool_size="1024"
+
+if [ "${OFF_HEAP_MEMORY%"G"}" != "$OFF_HEAP_MEMORY" ]
+then
+ off_heap_memory_size_in_mb=`expr ${OFF_HEAP_MEMORY%"G"} "*" 1024`
+else
+ off_heap_memory_size_in_mb=`expr ${OFF_HEAP_MEMORY%"M"}`
+fi
+
+# threads number for io
+IO_THREADS_NUMBER="1000"
# Max cached buffer size, Note: unit can only be B!
-# which equals OFF_HEAP_MEMORY / threads_number / temp_buffer_pool_size
-MAX_CACHED_BUFFER_SIZE=`expr $on_heap_memory_size_in_mb \* 1024 \* 1024 /
$threads_number / $temp_buffer_pool_size`
+# which equals OFF_HEAP_MEMORY / IO_THREADS_NUMBER
+MAX_CACHED_BUFFER_SIZE=`expr $off_heap_memory_size_in_mb \* 1024 \* 1024 /
$IO_THREADS_NUMBER`
#true or false
#DO NOT FORGET TO MODIFY THE PASSWORD FOR SECURITY (${IOTDB_CONF}/jmx.password
and ${IOTDB_CONF}/jmx.access)
