This is an automated email from the ASF dual-hosted git repository.
CRZbulabula pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new e123ada23ec Respect pre-set env vars and add cgroup memory detection
in env scripts (#17765)
e123ada23ec is described below
commit e123ada23ec4a203f5c6cadc68846aa44c2e1ef6
Author: Milea Robert Stefan
<[email protected]>
AuthorDate: Thu Jul 9 07:33:34 2026 +0300
Respect pre-set env vars and add cgroup memory detection in env scripts
(#17765)
---
scripts/conf/confignode-env.sh | 91 ++-----------------------
scripts/conf/datanode-env.sh | 86 ++---------------------
scripts/conf/iotdb-common.sh | 117 +++++++++++++++++++++++++++++++-
scripts/conf/windows/confignode-env.bat | 5 +-
scripts/conf/windows/datanode-env.bat | 3 +-
5 files changed, 129 insertions(+), 173 deletions(-)
diff --git a/scripts/conf/confignode-env.sh b/scripts/conf/confignode-env.sh
index 89be12c3e01..c428a825501 100644
--- a/scripts/conf/confignode-env.sh
+++ b/scripts/conf/confignode-env.sh
@@ -19,7 +19,8 @@
#
# You can set ConfigNode memory size, example '2G' or '2048M'
-MEMORY_SIZE=
+# If the MEMORY_SIZE environment variable is already set, its value will be
used.
+MEMORY_SIZE=${MEMORY_SIZE:-}
# You can put your env variable here
# export JAVA_HOME=$JAVA_HOME
@@ -63,91 +64,6 @@ esac
# whether we allow enable heap dump files
IOTDB_ALLOW_HEAP_DUMP="true"
-calculate_memory_sizes()
-{
- case "`uname`" in
- Linux)
- system_memory_in_mb=`free -m| sed -n '2p' | awk '{print $2}'`
- system_cpu_cores=`egrep -c 'processor([[:space:]]+):.*'
/proc/cpuinfo`
- ;;
- FreeBSD)
- system_memory_in_bytes=`sysctl hw.physmem | awk '{print $2}'`
- system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
- system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
- ;;
- SunOS)
- system_memory_in_mb=`prtconf | awk '/Memory size:/ {print $3}'`
- system_cpu_cores=`psrinfo | wc -l`
- ;;
- Darwin)
- system_memory_in_bytes=`sysctl hw.memsize | awk '{print $2}'`
- system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
- system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
- ;;
- *)
- # assume reasonable defaults for e.g. a modern desktop or
- # cheap server
- system_memory_in_mb="2048"
- system_cpu_cores="2"
- ;;
- esac
-
- # some systems like the raspberry pi don't report cores, use at least 1
- if [ "$system_cpu_cores" -lt "1" ]
- then
- system_cpu_cores="1"
- fi
-
- # suggest using memory, system memory 3 / 10
- suggest_using_memory_in_mb=`expr $system_memory_in_mb / 10 \* 3`
-
- if [ -n "$MEMORY_SIZE" ]
- then
- if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ] || [
"${MEMORY_SIZE%"M"}" != "$MEMORY_SIZE" ]
- then
- if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ]
- then
- memory_size_in_mb=`expr ${MEMORY_SIZE%"G"} "*" 1024`
- else
- memory_size_in_mb=`expr ${MEMORY_SIZE%"M"}`
- fi
- else
- echo "Invalid format of MEMORY_SIZE, please use the format like
2048M or 2G"
- exit 1
- fi
- else
- # set memory size to suggest using memory, if suggest using memory is
greater than 8GB, set memory size to 8GB
- if [ "$suggest_using_memory_in_mb" -gt "8192" ]
- then
- memory_size_in_mb="8192"
- else
- memory_size_in_mb=$suggest_using_memory_in_mb
- fi
- fi
-
- # set on heap memory size
- # when memory_size_in_mb is less than 4 * 1024, we will set on heap memory
size to memory_size_in_mb / 4 * 3
- # when memory_size_in_mb is greater than 4 * 1024 and less than 16 * 1024,
we will set on heap memory size to memory_size_in_mb / 5 * 4
- # when memory_size_in_mb is greater than 16 * 1024 and less than 128 *
1024, we will set on heap memory size to memory_size_in_mb / 8 * 7
- # when memory_size_in_mb is greater than 128 * 1024, we will set on heap
memory size to memory_size_in_mb - 16 * 1024
- if [ "$memory_size_in_mb" -lt "4096" ]
- then
- on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 4 \* 3`
- elif [ "$memory_size_in_mb" -lt "16384" ]
- then
- on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 5 \* 4`
- elif [ "$memory_size_in_mb" -lt "131072" ]
- then
- on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 8 \* 7`
- else
- on_heap_memory_size_in_mb=`expr $memory_size_in_mb - 16384`
- fi
- off_heap_memory_size_in_mb=`expr $memory_size_in_mb -
$on_heap_memory_size_in_mb`
-
- ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M"
- OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M"
-}
-
get_cn_system_dir() {
local config_file="$1"
local cn_system_dir=""
@@ -234,7 +150,8 @@ illegal_access_params="$illegal_access_params
--add-opens=java.base/java.io=ALL-
illegal_access_params="$illegal_access_params
--add-opens=java.base/java.net=ALL-UNNAMED"
-calculate_memory_sizes
+# ConfigNode: suggest 30% of system memory (3/10), cap at 8 GB.
+calculate_memory_sizes 3 10 8192
# on heap memory size
#ON_HEAP_MEMORY="2G"
diff --git a/scripts/conf/datanode-env.sh b/scripts/conf/datanode-env.sh
index be049774088..85e880634c4 100755
--- a/scripts/conf/datanode-env.sh
+++ b/scripts/conf/datanode-env.sh
@@ -19,7 +19,8 @@
#
# You can set DataNode memory size, example '2G' or '2048M'
-MEMORY_SIZE=
+# If the MEMORY_SIZE environment variable is already set, its value will be
used.
+MEMORY_SIZE=${MEMORY_SIZE:-}
# You can put your env variable here
@@ -64,86 +65,6 @@ esac
# whether we allow enable heap dump files
IOTDB_ALLOW_HEAP_DUMP="true"
-calculate_memory_sizes()
-{
- case "`uname`" in
- Linux)
- system_memory_in_mb=`free -m| sed -n '2p' | awk '{print $2}'`
- system_cpu_cores=`egrep -c 'processor([[:space:]]+):.*'
/proc/cpuinfo`
- ;;
- FreeBSD)
- system_memory_in_bytes=`sysctl hw.physmem | awk '{print $2}'`
- system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
- system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
- ;;
- SunOS)
- system_memory_in_mb=`prtconf | awk '/Memory size:/ {print $3}'`
- system_cpu_cores=`psrinfo | wc -l`
- ;;
- Darwin)
- system_memory_in_bytes=`sysctl hw.memsize | awk '{print $2}'`
- system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
- system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
- ;;
- *)
- # assume reasonable defaults for e.g. a modern desktop or
- # cheap server
- system_memory_in_mb="2048"
- system_cpu_cores="2"
- ;;
- esac
-
- # some systems like the raspberry pi don't report cores, use at least 1
- if [ "$system_cpu_cores" -lt "1" ]
- then
- system_cpu_cores="1"
- fi
-
- # suggest using memory, system memory 1 / 2
- suggest_using_memory_in_mb=`expr $system_memory_in_mb / 2`
-
- if [ -n "$MEMORY_SIZE" ]
- then
- if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ] || [
"${MEMORY_SIZE%"M"}" != "$MEMORY_SIZE" ]
- then
- if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ]
- then
- memory_size_in_mb=`expr ${MEMORY_SIZE%"G"} "*" 1024`
- else
- memory_size_in_mb=`expr ${MEMORY_SIZE%"M"}`
- fi
- else
- echo "Invalid format of MEMORY_SIZE, please use the format like
2048M or 2G"
- exit 1
- fi
- else
- memory_size_in_mb=$suggest_using_memory_in_mb
- fi
-
- # set on heap memory size
- # when memory_size_in_mb is less than 4 * 1024, we will set on heap memory
size to memory_size_in_mb / 4 * 3
- # when memory_size_in_mb is greater than 4 * 1024 and less than 16 * 1024,
we will set on heap memory size to memory_size_in_mb / 5 * 4
- # when memory_size_in_mb is greater than 16 * 1024 and less than 128 *
1024, we will set on heap memory size to memory_size_in_mb / 8 * 7
- # when memory_size_in_mb is greater than 128 * 1024, we will set on heap
memory size to memory_size_in_mb - 16 * 1024
- if [ "$memory_size_in_mb" -lt "4096" ]
- then
- on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 4 \* 3`
- elif [ "$memory_size_in_mb" -lt "16384" ]
- then
- on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 5 \* 4`
- elif [ "$memory_size_in_mb" -lt "131072" ]
- then
- on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 8 \* 7`
- else
- on_heap_memory_size_in_mb=`expr $memory_size_in_mb - 16384`
- fi
- off_heap_memory_size_in_mb=`expr $memory_size_in_mb -
$on_heap_memory_size_in_mb`
-
- ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M"
- OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M"
-}
-
-
# find first dir of dn_data_dirs from properties file
get_first_data_dir() {
local config_file="$1"
@@ -242,7 +163,8 @@ illegal_access_params="$illegal_access_params
--add-opens=java.base/java.nio=ALL
illegal_access_params="$illegal_access_params
--add-opens=java.base/java.io=ALL-UNNAMED"
illegal_access_params="$illegal_access_params
--add-opens=java.base/java.net=ALL-UNNAMED"
-calculate_memory_sizes
+# DataNode: suggest 50% of system memory (1/2), no cap.
+calculate_memory_sizes 1 2 0
# on heap memory size
#ON_HEAP_MEMORY="2G"
diff --git a/scripts/conf/iotdb-common.sh b/scripts/conf/iotdb-common.sh
index 057622f53f0..6a6b589e523 100755
--- a/scripts/conf/iotdb-common.sh
+++ b/scripts/conf/iotdb-common.sh
@@ -392,4 +392,119 @@ get_iotdb_include() {
esac
done
echo "$VARS"
-}
\ No newline at end of file
+}
+
+# Calculate ON_HEAP_MEMORY and OFF_HEAP_MEMORY from system resources.
+#
+# $1 numerator of the suggested memory fraction (e.g. 1 for DataNode, 3
for ConfigNode)
+# $2 denominator of the suggested memory fraction (e.g. 2 for DataNode, 10
for ConfigNode)
+# $3 max suggested memory in MB; 0 means no cap (e.g. 0 for DataNode, 8192
for ConfigNode)
+#
+# Respects the MEMORY_SIZE environment variable (e.g. '2G' or '2048M') when
set.
+calculate_memory_sizes()
+{
+ local suggest_numerator=${1:-1}
+ local suggest_denominator=${2:-2}
+ local max_suggest_mb=${3:-0}
+
+ case "`uname`" in
+ Linux)
+ system_memory_in_mb=`free -m| sed -n '2p' | awk '{print $2}'`
+ # When running in a container/pod, use cgroup memory limit instead
of host memory.
+ # This reads the root cgroup, which is what containers see; it
does NOT walk
+ # /proc/self/cgroup, so cgroup-restricted processes on bare metal
are not covered.
+ if [ -f /sys/fs/cgroup/memory.max ]; then
+ # cgroup v2
+ cgroup_mem=`cat /sys/fs/cgroup/memory.max`
+ if [ "$cgroup_mem" != "max" ]; then
+ cgroup_mem_in_mb=`expr $cgroup_mem / 1024 / 1024`
+ if [ "$cgroup_mem_in_mb" -lt "$system_memory_in_mb" ]; then
+ system_memory_in_mb=$cgroup_mem_in_mb
+ fi
+ fi
+ elif [ -f /sys/fs/cgroup/memory/memory.limit_in_bytes ]; then
+ # v1 has no "max" sentinel — when unset, this file holds a
huge integer (~PAGE_COUNTER_MAX);
+ # the -lt check below naturally rejects it, so no explicit
guard is needed.
+ cgroup_mem=`cat /sys/fs/cgroup/memory/memory.limit_in_bytes`
+ cgroup_mem_in_mb=`expr $cgroup_mem / 1024 / 1024`
+ if [ "$cgroup_mem_in_mb" -lt "$system_memory_in_mb" ]; then
+ system_memory_in_mb=$cgroup_mem_in_mb
+ fi
+ fi
+ system_cpu_cores=`egrep -c 'processor([[:space:]]+):.*'
/proc/cpuinfo`
+ ;;
+ FreeBSD)
+ system_memory_in_bytes=`sysctl hw.physmem | awk '{print $2}'`
+ system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
+ system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
+ ;;
+ SunOS)
+ system_memory_in_mb=`prtconf | awk '/Memory size:/ {print $3}'`
+ system_cpu_cores=`psrinfo | wc -l`
+ ;;
+ Darwin)
+ system_memory_in_bytes=`sysctl hw.memsize | awk '{print $2}'`
+ system_memory_in_mb=`expr $system_memory_in_bytes / 1024 / 1024`
+ system_cpu_cores=`sysctl hw.ncpu | awk '{print $2}'`
+ ;;
+ *)
+ # assume reasonable defaults for e.g. a modern desktop or
+ # cheap server
+ system_memory_in_mb="2048"
+ system_cpu_cores="2"
+ ;;
+ esac
+
+ # some systems like the raspberry pi don't report cores, use at least 1
+ if [ "$system_cpu_cores" -lt "1" ]
+ then
+ system_cpu_cores="1"
+ fi
+
+ suggest_using_memory_in_mb=`expr $system_memory_in_mb /
$suggest_denominator \* $suggest_numerator`
+
+ if [ -n "$MEMORY_SIZE" ]
+ then
+ if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ] || [
"${MEMORY_SIZE%"M"}" != "$MEMORY_SIZE" ]
+ then
+ if [ "${MEMORY_SIZE%"G"}" != "$MEMORY_SIZE" ]
+ then
+ memory_size_in_mb=`expr ${MEMORY_SIZE%"G"} "*" 1024`
+ else
+ memory_size_in_mb=`expr ${MEMORY_SIZE%"M"}`
+ fi
+ else
+ echo "Invalid format of MEMORY_SIZE, please use the format like
2048M or 2G"
+ exit 1
+ fi
+ else
+ if [ "$max_suggest_mb" -gt "0" ] && [ "$suggest_using_memory_in_mb"
-gt "$max_suggest_mb" ]
+ then
+ memory_size_in_mb=$max_suggest_mb
+ else
+ memory_size_in_mb=$suggest_using_memory_in_mb
+ fi
+ fi
+
+ # set on heap memory size
+ # when memory_size_in_mb is less than 4 * 1024, we will set on heap memory
size to memory_size_in_mb / 4 * 3
+ # when memory_size_in_mb is greater than 4 * 1024 and less than 16 * 1024,
we will set on heap memory size to memory_size_in_mb / 5 * 4
+ # when memory_size_in_mb is greater than 16 * 1024 and less than 128 *
1024, we will set on heap memory size to memory_size_in_mb / 8 * 7
+ # when memory_size_in_mb is greater than 128 * 1024, we will set on heap
memory size to memory_size_in_mb - 16 * 1024
+ if [ "$memory_size_in_mb" -lt "4096" ]
+ then
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 4 \* 3`
+ elif [ "$memory_size_in_mb" -lt "16384" ]
+ then
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 5 \* 4`
+ elif [ "$memory_size_in_mb" -lt "131072" ]
+ then
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb / 8 \* 7`
+ else
+ on_heap_memory_size_in_mb=`expr $memory_size_in_mb - 16384`
+ fi
+ off_heap_memory_size_in_mb=`expr $memory_size_in_mb -
$on_heap_memory_size_in_mb`
+
+ ON_HEAP_MEMORY="${on_heap_memory_size_in_mb}M"
+ OFF_HEAP_MEMORY="${off_heap_memory_size_in_mb}M"
+}
diff --git a/scripts/conf/windows/confignode-env.bat
b/scripts/conf/windows/confignode-env.bat
index baeed004667..9e6cc759bbd 100644
--- a/scripts/conf/windows/confignode-env.bat
+++ b/scripts/conf/windows/confignode-env.bat
@@ -19,8 +19,9 @@
@echo off
-@REM You can set datanode memory size, example '2G' or '2048M'
-set MEMORY_SIZE=
+@REM You can set confignode memory size, example '2G' or '2048M'
+@REM If the MEMORY_SIZE environment variable is already set, its value will be
used.
+if not defined MEMORY_SIZE set MEMORY_SIZE=
@REM true or false
@REM DO NOT FORGET TO MODIFY THE PASSWORD FOR SECURITY
(%CONFIGNODE_CONF%\jmx.password and %{CONFIGNODE_CONF%\jmx.access)
diff --git a/scripts/conf/windows/datanode-env.bat
b/scripts/conf/windows/datanode-env.bat
index ccaabf24bec..c9422670df5 100644
--- a/scripts/conf/windows/datanode-env.bat
+++ b/scripts/conf/windows/datanode-env.bat
@@ -20,7 +20,8 @@
@echo off
@REM You can set datanode memory size, example '2G' or '2048M'
-set MEMORY_SIZE=
+@REM If the MEMORY_SIZE environment variable is already set, its value will be
used.
+if not defined MEMORY_SIZE set MEMORY_SIZE=
@REM true or false
@REM DO NOT FORGET TO MODIFY THE PASSWORD FOR SECURITY
(%IOTDB_CONF%\jmx.password and %{IOTDB_CONF%\jmx.access)
