This is an automated email from the ASF dual-hosted git repository.
critas pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iotdb.git
The following commit(s) were added to refs/heads/master by this push:
new 665e18f6f63 docker: support configuring JVM memory environment
variables in docker-compose file:IOTDB_JMX_OPTS for
datanode,CONFIGNODE_JMX_OPTS for confignode (#15413)
665e18f6f63 is described below
commit 665e18f6f638f55458dfb1066c90a1b857c3192a
Author: changxue2022 <[email protected]>
AuthorDate: Sun Apr 27 09:38:41 2025 +0800
docker: support configuring JVM memory environment variables in
docker-compose file:IOTDB_JMX_OPTS for datanode,CONFIGNODE_JMX_OPTS for
confignode (#15413)
---
.../DockerCompose/docker-compose-host-3c3d.yml | 2 ++
.../DockerCompose/docker-compose-standalone.yml | 4 ++-
.../main/DockerCompose/replace-conf-from-env.sh | 11 +++----
.../src/assembly/resources/conf/confignode-env.sh | 31 ++++++++++++++-----
.../src/assembly/resources/conf/datanode-env.sh | 35 +++++++++++++++-------
5 files changed, 59 insertions(+), 24 deletions(-)
diff --git a/docker/src/main/DockerCompose/docker-compose-host-3c3d.yml
b/docker/src/main/DockerCompose/docker-compose-host-3c3d.yml
index ab88c54f254..242505d9d93 100644
--- a/docker/src/main/DockerCompose/docker-compose-host-3c3d.yml
+++ b/docker/src/main/DockerCompose/docker-compose-host-3c3d.yml
@@ -29,6 +29,7 @@ services:
- schema_replication_factor=3
-
schema_region_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus
-
config_node_consensus_protocol_class=org.apache.iotdb.consensus.ratis.RatisConsensus
+ - CONFIGNODE_JMX_OPTS=-Xms1G -Xmx1G -XX:MaxDirectMemorySize=256M
volumes:
- /etc/hosts:/etc/hosts:ro
- ./data/confignode:/iotdb/data
@@ -48,6 +49,7 @@ services:
- dn_data_region_consensus_port=10760
- data_replication_factor=3
-
data_region_consensus_protocol_class=org.apache.iotdb.consensus.iot.IoTConsensus
+ - IOTDB_JMX_OPTS=-Xms4G -Xmx4G -XX:MaxDirectMemorySize=1G
volumes:
- /etc/hosts:/etc/hosts:ro
- ./data/datanode:/iotdb/data/
diff --git a/docker/src/main/DockerCompose/docker-compose-standalone.yml
b/docker/src/main/DockerCompose/docker-compose-standalone.yml
index 7193acfdcde..e3c4a19b6be 100644
--- a/docker/src/main/DockerCompose/docker-compose-standalone.yml
+++ b/docker/src/main/DockerCompose/docker-compose-standalone.yml
@@ -36,7 +36,9 @@ services:
- dn_schema_region_consensus_port=10750
- dn_data_region_consensus_port=10760
- dn_seed_config_node=iotdb-service:10710
- volumes:
+ - IOTDB_JMX_OPTS=-Xms4G -Xmx4G -XX:MaxDirectMemorySize=1G
+ - CONFIGNODE_JMX_OPTS=-Xms1G -Xmx1G -XX:MaxDirectMemorySize=256M
+ volumes:
- ./data:/iotdb/data
- ./logs:/iotdb/logs
networks:
diff --git a/docker/src/main/DockerCompose/replace-conf-from-env.sh
b/docker/src/main/DockerCompose/replace-conf-from-env.sh
index eb183033119..00a48050d7e 100755
--- a/docker/src/main/DockerCompose/replace-conf-from-env.sh
+++ b/docker/src/main/DockerCompose/replace-conf-from-env.sh
@@ -34,17 +34,18 @@ function process_single(){
if [[ "${content:0:1}" != "#" ]]; then
sed -i "${line_no}d" ${filename}
fi
- sed -i "${line_no} i${key_value}" ${filename}
+ sed -i "${line_no}a${key_value}" ${filename}
else
- echo "append $key $filename"
- line_no=$(wc -l $filename)
- sed -i "${line_no} a${key_value}" ${filename}
+ echo "append $key $filename"
+ line_no=$(wc -l $filename|cut -d ' ' -f1)
+ sed -i "${line_no}a${key_value}" ${filename}
fi
}
function replace_configs(){
for v in $(env); do
- if [[ "${v}" =~ "=" && "${v}" =~ "_" && ! "${v}" =~ "JAVA_" ]]; then
+ key_name="${v%%=*}"
+ if [[ "${key_name}" == "${key_name,,}" && ! 2w$key_name =~ ^_ ]]; then
# echo "###### $v ####"
for f in ${target_files}; do
process_single $v ${conf_path}/$f
diff --git
a/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
b/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
index 065f08dee35..aa19da4ebea 100644
--- a/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
+++ b/iotdb-core/confignode/src/assembly/resources/conf/confignode-env.sh
@@ -20,6 +20,10 @@
# You can set ConfigNode memory size, example '2G' or '2048M'
MEMORY_SIZE=
+# on heap memory size
+#ON_HEAP_MEMORY="2G"
+# off heap memory size
+#OFF_HEAP_MEMORY="512M"
# You can put your env variable here
# export JAVA_HOME=$JAVA_HOME
@@ -250,12 +254,23 @@ else
fi
-calculate_memory_sizes
+if [[ "$CONFIGNODE_JMX_OPTS" =~ -Xms ]];then
+ item_arr=(${CONFIGNODE_JMX_OPTS})
+ for item in ${item_arr[@]};do
+ if [[ -n "$item" ]]; then
+ if [[ "$item" =~ -Xmx ]]; then
+ ON_HEAP_MEMORY=${item#*mx}
+ elif [[ "$item" =~ -XX:MaxDirectMemorySize= ]]; then
+ OFF_HEAP_MEMORY=${item#*=}
+ fi
+ fi
+ done
+elif [[ -n "$ON_HEAP_MEMORY" ]]; then
+ echo "ON_HEAP_MEMORY=$ON_HEAP_MEMORY"
+else
+ calculate_memory_sizes
+fi
-# on heap memory size
-#ON_HEAP_MEMORY="2G"
-# off heap memory size
-#OFF_HEAP_MEMORY="512M"
if [ "${OFF_HEAP_MEMORY%"G"}" != "$OFF_HEAP_MEMORY" ]
then
@@ -300,9 +315,9 @@ else
fi
CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS -Diotdb.jmx.local=$JMX_LOCAL"
-CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS -Xms${ON_HEAP_MEMORY}"
-CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS -Xmx${ON_HEAP_MEMORY}"
-CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS
-XX:MaxDirectMemorySize=${OFF_HEAP_MEMORY}"
+if [[ ! "$CONFIGNODE_JMX_OPTS" =~ -Xms ]]; then
CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS -Xms${ON_HEAP_MEMORY}"; fi
+if [[ ! "$CONFIGNODE_JMX_OPTS" =~ -Xmx ]]; then
CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS -Xmx${ON_HEAP_MEMORY}"; fi
+if [[ ! "$CONFIGNODE_JMX_OPTS" =~ -XX:MaxDirectMemorySize ]]; then
CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS
-XX:MaxDirectMemorySize=${OFF_HEAP_MEMORY}"; fi
CONFIGNODE_JMX_OPTS="$CONFIGNODE_JMX_OPTS
-Djdk.nio.maxCachedBufferSize=${MAX_CACHED_BUFFER_SIZE}"
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+CrashOnOutOfMemoryError"
# if you want to dump the heap memory while OOM happening, you can use the
following command, remember to replace
${heap_dump_dir}/confignode_heapdump.hprof with your own file path and the
folder where this file is located needs to be created in advance
diff --git a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
index e9e59f43405..b1e9e342485 100755
--- a/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
+++ b/iotdb-core/datanode/src/assembly/resources/conf/datanode-env.sh
@@ -20,6 +20,11 @@
# You can set DataNode memory size, example '2G' or '2048M'
MEMORY_SIZE=
+# on heap memory size
+#ON_HEAP_MEMORY="2G"
+# off heap memory size
+#OFF_HEAP_MEMORY="512M"
+
# You can put your env variable here
# export JAVA_HOME=$JAVA_HOME
@@ -261,12 +266,22 @@ else
fi
-calculate_memory_sizes
-
-# on heap memory size
-#ON_HEAP_MEMORY="2G"
-# off heap memory size
-#OFF_HEAP_MEMORY="512M"
+if [[ "$IOTDB_JMX_OPTS" =~ -Xms ]];then
+ item_arr=(${IOTDB_JMX_OPTS})
+ for item in ${item_arr[@]};do
+ if [[ -n "$item" ]]; then
+ if [[ "$item" =~ -Xmx ]]; then
+ ON_HEAP_MEMORY=${item#*mx}
+ elif [[ "$item" =~ -XX:MaxDirectMemorySize= ]]; then
+ OFF_HEAP_MEMORY=${item#*=}
+ fi
+ fi
+ done
+elif [[ -n "$ON_HEAP_MEMORY" ]]; then
+ echo "ON_HEAP_MEMORY=$ON_HEAP_MEMORY"
+else
+ calculate_memory_sizes
+fi
if [ "${OFF_HEAP_MEMORY%"G"}" != "$OFF_HEAP_MEMORY" ]
@@ -311,10 +326,11 @@ else
echo "setting local JMX..."
fi
+
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Diotdb.jmx.local=$JMX_LOCAL"
-IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xms${ON_HEAP_MEMORY}"
-IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xmx${ON_HEAP_MEMORY}"
-IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:MaxDirectMemorySize=${OFF_HEAP_MEMORY}"
+if [[ ! "$CONFIGNODE_JMX_OPTS" =~ -Xms ]]; then
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xms${ON_HEAP_MEMORY}"; fi
+if [[ ! "$CONFIGNODE_JMX_OPTS" =~ -Xmx ]]; then
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -Xmx${ON_HEAP_MEMORY}"; fi
+if [[ ! "$CONFIGNODE_JMX_OPTS" =~ -XX:MaxDirectMemorySize= ]]; then
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:MaxDirectMemorySize=${OFF_HEAP_MEMORY}"; fi
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS
-Djdk.nio.maxCachedBufferSize=${MAX_CACHED_BUFFER_SIZE}"
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+CrashOnOutOfMemoryError"
IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+UseAdaptiveSizePolicy"
@@ -345,7 +361,6 @@ IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+SafepointTimeout"
# if you want to dump the heap memory while OOM happening, you can use the
following command, remember to replace ${heap_dump_dir}/datanode_heapdump.hprof
with your own file path and the folder where this file is located needs to be
created in advance
# IOTDB_JMX_OPTS="$IOTDB_JMX_OPTS -XX:+HeapDumpOnOutOfMemoryError
-XX:HeapDumpPath=${heap_dump_dir}/datanode_heapdump.hprof"
-
echo "DataNode on heap memory size = ${ON_HEAP_MEMORY}B, off heap memory size
= ${OFF_HEAP_MEMORY}B"
echo "If you want to change this configuration, please check
conf/datanode-env.sh."
