This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new cfb8adea488 HDDS-11234. Manage Netty native memory consumption (#10354)
cfb8adea488 is described below
commit cfb8adea488da08a56f75acf46011574325482aa
Author: Siyao Meng <[email protected]>
AuthorDate: Thu May 28 09:11:00 2026 -0700
HDDS-11234. Manage Netty native memory consumption (#10354)
---
hadoop-hdds/common/src/main/conf/ozone-env.sh | 15 +++++++++++++++
hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh | 11 +++++++++++
2 files changed, 26 insertions(+)
diff --git a/hadoop-hdds/common/src/main/conf/ozone-env.sh
b/hadoop-hdds/common/src/main/conf/ozone-env.sh
index 5d2c93b7a04..6da8479812c 100644
--- a/hadoop-hdds/common/src/main/conf/ozone-env.sh
+++ b/hadoop-hdds/common/src/main/conf/ozone-env.sh
@@ -248,6 +248,21 @@ export OZONE_OS_TYPE=${OZONE_OS_TYPE:-$(uname -s)}
# Java property: hadoop.log.dir
# export OZONE_SECURE_LOG_DIR=${OZONE_LOG_DIR}
+###
+# Netty native (direct) memory caps (HDDS-11234)
+###
+# Both unshaded io.netty and the Ratis-shaded copy default their pooled
+# direct-memory ceiling to MaxDirectMemorySize (≈ -Xmx) per JVM, which
+# can let the resident size of a busy DataNode or S3 Gateway grow well
+# beyond the heap. To put a hard cap on each pool, export one or both
+# of the following before starting Ozone daemons. Values are raw byte
+# counts (suffixes like "m" or "g" are NOT supported by Netty's
+# property parser); for example, 536870912 = 512 MiB.
+#
+# For example, to cap each pool at 4 GiB on a DataNode with -Xmx16g:
+# export OZONE_NETTY_MAX_DIRECT_MEMORY=4294967296
+# export OZONE_RATIS_NETTY_MAX_DIRECT_MEMORY=4294967296
+
###
# Ozone Manager specific parameters
###
diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
index 325d6daa50b..4a4173f6d67 100755
--- a/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
+++ b/hadoop-ozone/dist/src/shell/ozone/ozone-functions.sh
@@ -1420,6 +1420,17 @@ function ozone_java_setup
RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true
${RATIS_OPTS}"
fi
+ # Opt-in caps on Netty's pooled direct-memory arena (HDDS-11234). Two
+ # properties are needed because Ozone runs both the unshaded io.netty
+ # *and* the Ratis-shaded copy in the same JVM, each with its own
+ # independent ceiling.
+ if [[ -n "${OZONE_NETTY_MAX_DIRECT_MEMORY:-}" ]]; then
+ OZONE_OPTS="-Dio.netty.maxDirectMemory=${OZONE_NETTY_MAX_DIRECT_MEMORY}
${OZONE_OPTS}"
+ fi
+ if [[ -n "${OZONE_RATIS_NETTY_MAX_DIRECT_MEMORY:-}" ]]; then
+
RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.maxDirectMemory=${OZONE_RATIS_NETTY_MAX_DIRECT_MEMORY}
${RATIS_OPTS}"
+ fi
+
ozone_set_module_access_args
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
