This is an automated email from the ASF dual-hosted git repository.
sumitagrawal pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e33fd2d7ca HDDS-10382. Optimize Netty memory allocation by avoiding
zero assignment in Java 9+ (#6975)
e33fd2d7ca is described below
commit e33fd2d7caa7a0d7380b1a2eef803ae64e3dad47
Author: Sarveksha Yeshavantha Raju
<[email protected]>
AuthorDate: Wed Jul 24 07:40:34 2024 +0530
HDDS-10382. Optimize Netty memory allocation by avoiding zero assignment in
Java 9+ (#6975)
---
hadoop-ozone/dist/src/shell/ozone/ozone | 37 ++++++++++++++++++++++-----------
1 file changed, 25 insertions(+), 12 deletions(-)
diff --git a/hadoop-ozone/dist/src/shell/ozone/ozone
b/hadoop-ozone/dist/src/shell/ozone/ozone
index dbb18db8e2..1de7adf83d 100755
--- a/hadoop-ozone/dist/src/shell/ozone/ozone
+++ b/hadoop-ozone/dist/src/shell/ozone/ozone
@@ -81,6 +81,19 @@ function ozonecmd_case
# Corresponding Ratis issue https://issues.apache.org/jira/browse/RATIS-534.
RATIS_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.allocator.useCacheForAllThreads=false
${RATIS_OPTS}"
+ # Get the version string
+ JAVA_VERSION_STRING=$(java -version 2>&1)
+
+ # Extract the major version number
+ JAVA_MAJOR_VERSION=$(echo "$JAVA_VERSION_STRING" | grep -oE
'[0-9]+(\.[0-9]+)*' | head -n 1 | awk -F. '{print ($1 == 1 ? $2 : $1)}')
+
+ # Add JVM parameter for Java 9+
+ # (org.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true) to
allow Netty unsafe memory allocation.
+ # Corresponding issue https://issues.apache.org/jira/browse/HDDS-10382.
+ if [[ "${JAVA_MAJOR_VERSION}" -ge "9" ]]; then
+
NETTY_OPTS="-Dorg.apache.ratis.thirdparty.io.netty.tryReflectionSetAccessible=true
${NETTY_OPTS}"
+ fi
+
case ${subcmd} in
auditparser)
OZONE_CLASSNAME=org.apache.hadoop.ozone.audit.parser.AuditParser
@@ -105,7 +118,7 @@ function ozonecmd_case
datanode)
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
ozone_deprecate_envvar HDDS_DN_OPTS OZONE_DATANODE_OPTS
- OZONE_DATANODE_OPTS="${RATIS_OPTS} ${OZONE_DATANODE_OPTS}"
+ OZONE_DATANODE_OPTS="${RATIS_OPTS} ${NETTY_OPTS} ${OZONE_DATANODE_OPTS}"
OZONE_DATANODE_OPTS="-Dlog4j.configurationFile=${OZONE_CONF_DIR}/dn-audit-log4j2.properties,${OZONE_CONF_DIR}/dn-container-log4j2.properties
${OZONE_DATANODE_OPTS}"
OZONE_DATANODE_OPTS="-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
${OZONE_DATANODE_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_CLASSNAME=org.apache.hadoop.ozone.HddsDatanodeService
@@ -125,7 +138,7 @@ function ozonecmd_case
;;
freon)
OZONE_CLASSNAME=org.apache.hadoop.ozone.freon.Freon
- OZONE_FREON_OPTS="${OZONE_FREON_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
+ OZONE_FREON_OPTS="${OZONE_FREON_OPTS} ${NETTY_OPTS}
${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
getconf)
@@ -136,7 +149,7 @@ function ozonecmd_case
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_CLASSNAME=org.apache.hadoop.ozone.om.OzoneManagerStarter
ozone_deprecate_envvar HDFS_OM_OPTS OZONE_OM_OPTS
- OZONE_OM_OPTS="${RATIS_OPTS} ${OZONE_OM_OPTS}"
+ OZONE_OM_OPTS="${RATIS_OPTS} ${NETTY_OPTS} ${OZONE_OM_OPTS}"
OZONE_OM_OPTS="${OZONE_OM_OPTS}
-Dlog4j.configurationFile=${OZONE_CONF_DIR}/om-audit-log4j2.properties"
OZONE_OM_OPTS="-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
${OZONE_OM_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-manager"
@@ -144,7 +157,7 @@ function ozonecmd_case
sh | shell)
OZONE_CLASSNAME=org.apache.hadoop.ozone.shell.OzoneShell
ozone_deprecate_envvar HDFS_OM_SH_OPTS OZONE_SH_OPTS
- OZONE_SH_OPTS="${OZONE_SH_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
+ OZONE_SH_OPTS="${OZONE_SH_OPTS} ${NETTY_OPTS}
${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
s3)
@@ -155,7 +168,7 @@ function ozonecmd_case
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_CLASSNAME='org.apache.hadoop.hdds.scm.server.StorageContainerManagerStarter'
ozone_deprecate_envvar HDFS_STORAGECONTAINERMANAGER_OPTS OZONE_SCM_OPTS
- OZONE_SCM_OPTS="${RATIS_OPTS} ${OZONE_SCM_OPTS}"
+ OZONE_SCM_OPTS="${RATIS_OPTS} ${NETTY_OPTS} ${OZONE_SCM_OPTS}"
OZONE_SCM_OPTS="${OZONE_SCM_OPTS}
-Dlog4j.configurationFile=${OZONE_CONF_DIR}/scm-audit-log4j2.properties"
OZONE_SCM_OPTS="-Dlog4j2.contextSelector=org.apache.logging.log4j.core.async.AsyncLoggerContextSelector
${OZONE_SCM_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="hdds-server-scm"
@@ -163,12 +176,12 @@ function ozonecmd_case
s3g)
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_CLASSNAME='org.apache.hadoop.ozone.s3.Gateway'
- OZONE_S3G_OPTS="${OZONE_S3G_OPTS}
-Dlog4j.configurationFile=${OZONE_CONF_DIR}/s3g-audit-log4j2.properties
${OZONE_MODULE_ACCESS_ARGS}"
+ OZONE_S3G_OPTS="${OZONE_S3G_OPTS} ${NETTY_OPTS}
-Dlog4j.configurationFile=${OZONE_CONF_DIR}/s3g-audit-log4j2.properties
${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-s3gateway"
;;
httpfs)
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
- OZONE_OPTS="${OZONE_OPTS} -Dhttpfs.home.dir=${OZONE_HOME}
-Dhttpfs.config.dir=${OZONE_CONF_DIR} -Dhttpfs.log.dir=${OZONE_HOME}/log
-Dhttpfs.temp.dir=${OZONE_HOME}/temp ${OZONE_MODULE_ACCESS_ARGS}"
+ OZONE_OPTS="${OZONE_OPTS} ${NETTY_OPTS} -Dhttpfs.home.dir=${OZONE_HOME}
-Dhttpfs.config.dir=${OZONE_CONF_DIR} -Dhttpfs.log.dir=${OZONE_HOME}/log
-Dhttpfs.temp.dir=${OZONE_HOME}/temp ${OZONE_MODULE_ACCESS_ARGS}"
OZONE_CLASSNAME='org.apache.ozone.fs.http.server.HttpFSServerWebServer'
OZONE_RUN_ARTIFACT_NAME="ozone-httpfsgateway"
;;
@@ -184,12 +197,12 @@ function ozonecmd_case
recon)
OZONE_SUBCMD_SUPPORTDAEMONIZATION="true"
OZONE_CLASSNAME='org.apache.hadoop.ozone.recon.ReconServer'
- OZONE_RECON_OPTS="${OZONE_RECON_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
+ OZONE_RECON_OPTS="${OZONE_RECON_OPTS} ${NETTY_OPTS}
${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-recon"
;;
fs)
OZONE_CLASSNAME=org.apache.hadoop.fs.ozone.OzoneFsShell
- OZONE_FS_OPTS="${OZONE_FS_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
+ OZONE_FS_OPTS="${OZONE_FS_OPTS} ${NETTY_OPTS}
${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
daemonlog)
@@ -214,17 +227,17 @@ function ozonecmd_case
;;
admin)
OZONE_CLASSNAME=org.apache.hadoop.hdds.cli.OzoneAdmin
- OZONE_ADMIN_OPTS="${OZONE_ADMIN_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
+ OZONE_ADMIN_OPTS="${OZONE_ADMIN_OPTS} ${NETTY_OPTS}
${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
debug)
OZONE_CLASSNAME=org.apache.hadoop.ozone.debug.OzoneDebug
- OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
+ OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${NETTY_OPTS}
${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
repair)
OZONE_CLASSNAME=org.apache.hadoop.ozone.repair.OzoneRepair
- OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${OZONE_MODULE_ACCESS_ARGS}"
+ OZONE_DEBUG_OPTS="${OZONE_DEBUG_OPTS} ${NETTY_OPTS}
${OZONE_MODULE_ACCESS_ARGS}"
OZONE_RUN_ARTIFACT_NAME="ozone-tools"
;;
checknative)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
