Repository: hadoop Updated Branches: refs/heads/HADOOP-13341 1fd60c834 -> 72b2b4da3
HADOOP-13358. Modify HDFS to use hadoop_subcommand_opts Signed-off-by: Allen Wittenauer <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72b2b4da Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72b2b4da Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72b2b4da Branch: refs/heads/HADOOP-13341 Commit: 72b2b4da34f372e88d9e69dd4cadc4ed4a1108c3 Parents: 1fd60c8 Author: Allen Wittenauer <[email protected]> Authored: Sun Aug 28 10:45:00 2016 -0700 Committer: Allen Wittenauer <[email protected]> Committed: Tue Aug 30 11:24:54 2016 -0700 ---------------------------------------------------------------------- .../src/main/bin/hadoop-functions.sh | 1 + .../hadoop-common/src/main/conf/hadoop-env.sh | 28 +++++++++---------- .../hadoop-hdfs/src/main/bin/hdfs | 29 ++++---------------- .../hadoop-hdfs/src/main/bin/hdfs-config.sh | 28 ++++++++++++------- 4 files changed, 38 insertions(+), 48 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/72b2b4da/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index 695fd41..db868bd 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -2048,6 +2048,7 @@ function hadoop_subcommand_opts return 0 fi } + ## @description Add custom (program)_(command)_SECURE_EXTRA_OPTS to HADOOP_OPTS. ## @description This *does not* handle the pre-3.x deprecated cases ## @audience public http://git-wip-us.apache.org/repos/asf/hadoop/blob/72b2b4da/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh index f4493f1..4656f4d 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh @@ -294,16 +294,16 @@ esac # and therefore may override any similar flags set in HADOOP_OPTS # # a) Set JMX options -# export HADOOP_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026" +# export HDFS_NAMENODE_OPTS="-Dcom.sun.management.jmxremote=true -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.port=1026" # # b) Set garbage collection logs -# export HADOOP_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" +# export HDFS_NAMENODE_OPTS="${HADOOP_GC_SETTINGS} -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" # # c) ... or set them directly -# export HADOOP_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" +# export HDFS_NAMENODE_OPTS="-verbose:gc -XX:+PrintGCDetails -XX:+PrintGCTimeStamps -XX:+PrintGCDateStamps -Xloggc:${HADOOP_LOG_DIR}/gc-rm.log-$(date +'%Y%m%d%H%M')" # this is the default: -# export HADOOP_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS" +# export HDFS_NAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS" ### # SecondaryNameNode specific parameters @@ -313,7 +313,7 @@ esac # and therefore may override any similar flags set in HADOOP_OPTS # # This is the default: -# export HADOOP_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS" +# export HDFS_SECONDARYNAMENODE_OPTS="-Dhadoop.security.logger=INFO,RFAS" ### # DataNode specific parameters @@ -323,7 +323,7 @@ esac # and therefore may override any similar flags set in HADOOP_OPTS # # This is the default: -# export HADOOP_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS" +# export HDFS_DATANODE_OPTS="-Dhadoop.security.logger=ERROR,RFAS" # On secure datanodes, user to run the datanode as after dropping privileges. # This **MUST** be uncommented to enable secure HDFS if using privileged ports @@ -336,7 +336,7 @@ esac # Supplemental options for secure datanodes # By default, Hadoop uses jsvc which needs to know to launch a # server jvm. -# export HADOOP_DN_SECURE_EXTRA_OPTS="-jvm server" +# export HDFS_DATANODE_SECURE_EXTRA_OPTS="-jvm server" # Where datanode log files are stored in the secure data environment. # This will replace the hadoop.log.dir Java property in secure mode. @@ -352,18 +352,18 @@ esac # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS # -# export HADOOP_NFS3_OPTS="" +# export HDFS_NFS3_OPTS="" # Specify the JVM options to be used when starting the Hadoop portmapper. # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS # -# export HADOOP_PORTMAP_OPTS="-Xmx512m" +# export HDFS_PORTMAP_OPTS="-Xmx512m" # Supplemental options for priviliged gateways # By default, Hadoop uses jsvc which needs to know to launch a # server jvm. -# export HADOOP_NFS3_SECURE_EXTRA_OPTS="-jvm server" +# export HDFS_NFS3_SECURE_EXTRA_OPTS="-jvm server" # On privileged gateways, user to run the gateway as after dropping privileges # This will replace the hadoop.id.str Java property in secure mode. @@ -376,7 +376,7 @@ esac # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS # -# export HADOOP_ZKFC_OPTS="" +# export HDFS_ZKFC_OPTS="" ### # QuorumJournalNode specific parameters @@ -385,7 +385,7 @@ esac # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS # -# export HADOOP_JOURNALNODE_OPTS="" +# export HDFS_JOURNALNODE_OPTS="" ### # HDFS Balancer specific parameters @@ -394,7 +394,7 @@ esac # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS # -# export HADOOP_BALANCER_OPTS="" +# export HDFS_BALANCER_OPTS="" ### # HDFS Mover specific parameters @@ -403,7 +403,7 @@ esac # These options will be appended to the options specified as HADOOP_OPTS # and therefore may override any similar flags set in HADOOP_OPTS # -# export HADOOP_MOVER_OPTS="" +# export HDFS_MOVER_OPTS="" ### # Advanced Users Only! http://git-wip-us.apache.org/repos/asf/hadoop/blob/72b2b4da/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 41b3c12..b704b00 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -79,8 +79,6 @@ function hdfscmd_case balancer) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.balancer.Balancer - hadoop_debug "Appending HADOOP_BALANCER_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_BALANCER_OPTS}" ;; cacheadmin) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.CacheAdmin @@ -103,13 +101,8 @@ function hdfscmd_case HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}" HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}" - hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS" - hadoop_debug "Appending HADOOP_DN_SECURE_EXTRA_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS}" HADOOP_CLASSNAME="org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter" else - hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS}" HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.datanode.DataNode' fi ;; @@ -157,8 +150,6 @@ function hdfscmd_case journalnode) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.hdfs.qjournal.server.JournalNode' - hadoop_debug "Appending HADOOP_JOURNALNODE_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_JOURNALNODE_OPTS}" ;; jmxget) HADOOP_CLASSNAME=org.apache.hadoop.hdfs.tools.JMXGet @@ -169,14 +160,10 @@ function hdfscmd_case mover) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.hdfs.server.mover.Mover - hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}" ;; namenode) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.NameNode' - hadoop_debug "Appending HADOOP_NAMENODE_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NAMENODE_OPTS}" hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}" ;; nfs3) @@ -189,13 +176,8 @@ function hdfscmd_case HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}" HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}" - hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS" - hadoop_debug "Appending HADOOP_NFS3_SECURE_EXTRA_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS}" HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.PrivilegedNfsGatewayStarter else - hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS}" HADOOP_CLASSNAME=org.apache.hadoop.hdfs.nfs.nfs3.Nfs3 fi ;; @@ -211,14 +193,10 @@ function hdfscmd_case portmap) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME=org.apache.hadoop.portmap.Portmap - hadoop_debug "Appending HADOOP_PORTMAP_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_PORTMAP_OPTS}" ;; secondarynamenode) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.hdfs.server.namenode.SecondaryNameNode' - hadoop_debug "Appending HADOOP_SECONDARYNAMENODE_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_SECONDARYNAMENODE_OPTS}" hadoop_add_param HADOOP_OPTS hdfs.audit.logger "-Dhdfs.audit.logger=${HDFS_AUDIT_LOGGER}" ;; snapshotDiff) @@ -233,8 +211,6 @@ function hdfscmd_case zkfc) HADOOP_SUBCMD_SUPPORTDAEMONIZATION="true" HADOOP_CLASSNAME='org.apache.hadoop.hdfs.tools.DFSZKFailoverController' - hadoop_debug "Appending HADOOP_ZKFC_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_ZKFC_OPTS}" ;; *) HADOOP_CLASSNAME="${subcmd}" @@ -288,8 +264,13 @@ if [[ ${HADOOP_WORKER_MODE} = true ]]; then exit $? fi +hadoop_subcommand_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" + if [[ "${HADOOP_SUBCMD_SECURESERVICE}" = true ]]; then HADOOP_SECURE_USER="${HADOOP_SUBCMD_SECUREUSER}" + + hadoop_subcommand_secure_opts "${HADOOP_SHELL_EXECNAME}" "${HADOOP_SUBCMD}" + hadoop_verify_secure_prereq hadoop_setup_secure_service priv_outfile="${HADOOP_LOG_DIR}/privileged-${HADOOP_IDENT_STRING}-${HADOOP_SUBCMD}-${HOSTNAME}.out" http://git-wip-us.apache.org/repos/asf/hadoop/blob/72b2b4da/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh old mode 100644 new mode 100755 index d440210..cba37a4 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs-config.sh @@ -26,7 +26,7 @@ function hadoop_subproject_init export HADOOP_HDFS_ENV_PROCESSED=true fi fi - + # at some point in time, someone thought it would be a good idea to # create separate vars for every subproject. *sigh* # let's perform some overrides and setup some defaults for bw compat @@ -42,23 +42,31 @@ function hadoop_subproject_init hadoop_deprecate_envvar HADOOP_HDFS_NICENESS HADOOP_NICENESS hadoop_deprecate_envvar HADOOP_HDFS_STOP_TIMEOUT HADOOP_STOP_TIMEOUT - + hadoop_deprecate_envvar HADOOP_HDFS_PID_DIR HADOOP_PID_DIR hadoop_deprecate_envvar HADOOP_HDFS_ROOT_LOGGER HADOOP_ROOT_LOGGER hadoop_deprecate_envvar HADOOP_HDFS_IDENT_STRING HADOOP_IDENT_STRING - + + hadoop_deprecate_envvar HADOOP_DN_SECURE_EXTRA_OPTS HDFS_DATANODE_SECURE_EXTRA_OPTS + + hadoop_deprecate_envvar HADOOP_NFS3_SECURE_EXTRA_OPTS HDFS_NFS3_SECURE_EXTRA_OPTS + + HADOOP_HDFS_HOME="${HADOOP_HDFS_HOME:-$HADOOP_HOME}" - + # turn on the defaults export HDFS_AUDIT_LOGGER=${HDFS_AUDIT_LOGGER:-INFO,NullAppender} - export HADOOP_NAMENODE_OPTS=${HADOOP_NAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"} - export HADOOP_SECONDARYNAMENODE_OPTS=${HADOOP_SECONDARYNAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"} - export HADOOP_DATANODE_OPTS=${HADOOP_DATANODE_OPTS:-"-Dhadoop.security.logger=ERROR,RFAS"} - export HADOOP_DN_SECURE_EXTRA_OPTS=${HADOOP_DN_SECURE_EXTRA_OPTS:-"-jvm server"} - export HADOOP_NFS3_SECURE_EXTRA_OPTS=${HADOOP_NFS3_SECURE_EXTRA_OPTS:-"-jvm server"} - export HADOOP_PORTMAP_OPTS=${HADOOP_PORTMAP_OPTS:-"-Xmx512m"} + export HDFS_NAMENODE_OPTS=${HDFS_NAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"} + export HDFS_SECONDARYNAMENODE_OPTS=${HDFS_SECONDARYNAMENODE_OPTS:-"-Dhadoop.security.logger=INFO,RFAS"} + export HDFS_DATANODE_OPTS=${HDFS_DATANODE_OPTS:-"-Dhadoop.security.logger=ERROR,RFAS"} + export HDFS_PORTMAP_OPTS=${HDFS_PORTMAP_OPTS:-"-Xmx512m"} + + # depending upon what is being used to start Java, these may need to be + # set empty. (thus no colon) + export HDFS_DATANODE_SECURE_EXTRA_OPTS=${HDFS_DATANODE_SECURE_EXTRA_OPTS-"-jvm server"} + export HDFS_NFS3_SECURE_EXTRA_OPTS=${HDFS_NFS3_SECURE_EXTRA_OPTS-"-jvm server"} } if [[ -z "${HADOOP_LIBEXEC_DIR}" ]]; then --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
