HADOOP-10979. Auto-entries in hadoop_usage (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ee36f4f9 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ee36f4f9 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ee36f4f9 Branch: refs/heads/YARN-1197 Commit: ee36f4f9b87b194c965a2c5ace0244ab11e1d2d6 Parents: 3540d5f Author: Allen Wittenauer <a...@apache.org> Authored: Thu Jul 16 16:58:11 2015 -0700 Committer: Allen Wittenauer <a...@apache.org> Committed: Thu Jul 16 16:58:11 2015 -0700 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 2 + .../hadoop-common/src/main/bin/hadoop | 52 +++++------ .../src/main/bin/hadoop-functions.sh | 90 ++++++++++++++++++++ .../hadoop-hdfs/src/main/bin/hdfs | 82 ++++++++---------- hadoop-mapreduce-project/bin/mapred | 33 +++---- hadoop-yarn-project/hadoop-yarn/bin/yarn | 54 +++++------- 6 files changed, 186 insertions(+), 127 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee36f4f9/hadoop-common-project/hadoop-common/CHANGES.txt ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt index b54688f..62703c3 100644 --- a/hadoop-common-project/hadoop-common/CHANGES.txt +++ b/hadoop-common-project/hadoop-common/CHANGES.txt @@ -232,6 +232,8 @@ Trunk (Unreleased) HADOOP-12149. copy all of test-patch BINDIR prior to re-exec (aw) + HADOOP-10979. Auto-entries in hadoop_usage (aw) + BUG FIXES HADOOP-11473. test-patch says "-1 overall" even when all checks are +1 http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee36f4f9/hadoop-common-project/hadoop-common/src/main/bin/hadoop ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop index 9e7b3ef..c5444d6 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -15,47 +15,35 @@ # See the License for the specific language governing permissions and # limitations under the License. +MYNAME="${BASH_SOURCE-$0}" + function hadoop_usage() { - echo "Usage: hadoop [--config confdir] [COMMAND | CLASSNAME]" - echo " CLASSNAME run the class named CLASSNAME" - echo " or" - echo " where COMMAND is one of:" - echo " archive -archiveName NAME -p <parent path> <src>* <dest>" - echo " create a Hadoop archive" - echo " checknative [-a|-h] check native Hadoop and compression " - echo " libraries availability" - echo " classpath prints the class path needed to get the" - echo " Hadoop jar and the required libraries" - echo " conftest validate configuration XML files" - echo " credential interact with credential providers" - echo " daemonlog get/set the log level for each daemon" - echo " distch path:owner:group:permisson" - echo " distributed metadata changer" - echo " distcp <srcurl> <desturl> " - echo " copy file or directories recursively" - echo " fs run a generic filesystem user client" - echo " jar <jar> run a jar file" - echo " note: please use \"yarn jar\" to launch" - echo " YARN applications, not this command." - echo " jnipath prints the java.library.path" - echo " kerbname show auth_to_local principal conversion" - echo " key manage keys via the KeyProvider" - echo " trace view and modify Hadoop tracing settings" - echo " version print the version" - echo "" - echo "Most commands print help when invoked w/o parameters." + hadoop_add_subcommand "archive" "create a Hadoop archive" + hadoop_add_subcommand "checknative" "check native Hadoop and compression libraries availability" + hadoop_add_subcommand "classpath" "prints the class path needed to get the Hadoop jar and the required libraries" + hadoop_add_subcommand "conftest" "validate configuration XML files" + hadoop_add_subcommand "credential" "interact with credential providers" + hadoop_add_subcommand "daemonlog" "get/set the log level for each daemon" + hadoop_add_subcommand "distch" "distributed metadata changer" + hadoop_add_subcommand "distcp" "copy file or directories recursively" + hadoop_add_subcommand "fs" "run a generic filesystem user client" + hadoop_add_subcommand "jar <jar>" "run a jar file. NOTE: please use \"yarn jar\" to launch YARN applications, not this command." + hadoop_add_subcommand "jnipath" "prints the java.library.path" + hadoop_add_subcommand "kerbname" "show auth_to_local principal conversion" + hadoop_add_subcommand "key" "manage keys via the KeyProvider" + hadoop_add_subcommand "trace" "view and modify Hadoop tracing settings" + hadoop_add_subcommand "version" "print the version" + hadoop_generate_usage "${MYNAME}" true } - # This script runs the hadoop core commands. # let's locate libexec... if [[ -n "${HADOOP_PREFIX}" ]]; then DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" else - this="${BASH_SOURCE-$0}" - bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi @@ -98,7 +86,7 @@ case ${COMMAND} in exit 1 fi ;; - + #mapred commands for backwards compatibility pipes|job|queue|mrgroups|mradmin|jobtracker|tasktracker) hadoop_error "WARNING: Use of this script to execute ${COMMAND} is deprecated." http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee36f4f9/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh index be65d7f..6ebbee1 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop-functions.sh @@ -14,6 +14,10 @@ # See the License for the specific language governing permissions and # limitations under the License. +# we need to declare this globally as an array, which can only +# be done outside of a function +declare -a HADOOP_USAGE=() + ## @description Print a message to stderr ## @audience public ## @stability stable @@ -36,6 +40,89 @@ function hadoop_debug fi } +## @description Add a subcommand to the usage output +## @audience private +## @stability evolving +## @replaceable no +## @param subcommand +## @param subcommanddesc +function hadoop_add_subcommand +{ + local option=$1 + local text=$2 + + HADOOP_USAGE[${HADOOP_USAGE_COUNTER}]="${option}@${text}" + ((HADOOP_USAGE_COUNTER=HADOOP_USAGE_COUNTER+1)) +} + +## @description generate standard usage output +## @description and optionally takes a class +## @audience private +## @stability evolving +## @replaceable no +## @param execname +## @param [true|false] +function hadoop_generate_usage +{ + local cmd=$1 + local takesclass=$2 + local i + local counter + local line + local option + local giventext + local maxoptsize + local foldsize=75 + declare -a tmpa + + cmd=${cmd##*/} + + echo "Usage: ${cmd} [OPTIONS] SUBCOMMAND [SUBCOMMAND OPTIONS]" + if [[ ${takesclass} = true ]]; then + echo " or ${cmd} [OPTIONS] CLASSNAME [CLASSNAME OPTIONS]" + echo " where CLASSNAME is a user-provided Java class" + fi + echo "" + echo " OPTIONS is none or any of:" + echo " --config confdir" + echo " --daemon (start|stop|status)" + echo " --debug" + echo " --hostnames list[,of,host,names]" + echo " --hosts filename" + echo " --loglevel loglevel" + echo " --slaves" + echo "" + echo " SUBCOMMAND is one of:" + + counter=0 + while read -r line; do + tmpa[${counter}]=${line} + ((counter=counter+1)) + option=$(echo "${line}" | cut -f1 -d'@') + if [[ ${#option} -gt ${maxoptsize} ]]; then + maxoptsize=${#option} + fi + done < <(for i in "${HADOOP_USAGE[@]}"; do + echo "${i}" + done | sort) + + i=0 + ((foldsize=75-maxoptsize)) + + until [[ $i -eq ${#tmpa[@]} ]]; do + option=$(echo "${tmpa[$i]}" | cut -f1 -d'@') + giventext=$(echo "${tmpa[$i]}" | cut -f2 -d'@') + + while read -r line; do + printf "%-${maxoptsize}s %-s\n" "${option}" "${line}" + option=" " + done < <(echo "${giventext}"| fold -s -w ${foldsize}) + ((i=i+1)) + done + echo "" + echo "Most subcommands print help when invoked w/o parameters or with -h." +} + ## @description Replace `oldvar` with `newvar` if `oldvar` exists. ## @audience public ## @stability stable @@ -101,6 +188,9 @@ function hadoop_bootstrap # setup a default TOOL_PATH TOOL_PATH=${TOOL_PATH:-${HADOOP_PREFIX}/share/hadoop/tools/lib/*} + # usage output set to zero + HADOOP_USAGE_COUNTER=0 + export HADOOP_OS_TYPE=${HADOOP_OS_TYPE:-$(uname -s)} # defaults http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee36f4f9/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 2e09cef..a996a80 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -15,55 +15,47 @@ # See the License for the specific language governing permissions and # limitations under the License. +MYNAME="${BASH_SOURCE-$0}" + function hadoop_usage { - echo "Usage: hdfs [--config confdir] [--daemon (start|stop|status)]" - echo " [--loglevel loglevel] COMMAND" - echo " where COMMAND is one of:" - echo " balancer run a cluster balancing utility" - echo " cacheadmin configure the HDFS cache" - echo " classpath prints the class path needed to get the" - echo " Hadoop jar and the required libraries" - echo " crypto configure HDFS encryption zones" - echo " datanode run a DFS datanode" - echo " debug run a Debug Admin to execute HDFS debug commands" - echo " dfs run a filesystem command on the file system" - echo " dfsadmin run a DFS admin client" - echo " fetchdt fetch a delegation token from the NameNode" - echo " fsck run a DFS filesystem checking utility" - echo " getconf get config values from configuration" - echo " groups get the groups which users belong to" - echo " haadmin run a DFS HA admin client" - echo " jmxget get JMX exported values from NameNode or DataNode." - echo " journalnode run the DFS journalnode" - echo " lsSnapshottableDir list all snapshottable dirs owned by the current user" - echo " Use -help to see options" - echo " mover run a utility to move block replicas across" - echo " storage types" - echo " namenode run the DFS namenode" - echo " Use -format to initialize the DFS filesystem" - echo " nfs3 run an NFS version 3 gateway" - echo " oev apply the offline edits viewer to an edits file" - echo " oiv apply the offline fsimage viewer to an fsimage" - echo " oiv_legacy apply the offline fsimage viewer to a legacy fsimage" - echo " portmap run a portmap service" - echo " secondarynamenode run the DFS secondary namenode" - echo " snapshotDiff diff two snapshots of a directory or diff the" - echo " current directory contents with a snapshot" - echo " storagepolicies list/get/set block storage policies" - echo " version print the version" - echo " zkfc run the ZK Failover Controller daemon" - echo "" - echo "Most commands print help when invoked w/o parameters." - # There are also debug commands, but they don't show up in this listing. + hadoop_add_subcommand "balancer" "run a cluster balancing utility" + hadoop_add_subcommand "cacheadmin" "configure the HDFS cache" + hadoop_add_subcommand "classpath" "prints the class path needed to get the hadoop jar and the required libraries" + hadoop_add_subcommand "crypto" "configure HDFS encryption zones" + hadoop_add_subcommand "datanode" "run a DFS datanode" + hadoop_add_subcommand "debug" "run a Debug Admin to execute HDFS debug commands" + hadoop_add_subcommand "dfs" "run a filesystem command on the file system" + hadoop_add_subcommand "dfsadmin" "run a DFS admin client" + hadoop_add_subcommand "fetchdt" "fetch a delegation token from the NameNode" + hadoop_add_subcommand "fsck" "run a DFS filesystem checking utility" + hadoop_add_subcommand "getconf" "get config values from configuration" + hadoop_add_subcommand "groups" "get the groups which users belong to" + hadoop_add_subcommand "haadmin" "run a DFS HA admin client" + hadoop_add_subcommand "jmxget" "get JMX exported values from NameNode or DataNode." + hadoop_add_subcommand "journalnode" "run the DFS journalnode" + hadoop_add_subcommand "lsSnapshottableDir" "list all snapshottable dirs owned by the current user" + hadoop_add_subcommand "mover" "run a utility to move block replicas across storage types" + hadoop_add_subcommand "namenode" "run the DFS namenode" + hadoop_add_subcommand "nfs3" "run an NFS version 3 gateway" + hadoop_add_subcommand "oev" "apply the offline edits viewer to an edits file" + hadoop_add_subcommand "oiv" "apply the offline fsimage viewer to an fsimage" + hadoop_add_subcommand "oiv_legacy" "apply the offline fsimage viewer to a legacy fsimage" + hadoop_add_subcommand "portmap" "run a portmap service" + hadoop_add_subcommand "secondarynamenode" "run the DFS secondary namenode" + hadoop_add_subcommand "snapshotDiff" "diff two snapshots of a directory or diff the current directory contents with a snapshot" + hadoop_add_subcommand "storagepolicies" "list/get/set block storage policies" + hadoop_add_subcommand "version" "print the version" + hadoop_add_subcommand "zkfc" "run the ZK Failover Controller daemon" + hadoop_generate_usage "${MYNAME}" + } # let's locate libexec... if [[ -n "${HADOOP_PREFIX}" ]]; then DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" else - this="${BASH_SOURCE-$0}" - bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi @@ -107,11 +99,11 @@ case ${COMMAND} in if [[ -n "${HADOOP_SECURE_DN_USER}" ]]; then secure_service="true" secure_user="${HADOOP_SECURE_DN_USER}" - + # backward compatiblity HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_DN_PID_DIR}" HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_DN_LOG_DIR}" - + hadoop_debug "Appending HADOOP_DATANODE_OPTS onto HADOOP_OPTS" hadoop_debug "Appending HADOOP_DN_SECURE_EXTRA_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_DATANODE_OPTS} ${HADOOP_DN_SECURE_EXTRA_OPTS}" @@ -186,11 +178,11 @@ case ${COMMAND} in if [[ -n "${HADOOP_PRIVILEGED_NFS_USER}" ]]; then secure_service="true" secure_user="${HADOOP_PRIVILEGED_NFS_USER}" - + # backward compatiblity HADOOP_SECURE_PID_DIR="${HADOOP_SECURE_PID_DIR:-$HADOOP_SECURE_NFS3_PID_DIR}" HADOOP_SECURE_LOG_DIR="${HADOOP_SECURE_LOG_DIR:-$HADOOP_SECURE_NFS3_LOG_DIR}" - + hadoop_debug "Appending HADOOP_NFS3_OPTS onto HADOOP_OPTS" hadoop_debug "Appending HADOOP_NFS3_SECURE_EXTRA_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_NFS3_OPTS} ${HADOOP_NFS3_SECURE_EXTRA_OPTS}" http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee36f4f9/hadoop-mapreduce-project/bin/mapred ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index 5afe02e..8c16369 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -15,29 +15,24 @@ # See the License for the specific language governing permissions and # limitations under the License. +MYNAME="${BASH_SOURCE-$0}" + function hadoop_usage { - echo "Usage: mapred [--config confdir] [--daemon (start|stop|status)]" - echo " [--loglevel loglevel] COMMAND" - echo " where COMMAND is one of:" - - echo " archive -archiveName NAME -p <parent path> <src>* <dest> create a hadoop archive" - echo " classpath prints the class path needed for running" - echo " mapreduce subcommands" - echo " distcp <srcurl> <desturl> copy file or directories recursively" - echo " historyserver run job history servers as a standalone daemon" - echo " hsadmin job history server admin interface" - echo " job manipulate MapReduce jobs" - echo " pipes run a Pipes job" - echo " queue get information regarding JobQueues" - echo " sampler sampler" - echo " version print the version" - echo "" - echo "Most commands print help when invoked w/o parameters." + hadoop_add_subcommand "archive" "create a hadoop archive" + hadoop_add_subcommand "classpath" "prints the class path needed for running mapreduce subcommands" + hadoop_add_subcommand "distcp" "copy file or directories recursively" + hadoop_add_subcommand "historyserver" "run job history servers as a standalone daemon" + hadoop_add_subcommand "hsadmin" "job history server admin interface" + hadoop_add_subcommand "job" "manipulate MapReduce jobs" + hadoop_add_subcommand "pipes" "run a Pipes job" + hadoop_add_subcommand "queue" "get information regarding JobQueues" + hadoop_add_subcommand "sampler" "sampler" + hadoop_add_subcommand "version" "print the version" + hadoop_generate_usage "${MYNAME}" } -this="${BASH_SOURCE-$0}" -bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) +bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) # let's locate libexec... if [[ -n "${HADOOP_PREFIX}" ]]; then http://git-wip-us.apache.org/repos/asf/hadoop/blob/ee36f4f9/hadoop-yarn-project/hadoop-yarn/bin/yarn ---------------------------------------------------------------------- diff --git a/hadoop-yarn-project/hadoop-yarn/bin/yarn b/hadoop-yarn-project/hadoop-yarn/bin/yarn old mode 100644 new mode 100755 index 12e9ef6..50607c8 --- a/hadoop-yarn-project/hadoop-yarn/bin/yarn +++ b/hadoop-yarn-project/hadoop-yarn/bin/yarn @@ -15,37 +15,30 @@ # See the License for the specific language governing permissions and # limitations under the License. +MYNAME="${BASH_SOURCE-$0}" + function hadoop_usage { - echo "Usage: yarn [--config confdir] [COMMAND | CLASSNAME]" - echo " CLASSNAME run the class named CLASSNAME" - echo " or" - echo " where COMMAND is one of:" - echo " application prints application(s) report/kill application" - echo " applicationattempt prints applicationattempt(s) report" - echo " classpath prints the class path needed to get the" - echo " Hadoop jar and the required libraries" - echo " cluster prints cluster information" - echo " container prints container(s) report" - echo " daemonlog get/set the log level for each daemon" - echo " jar <jar> run a jar file" - echo " logs dump container logs" - echo " node prints node report(s)" - echo " nodemanager run a nodemanager on each slave" - echo " proxyserver run the web app proxy server" - echo " queue prints queue information" - echo " resourcemanager run the ResourceManager" - echo " Use -format-state-store for deleting the RMStateStore." - echo " Use -remove-application-from-state-store <appId> for " - echo " removing application from RMStateStore." - echo " rmadmin admin tools" - echo " scmadmin SharedCacheManager admin tools" - echo " sharedcachemanager run the SharedCacheManager daemon" - echo " timelineserver run the timeline server" - echo " top view cluster information" - echo " version print the version" - echo "" - echo "Most commands print help when invoked w/o parameters." + hadoop_add_subcommand "application" "prints application(s) report/kill application" + hadoop_add_subcommand "applicationattempt" "prints applicationattempt(s) report" + hadoop_add_subcommand "classpath" "prints the class path needed to get the hadoop jar and the required libraries" + hadoop_add_subcommand "cluster" "prints cluster information" + hadoop_add_subcommand "container" "prints container(s) report" + hadoop_add_subcommand "daemonlog" "get/set the log level for each daemon" + hadoop_add_subcommand "jar <jar>" "run a jar file" + hadoop_add_subcommand "logs" "dump container logs" + hadoop_add_subcommand "node" "prints node report(s)" + hadoop_add_subcommand "nodemanager" "run a nodemanager on each slave" + hadoop_add_subcommand "proxyserver" "run the web app proxy server" + hadoop_add_subcommand "queue" "prints queue information" + hadoop_add_subcommand "resourcemanager" "run the ResourceManager" + hadoop_add_subcommand "rmadmin" "admin tools" + hadoop_add_subcommand "scmadmin" "SharedCacheManager admin tools" + hadoop_add_subcommand "sharedcachemanager" "run the SharedCacheManager daemon" + hadoop_add_subcommand "timelineserver" "run the timeline server" + hadoop_add_subcommand "top" "view cluster information" + hadoop_add_subcommand "version" "print the version" + hadoop_generate_usage "${MYNAME}" true } @@ -53,8 +46,7 @@ function hadoop_usage if [[ -n "${HADOOP_PREFIX}" ]]; then DEFAULT_LIBEXEC_DIR="${HADOOP_PREFIX}/libexec" else - this="${BASH_SOURCE-$0}" - bin=$(cd -P -- "$(dirname -- "${this}")" >/dev/null && pwd -P) + bin=$(cd -P -- "$(dirname -- "${MYNAME}")" >/dev/null && pwd -P) DEFAULT_LIBEXEC_DIR="${bin}/../libexec" fi
