HADOOP-12936. modify hadoop-tools to take advantage of dynamic subcommands
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b67da297 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b67da297 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b67da297 Branch: refs/heads/HADOOP-12930 Commit: b67da297df9dcccc2422a14d53ab4fd8466b8609 Parents: 03699e0 Author: Allen Wittenauer <a...@apache.org> Authored: Wed May 4 07:21:36 2016 -0700 Committer: Allen Wittenauer <a...@apache.org> Committed: Wed May 4 20:43:21 2016 -0700 ---------------------------------------------------------------------- .../main/resources/assemblies/hadoop-tools.xml | 32 ++++++++++++++ .../hadoop-common/src/main/bin/hadoop | 45 +++++++------------- hadoop-mapreduce-project/bin/mapred | 21 --------- .../main/shellprofile.d/hadoop-archive-logs.sh | 33 ++++++++++++++ .../src/main/shellprofile.d/hadoop-archives.sh | 41 ++++++++++++++++++ .../src/main/shellprofile.d/hadoop-distcp.sh | 43 +++++++++++++++++++ .../src/main/shellprofile.d/hadoop-extras.sh | 33 ++++++++++++++ 7 files changed, 197 insertions(+), 51 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67da297/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml ---------------------------------------------------------------------- diff --git a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml index f8ba48e..c5ea6ad 100644 --- a/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml +++ b/hadoop-assemblies/src/main/resources/assemblies/hadoop-tools.xml @@ -24,6 +24,38 @@ <includeBaseDirectory>false</includeBaseDirectory> <fileSets> <fileSet> + <directory>../hadoop-archive-logs/src/main/shellprofile.d</directory> + <includes> + <include>*</include> + </includes> + <outputDirectory>/libexec/shellprofile.d</outputDirectory> + <fileMode>0755</fileMode> + </fileSet> + <fileSet> + <directory>../hadoop-archives/src/main/shellprofile.d</directory> + <includes> + <include>*</include> + </includes> + <outputDirectory>/libexec/shellprofile.d</outputDirectory> + <fileMode>0755</fileMode> + </fileSet> + <fileSet> + <directory>../hadoop-distcp/src/main/shellprofile.d</directory> + <includes> + <include>*</include> + </includes> + <outputDirectory>/libexec/shellprofile.d</outputDirectory> + <fileMode>0755</fileMode> + </fileSet> + <fileSet> + <directory>../hadoop-extras/src/main/shellprofile.d</directory> + <includes> + <include>*</include> + </includes> + <outputDirectory>/libexec/shellprofile.d</outputDirectory> + <fileMode>0755</fileMode> + </fileSet> + <fileSet> <directory>../hadoop-pipes/src/main/native/pipes/api/hadoop</directory> <includes> <include>*.hh</include> http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67da297/hadoop-common-project/hadoop-common/src/main/bin/hadoop ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/bin/hadoop b/hadoop-common-project/hadoop-common/src/main/bin/hadoop index 81bff75..bb4b041 100755 --- a/hadoop-common-project/hadoop-common/src/main/bin/hadoop +++ b/hadoop-common-project/hadoop-common/src/main/bin/hadoop @@ -30,14 +30,11 @@ function hadoop_usage hadoop_add_option "hosts filename" "list of hosts to use in slave mode" hadoop_add_option "slaves" "turn on slave mode" - hadoop_add_subcommand "archive" "create a Hadoop archive" hadoop_add_subcommand "checknative" "check native Hadoop and compression libraries availability" hadoop_add_subcommand "classpath" "prints the class path needed to get the Hadoop jar and the required libraries" hadoop_add_subcommand "conftest" "validate configuration XML files" hadoop_add_subcommand "credential" "interact with credential providers" hadoop_add_subcommand "daemonlog" "get/set the log level for each daemon" - hadoop_add_subcommand "distch" "distributed metadata changer" - hadoop_add_subcommand "distcp" "copy file or directories recursively" hadoop_add_subcommand "dtutil" "operations related to delegation tokens" hadoop_add_subcommand "envvars" "display computed Hadoop environment variables" hadoop_add_subcommand "fs" "run a generic filesystem user client" @@ -101,35 +98,23 @@ function hadoopcmd_case exit 1 fi ;; - archive) - HADOOP_CLASS=org.apache.hadoop.tools.HadoopArchives - hadoop_add_to_classpath_tools hadoop-archives - ;; checknative) - HADOOP_CLASS=org.apache.hadoop.util.NativeLibraryChecker + HADOOP_CLASSNAME=org.apache.hadoop.util.NativeLibraryChecker ;; classpath) - hadoop_do_classpath_subcommand HADOOP_CLASS "$@" + hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@" ;; conftest) - HADOOP_CLASS=org.apache.hadoop.util.ConfTest + HADOOP_CLASSNAME=org.apache.hadoop.util.ConfTest ;; credential) - HADOOP_CLASS=org.apache.hadoop.security.alias.CredentialShell + HADOOP_CLASSNAME=org.apache.hadoop.security.alias.CredentialShell ;; daemonlog) - HADOOP_CLASS=org.apache.hadoop.log.LogLevel - ;; - distch) - HADOOP_CLASS=org.apache.hadoop.tools.DistCh - hadoop_add_to_classpath_tools hadoop-extras - ;; - distcp) - HADOOP_CLASS=org.apache.hadoop.tools.DistCp - hadoop_add_to_classpath_tools hadoop-distcp + HADOOP_CLASSNAME=org.apache.hadoop.log.LogLevel ;; dtutil) - HADOOP_CLASS=org.apache.hadoop.security.token.DtUtilShell + HADOOP_CLASSNAME=org.apache.hadoop.security.token.DtUtilShell ;; envvars) echo "JAVA_HOME='${JAVA_HOME}'" @@ -144,13 +129,13 @@ function hadoopcmd_case exit 0 ;; fs) - HADOOP_CLASS=org.apache.hadoop.fs.FsShell + HADOOP_CLASSNAME=org.apache.hadoop.fs.FsShell ;; jar) if [[ -n "${YARN_OPTS}" ]] || [[ -n "${YARN_CLIENT_OPTS}" ]]; then hadoop_error "WARNING: Use \"yarn jar\" to launch YARN applications." fi - HADOOP_CLASS=org.apache.hadoop.util.RunJar + HADOOP_CLASSNAME=org.apache.hadoop.util.RunJar ;; jnipath) hadoop_finalize @@ -158,20 +143,20 @@ function hadoopcmd_case exit 0 ;; kerbname) - HADOOP_CLASS=org.apache.hadoop.security.HadoopKerberosName + HADOOP_CLASSNAME=org.apache.hadoop.security.HadoopKerberosName ;; key) - HADOOP_CLASS=org.apache.hadoop.crypto.key.KeyShell + HADOOP_CLASSNAME=org.apache.hadoop.crypto.key.KeyShell ;; trace) - HADOOP_CLASS=org.apache.hadoop.tracing.TraceAdmin + HADOOP_CLASSNAME=org.apache.hadoop.tracing.TraceAdmin ;; version) - HADOOP_CLASS=org.apache.hadoop.util.VersionInfo + HADOOP_CLASSNAME=org.apache.hadoop.util.VersionInfo ;; *) - HADOOP_CLASS="${subcmd}" - if ! hadoop_validate_classname "${HADOOP_CLASS}"; then + HADOOP_CLASSNAME="${subcmd}" + if ! hadoop_validate_classname "${HADOOP_CLASSNAME}"; then hadoop_exit_with_usage 1 fi ;; @@ -223,5 +208,5 @@ hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" hadoop_finalize -hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASS}" "$@" +hadoop_java_exec "${HADOOP_SUBCMD}" "${HADOOP_CLASSNAME}" "$@" http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67da297/hadoop-mapreduce-project/bin/mapred ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/bin/mapred b/hadoop-mapreduce-project/bin/mapred index 69b00b3..84f03e4 100755 --- a/hadoop-mapreduce-project/bin/mapred +++ b/hadoop-mapreduce-project/bin/mapred @@ -24,10 +24,7 @@ HADOOP_SHELL_EXECNAME="${MYNAME##*/}" ## @replaceable no function hadoop_usage { - hadoop_add_subcommand "archive" "create a hadoop archive" - hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives" hadoop_add_subcommand "classpath" "prints the class path needed for running mapreduce subcommands" - hadoop_add_subcommand "distcp" "copy file or directories recursively" hadoop_add_subcommand "envvars" "display computed Hadoop environment variables" hadoop_add_subcommand "historyserver" "run job history servers as a standalone daemon" hadoop_add_subcommand "hsadmin" "job history server admin interface" @@ -55,27 +52,9 @@ function mapredcmd_case hadoop_error "You may find similar functionality with the \"yarn\" shell command." hadoop_exit_with_usage 1 ;; - archive) - HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchives - hadoop_add_to_classpath_tools hadoop-archives - hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" - ;; - archive-logs) - HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs - hadoop_add_to_classpath_tools hadoop-archive-logs - hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" - ;; classpath) hadoop_do_classpath_subcommand HADOOP_CLASSNAME "$@" ;; - distcp) - HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp - hadoop_add_to_classpath_tools hadoop-distcp - hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" - HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" - ;; envvars) echo "JAVA_HOME='${JAVA_HOME}'" echo "HADOOP_MAPRED_HOME='${HADOOP_MAPRED_HOME}'" http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67da297/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh b/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh new file mode 100755 index 0000000..d37411e --- /dev/null +++ b/hadoop-tools/hadoop-archive-logs/src/main/shellprofile.d/hadoop-archive-logs.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then + hadoop_add_subcommand "archive-logs" "combine aggregated logs into hadoop archives" +fi + +## @description archive-logs command for mapred +## @audience public +## @stability stable +## @replaceable no +function mapred_subcommand_archive-logs +{ + # shellcheck disable=SC2034 + HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchiveLogs + hadoop_add_to_classpath_tools hadoop-archive-logs + hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67da297/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh b/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh new file mode 100755 index 0000000..b85ff25 --- /dev/null +++ b/hadoop-tools/hadoop-archives/src/main/shellprofile.d/hadoop-archives.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop + || "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then + hadoop_add_subcommand "archive" "create a Hadoop archive" +fi + +## @description archive command for hadoop (and mapred) +## @audience public +## @stability stable +## @replaceable no +function hadoop_subcommand_archive +{ + # shellcheck disable=SC2034 + HADOOP_CLASSNAME=org.apache.hadoop.tools.HadoopArchives + hadoop_add_to_classpath_tools hadoop-archives +} + +## @description archive-logs command for mapred (calls hadoop version) +## @audience public +## @stability stable +## @replaceable no +function mapred_subcommand_archive +{ + hadoop_subcommand_archive +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67da297/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh b/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh new file mode 100755 index 0000000..6cb13fc --- /dev/null +++ b/hadoop-tools/hadoop-distcp/src/main/shellprofile.d/hadoop-distcp.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop + || "${HADOOP_SHELL_EXECNAME}" = mapred ]]; then + hadoop_add_subcommand "distcp" "copy file or directories recursively" +fi + +## @description distcp command for hadoop +## @audience public +## @stability stable +## @replaceable no +function hadoop_subcommand_distcp +{ + # shellcheck disable=SC2034 + HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCp + hadoop_add_to_classpath_tools hadoop-distcp + hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" +} + +## @description distcp command for mapred (calls hadoop) +## @audience public +## @stability stable +## @replaceable no +function mapred_subcommand_distcp +{ + hadoop_subcommand_distcp +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/b67da297/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh b/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh new file mode 100755 index 0000000..e599334 --- /dev/null +++ b/hadoop-tools/hadoop-extras/src/main/shellprofile.d/hadoop-extras.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +if [[ "${HADOOP_SHELL_EXECNAME}" = hadoop ]]; then + hadoop_add_subcommand "distch" "distributed metadata changer" +fi + +## @description distch command for hadoop +## @audience public +## @stability stable +## @replaceable no +function hadoop_subcommand_distch +{ + # shellcheck disable=SC2034 + HADOOP_CLASSNAME=org.apache.hadoop.tools.DistCh + hadoop_add_to_classpath_tools hadoop-extras + hadoop_debug "Appending HADOOP_CLIENT_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_CLIENT_OPTS}" +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
