HDFS-7052. Archival Storage: Add Mover into hdfs script. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2e4033e8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2e4033e8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2e4033e8 Branch: refs/heads/HDFS-6581 Commit: 2e4033e872331e08d032782ce2b39feb4a4d7e77 Parents: ece3eca Author: Jing Zhao <j...@hortonworks.com> Authored: Tue Sep 16 17:52:28 2014 -0700 Committer: Jing Zhao <j...@hortonworks.com> Committed: Tue Sep 16 17:52:28 2014 -0700 ---------------------------------------------------------------------- .../hadoop-common/src/main/conf/hadoop-env.sh | 9 ++++++ .../hadoop-hdfs/src/main/bin/hdfs | 7 +++++ .../hadoop-hdfs/src/main/bin/hdfs.cmd | 8 ++++- .../apache/hadoop/hdfs/server/mover/Mover.java | 32 ++++++++++---------- 4 files changed, 39 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e4033e8/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh ---------------------------------------------------------------------- diff --git a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh index eda47c9..29ce8b1 100644 --- a/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh +++ b/hadoop-common-project/hadoop-common/src/main/conf/hadoop-env.sh @@ -329,6 +329,15 @@ esac # export HADOOP_BALANCER_OPTS="" ### +# HDFS Mover specific parameters +### +# Specify the JVM options to be used when starting the HDFS Mover. +# These options will be appended to the options specified as HADOOP_OPTS +# and therefore may override any similar flags set in HADOOP_OPTS +# +# export HADOOP_MOVER_OPTS="" + +### # Advanced Users Only! ### http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e4033e8/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs index 2ba6d1b..22a0f0f 100755 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs @@ -36,6 +36,8 @@ function hadoop_usage echo " journalnode run the DFS journalnode" echo " lsSnapshottableDir list all snapshottable dirs owned by the current user" echo " Use -help to see options" + echo " mover run a utility to move block replicas across" + echo " storage types" echo " namenode run the DFS namenode" echo " Use -format to initialize the DFS filesystem" echo " nfs3 run an NFS version 3 gateway" @@ -159,6 +161,11 @@ case ${COMMAND} in lsSnapshottableDir) CLASS=org.apache.hadoop.hdfs.tools.snapshot.LsSnapshottableDir ;; + mover) + CLASS=org.apache.hadoop.hdfs.server.mover.Mover + hadoop_debug "Appending HADOOP_MOVER_OPTS onto HADOOP_OPTS" + HADOOP_OPTS="${HADOOP_OPTS} ${HADOOP_MOVER_OPTS}" + ;; namenode) daemon="true" CLASS='org.apache.hadoop.hdfs.server.namenode.NameNode' http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e4033e8/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd index f5f77f0..9fb8426 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs.cmd @@ -47,7 +47,7 @@ if "%1" == "--config" ( goto print_usage ) - set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin + set hdfscommands=dfs namenode secondarynamenode journalnode zkfc datanode dfsadmin haadmin fsck balancer jmxget oiv oev fetchdt getconf groups snapshotDiff lsSnapshottableDir cacheadmin mover for %%i in ( %hdfscommands% ) do ( if %hdfs-command% == %%i set hdfscommand=true ) @@ -150,6 +150,11 @@ goto :eof set CLASS=org.apache.hadoop.hdfs.tools.CacheAdmin goto :eof +:mover + set CLASS=org.apache.hadoop.hdfs.server.mover.Mover + set HADOOP_OPTS=%HADOOP_OPTS% %HADOOP_MOVER_OPTS% + goto :eof + @rem This changes %1, %2 etc. Hence those cannot be used after calling this. :make_command_arguments if "%1" == "--config" ( @@ -198,6 +203,7 @@ goto :eof @echo lsSnapshottableDir list all snapshottable dirs owned by the current user @echo Use -help to see options @echo cacheadmin configure the HDFS cache + @echo mover run a utility to move block replicas across storage types @echo. @echo Most commands print help when invoked w/o parameters. http://git-wip-us.apache.org/repos/asf/hadoop/blob/2e4033e8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java ---------------------------------------------------------------------- diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java index e336ebc..0812c03 100644 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/mover/Mover.java @@ -621,24 +621,24 @@ public class Mover { System.out.println("Mover took " + StringUtils.formatTime(Time.monotonicNow()-startTime)); } } + } - /** - * Run a Mover in command line. - * - * @param args Command line arguments - */ - public static void main(String[] args) { - if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) { - System.exit(0); - } + /** + * Run a Mover in command line. + * + * @param args Command line arguments + */ + public static void main(String[] args) { + if (DFSUtil.parseHelpArgument(args, Cli.USAGE, System.out, true)) { + System.exit(0); + } - try { - System.exit(ToolRunner.run(new HdfsConfiguration(), new Cli(), args)); - } catch (Throwable e) { - LOG.error("Exiting " + Mover.class.getSimpleName() - + " due to an exception", e); - System.exit(-1); - } + try { + System.exit(ToolRunner.run(new HdfsConfiguration(), new Cli(), args)); + } catch (Throwable e) { + LOG.error("Exiting " + Mover.class.getSimpleName() + + " due to an exception", e); + System.exit(-1); } } } \ No newline at end of file
