Author: cdouglas
Date: Mon May 11 18:50:11 2009
New Revision: 773655
URL: http://svn.apache.org/viewvc?rev=773655&view=rev
Log:
HADOOP-5675. Do not launch a job if DistCp has no work to do. Contributed by
Tsz Wo (Nicholas), SZE
Modified:
hadoop/core/trunk/CHANGES.txt
hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
Modified: hadoop/core/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/CHANGES.txt?rev=773655&r1=773654&r2=773655&view=diff
==============================================================================
--- hadoop/core/trunk/CHANGES.txt (original)
+++ hadoop/core/trunk/CHANGES.txt Mon May 11 18:50:11 2009
@@ -549,6 +549,9 @@
HADOOP-5476. Close the underlying InputStream in SequenceFile::Reader when
the constructor throws an exception. (Michael Tamm via cdouglas)
+ HADOOP-5675. Do not launch a job if DistCp has no work to do. (Tsz Wo
+ (Nicholas), SZE via cdouglas)
+
Release 0.20.1 - Unreleased
INCOMPATIBLE CHANGES
Modified: hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java
URL:
http://svn.apache.org/viewvc/hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java?rev=773655&r1=773654&r2=773655&view=diff
==============================================================================
--- hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java (original)
+++ hadoop/core/trunk/src/tools/org/apache/hadoop/tools/DistCp.java Mon May 11
18:50:11 2009
@@ -647,8 +647,9 @@
//Initialize the mapper
try {
- setup(conf, job, args);
- JobClient.runJob(job);
+ if (setup(conf, job, args)) {
+ JobClient.runJob(job);
+ }
finalize(conf, job, args.dst, args.preservedAttributes);
} finally {
//delete tmp
@@ -968,8 +969,9 @@
* @param conf : The dfs/mapred configuration.
* @param jobConf : The handle to the jobConf object to be initialized.
* @param args Arguments
+ * @return true if it is necessary to launch a job.
*/
- private static void setup(Configuration conf, JobConf jobConf,
+ private static boolean setup(Configuration conf, JobConf jobConf,
final Arguments args)
throws IOException {
jobConf.set(DST_DIR_LABEL, args.dst.toUri().toString());
@@ -1139,10 +1141,13 @@
(dstExists && !dstIsDir) || (!dstExists && srcCount == 1)?
args.dst.getParent(): args.dst, "_distcp_tmp_" + randomId);
jobConf.set(TMP_DIR_LABEL, tmpDir.toUri().toString());
- LOG.info("srcCount=" + srcCount);
+ LOG.info("sourcePathsCount=" + srcCount);
+ LOG.info("filesToCopyCount=" + fileCount);
+ LOG.info("bytesToCopyCount=" + StringUtils.humanReadableInt(byteCount));
jobConf.setInt(SRC_COUNT_LABEL, srcCount);
jobConf.setLong(TOTAL_SIZE_LABEL, byteCount);
setMapCount(byteCount, jobConf);
+ return fileCount > 0;
}
/**
