Author: cutting Date: Tue Jun 13 14:56:36 2006 New Revision: 413984 URL: http://svn.apache.org/viewvc?rev=413984&view=rev Log: HADOOP-298. Improved progress reports for CopyFiles, the distributed file copier. Contributed by Owen.
Modified: lucene/hadoop/trunk/CHANGES.txt lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java Modified: lucene/hadoop/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/CHANGES.txt?rev=413984&r1=413983&r2=413984&view=diff ============================================================================== --- lucene/hadoop/trunk/CHANGES.txt (original) +++ lucene/hadoop/trunk/CHANGES.txt Tue Jun 13 14:56:36 2006 @@ -7,6 +7,9 @@ selector, and a thread per connection is no longer required. This should permit larger clusters. (Devaraj Das via cutting) + 2. HADOOP-298. Improved progress reports for CopyFiles utility, the + distributed file copier. (omalley via cutting) + Release 0.3.2 - 2006-06-09 Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java?rev=413984&r1=413983&r2=413984&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/CopyFiles.java Tue Jun 13 14:56:36 2006 @@ -19,9 +19,9 @@ import java.io.IOException; import java.net.URI; import java.net.URISyntaxException; +import java.text.DecimalFormat; import java.util.ArrayList; import java.util.Collections; -import java.util.Date; import java.util.Enumeration; import java.util.Iterator; import java.util.StringTokenizer; @@ -75,11 +75,13 @@ private static final long reportInterval = 1L << 25; private long bytesSinceLastReport = 0L; private long totalBytesCopied = 0L; + private static DecimalFormat percentFormat = new DecimalFormat("0.00"); private void copy(String src, Reporter reporter) throws IOException { // open source file Path srcFile = new Path(srcPath, src); FSDataInputStream in = srcFileSys.open(srcFile); + long totalBytes = srcFileSys.getLength(srcFile); // create directories to hold destination file and create destFile Path destFile = new Path(destPath, src); @@ -96,7 +98,12 @@ if (bytesSinceLastReport > reportInterval) { totalBytesCopied += bytesSinceLastReport; bytesSinceLastReport = 0L; - reporter.setStatus("Total bytes copied: "+totalBytesCopied); + reporter.setStatus("Copy "+ src + ": " + + percentFormat.format(100.0 * totalBytesCopied / + totalBytes) + + "% and " + + StringUtils.humanReadableInt(totalBytesCopied) + + " bytes"); } } @@ -105,7 +112,8 @@ // report at least once for each file totalBytesCopied += bytesSinceLastReport; bytesSinceLastReport = 0L; - reporter.setStatus("Total bytes copied: "+totalBytesCopied); + reporter.setStatus("Finished. Bytes copied: " + + StringUtils.humanReadableInt(totalBytesCopied)); } /** Mapper configuration. Modified: lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java URL: http://svn.apache.org/viewvc/lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java?rev=413984&r1=413983&r2=413984&view=diff ============================================================================== --- lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java (original) +++ lucene/hadoop/trunk/src/java/org/apache/hadoop/util/StringUtils.java Tue Jun 13 14:56:36 2006 @@ -18,6 +18,7 @@ import java.io.PrintWriter; import java.io.StringWriter; +import java.text.DecimalFormat; /** * General string utils @@ -51,4 +52,31 @@ return fullHostname; } + private static DecimalFormat numFormat = new DecimalFormat("0.0"); + + /** + * Given an integer, return a string that is in an approximate, but human + * readable format. + * It uses the bases 'k', 'm', and 'g' for 1024, 1024**2, and 1024**3. + * @param number the number to format + * @return a human readable form of the integer + */ + public static String humanReadableInt(long number) { + long absNumber = Math.abs(number); + double result = number; + String suffix = ""; + if (absNumber < 1024) { + // nothing + } else if (absNumber < 1024 * 1024) { + result = number / 1024.0; + suffix = "k"; + } else if (absNumber < 1024 * 1024 * 1024) { + result = number / (1024.0 * 1024); + suffix = "m"; + } else { + result = number / (1024.0 * 1024 * 1024); + suffix = "g"; + } + return numFormat.format(result) + suffix; + } }