Author: szetszwo
Date: Mon Apr 30 23:01:07 2012
New Revision: 1332461
URL: http://svn.apache.org/viewvc?rev=1332461&view=rev
Log:
Merge r1329944 through r1332459 from trunk.
Added:
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
- copied unchanged from r1332459,
hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
Modified:
hadoop/common/branches/HDFS-3092/ (props changed)
hadoop/common/branches/HDFS-3092/BUILDING.txt
hadoop/common/branches/HDFS-3092/dev-support/test-patch.sh
hadoop/common/branches/HDFS-3092/hadoop-project/src/site/site.xml
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
Propchange: hadoop/common/branches/HDFS-3092/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk:r1329944-1332459
Modified: hadoop/common/branches/HDFS-3092/BUILDING.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/BUILDING.txt?rev=1332461&r1=1332460&r2=1332461&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/BUILDING.txt (original)
+++ hadoop/common/branches/HDFS-3092/BUILDING.txt Mon Apr 30 23:01:07 2012
@@ -87,4 +87,8 @@ Create source and binary distributions w
$ mvn package -Pdist,native,docs,src -DskipTests -Dtar
+Create a local staging version of the website (in /tmp/hadoop-site)
+
+ $ mvn clean site; mvn site:stage -DstagingDirectory=/tmp/hadoop-site
+
----------------------------------------------------------------------------------
Modified: hadoop/common/branches/HDFS-3092/dev-support/test-patch.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/dev-support/test-patch.sh?rev=1332461&r1=1332460&r2=1332461&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/dev-support/test-patch.sh (original)
+++ hadoop/common/branches/HDFS-3092/dev-support/test-patch.sh Mon Apr 30
23:01:07 2012
@@ -39,6 +39,7 @@ WGET=${WGET:-wget}
SVN=${SVN:-svn}
GREP=${GREP:-grep}
PATCH=${PATCH:-patch}
+DIFF=${DIFF:-diff}
JIRACLI=${JIRA:-jira}
FINDBUGS_HOME=${FINDBUGS_HOME}
FORREST_HOME=${FORREST_HOME}
@@ -61,6 +62,7 @@ printUsage() {
echo "--svn-cmd=<cmd> The 'svn' command to use (default 'svn')"
echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
+ echo "--diff-cmd=<cmd> The 'diff' command to use (default 'diff')"
echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME
environment variable)"
echo "--forrest-home=<path> Forrest home directory (default FORREST_HOME
environment variable)"
echo "--dirty-workspace Allow the local SVN workspace to have
uncommitted changes"
@@ -113,6 +115,9 @@ parseArgs() {
--patch-cmd=*)
PATCH=${i#*=}
;;
+ --diff-cmd=*)
+ DIFF=${i#*=}
+ ;;
--jira-cmd=*)
JIRACLI=${i#*=}
;;
@@ -430,14 +435,21 @@ checkJavacWarnings () {
fi
### Compare trunk and patch javac warning numbers
if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then
- trunkJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt
| $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
- patchJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt
| $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
+ $GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt >
$PATCH_DIR/filteredTrunkJavacWarnings.txt
+ $GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt >
$PATCH_DIR/filteredPatchJavacWarnings.txt
+ trunkJavacWarnings=`cat $PATCH_DIR/filteredTrunkJavacWarnings.txt | $AWK
'BEGIN {total = 0} {total += 1} END {print total}'`
+ patchJavacWarnings=`cat $PATCH_DIR/filteredPatchJavacWarnings.txt | $AWK
'BEGIN {total = 0} {total += 1} END {print total}'`
echo "There appear to be $trunkJavacWarnings javac compiler warnings
before the patch and $patchJavacWarnings javac compiler warnings after applying
the patch."
if [[ $patchJavacWarnings != "" && $trunkJavacWarnings != "" ]] ; then
if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
-1 javac. The applied patch generated $patchJavacWarnings javac compiler
warnings (more than the trunk's current $trunkJavacWarnings warnings)."
+
+ $DIFF $PATCH_DIR/filteredTrunkJavacWarnings.txt
$PATCH_DIR/filteredPatchJavacWarnings.txt > $PATCH_DIR/diffJavacWarnings.txt
+ JIRA_COMMENT_FOOTER="Javac warnings:
$BUILD_URL/artifact/trunk/$(basename
$BASEDIR)/patchprocess/diffJavacWarnings.txt
+$JIRA_COMMENT_FOOTER"
+
return 1
fi
fi
Modified: hadoop/common/branches/HDFS-3092/hadoop-project/src/site/site.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-project/src/site/site.xml?rev=1332461&r1=1332460&r2=1332461&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3092/hadoop-project/src/site/site.xml (original)
+++ hadoop/common/branches/HDFS-3092/hadoop-project/src/site/site.xml Mon Apr
30 23:01:07 2012
@@ -95,7 +95,7 @@
<menu name="Configuration" inherit="top">
<item name="core-default.xml"
href="hadoop-project-dist/hadoop-common/core-default.xml"/>
<item name="hdfs-default.xml"
href="hadoop-project-dist/hadoop-hdfs/hdfs-default.xml"/>
- <item name="yarn-default.xml"
href="hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml"/>
+ <item name="yarn-default.xml"
href="hadoop-yarn/hadoop-yarn-common/yarn-default.xml"/>
<item name="mapred-default.xml"
href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml"/>
<item name="Deprecated Properties"
href="hadoop-project-dist/hadoop-common/DeprecatedProperties.html"/>
</menu>
Modified:
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java?rev=1332461&r1=1332460&r2=1332461&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
(original)
+++
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
Mon Apr 30 23:01:07 2012
@@ -613,7 +613,7 @@ public class HadoopArchives implements T
destFs.delete(tmpOutput, false);
}
partStream = destFs.create(tmpOutput, false,
conf.getInt("io.file.buffer.size", 4096),
- destFs.getDefaultReplication(), blockSize);
+ destFs.getDefaultReplication(tmpOutput), blockSize);
} catch(IOException ie) {
throw new RuntimeException("Unable to open output file " + tmpOutput,
ie);
}
Modified:
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java?rev=1332461&r1=1332460&r2=1332461&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
(original)
+++
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
Mon Apr 30 23:01:07 2012
@@ -107,8 +107,8 @@ public class RetriableFileCopyCommand ex
throws IOException {
OutputStream outStream = new BufferedOutputStream(targetFS.create(
tmpTargetPath, true, BUFFER_SIZE,
- getReplicationFactor(fileAttributes, sourceFileStatus, targetFS),
- getBlockSize(fileAttributes, sourceFileStatus, targetFS),
context));
+ getReplicationFactor(fileAttributes, sourceFileStatus, targetFS,
tmpTargetPath),
+ getBlockSize(fileAttributes, sourceFileStatus, targetFS,
tmpTargetPath), context));
return copyBytes(sourceFileStatus, outStream, BUFFER_SIZE, true, context);
}
@@ -218,16 +218,16 @@ public class RetriableFileCopyCommand ex
private static short getReplicationFactor(
EnumSet<FileAttribute> fileAttributes,
- FileStatus sourceFile, FileSystem targetFS) {
+ FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
return fileAttributes.contains(FileAttribute.REPLICATION)?
- sourceFile.getReplication() : targetFS.getDefaultReplication();
+ sourceFile.getReplication() :
targetFS.getDefaultReplication(tmpTargetPath);
}
private static long getBlockSize(
EnumSet<FileAttribute> fileAttributes,
- FileStatus sourceFile, FileSystem targetFS) {
+ FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
return fileAttributes.contains(FileAttribute.BLOCKSIZE)?
- sourceFile.getBlockSize() : targetFS.getDefaultBlockSize();
+ sourceFile.getBlockSize() :
targetFS.getDefaultBlockSize(tmpTargetPath);
}
/**
Modified:
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java?rev=1332461&r1=1332460&r2=1332461&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
(original)
+++
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
Mon Apr 30 23:01:07 2012
@@ -110,9 +110,9 @@ public class TestDistCp {
fs = cluster.getFileSystem();
final Path qualifiedPath = new Path(path).makeQualified(fs.getUri(),
fs.getWorkingDirectory());
- final long blockSize = fs.getDefaultBlockSize() * 2;
+ final long blockSize = fs.getDefaultBlockSize(new Path(path)) * 2;
outputStream = fs.create(qualifiedPath, true, 0,
- (short)(fs.getDefaultReplication()*2),
+ (short)(fs.getDefaultReplication(new Path(path))*2),
blockSize);
outputStream.write(new byte[FILE_SIZE]);
pathList.add(qualifiedPath);
Modified:
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java?rev=1332461&r1=1332460&r2=1332461&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
(original)
+++
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
Mon Apr 30 23:01:07 2012
@@ -127,9 +127,9 @@ public class TestCopyMapper {
fs = cluster.getFileSystem();
final Path qualifiedPath = new Path(path).makeQualified(fs.getUri(),
fs.getWorkingDirectory());
- final long blockSize = fs.getDefaultBlockSize() * 2;
+ final long blockSize = fs.getDefaultBlockSize(qualifiedPath) * 2;
outputStream = fs.create(qualifiedPath, true, 0,
- (short)(fs.getDefaultReplication()*2),
+ (short)(fs.getDefaultReplication(qualifiedPath)*2),
blockSize);
outputStream.write(new byte[FILE_SIZE]);
pathList.add(qualifiedPath);
Modified:
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java?rev=1332461&r1=1332460&r2=1332461&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
(original)
+++
hadoop/common/branches/HDFS-3092/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
Mon Apr 30 23:01:07 2012
@@ -374,9 +374,9 @@ public class DistCp implements Tool {
FsPermission permission = preseved.contains(FileAttribute.PERMISSION)?
srcstat.getPermission(): null;
short replication = preseved.contains(FileAttribute.REPLICATION)?
- srcstat.getReplication(): destFileSys.getDefaultReplication();
+ srcstat.getReplication(): destFileSys.getDefaultReplication(f);
long blockSize = preseved.contains(FileAttribute.BLOCK_SIZE)?
- srcstat.getBlockSize(): destFileSys.getDefaultBlockSize();
+ srcstat.getBlockSize(): destFileSys.getDefaultBlockSize(f);
return destFileSys.create(f, permission, true, sizeBuf, replication,
blockSize, reporter);
}
