Author: todd
Date: Thu May 3 02:14:01 2012
New Revision: 1333291
URL: http://svn.apache.org/viewvc?rev=1333291&view=rev
Log:
Merge trunk into auto-HA branch
Added:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
- copied unchanged from r1333288,
hadoop/common/trunk/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpViewFs.java
Removed:
hadoop/common/branches/HDFS-3042/dev-support/test-patch.properties
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestUlimit.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/UlimitApp.java
Modified:
hadoop/common/branches/HDFS-3042/ (props changed)
hadoop/common/branches/HDFS-3042/BUILDING.txt
hadoop/common/branches/HDFS-3042/dev-support/test-patch.sh
hadoop/common/branches/HDFS-3042/hadoop-client/pom.xml
hadoop/common/branches/HDFS-3042/hadoop-project/pom.xml
hadoop/common/branches/HDFS-3042/hadoop-project/src/site/site.xml
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/mapred/tools/package-info.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/CurrentJHParser.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/package-info.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
hadoop/common/branches/HDFS-3042/pom.xml
Propchange: hadoop/common/branches/HDFS-3042/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk:r1327719-1333290
Modified: hadoop/common/branches/HDFS-3042/BUILDING.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/BUILDING.txt?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/BUILDING.txt (original)
+++ hadoop/common/branches/HDFS-3042/BUILDING.txt Thu May 3 02:14:01 2012
@@ -87,4 +87,8 @@ Create source and binary distributions w
$ mvn package -Pdist,native,docs,src -DskipTests -Dtar
+Create a local staging version of the website (in /tmp/hadoop-site)
+
+ $ mvn clean site; mvn site:stage -DstagingDirectory=/tmp/hadoop-site
+
----------------------------------------------------------------------------------
Modified: hadoop/common/branches/HDFS-3042/dev-support/test-patch.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/dev-support/test-patch.sh?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/dev-support/test-patch.sh (original)
+++ hadoop/common/branches/HDFS-3042/dev-support/test-patch.sh Thu May 3
02:14:01 2012
@@ -39,6 +39,7 @@ WGET=${WGET:-wget}
SVN=${SVN:-svn}
GREP=${GREP:-grep}
PATCH=${PATCH:-patch}
+DIFF=${DIFF:-diff}
JIRACLI=${JIRA:-jira}
FINDBUGS_HOME=${FINDBUGS_HOME}
FORREST_HOME=${FORREST_HOME}
@@ -61,6 +62,7 @@ printUsage() {
echo "--svn-cmd=<cmd> The 'svn' command to use (default 'svn')"
echo "--grep-cmd=<cmd> The 'grep' command to use (default 'grep')"
echo "--patch-cmd=<cmd> The 'patch' command to use (default 'patch')"
+ echo "--diff-cmd=<cmd> The 'diff' command to use (default 'diff')"
echo "--findbugs-home=<path> Findbugs home directory (default FINDBUGS_HOME
environment variable)"
echo "--forrest-home=<path> Forrest home directory (default FORREST_HOME
environment variable)"
echo "--dirty-workspace Allow the local SVN workspace to have
uncommitted changes"
@@ -113,6 +115,9 @@ parseArgs() {
--patch-cmd=*)
PATCH=${i#*=}
;;
+ --diff-cmd=*)
+ DIFF=${i#*=}
+ ;;
--jira-cmd=*)
JIRACLI=${i#*=}
;;
@@ -235,15 +240,6 @@ setup () {
cleanupAndExit 0
fi
fi
- . $BASEDIR/dev-support/test-patch.properties
- ### exit if warnings are NOT defined in the properties file
- if [ -z "$OK_FINDBUGS_WARNINGS" ] || [[ -z "$OK_JAVADOC_WARNINGS" ]] || [[
-z $OK_RELEASEAUDIT_WARNINGS ]]; then
- echo "Please define the following properties in test-patch.properties file"
- echo "OK_FINDBUGS_WARNINGS"
- echo "OK_RELEASEAUDIT_WARNINGS"
- echo "OK_JAVADOC_WARNINGS"
- cleanupAndExit 1
- fi
echo ""
echo ""
echo "======================================================================"
@@ -384,10 +380,10 @@ checkJavadocWarnings () {
echo ""
echo "$MVN clean test javadoc:javadoc -DskipTests -Pdocs
-D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1"
if [ -d hadoop-project ]; then
- (cd hadoop-project; $MVN install)
+ (cd hadoop-project; $MVN install > /dev/null 2>&1)
fi
if [ -d hadoop-common-project/hadoop-annotations ]; then
- (cd hadoop-common-project/hadoop-annotations; $MVN install)
+ (cd hadoop-common-project/hadoop-annotations; $MVN install > /dev/null
2>&1)
fi
$MVN clean test javadoc:javadoc -DskipTests -Pdocs
-D${PROJECT_NAME}PatchProcess > $PATCH_DIR/patchJavadocWarnings.txt 2>&1
javadocWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavadocWarnings.txt |
$AWK '/Javadoc Warnings/,EOF' | $GREP warning | $AWK 'BEGIN {total = 0} {total
+= 1} END {print total}'`
@@ -395,8 +391,10 @@ checkJavadocWarnings () {
echo ""
echo "There appear to be $javadocWarnings javadoc warnings generated by the
patched build."
+ #There are 6 warnings that are caused by things that are caused by using sun
internal APIs.
+ OK_JAVADOC_WARNINGS=6;
### if current warnings greater than OK_JAVADOC_WARNINGS
- if [[ $javadocWarnings -gt $OK_JAVADOC_WARNINGS ]] ; then
+ if [[ $javadocWarnings -ne $OK_JAVADOC_WARNINGS ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
-1 javadoc. The javadoc tool appears to have generated `expr
$(($javadocWarnings-$OK_JAVADOC_WARNINGS))` warning messages."
@@ -430,14 +428,21 @@ checkJavacWarnings () {
fi
### Compare trunk and patch javac warning numbers
if [[ -f $PATCH_DIR/patchJavacWarnings.txt ]] ; then
- trunkJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt
| $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
- patchJavacWarnings=`$GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt
| $AWK 'BEGIN {total = 0} {total += 1} END {print total}'`
+ $GREP '\[WARNING\]' $PATCH_DIR/trunkJavacWarnings.txt >
$PATCH_DIR/filteredTrunkJavacWarnings.txt
+ $GREP '\[WARNING\]' $PATCH_DIR/patchJavacWarnings.txt >
$PATCH_DIR/filteredPatchJavacWarnings.txt
+ trunkJavacWarnings=`cat $PATCH_DIR/filteredTrunkJavacWarnings.txt | $AWK
'BEGIN {total = 0} {total += 1} END {print total}'`
+ patchJavacWarnings=`cat $PATCH_DIR/filteredPatchJavacWarnings.txt | $AWK
'BEGIN {total = 0} {total += 1} END {print total}'`
echo "There appear to be $trunkJavacWarnings javac compiler warnings
before the patch and $patchJavacWarnings javac compiler warnings after applying
the patch."
if [[ $patchJavacWarnings != "" && $trunkJavacWarnings != "" ]] ; then
if [[ $patchJavacWarnings -gt $trunkJavacWarnings ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
-1 javac. The applied patch generated $patchJavacWarnings javac compiler
warnings (more than the trunk's current $trunkJavacWarnings warnings)."
+
+ $DIFF $PATCH_DIR/filteredTrunkJavacWarnings.txt
$PATCH_DIR/filteredPatchJavacWarnings.txt > $PATCH_DIR/diffJavacWarnings.txt
+ JIRA_COMMENT_FOOTER="Javac warnings:
$BUILD_URL/artifact/trunk/$(basename
$BASEDIR)/patchprocess/diffJavacWarnings.txt
+$JIRA_COMMENT_FOOTER"
+
return 1
fi
fi
@@ -460,8 +465,8 @@ checkReleaseAuditWarnings () {
echo "======================================================================"
echo ""
echo ""
- echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1"
- $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess 2>&1
+ echo "$MVN apache-rat:check -D${PROJECT_NAME}PatchProcess >
$PATCH_DIR/patchReleaseAuditOutput.txt 2>&1"
+ $MVN apache-rat:check -D${PROJECT_NAME}PatchProcess >
$PATCH_DIR/patchReleaseAuditOutput.txt 2>&1
find $BASEDIR -name rat.txt | xargs cat >
$PATCH_DIR/patchReleaseAuditWarnings.txt
### Compare trunk and patch release audit warning numbers
@@ -469,12 +474,12 @@ checkReleaseAuditWarnings () {
patchReleaseAuditWarnings=`$GREP -c '\!?????'
$PATCH_DIR/patchReleaseAuditWarnings.txt`
echo ""
echo ""
- echo "There appear to be $OK_RELEASEAUDIT_WARNINGS release audit warnings
before the patch and $patchReleaseAuditWarnings release audit warnings after
applying the patch."
- if [[ $patchReleaseAuditWarnings != "" && $OK_RELEASEAUDIT_WARNINGS != ""
]] ; then
- if [[ $patchReleaseAuditWarnings -gt $OK_RELEASEAUDIT_WARNINGS ]] ; then
+ echo "There appear to be $patchReleaseAuditWarnings release audit warnings
after applying the patch."
+ if [[ $patchReleaseAuditWarnings != "" ]] ; then
+ if [[ $patchReleaseAuditWarnings -gt 0 ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
- -1 release audit. The applied patch generated $patchReleaseAuditWarnings
release audit warnings (more than the trunk's current $OK_RELEASEAUDIT_WARNINGS
warnings)."
+ -1 release audit. The applied patch generated $patchReleaseAuditWarnings
release audit warnings."
$GREP '\!?????' $PATCH_DIR/patchReleaseAuditWarnings.txt >
$PATCH_DIR/patchReleaseAuditProblems.txt
echo "Lines that start with ????? in the release audit report indicate
files that do not have an Apache license header." >>
$PATCH_DIR/patchReleaseAuditProblems.txt
JIRA_COMMENT_FOOTER="Release audit warnings:
$BUILD_URL/artifact/trunk/patchprocess/patchReleaseAuditProblems.txt
@@ -536,10 +541,21 @@ checkFindbugsWarnings () {
echo "======================================================================"
echo ""
echo ""
- echo "$MVN clean test findbugs:findbugs -DskipTests
-D${PROJECT_NAME}PatchProcess"
- $MVN clean test findbugs:findbugs -DskipTests -D${PROJECT_NAME}PatchProcess
< /dev/null
+
+ modules=$(findModules)
+ rc=0
+ for module in $modules;
+ do
+ cd $module
+ echo " Running findbugs in $module"
+ module_suffix=`basename ${module}`
+ echo "$MVN clean test findbugs:findbugs -DskipTests
-D${PROJECT_NAME}PatchProcess < /dev/null >
$PATCH_DIR/patchFindBugsOutput${module_suffix}.txt 2>&1"
+ $MVN clean test findbugs:findbugs -DskipTests
-D${PROJECT_NAME}PatchProcess < /dev/null >
$PATCH_DIR/patchFindBugsOutput${module_suffix}.txt 2>&1
+ (( rc = rc + $? ))
+ cd -
+ done
- if [ $? != 0 ] ; then
+ if [ $rc != 0 ] ; then
JIRA_COMMENT="$JIRA_COMMENT
-1 findbugs. The patch appears to cause Findbugs (version
${findbugs_version}) to fail."
@@ -572,11 +588,10 @@ $JIRA_COMMENT_FOOTER"
fi
done
- ### if current warnings greater than OK_FINDBUGS_WARNINGS
- if [[ $findbugsWarnings -gt $OK_FINDBUGS_WARNINGS ]] ; then
+ if [[ $findbugsWarnings -gt 0 ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
- -1 findbugs. The patch appears to introduce `expr
$(($findbugsWarnings-$OK_FINDBUGS_WARNINGS))` new Findbugs (version
${findbugs_version}) warnings."
+ -1 findbugs. The patch appears to introduce $findbugsWarnings new
Findbugs (version ${findbugs_version}) warnings."
return 1
fi
JIRA_COMMENT="$JIRA_COMMENT
@@ -598,8 +613,8 @@ checkEclipseGeneration () {
echo ""
echo ""
- echo "$MVN eclipse:eclipse -D${PROJECT_NAME}PatchProcess"
- $MVN eclipse:eclipse -D${PROJECT_NAME}PatchProcess
+ echo "$MVN eclipse:eclipse -D${PROJECT_NAME}PatchProcess >
$PATCH_DIR/patchEclipseOutput.txt 2>&1"
+ $MVN eclipse:eclipse -D${PROJECT_NAME}PatchProcess >
$PATCH_DIR/patchEclipseOutput.txt 2>&1
if [[ $? != 0 ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
@@ -627,16 +642,28 @@ runTests () {
echo ""
echo ""
- echo "$MVN clean install -fn -Pnative -D${PROJECT_NAME}PatchProcess"
- $MVN clean install -fn -Pnative -D${PROJECT_NAME}PatchProcess
- failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E
"<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|
|g" | sed -e "s|\.xml||g"`
- # With -fn mvn always exits with a 0 exit code. Because of this we need to
- # find the errors instead of using the exit code. We assume that if the
build
- # failed a -1 is already given for that case
+ failed_tests=""
+ modules=$(findModules)
+ for module in $modules;
+ do
+ cd $module
+ echo " Running tests in $module"
+ echo " $MVN clean install -fn -Pnative -D${PROJECT_NAME}PatchProcess"
+ $MVN clean install -fn -Pnative -D${PROJECT_NAME}PatchProcess
+ module_failed_tests=`find . -name 'TEST*.xml' | xargs $GREP -l -E
"<failure|<error" | sed -e "s|.*target/surefire-reports/TEST-|
|g" | sed -e "s|\.xml||g"`
+ # With -fn mvn always exits with a 0 exit code. Because of this we need to
+ # find the errors instead of using the exit code. We assume that if the
build
+ # failed a -1 is already given for that case
+ if [[ -n "$module_failed_tests" ]] ; then
+ failed_tests="${failed_tests}
+${module_failed_tests}"
+ fi
+ cd -
+ done
if [[ -n "$failed_tests" ]] ; then
JIRA_COMMENT="$JIRA_COMMENT
- -1 core tests. The patch failed these unit tests:
+ -1 core tests. The patch failed these unit tests in $modules:
$failed_tests"
return 1
fi
@@ -647,6 +674,51 @@ $failed_tests"
}
###############################################################################
+# Find the maven module containing the given file.
+findModule (){
+ dir=`dirname $1`
+ while [ 1 ]
+ do
+ if [ -f "$dir/pom.xml" ]
+ then
+ echo $dir
+ return
+ else
+ dir=`dirname $dir`
+ fi
+ done
+}
+
+findModules () {
+ # Come up with a list of changed files into $TMP
+ TMP=/tmp/tmp.paths.$$
+ $GREP '^+++\|^---' $PATCH_DIR/patch | cut -c '5-' | $GREP -v /dev/null |
sort | uniq > $TMP
+
+ # if all of the lines start with a/ or b/, then this is a git patch that
+ # was generated without --no-prefix
+ if ! $GREP -qv '^a/\|^b/' $TMP ; then
+ sed -i -e 's,^[ab]/,,' $TMP
+ fi
+
+ # Now find all the modules that were changed
+ TMP_MODULES=/tmp/tmp.modules.$$
+ for file in $(cut -f 1 $TMP | sort | uniq); do
+ echo $(findModule $file) >> $TMP_MODULES
+ done
+ rm $TMP
+
+ # Filter out modules without code
+ CHANGED_MODULES=""
+ for module in $(cat $TMP_MODULES | sort | uniq); do
+ $GREP "<packaging>pom</packaging>" $module/pom.xml > /dev/null
+ if [ "$?" != 0 ]; then
+ CHANGED_MODULES="$CHANGED_MODULES $module"
+ fi
+ done
+ rm $TMP_MODULES
+ echo $CHANGED_MODULES
+}
+###############################################################################
### Run the test-contrib target
runContribTests () {
echo ""
Modified: hadoop/common/branches/HDFS-3042/hadoop-client/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-client/pom.xml?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-client/pom.xml (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-client/pom.xml Thu May 3 02:14:01
2012
@@ -172,10 +172,6 @@
<artifactId>junit</artifactId>
</exclusion>
<exclusion>
- <groupId>com.cenqua.clover</groupId>
- <artifactId>clover</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.apache.avro</groupId>
<artifactId>avro</artifactId>
</exclusion>
@@ -212,10 +208,6 @@
<artifactId>jersey-guice</artifactId>
</exclusion>
<exclusion>
- <groupId>com.cenqua.clover</groupId>
- <artifactId>clover</artifactId>
- </exclusion>
- <exclusion>
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
</exclusion>
@@ -263,10 +255,6 @@
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
</exclusion>
- <exclusion>
- <groupId>com.cenqua.clover</groupId>
- <artifactId>clover</artifactId>
- </exclusion>
</exclusions>
</dependency>
@@ -291,10 +279,6 @@
<groupId>com.google.inject.extensions</groupId>
<artifactId>guice-servlet</artifactId>
</exclusion>
- <exclusion>
- <groupId>com.cenqua.clover</groupId>
- <artifactId>clover</artifactId>
- </exclusion>
</exclusions>
</dependency>
</dependencies>
Modified: hadoop/common/branches/HDFS-3042/hadoop-project/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-project/pom.xml?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-project/pom.xml (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-project/pom.xml Thu May 3 02:14:01
2012
@@ -604,12 +604,6 @@
</dependency>
<dependency>
- <groupId>com.cenqua.clover</groupId>
- <artifactId>clover</artifactId>
- <version>3.0.2</version>
- </dependency>
-
- <dependency>
<groupId>org.apache.zookeeper</groupId>
<artifactId>zookeeper</artifactId>
<version>3.4.2</version>
@@ -687,11 +681,6 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-assembly-plugin</artifactId>
- <version>2.2.1</version>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.8.1</version>
</plugin>
@@ -902,6 +891,7 @@
<groupId>com.atlassian.maven.plugins</groupId>
<artifactId>maven-clover2-plugin</artifactId>
<configuration>
+ <includesAllSourceRoots>true</includesAllSourceRoots>
<includesTestSourceRoots>true</includesTestSourceRoots>
<licenseLocation>${cloverLicenseLocation}</licenseLocation>
<cloverDatabase>${cloverDatabase}</cloverDatabase>
@@ -912,8 +902,8 @@
</configuration>
<executions>
<execution>
- <id>setup</id>
- <phase>generate-sources</phase>
+ <id>clover-setup</id>
+ <phase>process-sources</phase>
<goals>
<goal>setup</goal>
</goals>
Modified: hadoop/common/branches/HDFS-3042/hadoop-project/src/site/site.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-project/src/site/site.xml?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/hadoop-project/src/site/site.xml (original)
+++ hadoop/common/branches/HDFS-3042/hadoop-project/src/site/site.xml Thu May
3 02:14:01 2012
@@ -95,7 +95,7 @@
<menu name="Configuration" inherit="top">
<item name="core-default.xml"
href="hadoop-project-dist/hadoop-common/core-default.xml"/>
<item name="hdfs-default.xml"
href="hadoop-project-dist/hadoop-hdfs/hdfs-default.xml"/>
- <item name="yarn-default.xml"
href="hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml"/>
+ <item name="yarn-default.xml"
href="hadoop-yarn/hadoop-yarn-common/yarn-default.xml"/>
<item name="mapred-default.xml"
href="hadoop-mapreduce-client/hadoop-mapreduce-client-core/mapred-default.xml"/>
<item name="Deprecated Properties"
href="hadoop-project-dist/hadoop-common/DeprecatedProperties.html"/>
</menu>
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-archives/src/main/java/org/apache/hadoop/tools/HadoopArchives.java
Thu May 3 02:14:01 2012
@@ -613,7 +613,7 @@ public class HadoopArchives implements T
destFs.delete(tmpOutput, false);
}
partStream = destFs.create(tmpOutput, false,
conf.getInt("io.file.buffer.size", 4096),
- destFs.getDefaultReplication(), blockSize);
+ destFs.getDefaultReplication(tmpOutput), blockSize);
} catch(IOException ie) {
throw new RuntimeException("Unable to open output file " + tmpOutput,
ie);
}
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/mapred/RetriableFileCopyCommand.java
Thu May 3 02:14:01 2012
@@ -107,8 +107,8 @@ public class RetriableFileCopyCommand ex
throws IOException {
OutputStream outStream = new BufferedOutputStream(targetFS.create(
tmpTargetPath, true, BUFFER_SIZE,
- getReplicationFactor(fileAttributes, sourceFileStatus, targetFS),
- getBlockSize(fileAttributes, sourceFileStatus, targetFS),
context));
+ getReplicationFactor(fileAttributes, sourceFileStatus, targetFS,
tmpTargetPath),
+ getBlockSize(fileAttributes, sourceFileStatus, targetFS,
tmpTargetPath), context));
return copyBytes(sourceFileStatus, outStream, BUFFER_SIZE, true, context);
}
@@ -218,16 +218,16 @@ public class RetriableFileCopyCommand ex
private static short getReplicationFactor(
EnumSet<FileAttribute> fileAttributes,
- FileStatus sourceFile, FileSystem targetFS) {
+ FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
return fileAttributes.contains(FileAttribute.REPLICATION)?
- sourceFile.getReplication() : targetFS.getDefaultReplication();
+ sourceFile.getReplication() :
targetFS.getDefaultReplication(tmpTargetPath);
}
private static long getBlockSize(
EnumSet<FileAttribute> fileAttributes,
- FileStatus sourceFile, FileSystem targetFS) {
+ FileStatus sourceFile, FileSystem targetFS, Path tmpTargetPath) {
return fileAttributes.contains(FileAttribute.BLOCKSIZE)?
- sourceFile.getBlockSize() : targetFS.getDefaultBlockSize();
+ sourceFile.getBlockSize() :
targetFS.getDefaultBlockSize(tmpTargetPath);
}
/**
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCp.java
Thu May 3 02:14:01 2012
@@ -110,9 +110,9 @@ public class TestDistCp {
fs = cluster.getFileSystem();
final Path qualifiedPath = new Path(path).makeQualified(fs.getUri(),
fs.getWorkingDirectory());
- final long blockSize = fs.getDefaultBlockSize() * 2;
+ final long blockSize = fs.getDefaultBlockSize(new Path(path)) * 2;
outputStream = fs.create(qualifiedPath, true, 0,
- (short)(fs.getDefaultReplication()*2),
+ (short)(fs.getDefaultReplication(new Path(path))*2),
blockSize);
outputStream.write(new byte[FILE_SIZE]);
pathList.add(qualifiedPath);
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/mapred/TestCopyMapper.java
Thu May 3 02:14:01 2012
@@ -127,9 +127,9 @@ public class TestCopyMapper {
fs = cluster.getFileSystem();
final Path qualifiedPath = new Path(path).makeQualified(fs.getUri(),
fs.getWorkingDirectory());
- final long blockSize = fs.getDefaultBlockSize() * 2;
+ final long blockSize = fs.getDefaultBlockSize(qualifiedPath) * 2;
outputStream = fs.create(qualifiedPath, true, 0,
- (short)(fs.getDefaultReplication()*2),
+ (short)(fs.getDefaultReplication(qualifiedPath)*2),
blockSize);
outputStream.write(new byte[FILE_SIZE]);
pathList.add(qualifiedPath);
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/mapred/tools/package-info.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/mapred/tools/package-info.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/mapred/tools/package-info.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/mapred/tools/package-info.java
Thu May 3 02:14:01 2012
@@ -17,7 +17,6 @@
*/
/**
- * Command-line tools associated with the {@link org.apache.hadoop.mapred}
- * package.
+ * Command-line tools associated with MapReduce.
*/
package org.apache.hadoop.mapred.tools;
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-extras/src/main/java/org/apache/hadoop/tools/DistCp.java
Thu May 3 02:14:01 2012
@@ -374,9 +374,9 @@ public class DistCp implements Tool {
FsPermission permission = preseved.contains(FileAttribute.PERMISSION)?
srcstat.getPermission(): null;
short replication = preseved.contains(FileAttribute.REPLICATION)?
- srcstat.getReplication(): destFileSys.getDefaultReplication();
+ srcstat.getReplication(): destFileSys.getDefaultReplication(f);
long blockSize = preseved.contains(FileAttribute.BLOCK_SIZE)?
- srcstat.getBlockSize(): destFileSys.getDefaultBlockSize();
+ srcstat.getBlockSize(): destFileSys.getDefaultBlockSize(f);
return destFileSys.create(f, permission, true, sizeBuf, replication,
blockSize, reporter);
}
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/CurrentJHParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/CurrentJHParser.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/CurrentJHParser.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/CurrentJHParser.java
Thu May 3 02:14:01 2012
@@ -26,9 +26,7 @@ import org.apache.hadoop.mapreduce.jobhi
import org.apache.hadoop.mapreduce.v2.hs.JobHistory;
/**
- * {@link JobHistoryParser} that parses {@link JobHistory} files produced by
- * {@link org.apache.hadoop.mapreduce.jobhistory.JobHistory} in the same source
- * code tree as rumen.
+ * {@link JobHistoryParser} that parses JobHistory files.
*/
public class CurrentJHParser implements JobHistoryParser {
private EventReader reader;
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/LoggedTaskAttempt.java
Thu May 3 02:14:01 2012
@@ -170,7 +170,7 @@ public class LoggedTaskAttempt implement
/**
*
- * @returns a list of all splits vectors, ordered in enumeral order
+ * @return a list of all splits vectors, ordered in enumeral order
* within {@link SplitVectorKind} . Do NOT use hard-coded
* indices within the return for this with a hard-coded
* index to get individual values; use
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/datatypes/util/MapReduceJobPropertiesParser.java
Thu May 3 02:14:01 2012
@@ -43,7 +43,7 @@ import org.apache.log4j.Logger;
* across versions. {@link MapReduceJobPropertiesParser} is a utility class
that
* parses MapReduce job configuration properties and converts the value into a
* well defined {@link DataType}. Users can use the
- * {@link MapReduceJobPropertiesParser#parseJobProperty()} API to process job
+ * {@link #parseJobProperty(String, String)} API to process job
* configuration parameters. This API will parse a job property represented as
a
* key-value pair and return the value wrapped inside a {@link DataType}.
* Callers can then use the returned {@link DataType} for further processing.
@@ -61,12 +61,12 @@ import org.apache.log4j.Logger;
* {@link DefaultDataType}. Currently only '-Xmx' and '-Xms' settings are
* considered while the rest are ignored.
*
- * Note that the {@link MapReduceJobPropertiesParser#parseJobProperty()} API
+ * Note that the {@link #parseJobProperty(String, String)} API
* maps the keys to a configuration parameter listed in
* {@link MRJobConfig}. This not only filters non-framework specific keys thus
* ignoring user-specific and hard-to-parse keys but also provides a consistent
* view for all possible inputs. So if users invoke the
- * {@link MapReduceJobPropertiesParser#parseJobProperty()} API with either
+ * {@link #parseJobProperty(String, String)} API with either
* <"mapreduce.job.user.name", "bob"> or <"user.name", "bob">, then the result
* would be a {@link UserName} {@link DataType} wrapping the user-name "bob".
*/
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/package-info.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/package-info.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/package-info.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-rumen/src/main/java/org/apache/hadoop/tools/rumen/package-info.java
Thu May 3 02:14:01 2012
@@ -181,8 +181,8 @@
* <li>
* {@link org.apache.hadoop.tools.rumen.JobBuilder}<br>
* Summarizes a job history file.
- * {@link org.apache.hadoop.tools.rumen.TraceBuilder} provides
- * {@link
org.apache.hadoop.tools.rumen.TraceBuilder#extractJobID(String)}
+ * {@link org.apache.hadoop.tools.rumen.JobHistoryUtils} provides
+ * {@link
org.apache.hadoop.tools.rumen.JobHistoryUtils#extractJobID(String)}
* API for extracting job id from job history or job configuration files
* which can be used for instantiating {@link
org.apache.hadoop.tools.rumen.JobBuilder}.
* {@link org.apache.hadoop.tools.rumen.JobBuilder} generates a
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestFileArgs.java
Thu May 3 02:14:01 2012
@@ -20,12 +20,13 @@ package org.apache.hadoop.streaming;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.util.Map;
+
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.MiniMRCluster;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.junit.After;
import org.junit.Before;
@@ -38,8 +39,6 @@ public class TestFileArgs extends TestSt
private MiniDFSCluster dfs = null;
private MiniMRCluster mr = null;
private FileSystem fileSys = null;
- private String strJobTracker = null;
- private String strNamenode = null;
private String namenode = null;
private Configuration conf = null;
@@ -56,8 +55,6 @@ public class TestFileArgs extends TestSt
fileSys = dfs.getFileSystem();
namenode = fileSys.getUri().getAuthority();
mr = new MiniMRCluster(1, namenode, 1);
- strJobTracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" +
mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS);
- strNamenode = "fs.default.name=" +
mr.createJobConf().get("fs.default.name");
map = LS_PATH;
FileSystem.setDefaultUri(conf, "hdfs://" + namenode);
@@ -100,18 +97,16 @@ public class TestFileArgs extends TestSt
@Override
protected String[] genArgs() {
+ for (Map.Entry<String, String> entry : mr.createJobConf()) {
+ args.add("-jobconf");
+ args.add(entry.getKey() + "=" + entry.getValue());
+ }
args.add("-file");
args.add(new java.io.File("target/sidefile").getAbsolutePath());
args.add("-numReduceTasks");
args.add("0");
args.add("-jobconf");
- args.add(strNamenode);
- args.add("-jobconf");
- args.add(strJobTracker);
- args.add("-jobconf");
args.add("mapred.jar=" + STREAMING_JAR);
- args.add("-jobconf");
- args.add("mapreduce.framework.name=yarn");
args.add("-verbose");
return super.genArgs();
}
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleArchiveFiles.java
Thu May 3 02:14:01 2012
@@ -19,14 +19,10 @@
package org.apache.hadoop.streaming;
import java.io.File;
-import java.io.FileOutputStream;
import java.io.IOException;
import java.io.DataOutputStream;
-import java.io.InputStreamReader;
-import java.io.BufferedReader;
-import java.util.Arrays;
+import java.util.Map;
import java.util.zip.ZipEntry;
-import java.util.jar.JarOutputStream;
import java.util.zip.ZipOutputStream;
import org.apache.commons.logging.Log;
@@ -37,12 +33,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.mapred.*;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.apache.hadoop.util.StringUtils;
-
-import org.junit.Test;
-import static org.junit.Assert.*;
/**
* This class tests cacheArchive option of streaming
@@ -66,8 +57,6 @@ public class TestMultipleArchiveFiles ex
private MiniDFSCluster dfs = null;
private MiniMRCluster mr = null;
private FileSystem fileSys = null;
- private String strJobTracker = null;
- private String strNamenode = null;
private String namenode = null;
public TestMultipleArchiveFiles() throws Exception {
@@ -80,8 +69,6 @@ public class TestMultipleArchiveFiles ex
fileSys = dfs.getFileSystem();
namenode = fileSys.getUri().getAuthority();
mr = new MiniMRCluster(1, namenode, 1);
- strJobTracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" +
mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS);
- strNamenode = "fs.default.name=" +
mr.createJobConf().get("fs.default.name");
map = "xargs cat";
reduce = "cat";
@@ -123,6 +110,10 @@ public class TestMultipleArchiveFiles ex
String cache1 = workDir + CACHE_ARCHIVE_1 + "#symlink1";
String cache2 = workDir + CACHE_ARCHIVE_2 + "#symlink2";
+ for (Map.Entry<String, String> entry : mr.createJobConf()) {
+ args.add("-jobconf");
+ args.add(entry.getKey() + "=" + entry.getValue());
+ }
args.add("-jobconf");
args.add("mapreduce.job.reduces=1");
args.add("-cacheArchive");
@@ -130,13 +121,7 @@ public class TestMultipleArchiveFiles ex
args.add("-cacheArchive");
args.add(cache2);
args.add("-jobconf");
- args.add(strNamenode);
- args.add("-jobconf");
- args.add(strJobTracker);
- args.add("-jobconf");
args.add("mapred.jar=" + STREAMING_JAR);
- args.add("-jobconf");
- args.add("mapreduce.framework.name=yarn");
return super.genArgs();
}
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestMultipleCachefiles.java
Thu May 3 02:14:01 2012
@@ -22,8 +22,9 @@ import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
-import java.io.PrintWriter;
-import java.io.StringWriter;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -36,7 +37,7 @@ import org.apache.hadoop.hdfs.MiniDFSClu
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.MiniMRCluster;
import org.apache.hadoop.mapred.Utils;
-import org.apache.hadoop.mapreduce.server.jobtracker.JTConfig;
+
/**
* This test case tests the symlink creation
* utility provided by distributed caching
@@ -73,15 +74,18 @@ public class TestMultipleCachefiles
String namenode = fileSys.getUri().toString();
mr = new MiniMRCluster(1, namenode, 3);
- String strJobtracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" +
mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS);
- String strNamenode = "fs.default.name=" +
mr.createJobConf().get("fs.default.name");
+
+ List<String> args = new ArrayList<String>();
+ for (Map.Entry<String, String> entry : mr.createJobConf()) {
+ args.add("-jobconf");
+ args.add(entry.getKey() + "=" + entry.getValue());
+ }
+
String argv[] = new String[] {
"-input", INPUT_FILE,
"-output", OUTPUT_DIR,
"-mapper", map,
"-reducer", reduce,
- "-jobconf", strNamenode,
- "-jobconf", strJobtracker,
"-jobconf",
"stream.tmpdir="+System.getProperty("test.build.data","/tmp"),
"-jobconf",
JobConf.MAPRED_MAP_TASK_JAVA_OPTS + "=" +
@@ -98,9 +102,13 @@ public class TestMultipleCachefiles
"-cacheFile", fileSys.getUri() + CACHE_FILE + "#" + mapString,
"-cacheFile", fileSys.getUri() + CACHE_FILE_2 + "#" + mapString2,
"-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR,
- "-jobconf", "mapreduce.framework.name=yarn"
};
+ for (String arg : argv) {
+ args.add(arg);
+ }
+ argv = args.toArray(new String[args.size()]);
+
fileSys.delete(new Path(OUTPUT_DIR), true);
DataOutputStream file = fileSys.create(new Path(INPUT_FILE));
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestStreamingTaskLog.java
Thu May 3 02:14:01 2012
@@ -19,6 +19,9 @@
package org.apache.hadoop.streaming;
import java.io.*;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -47,20 +50,30 @@ public class TestStreamingTaskLog {
final long USERLOG_LIMIT_KB = 5;//consider 5kb as logSize
String[] genArgs() {
- return new String[] {
+
+ List<String> args = new ArrayList<String>();
+ for (Map.Entry<String, String> entry : mr.createJobConf()) {
+ args.add("-jobconf");
+ args.add(entry.getKey() + "=" + entry.getValue());
+ }
+
+ String[] argv = new String[] {
"-input", inputPath.toString(),
"-output", outputPath.toString(),
"-mapper", map,
"-reducer", StreamJob.REDUCE_NONE,
- "-jobconf", "mapred.job.tracker=" +
mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS),
- "-jobconf", "fs.default.name=" + fs.getUri().toString(),
"-jobconf", "mapred.map.tasks=1",
"-jobconf", "keep.failed.task.files=true",
"-jobconf", "mapreduce.task.userlog.limit.kb=" + USERLOG_LIMIT_KB,
"-jobconf",
"stream.tmpdir="+System.getProperty("test.build.data","/tmp"),
"-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR,
- "-jobconf", "mapreduce.framework.name=yarn"
};
+
+ for (String arg : argv) {
+ args.add(arg);
+ }
+ argv = args.toArray(new String[args.size()]);
+ return argv;
}
/**
Modified:
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
---
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
(original)
+++
hadoop/common/branches/HDFS-3042/hadoop-tools/hadoop-streaming/src/test/java/org/apache/hadoop/streaming/TestSymLink.java
Thu May 3 02:14:01 2012
@@ -21,6 +21,9 @@ package org.apache.hadoop.streaming;
import java.io.BufferedReader;
import java.io.DataOutputStream;
import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
import org.junit.Test;
import static org.junit.Assert.*;
@@ -62,17 +65,20 @@ public class TestSymLink
FileSystem fileSys = dfs.getFileSystem();
String namenode = fileSys.getUri().toString();
mr = new MiniMRCluster(1, namenode, 3);
+
+ List<String> args = new ArrayList<String>();
+ for (Map.Entry<String, String> entry : mr.createJobConf()) {
+ args.add("-jobconf");
+ args.add(entry.getKey() + "=" + entry.getValue());
+ }
+
// During tests, the default Configuration will use a local mapred
// So don't specify -config or -cluster
- String strJobtracker = JTConfig.JT_IPC_ADDRESS + "=localhost:" +
mr.createJobConf().get(JTConfig.JT_IPC_ADDRESS);
- String strNamenode = "fs.default.name=" +
mr.createJobConf().get("fs.default.name");
String argv[] = new String[] {
"-input", INPUT_FILE,
"-output", OUTPUT_DIR,
"-mapper", map,
"-reducer", reduce,
- "-jobconf", strNamenode,
- "-jobconf", strJobtracker,
"-jobconf",
"stream.tmpdir="+System.getProperty("test.build.data","/tmp"),
"-jobconf",
JobConf.MAPRED_MAP_TASK_JAVA_OPTS+ "=" +
@@ -88,9 +94,13 @@ public class TestSymLink
conf.get(JobConf.MAPRED_TASK_JAVA_OPTS, "")),
"-cacheFile", fileSys.getUri() + CACHE_FILE + "#testlink",
"-jobconf", "mapred.jar=" + TestStreaming.STREAMING_JAR,
- "-jobconf", "mapreduce.framework.name=yarn"
};
+ for (String arg : argv) {
+ args.add(arg);
+ }
+ argv = args.toArray(new String[args.size()]);
+
fileSys.delete(new Path(OUTPUT_DIR), true);
DataOutputStream file = fileSys.create(new Path(INPUT_FILE));
Modified: hadoop/common/branches/HDFS-3042/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/HDFS-3042/pom.xml?rev=1333291&r1=1333290&r2=1333291&view=diff
==============================================================================
--- hadoop/common/branches/HDFS-3042/pom.xml (original)
+++ hadoop/common/branches/HDFS-3042/pom.xml Thu May 3 02:14:01 2012
@@ -103,6 +103,11 @@
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-assembly-plugin</artifactId>
+ <version>2.3</version>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.5</version>
</plugin>
