svn commit: r1412294 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/main/java/org/apache/hadoop/util/StringUtils.java
Author: tgraves
Date: Wed Nov 21 21:03:01 2012
New Revision: 1412294
URL: http://svn.apache.org/viewvc?rev=1412294view=rev
Log:
HADOOP-8931. Add Java version to startup message. (eli)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1412294r1=1412293r2=1412294view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Nov 21 21:03:01 2012
@@ -8,6 +8,8 @@ Release 0.23.6 - UNRELEASED
IMPROVEMENTS
+HADOOP-8931. Add Java version to startup message. (eli)
+
OPTIMIZATIONS
BUG FIXES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java?rev=1412294r1=1412293r2=1412294view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringUtils.java
Wed Nov 21 21:03:01 2012
@@ -604,7 +604,8 @@ public class StringUtils {
build = + VersionInfo.getUrl() + -r
+ VersionInfo.getRevision()
+ ; compiled by ' + VersionInfo.getUser()
- + ' on + VersionInfo.getDate()}
+ + ' on + VersionInfo.getDate(),
+ java = + System.getProperty(java.version) }
)
);
svn commit: r1411884 - /hadoop/common/trunk/pom.xml
Author: tgraves
Date: Tue Nov 20 21:22:20 2012
New Revision: 1411884
URL: http://svn.apache.org/viewvc?rev=1411884view=rev
Log:
HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins (Robert Parker
via tgraves)
Modified:
hadoop/common/trunk/pom.xml
Modified: hadoop/common/trunk/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/pom.xml?rev=1411884r1=1411883r2=1411884view=diff
==
--- hadoop/common/trunk/pom.xml (original)
+++ hadoop/common/trunk/pom.xml Tue Nov 20 21:22:20 2012
@@ -517,7 +517,7 @@ xsi:schemaLocation=http://maven.apache.
groupIdcom.atlassian.maven.plugins/groupId
artifactIdmaven-clover2-plugin/artifactId
configuration
- includesAllSourceRootstrue/includesAllSourceRoots
+ includesAllSourceRootsfalse/includesAllSourceRoots
includesTestSourceRootstrue/includesTestSourceRoots
licenseLocation${cloverLicenseLocation}/licenseLocation
cloverDatabase${cloverDatabase}/cloverDatabase
svn commit: r1411884 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tgraves Date: Tue Nov 20 21:22:20 2012 New Revision: 1411884 URL: http://svn.apache.org/viewvc?rev=1411884view=rev Log: HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins (Robert Parker via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1411884r1=1411883r2=1411884view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Nov 20 21:22:20 2012 @@ -1142,6 +1142,9 @@ Release 0.23.6 - UNRELEASED BUG FIXES +HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins +(Robert Parker via tgraves) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES
svn commit: r1411891 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tgraves Date: Tue Nov 20 21:28:46 2012 New Revision: 1411891 URL: http://svn.apache.org/viewvc?rev=1411891view=rev Log: merge -r 1411883:1411884 from trunk. FIXES: HADOOP-9072 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1411891r1=1411890r2=1411891view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Tue Nov 20 21:28:46 2012 @@ -854,6 +854,9 @@ Release 0.23.6 - UNRELEASED BUG FIXES +HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins +(Robert Parker via tgraves) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES
svn commit: r1411891 - /hadoop/common/branches/branch-2/pom.xml
Author: tgraves
Date: Tue Nov 20 21:28:46 2012
New Revision: 1411891
URL: http://svn.apache.org/viewvc?rev=1411891view=rev
Log:
merge -r 1411883:1411884 from trunk. FIXES: HADOOP-9072
Modified:
hadoop/common/branches/branch-2/pom.xml
Modified: hadoop/common/branches/branch-2/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/pom.xml?rev=1411891r1=1411890r2=1411891view=diff
==
--- hadoop/common/branches/branch-2/pom.xml (original)
+++ hadoop/common/branches/branch-2/pom.xml Tue Nov 20 21:28:46 2012
@@ -410,7 +410,7 @@ xsi:schemaLocation=http://maven.apache.
groupIdcom.atlassian.maven.plugins/groupId
artifactIdmaven-clover2-plugin/artifactId
configuration
- includesAllSourceRootstrue/includesAllSourceRoots
+ includesAllSourceRootsfalse/includesAllSourceRoots
includesTestSourceRootstrue/includesTestSourceRoots
licenseLocation${cloverLicenseLocation}/licenseLocation
cloverDatabase${cloverDatabase}/cloverDatabase
svn commit: r1411892 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Author: tgraves Date: Tue Nov 20 21:28:54 2012 New Revision: 1411892 URL: http://svn.apache.org/viewvc?rev=1411892view=rev Log: merge -r 1411883:1411884 from trunk. FIXES: HADOOP-9072 Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1411892r1=1411891r2=1411892view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt Tue Nov 20 21:28:54 2012 @@ -12,6 +12,9 @@ Release 0.23.6 - UNRELEASED BUG FIXES +HADOOP-9072. Hadoop-Common-0.23-Build Fails to build in Jenkins +(Robert Parker via tgraves) + Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES
svn commit: r1411892 - /hadoop/common/branches/branch-0.23/pom.xml
Author: tgraves
Date: Tue Nov 20 21:28:54 2012
New Revision: 1411892
URL: http://svn.apache.org/viewvc?rev=1411892view=rev
Log:
merge -r 1411883:1411884 from trunk. FIXES: HADOOP-9072
Modified:
hadoop/common/branches/branch-0.23/pom.xml
Modified: hadoop/common/branches/branch-0.23/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/pom.xml?rev=1411892r1=1411891r2=1411892view=diff
==
--- hadoop/common/branches/branch-0.23/pom.xml (original)
+++ hadoop/common/branches/branch-0.23/pom.xml Tue Nov 20 21:28:54 2012
@@ -398,7 +398,7 @@
groupIdcom.atlassian.maven.plugins/groupId
artifactIdmaven-clover2-plugin/artifactId
configuration
- includesAllSourceRootstrue/includesAllSourceRoots
+ includesAllSourceRootsfalse/includesAllSourceRoots
includesTestSourceRootstrue/includesTestSourceRoots
licenseLocation${cloverLicenseLocation}/licenseLocation
cloverDatabase${cloverDatabase}/cloverDatabase
svn commit: r1411441 - in /hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project: hadoop-hdfs-httpfs/pom.xml hadoop-hdfs/CHANGES.txt hadoop-hdfs/pom.xml pom.xml
Author: tgraves Date: Mon Nov 19 21:53:12 2012 New Revision: 1411441 URL: http://svn.apache.org/viewvc?rev=1411441view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/pom.xml Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml Mon Nov 19 21:53:12 2012 @@ -22,12 +22,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-hdfs-httpfs/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version packagingwar/packaging nameApache Hadoop HttpFS/name Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Mon Nov 19 21:53:12 2012 @@ -1,17 +1,5 @@ Hadoop HDFS Change Log -Release 0.23.6 - UNRELEASED - - INCOMPATIBLE CHANGES - - NEW FEATURES - - IMPROVEMENTS - - OPTIMIZATIONS - - BUG FIXES - Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/pom.xml Mon Nov 19 21:53:12 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project-dist/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../../hadoop-project-dist/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-hdfs/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version descriptionApache Hadoop HDFS/description nameApache Hadoop HDFS/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/pom.xml Mon Nov 19 21:53:12 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-hdfs-project/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version descriptionApache Hadoop HDFS Project/description nameApache Hadoop HDFS Project/name packagingpom/packaging
svn commit: r1411470 - /hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml
Author: tgraves
Date: Mon Nov 19 22:41:55 2012
New Revision: 1411470
URL: http://svn.apache.org/viewvc?rev=1411470view=rev
Log:
Preparing 0.23.5 release
Modified:
hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml
Modified:
hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml?rev=1411470r1=1411469r2=1411470view=diff
==
---
hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml
(original)
+++
hadoop/common/branches/branch-0.23.5/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/build/aop.xml
Mon Nov 19 22:41:55 2012
@@ -21,7 +21,7 @@
property name=aspectversion value=1.6.5/
!-- TODO this has to be changed synchronously with build.xml version
prop.--
!-- this workarounds of test-patch setting its own 'version' --
- property name=project.version value=0.23.6-SNAPSHOT/
+ property name=project.version value=0.23.5/
!-- Properties common for all fault injections --
property name=build-fi.dir value=${basedir}/build-fi/
svn commit: r1411441 - in /hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/ hadoop-mapreduce-client/hadoop-mapreduce-client-app/ hadoop-mapreduce-client/hadoo
Author: tgraves Date: Mon Nov 19 21:53:12 2012 New Revision: 1411441 URL: http://svn.apache.org/viewvc?rev=1411441view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-shuffle/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-examples/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/pom.xml Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/CHANGES.txt?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/CHANGES.txt Mon Nov 19 21:53:12 2012 @@ -1,20 +1,5 @@ Hadoop MapReduce Change Log -Release 0.23.6 - UNRELEASED - - INCOMPATIBLE CHANGES - - NEW FEATURES - - IMPROVEMENTS - - OPTIMIZATIONS - - BUG FIXES - -MAPREDUCE-4802. Takes a long time to load the task list on the AM for -large jobs (Ravi Prakash via bobby) - Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -123,6 +108,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4801. ShuffleHandler can generate large logs due to prematurely closed channels (jlowe via bobby) + +MAPREDUCE-4802. Takes a long time to load the task list on the AM for +large jobs (Ravi Prakash via bobby) Release 0.23.4 Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml Mon Nov 19 21:53:12 2012 @@ -19,12 +19,12 @@ parent artifactIdhadoop-mapreduce-client/artifactId groupIdorg.apache.hadoop/groupId -version0.23.6-SNAPSHOT/version +version0.23.5/version /parent modelVersion4.0.0/modelVersion groupIdorg.apache.hadoop/groupId artifactIdhadoop-mapreduce-client-app/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version namehadoop-mapreduce-client-app/name properties Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml Mon Nov 19 21:53:12 2012 @@ -19,12 +19,12 @@ parent artifactIdhadoop-mapreduce-client/artifactId groupIdorg.apache.hadoop/groupId -version0.23.6-SNAPSHOT/version +version0.23.5/version /parent modelVersion4.0.0/modelVersion groupIdorg.apache.hadoop/groupId artifactIdhadoop-mapreduce-client-common/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version namehadoop-mapreduce-client-common/name properties Modified: hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client
svn commit: r1411442 - /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
Author: tgraves Date: Mon Nov 19 21:54:42 2012 New Revision: 1411442 URL: http://svn.apache.org/viewvc?rev=1411442view=rev Log: Fixing CHANGES.txt to show jiras in re-branched 0.23.5 release Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1411442r1=1411441r2=1411442view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon Nov 19 21:54:42 2012 @@ -12,9 +12,6 @@ Release 0.23.6 - UNRELEASED BUG FIXES -MAPREDUCE-4802. Takes a long time to load the task list on the AM for -large jobs (Ravi Prakash via bobby) - Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -123,6 +120,9 @@ Release 0.23.5 - UNRELEASED MAPREDUCE-4801. ShuffleHandler can generate large logs due to prematurely closed channels (jlowe via bobby) + +MAPREDUCE-4802. Takes a long time to load the task list on the AM for +large jobs (Ravi Prakash via bobby) Release 0.23.4
svn commit: r1411470 - /hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
Author: tgraves
Date: Mon Nov 19 22:41:55 2012
New Revision: 1411470
URL: http://svn.apache.org/viewvc?rev=1411470view=rev
Log:
Preparing 0.23.5 release
Modified:
hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
Modified:
hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml?rev=1411470r1=1411469r2=1411470view=diff
==
---
hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
(original)
+++
hadoop/common/branches/branch-0.23.5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
Mon Nov 19 22:41:55 2012
@@ -105,7 +105,7 @@
target
symlink link=${applink.base}.jar
resource=mr-app.jar failonerror=false/
-symlink link=${applink.base}-0.23.6-SNAPSHOT.jar
+symlink link=${applink.base}-0.23.5.jar
resource=mr-app.jar failonerror=false/
/target
/configuration
svn commit: r1411441 - in /hadoop/common/branches/branch-0.23.5/hadoop-yarn-project: ./ hadoop-yarn/ hadoop-yarn/hadoop-yarn-api/ hadoop-yarn/hadoop-yarn-applications/ hadoop-yarn/hadoop-yarn-applicat
Author: tgraves Date: Mon Nov 19 21:53:12 2012 New Revision: 1411441 URL: http://svn.apache.org/viewvc?rev=1411441view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/CHANGES.txt hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-web-proxy/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/pom.xml Modified: hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/CHANGES.txt?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/CHANGES.txt Mon Nov 19 21:53:12 2012 @@ -1,26 +1,5 @@ Hadoop YARN Change Log -Release 0.23.6 - UNRELEASED - - INCOMPATIBLE CHANGES - - NEW FEATURES - - IMPROVEMENTS - - OPTIMIZATIONS - - BUG FIXES - -YARN-188. Coverage fixing for CapacityScheduler (Aleksey Gorshkov via -bobby) - -YARN-214. RMContainerImpl does not handle event EXPIRE at state RUNNING -(jeagles via bobby) - -YARN-151. Browser thinks RM main page JS is taking too long -(Ravi Prakash via bobby) - Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -111,6 +90,15 @@ Release 0.23.5 - UNRELEASED YARN-219. NM should aggregate logs when application finishes. (bobby) +YARN-188. Coverage fixing for CapacityScheduler (Aleksey Gorshkov via +bobby) + +YARN-214. RMContainerImpl does not handle event EXPIRE at state RUNNING +(jeagles via bobby) + +YARN-151. Browser thinks RM main page JS is taking too long +(Ravi Prakash via bobby) + Release 0.23.4 INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/pom.xml Mon Nov 19 21:53:12 2012 @@ -19,12 +19,12 @@ parent artifactIdhadoop-yarn/artifactId groupIdorg.apache.hadoop/groupId -version0.23.6-SNAPSHOT/version +version0.23.5/version /parent modelVersion4.0.0/modelVersion groupIdorg.apache.hadoop/groupId artifactIdhadoop-yarn-api/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version namehadoop-yarn-api/name properties Modified: hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/pom.xml Mon Nov 19 21:53:12 2012 @@ -19,12 +19,12 @@ parent artifactIdhadoop-yarn-applications
svn commit: r1411442 - /hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
Author: tgraves Date: Mon Nov 19 21:54:42 2012 New Revision: 1411442 URL: http://svn.apache.org/viewvc?rev=1411442view=rev Log: Fixing CHANGES.txt to show jiras in re-branched 0.23.5 release Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1411442r1=1411441r2=1411442view=diff == --- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Mon Nov 19 21:54:42 2012 @@ -12,15 +12,6 @@ Release 0.23.6 - UNRELEASED BUG FIXES -YARN-188. Coverage fixing for CapacityScheduler (Aleksey Gorshkov via -bobby) - -YARN-214. RMContainerImpl does not handle event EXPIRE at state RUNNING -(jeagles via bobby) - -YARN-151. Browser thinks RM main page JS is taking too long -(Ravi Prakash via bobby) - Release 0.23.5 - UNRELEASED INCOMPATIBLE CHANGES @@ -111,6 +102,16 @@ Release 0.23.5 - UNRELEASED YARN-219. NM should aggregate logs when application finishes. (bobby) +YARN-188. Coverage fixing for CapacityScheduler (Aleksey Gorshkov via +bobby) + +YARN-214. RMContainerImpl does not handle event EXPIRE at state RUNNING +(jeagles via bobby) + +YARN-151. Browser thinks RM main page JS is taking too long +(Ravi Prakash via bobby) + + Release 0.23.4 INCOMPATIBLE CHANGES
svn commit: r1411429 - /hadoop/common/branches/branch-0.23.5/
Author: tgraves Date: Mon Nov 19 21:33:35 2012 New Revision: 1411429 URL: http://svn.apache.org/viewvc?rev=1411429view=rev Log: Remove branch-0.23.5. It is going to be re-branched to pick up a couple more jira. Removed: hadoop/common/branches/branch-0.23.5/
svn commit: r1411435 - /hadoop/common/branches/branch-0.23.5/
Author: tgraves Date: Mon Nov 19 21:38:34 2012 New Revision: 1411435 URL: http://svn.apache.org/viewvc?rev=1411435view=rev Log: Branching 0.23.5 release Added: hadoop/common/branches/branch-0.23.5/ (props changed) - copied from r1411434, hadoop/common/branches/branch-0.23/ Propchange: hadoop/common/branches/branch-0.23.5/ -- --- svn:ignore (added) +++ svn:ignore Mon Nov 19 21:38:34 2012 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/branches/branch-0.23.5/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Mon Nov 19 21:38:34 2012 @@ -0,0 +1 @@ +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1304099,1351818,1373683
svn commit: r1411441 - in /hadoop/common/branches/branch-0.23.5: ./ hadoop-assemblies/ hadoop-client/ hadoop-dist/ hadoop-minicluster/ hadoop-project-dist/ hadoop-project/ hadoop-tools/ hadoop-tools/h
Author: tgraves Date: Mon Nov 19 21:53:12 2012 New Revision: 1411441 URL: http://svn.apache.org/viewvc?rev=1411441view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-assemblies/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-client/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-dist/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-minicluster/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-project-dist/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-project/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-archives/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-datajoin/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-distcp/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-extras/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-gridmix/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-pipes/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-rumen/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-streaming/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/hadoop-tools-dist/pom.xml hadoop/common/branches/branch-0.23.5/hadoop-tools/pom.xml hadoop/common/branches/branch-0.23.5/pom.xml Modified: hadoop/common/branches/branch-0.23.5/hadoop-assemblies/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-assemblies/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-assemblies/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-assemblies/pom.xml Mon Nov 19 21:53:12 2012 @@ -23,12 +23,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-assemblies/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version nameApache Hadoop Assemblies/name descriptionApache Hadoop Assemblies/description Modified: hadoop/common/branches/branch-0.23.5/hadoop-client/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-client/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-client/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-client/pom.xml Mon Nov 19 21:53:12 2012 @@ -18,12 +18,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-client/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version packagingjar/packaging descriptionApache Hadoop Client/description Modified: hadoop/common/branches/branch-0.23.5/hadoop-dist/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-dist/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-dist/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-dist/pom.xml Mon Nov 19 21:53:12 2012 @@ -20,12 +20,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-dist/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version descriptionApache Hadoop Distribution/description nameApache Hadoop Distribution/name packagingjar/packaging Modified: hadoop/common/branches/branch-0.23.5/hadoop-minicluster/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23.5/hadoop-minicluster/pom.xml?rev=1411441r1=1411440r2=1411441view=diff == --- hadoop/common/branches/branch-0.23.5/hadoop-minicluster/pom.xml (original) +++ hadoop/common/branches/branch-0.23.5/hadoop-minicluster/pom.xml Mon Nov 19 21:53:12 2012 @@ -18,12 +18,12 @@ parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-project/artifactId -version0.23.6-SNAPSHOT/version +version0.23.5/version relativePath../hadoop-project/relativePath /parent groupIdorg.apache.hadoop/groupId artifactIdhadoop-minicluster/artifactId - version0.23.6-SNAPSHOT/version + version0.23.5/version packagingjar/packaging descriptionApache Hadoop Mini-Cluster
svn commit: r1411470 [1/2] - /hadoop/common/branches/branch-0.23.5/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
Author: tgraves Date: Mon Nov 19 22:41:55 2012 New Revision: 1411470 URL: http://svn.apache.org/viewvc?rev=1411470view=rev Log: Preparing 0.23.5 release Modified: hadoop/common/branches/branch-0.23.5/hadoop-common-project/hadoop-common/src/main/docs/releasenotes.html
svn commit: r1409032 - in /hadoop/common/trunk/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainer
Author: tgraves
Date: Tue Nov 13 23:47:57 2012
New Revision: 1409032
URL: http://svn.apache.org/viewvc?rev=1409032view=rev
Log:
MAPREDUCE-4517. Too many INFO messages written out during AM to RM heartbeat
(Jason Lowe via tgraves)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1409032r1=1409031r2=1409032view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Nov 13
23:47:57 2012
@@ -660,6 +660,9 @@ Release 0.23.5 - UNRELEASED
MAPREDUCE-4786. Job End Notification retry interval is 5 milliseconds by
default (Ravi Prakash via bobby)
+
+MAPREDUCE-4517. Too many INFO messages written out during AM to RM
heartbeat
+(Jason Lowe via tgraves)
Release 0.23.4 - UNRELEASED
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1409032r1=1409031r2=1409032view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Tue Nov 13 23:47:57 2012
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -145,6 +146,8 @@ public class RMContainerAllocator extend
BlockingQueueContainerAllocatorEvent eventQueue
= new LinkedBlockingQueueContainerAllocatorEvent();
+ private ScheduleStats scheduleStats = new ScheduleStats();
+
public RMContainerAllocator(ClientService clientService, AppContext context)
{
super(clientService, context);
this.stopped = new AtomicBoolean(false);
@@ -208,13 +211,10 @@ public class RMContainerAllocator extend
@Override
protected synchronized void heartbeat() throws Exception {
-LOG.info(Before Scheduling: + getStat());
+scheduleStats.updateAndLogIfChanged(Before Scheduling: );
ListContainer allocatedContainers = getResources();
-LOG.info(After Scheduling: + getStat());
if (allocatedContainers.size() 0) {
- LOG.info(Before Assign: + getStat());
scheduledRequests.assign(allocatedContainers);
- LOG.info(After Assign: + getStat());
}
int completedMaps = getJob().getCompletedMaps();
@@ -235,6 +235,8 @@ public class RMContainerAllocator extend
maxReduceRampupLimit, reduceSlowStart);
recalculateReduceSchedule = false;
}
+
+scheduleStats.updateAndLogIfChanged(After Scheduling: );
}
@Override
@@ -245,7 +247,7 @@ public class RMContainerAllocator extend
}
eventHandlingThread.interrupt();
super.stop();
-LOG.info(Final Stats: + getStat());
+scheduleStats.log(Final Stats: );
}
public boolean getIsReduceStarted() {
@@ -427,7 +429,9 @@ public class RMContainerAllocator extend
return;
}
-LOG.info(Recalculating schedule...);
+int headRoom = getAvailableResources() != null ?
+getAvailableResources().getMemory() : 0;
+LOG.info(Recalculating schedule, headroom= + headRoom);
//check for slow start
if (!getIsReduceStarted()) {//not set yet
@@ -536,24 +540,6 @@ public class RMContainerAllocator extend
}
}
- /**
- * Synchronized to avoid findbugs warnings
- */
- private synchronized String getStat() {
-return PendingReduces: + pendingReduces.size() +
- ScheduledMaps: + scheduledRequests.maps.size() +
- ScheduledReduces: + scheduledRequests.reduces.size() +
- AssignedMaps: + assignedRequests.maps.size() +
- AssignedReduces: + assignedRequests.reduces.size() +
- completedMaps: + getJob().getCompletedMaps() +
- completedReduces
svn commit: r1409034 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm
Author: tgraves
Date: Tue Nov 13 23:49:04 2012
New Revision: 1409034
URL: http://svn.apache.org/viewvc?rev=1409034view=rev
Log:
merge -r 1409031:1409032 from trunk. FIXES: MAPREDUCE-4517
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1409034r1=1409033r2=1409034view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue
Nov 13 23:49:04 2012
@@ -513,6 +513,9 @@ Release 0.23.5 - UNRELEASED
MAPREDUCE-4786. Job End Notification retry interval is 5 milliseconds by
default (Ravi Prakash via bobby)
+
+MAPREDUCE-4517. Too many INFO messages written out during AM to RM
heartbeat
+(Jason Lowe via tgraves)
Release 0.23.4 - UNRELEASED
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1409034r1=1409033r2=1409034view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Tue Nov 13 23:49:04 2012
@@ -67,6 +67,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.NodeState;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -145,6 +146,8 @@ public class RMContainerAllocator extend
BlockingQueueContainerAllocatorEvent eventQueue
= new LinkedBlockingQueueContainerAllocatorEvent();
+ private ScheduleStats scheduleStats = new ScheduleStats();
+
public RMContainerAllocator(ClientService clientService, AppContext context)
{
super(clientService, context);
this.stopped = new AtomicBoolean(false);
@@ -208,13 +211,10 @@ public class RMContainerAllocator extend
@Override
protected synchronized void heartbeat() throws Exception {
-LOG.info(Before Scheduling: + getStat());
+scheduleStats.updateAndLogIfChanged(Before Scheduling: );
ListContainer allocatedContainers = getResources();
-LOG.info(After Scheduling: + getStat());
if (allocatedContainers.size() 0) {
- LOG.info(Before Assign: + getStat());
scheduledRequests.assign(allocatedContainers);
- LOG.info(After Assign: + getStat());
}
int completedMaps = getJob().getCompletedMaps();
@@ -235,6 +235,8 @@ public class RMContainerAllocator extend
maxReduceRampupLimit, reduceSlowStart);
recalculateReduceSchedule = false;
}
+
+scheduleStats.updateAndLogIfChanged(After Scheduling: );
}
@Override
@@ -245,7 +247,7 @@ public class RMContainerAllocator extend
}
eventHandlingThread.interrupt();
super.stop();
-LOG.info(Final Stats: + getStat());
+scheduleStats.log(Final Stats: );
}
public boolean getIsReduceStarted() {
@@ -427,7 +429,9 @@ public class RMContainerAllocator extend
return;
}
-LOG.info(Recalculating schedule...);
+int headRoom = getAvailableResources() != null ?
+getAvailableResources().getMemory() : 0;
+LOG.info(Recalculating schedule, headroom= + headRoom);
//check for slow start
if (!getIsReduceStarted()) {//not set yet
@@ -536,24 +540,6 @@ public class RMContainerAllocator extend
}
}
- /**
- * Synchronized to avoid findbugs warnings
- */
- private synchronized String getStat() {
-return PendingReduces: + pendingReduces.size() +
- ScheduledMaps: + scheduledRequests.maps.size() +
- ScheduledReduces: + scheduledRequests.reduces.size() +
- AssignedMaps: + assignedRequests.maps.size() +
- AssignedReduces: + assignedRequests.reduces.size
svn commit: r1409035 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: CHANGES.txt hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app
Author: tgraves
Date: Tue Nov 13 23:50:23 2012
New Revision: 1409035
URL: http://svn.apache.org/viewvc?rev=1409035view=rev
Log:
merge -r 1409031:1409032 from trunk. FIXES: MAPREDUCE-4517
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1409035r1=1409034r2=1409035view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue
Nov 13 23:50:23 2012
@@ -96,6 +96,9 @@ Release 0.23.5 - UNRELEASED
MAPREDUCE-4786. Job End Notification retry interval is 5 milliseconds by
default (Ravi Prakash via bobby)
+
+MAPREDUCE-4517. Too many INFO messages written out during AM to RM
heartbeat
+(Jason Lowe via tgraves)
Release 0.23.4 - UNRELEASED
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java?rev=1409035r1=1409034r2=1409035view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMContainerAllocator.java
Tue Nov 13 23:50:23 2012
@@ -62,6 +62,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.Priority;
+import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.util.RackResolver;
@@ -140,6 +141,8 @@ public class RMContainerAllocator extend
BlockingQueueContainerAllocatorEvent eventQueue
= new LinkedBlockingQueueContainerAllocatorEvent();
+ private ScheduleStats scheduleStats = new ScheduleStats();
+
public RMContainerAllocator(ClientService clientService, AppContext context)
{
super(clientService, context);
this.stopped = new AtomicBoolean(false);
@@ -203,13 +206,10 @@ public class RMContainerAllocator extend
@Override
protected synchronized void heartbeat() throws Exception {
-LOG.info(Before Scheduling: + getStat());
+scheduleStats.updateAndLogIfChanged(Before Scheduling: );
ListContainer allocatedContainers = getResources();
-LOG.info(After Scheduling: + getStat());
if (allocatedContainers.size() 0) {
- LOG.info(Before Assign: + getStat());
scheduledRequests.assign(allocatedContainers);
- LOG.info(After Assign: + getStat());
}
int completedMaps = getJob().getCompletedMaps();
@@ -230,6 +230,8 @@ public class RMContainerAllocator extend
maxReduceRampupLimit, reduceSlowStart);
recalculateReduceSchedule = false;
}
+
+scheduleStats.updateAndLogIfChanged(After Scheduling: );
}
@Override
@@ -240,7 +242,7 @@ public class RMContainerAllocator extend
}
eventHandlingThread.interrupt();
super.stop();
-LOG.info(Final Stats: + getStat());
+scheduleStats.log(Final Stats: );
}
public boolean getIsReduceStarted() {
@@ -422,7 +424,9 @@ public class RMContainerAllocator extend
return;
}
-LOG.info(Recalculating schedule...);
+int headRoom = getAvailableResources() != null ?
+getAvailableResources().getMemory() : 0;
+LOG.info(Recalculating schedule, headroom= + headRoom);
//check for slow start
if (!getIsReduceStarted()) {//not set yet
@@ -531,24 +535,6 @@ public class RMContainerAllocator extend
}
}
- /**
- * Synchronized to avoid findbugs warnings
- */
- private synchronized String getStat() {
-return PendingReduces: + pendingReduces.size() +
- ScheduledMaps: + scheduledRequests.maps.size() +
- ScheduledReduces: + scheduledRequests.reduces.size() +
- AssignedMaps: + assignedRequests.maps.size() +
- AssignedReduces
svn commit: r1407590 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: ./ src/main/java/org/apache/hadoop/hdfs/server/common/ src/main/java/org/apache/hadoop/hdfs/server/datano
Author: tgraves
Date: Fri Nov 9 18:51:43 2012
New Revision: 1407590
URL: http://svn.apache.org/viewvc?rev=1407590view=rev
Log:
HDFS-4162. Some malformed and unquoted HTML strings are returned from datanode
web ui. (Darek Dagit via tgraves)
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/common/TestJspHelper.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1407590r1=1407589r2=1407590view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri Nov 9 18:51:43 2012
@@ -45,6 +45,9 @@ Release 0.23.5 - UNRELEASED
HDFS-4090. getFileChecksum() result incompatible when called against
zero-byte files. (Kihwal Lee via daryn)
+HDFS-4162. Some malformed and unquoted HTML strings are returned from
+datanode web ui. (Darek Dagit via tgraves)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java?rev=1407590r1=1407589r2=1407590view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/common/JspHelper.java
Fri Nov 9 18:51:43 2012
@@ -387,15 +387,15 @@ public class JspHelper {
if (!parts[i].equals()) {
tempPath.append(parts[i]);
out.print(a href=\browseDirectory.jsp + ?dir=
- + tempPath.toString() + namenodeInfoPort= + namenodeInfoPort
+ + HtmlQuoting.quoteHtmlChars(tempPath.toString()) +
namenodeInfoPort= + namenodeInfoPort
+ getDelegationTokenUrlParam(tokenString)
+ getUrlParam(NAMENODE_ADDRESS, nnAddress));
- out.print(\ + parts[i] + /a + Path.SEPARATOR);
+ out.print(\ + HtmlQuoting.quoteHtmlChars(parts[i]) + /a +
Path.SEPARATOR);
tempPath.append(Path.SEPARATOR);
}
}
if(parts.length 0) {
-out.print(parts[parts.length-1]);
+out.print(HtmlQuoting.quoteHtmlChars(parts[parts.length-1]));
}
}
catch (UnsupportedEncodingException ex) {
@@ -410,16 +410,16 @@ public class JspHelper {
String nnAddress) throws IOException {
out.print(form action=\browseDirectory.jsp\ method=\get\
name=\goto\);
out.print(Goto : );
-out.print(input name=\dir\ type=\text\ width=\50\ id\dir\
value=\+ file+\);
-out.print(input name=\go\ type=\submit\ value=\go\);
+out.print(input name=\dir\ type=\text\ width=\50\ id=\dir\
value=\+ HtmlQuoting.quoteHtmlChars(file)+\/);
+out.print(input name=\go\ type=\submit\ value=\go\/);
out.print(input name=\namenodeInfoPort\ type=\hidden\
-+ value=\ + namenodeInfoPort + \);
++ value=\ + namenodeInfoPort + \/);
if (UserGroupInformation.isSecurityEnabled()) {
out.print(input name=\ + DELEGATION_PARAMETER_NAME
- + \ type=\hidden\ value=\ + tokenString + \);
+ + \ type=\hidden\ value=\ + tokenString + \/);
}
out.print(input name=\+ NAMENODE_ADDRESS +\ type=\hidden\
-+ value=\ + nnAddress + \);
++ value=\ + nnAddress + \/);
out.print(/form);
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java?rev=1407590r1=1407589r2=1407590view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DatanodeJspHelper.java
(original)
+++
hadoop
svn commit: r1404177 - /hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
Author: tgraves
Date: Wed Oct 31 14:57:12 2012
New Revision: 1404177
URL: http://svn.apache.org/viewvc?rev=1404177view=rev
Log:
MAPREDUCE-4752. Reduce MR AM memory usage through String Interning (Robert
Evans via tgraves)
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1404177r1=1404176r2=1404177view=diff
==
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
Wed Oct 31 14:57:12 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.util;
import java.util.Iterator;
import java.util.Map;
+import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -109,6 +110,7 @@ public class Apps {
} else {
val = val + SYSTEM_PATH_SEPARATOR + value;
}
-environment.put(variable, val);
+environment.put(StringInterner.weakIntern(variable),
+StringInterner.weakIntern(val));
}
}
svn commit: r1404179 - /hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
Author: tgraves
Date: Wed Oct 31 14:59:47 2012
New Revision: 1404179
URL: http://svn.apache.org/viewvc?rev=1404179view=rev
Log:
MAPREDUCE-4752. Reduce MR AM memory usage through String Interning (Robert
Evans via tgraves)
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java?rev=1404179r1=1404178r2=1404179view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
Wed Oct 31 14:59:47 2012
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.util;
import java.util.Iterator;
import java.util.Map;
+import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.yarn.YarnException;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
@@ -109,6 +110,7 @@ public class Apps {
} else {
val = val + SYSTEM_PATH_SEPARATOR + value;
}
-environment.put(variable, val);
+environment.put(StringInterner.weakIntern(variable),
+StringInterner.weakIntern(val));
}
}
svn commit: r1404177 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop: conf/Configuration.java util/StringInterner.java
Author: tgraves
Date: Wed Oct 31 14:57:12 2012
New Revision: 1404177
URL: http://svn.apache.org/viewvc?rev=1404177view=rev
Log:
MAPREDUCE-4752. Reduce MR AM memory usage through String Interning (Robert
Evans via tgraves)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1404177r1=1404176r2=1404177view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
Wed Oct 31 14:57:12 2012
@@ -75,6 +75,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
@@ -2002,13 +2003,16 @@ public class Configuration implements It
continue;
Element field = (Element)fieldNode;
if (name.equals(field.getTagName()) field.hasChildNodes())
-attr = ((Text)field.getFirstChild()).getData().trim();
+attr = StringInterner.weakIntern(
+((Text)field.getFirstChild()).getData().trim());
if (value.equals(field.getTagName()) field.hasChildNodes())
-value = ((Text)field.getFirstChild()).getData();
+value = StringInterner.weakIntern(
+((Text)field.getFirstChild()).getData());
if (final.equals(field.getTagName()) field.hasChildNodes())
finalParameter =
true.equals(((Text)field.getFirstChild()).getData());
if (source.equals(field.getTagName()) field.hasChildNodes())
-source.add(((Text)field.getFirstChild()).getData());
+source.add(StringInterner.weakIntern(
+((Text)field.getFirstChild()).getData()));
}
source.add(name);
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java?rev=1404177r1=1404176r2=1404177view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
Wed Oct 31 14:57:12 2012
@@ -59,6 +59,9 @@ public class StringInterner {
* @return strong reference to interned string instance
*/
public static String strongIntern(String sample) {
+if (sample == null) {
+ return null;
+}
return strongInterner.intern(sample);
}
@@ -72,6 +75,9 @@ public class StringInterner {
* @return weak reference to interned string instance
*/
public static String weakIntern(String sample) {
+if (sample == null) {
+ return null;
+}
return weakInterner.intern(sample);
}
svn commit: r1404179 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop: conf/Configuration.java util/StringInterner.java
Author: tgraves
Date: Wed Oct 31 14:59:47 2012
New Revision: 1404179
URL: http://svn.apache.org/viewvc?rev=1404179view=rev
Log:
MAPREDUCE-4752. Reduce MR AM memory usage through String Interning (Robert
Evans via tgraves)
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1404179r1=1404178r2=1404179view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
Wed Oct 31 14:59:47 2012
@@ -74,6 +74,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
@@ -1916,13 +1917,16 @@ public class Configuration implements It
continue;
Element field = (Element)fieldNode;
if (name.equals(field.getTagName()) field.hasChildNodes())
-attr = ((Text)field.getFirstChild()).getData().trim();
+attr = StringInterner.weakIntern(
+((Text)field.getFirstChild()).getData().trim());
if (value.equals(field.getTagName()) field.hasChildNodes())
-value = ((Text)field.getFirstChild()).getData();
+value = StringInterner.weakIntern(
+((Text)field.getFirstChild()).getData());
if (final.equals(field.getTagName()) field.hasChildNodes())
finalParameter =
true.equals(((Text)field.getFirstChild()).getData());
if (source.equals(field.getTagName()) field.hasChildNodes())
-source.add(((Text)field.getFirstChild()).getData());
+source.add(StringInterner.weakIntern(
+((Text)field.getFirstChild()).getData()));
}
source.add(name);
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java?rev=1404179r1=1404178r2=1404179view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
Wed Oct 31 14:59:47 2012
@@ -59,6 +59,9 @@ public class StringInterner {
* @return strong reference to interned string instance
*/
public static String strongIntern(String sample) {
+if (sample == null) {
+ return null;
+}
return strongInterner.intern(sample);
}
@@ -72,6 +75,9 @@ public class StringInterner {
* @return weak reference to interned string instance
*/
public static String weakIntern(String sample) {
+if (sample == null) {
+ return null;
+}
return weakInterner.intern(sample);
}
svn commit: r1404182 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop: conf/Configuration.java util/StringInterner.java
Author: tgraves
Date: Wed Oct 31 15:02:59 2012
New Revision: 1404182
URL: http://svn.apache.org/viewvc?rev=1404182view=rev
Log:
merge -r 1404176:1404177 from trunk. FIXES: MAPREDUCE-4752
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java?rev=1404182r1=1404181r2=1404182view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
Wed Oct 31 15:02:59 2012
@@ -75,6 +75,7 @@ import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableUtils;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.hadoop.util.StringInterner;
import org.apache.hadoop.util.StringUtils;
import org.codehaus.jackson.JsonFactory;
import org.codehaus.jackson.JsonGenerator;
@@ -1964,13 +1965,16 @@ public class Configuration implements It
continue;
Element field = (Element)fieldNode;
if (name.equals(field.getTagName()) field.hasChildNodes())
-attr = ((Text)field.getFirstChild()).getData().trim();
+attr = StringInterner.weakIntern(
+((Text)field.getFirstChild()).getData().trim());
if (value.equals(field.getTagName()) field.hasChildNodes())
-value = ((Text)field.getFirstChild()).getData();
+value = StringInterner.weakIntern(
+((Text)field.getFirstChild()).getData());
if (final.equals(field.getTagName()) field.hasChildNodes())
finalParameter =
true.equals(((Text)field.getFirstChild()).getData());
if (source.equals(field.getTagName()) field.hasChildNodes())
-source.add(((Text)field.getFirstChild()).getData());
+source.add(StringInterner.weakIntern(
+((Text)field.getFirstChild()).getData()));
}
source.add(name);
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java?rev=1404182r1=1404181r2=1404182view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/StringInterner.java
Wed Oct 31 15:02:59 2012
@@ -59,6 +59,9 @@ public class StringInterner {
* @return strong reference to interned string instance
*/
public static String strongIntern(String sample) {
+if (sample == null) {
+ return null;
+}
return strongInterner.intern(sample);
}
@@ -72,6 +75,9 @@ public class StringInterner {
* @return weak reference to interned string instance
*/
public static String weakIntern(String sample) {
+if (sample == null) {
+ return null;
+}
return weakInterner.intern(sample);
}
svn commit: r1402660 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
Author: tgraves
Date: Fri Oct 26 21:03:29 2012
New Revision: 1402660
URL: http://svn.apache.org/viewvc?rev=1402660view=rev
Log:
HADOOP-8713. TestRPCCompatibility fails intermittently with JDK7 Trevor
Robinson via tgraves)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1402660r1=1402659r2=1402660view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Oct
26 21:03:29 2012
@@ -383,6 +383,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8951. RunJar to fail with user-comprehensible error
message if jar missing. (stevel via suresh)
+HADOOP-8713. TestRPCCompatibility fails intermittently with JDK7
+(Trevor Robinson via tgraves)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java?rev=1402660r1=1402659r2=1402660view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
Fri Oct 26 21:03:29 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.ipc.protobuf.Pr
import
org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
import org.apache.hadoop.net.NetUtils;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
/** Unit test for supporting method-name based compatible RPCs. */
@@ -114,6 +115,11 @@ public class TestRPCCompatibility {
}
}
+
+ @Before
+ public void setUp() {
+ProtocolSignature.resetCache();
+ }
@After
public void tearDown() throws IOException {
@@ -219,7 +225,6 @@ System.out.println(echo int is NOT supp
@Test // equal version client and server
public void testVersion2ClientVersion2Server() throws Exception {
-ProtocolSignature.resetCache();
// create a server with two handlers
TestImpl2 impl = new TestImpl2();
server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
svn commit: r1402662 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
Author: tgraves
Date: Fri Oct 26 21:04:57 2012
New Revision: 1402662
URL: http://svn.apache.org/viewvc?rev=1402662view=rev
Log:
merge -r 1402659:1402660 from trunk. FIXES: HADOOP-8713
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1402662r1=1402661r2=1402662view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Oct 26 21:04:57 2012
@@ -113,6 +113,9 @@ Release 2.0.3-alpha - Unreleased
HADOOP-8951. RunJar to fail with user-comprehensible error
message if jar missing. (stevel via suresh)
+HADOOP-8713. TestRPCCompatibility fails intermittently with JDK7
+(Trevor Robinson via tgraves)
+
Release 2.0.2-alpha - 2012-09-07
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java?rev=1402662r1=1402661r2=1402662view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/ipc/TestRPCCompatibility.java
Fri Oct 26 21:04:57 2012
@@ -36,6 +36,7 @@ import org.apache.hadoop.ipc.protobuf.Pr
import
org.apache.hadoop.ipc.protobuf.ProtocolInfoProtos.ProtocolSignatureProto;
import org.apache.hadoop.net.NetUtils;
import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
/** Unit test for supporting method-name based compatible RPCs. */
@@ -114,6 +115,11 @@ public class TestRPCCompatibility {
}
}
+
+ @Before
+ public void setUp() {
+ProtocolSignature.resetCache();
+ }
@After
public void tearDown() throws IOException {
@@ -219,7 +225,6 @@ System.out.println(echo int is NOT supp
@Test // equal version client and server
public void testVersion2ClientVersion2Server() throws Exception {
-ProtocolSignature.resetCache();
// create a server with two handlers
TestImpl2 impl = new TestImpl2();
server = new RPC.Builder(conf).setProtocol(TestProtocol2.class)
svn commit: r1401668 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/sch
Author: tgraves
Date: Wed Oct 24 13:21:09 2012
New Revision: 1401668
URL: http://svn.apache.org/viewvc?rev=1401668view=rev
Log:
YARN-177. CapacityScheduler - adding a queue while the RM is running has wacky
results (acmurthy vai tgraves)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1401668r1=1401667r2=1401668view=diff
==
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Wed Oct 24 13:21:09 2012
@@ -174,6 +174,9 @@ Release 0.23.5 - UNRELEASED
YARN-174. Modify NodeManager to pass the user's configuration even when
rebooting. (vinodkv)
+YARN-177. CapacityScheduler - adding a queue while the RM is running has
+wacky results (acmurthy vai tgraves)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1401668r1=1401667r2=1401668view=diff
==
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
Wed Oct 24 13:21:09 2012
@@ -51,6 +51,12 @@ extends org.apache.hadoop.yarn.server.re
public CSQueue getParent();
/**
+ * Set the parent codeQueue/code.
+ * @param newParentQueue new parent queue
+ */
+ public void setParent(CSQueue newParentQueue);
+
+ /**
* Get the queue name.
* @return the queue name
*/
@@ -195,10 +201,10 @@ extends org.apache.hadoop.yarn.server.re
/**
* Reinitialize the queue.
- * @param queue new queue to re-initalize from
+ * @param newlyParsedQueue new queue to re-initalize from
* @param clusterResource resources in the cluster
*/
- public void reinitialize(CSQueue queue, Resource clusterResource)
+ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException;
/**
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1401668r1=1401667r2=1401668view=diff
==
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Wed Oct 24 13:21:09 2012
@@ -223,7 +223,7 @@ public class LeafQueue implements CSQueu
{
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity);
-float absCapacity = parent.getAbsoluteCapacity() * capacity;
+float absCapacity = getParent().getAbsoluteCapacity() * capacity
svn commit: r1401669 - in /hadoop/common/branches/branch-2/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourc
Author: tgraves
Date: Wed Oct 24 13:25:19 2012
New Revision: 1401669
URL: http://svn.apache.org/viewvc?rev=1401669view=rev
Log:
merge -r 1401667:1401668 from trunk. FIXES: YARN-177
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/ParentQueue.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1401669r1=1401668r2=1401669view=diff
==
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Wed Oct 24
13:25:19 2012
@@ -157,6 +157,9 @@ Release 0.23.5 - UNRELEASED
YARN-174. Modify NodeManager to pass the user's configuration even when
rebooting. (vinodkv)
+YARN-177. CapacityScheduler - adding a queue while the RM is running has
+wacky results (acmurthy vai tgraves)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java?rev=1401669r1=1401668r2=1401669view=diff
==
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CSQueue.java
Wed Oct 24 13:25:19 2012
@@ -51,6 +51,12 @@ extends org.apache.hadoop.yarn.server.re
public CSQueue getParent();
/**
+ * Set the parent codeQueue/code.
+ * @param newParentQueue new parent queue
+ */
+ public void setParent(CSQueue newParentQueue);
+
+ /**
* Get the queue name.
* @return the queue name
*/
@@ -195,10 +201,10 @@ extends org.apache.hadoop.yarn.server.re
/**
* Reinitialize the queue.
- * @param queue new queue to re-initalize from
+ * @param newlyParsedQueue new queue to re-initalize from
* @param clusterResource resources in the cluster
*/
- public void reinitialize(CSQueue queue, Resource clusterResource)
+ public void reinitialize(CSQueue newlyParsedQueue, Resource clusterResource)
throws IOException;
/**
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java?rev=1401669r1=1401668r2=1401669view=diff
==
---
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
Wed Oct 24 13:25:19 2012
@@ -223,7 +223,7 @@ public class LeafQueue implements CSQueu
{
// Sanity check
CSQueueUtils.checkMaxCapacity(getQueueName(), capacity, maximumCapacity
svn commit: r1396705 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Author: tgraves
Date: Wed Oct 10 17:35:18 2012
New Revision: 1396705
URL: http://svn.apache.org/viewvc?rev=1396705view=rev
Log:
HDFS-3824. TestHftpDelegationToken fails intermittently with JDK7 (Trevor
Robinson via tgraves)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1396705r1=1396704r2=1396705view=diff
==
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Wed Oct 10
17:35:18 2012
@@ -1700,6 +1700,9 @@ Release 0.23.5 - UNRELEASED
HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor
Robinson via tgraves)
+HDFS-3824. TestHftpDelegationToken fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1396705r1=1396704r2=1396705view=diff
==
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Wed Oct 10 17:35:18 2012
@@ -41,6 +41,8 @@ public class TestHftpDelegationToken {
@Test
public void testHdfsDelegationToken() throws Exception {
+SecurityUtilTestHelper.setTokenServiceUseIp(true);
+
final Configuration conf = new Configuration();
conf.set(HADOOP_SECURITY_AUTHENTICATION, kerberos);
UserGroupInformation.setConfiguration(conf);
@@ -265,4 +267,4 @@ public class TestHftpDelegationToken {
@Override
protected void initDelegationToken() throws IOException {}
}
-}
\ No newline at end of file
+}
svn commit: r1396706 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Author: tgraves
Date: Wed Oct 10 17:37:54 2012
New Revision: 1396706
URL: http://svn.apache.org/viewvc?rev=1396706view=rev
Log:
merge -r 1396704:1396705 from trunk. FIXES: HDFS-3824
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1396706r1=1396705r2=1396706view=diff
==
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Wed Oct 10 17:37:54 2012
@@ -1545,6 +1545,9 @@ Release 0.23.5 - UNRELEASED
HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor
Robinson via tgraves)
+HDFS-3824. TestHftpDelegationToken fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1396706r1=1396705r2=1396706view=diff
==
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Wed Oct 10 17:37:54 2012
@@ -41,6 +41,8 @@ public class TestHftpDelegationToken {
@Test
public void testHdfsDelegationToken() throws Exception {
+SecurityUtilTestHelper.setTokenServiceUseIp(true);
+
final Configuration conf = new Configuration();
conf.set(HADOOP_SECURITY_AUTHENTICATION, kerberos);
UserGroupInformation.setConfiguration(conf);
@@ -265,4 +267,4 @@ public class TestHftpDelegationToken {
@Override
protected void initDelegationToken() throws IOException {}
}
-}
\ No newline at end of file
+}
svn commit: r1396707 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Author: tgraves
Date: Wed Oct 10 17:41:05 2012
New Revision: 1396707
URL: http://svn.apache.org/viewvc?rev=1396707view=rev
Log:
merge -r 1396705:1396706 from branch-2. FIXES: HDFS-3824
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1396707r1=1396706r2=1396707view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Wed Oct 10 17:41:05 2012
@@ -21,6 +21,9 @@ Release 0.23.5 - UNRELEASED
HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor
Robinson via tgraves)
+HDFS-3824. TestHftpDelegationToken fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java?rev=1396707r1=1396706r2=1396707view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpDelegationToken.java
Wed Oct 10 17:41:05 2012
@@ -43,6 +43,8 @@ public class TestHftpDelegationToken {
@Test
public void testHdfsDelegationToken() throws Exception {
+SecurityUtilTestHelper.setTokenServiceUseIp(true);
+
final Configuration conf = new Configuration();
conf.set(HADOOP_SECURITY_AUTHENTICATION, kerberos);
UserGroupInformation.setConfiguration(conf);
svn commit: r1394609 - in /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Author: tgraves
Date: Fri Oct 5 15:51:10 2012
New Revision: 1394609
URL: http://svn.apache.org/viewvc?rev=1394609view=rev
Log:
HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor Robinson
via tgraves)
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Modified: hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1394609r1=1394608r2=1394609view=diff
==
--- hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Fri Oct 5
15:51:10 2012
@@ -1677,6 +1677,9 @@ Release 0.23.5 - UNRELEASED
BUG FIXES
+HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java?rev=1394609r1=1394608r2=1394609view=diff
==
---
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
(original)
+++
hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Fri Oct 5 15:51:10 2012
@@ -53,19 +53,23 @@ public class TestHftpURLTimeouts {
boolean timedout = false;
HftpFileSystem fs = (HftpFileSystem)FileSystem.get(uri, conf);
-HttpURLConnection conn = fs.openConnection(/, );
-timedout = false;
try {
- // this will consume the only slot in the backlog
- conn.getInputStream();
-} catch (SocketTimeoutException ste) {
- timedout = true;
- assertEquals(Read timed out, ste.getMessage());
+ HttpURLConnection conn = fs.openConnection(/, );
+ timedout = false;
+ try {
+// this will consume the only slot in the backlog
+conn.getInputStream();
+ } catch (SocketTimeoutException ste) {
+timedout = true;
+assertEquals(Read timed out, ste.getMessage());
+ } finally {
+if (conn != null) conn.disconnect();
+ }
+ assertTrue(read timedout, timedout);
+ assertTrue(connect timedout, checkConnectTimeout(fs, false));
} finally {
- if (conn != null) conn.disconnect();
+ fs.close();
}
-assertTrue(read timedout, timedout);
-assertTrue(connect timedout, checkConnectTimeout(fs, false));
}
@Test
@@ -79,20 +83,24 @@ public class TestHftpURLTimeouts {
boolean timedout = false;
HsftpFileSystem fs = (HsftpFileSystem)FileSystem.get(uri, conf);
-HttpURLConnection conn = null;
-timedout = false;
try {
- // this will consume the only slot in the backlog
- conn = fs.openConnection(/, );
-} catch (SocketTimeoutException ste) {
- // SSL expects a negotiation, so it will timeout on read, unlike hftp
- timedout = true;
- assertEquals(Read timed out, ste.getMessage());
+ HttpURLConnection conn = null;
+ timedout = false;
+ try {
+// this will consume the only slot in the backlog
+conn = fs.openConnection(/, );
+ } catch (SocketTimeoutException ste) {
+// SSL expects a negotiation, so it will timeout on read, unlike hftp
+timedout = true;
+assertEquals(Read timed out, ste.getMessage());
+ } finally {
+if (conn != null) conn.disconnect();
+ }
+ assertTrue(ssl read connect timedout, timedout);
+ assertTrue(connect timedout, checkConnectTimeout(fs, true));
} finally {
- if (conn != null) conn.disconnect();
+ fs.close();
}
-assertTrue(ssl read connect timedout, timedout);
-assertTrue(connect timedout, checkConnectTimeout(fs, true));
}
private boolean checkConnectTimeout(HftpFileSystem fs, boolean
ignoreReadTimeout)
svn commit: r1394610 - in /hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Author: tgraves
Date: Fri Oct 5 15:52:18 2012
New Revision: 1394610
URL: http://svn.apache.org/viewvc?rev=1394610view=rev
Log:
merge -r 1394608:1394609 from trunk. FIXES: HDFS-3829
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1394610r1=1394609r2=1394610view=diff
==
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri Oct 5 15:52:18 2012
@@ -1525,6 +1525,9 @@ Release 0.23.5 - UNRELEASED
BUG FIXES
+HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java?rev=1394610r1=1394609r2=1394610view=diff
==
---
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Fri Oct 5 15:52:18 2012
@@ -53,19 +53,23 @@ public class TestHftpURLTimeouts {
boolean timedout = false;
HftpFileSystem fs = (HftpFileSystem)FileSystem.get(uri, conf);
-HttpURLConnection conn = fs.openConnection(/, );
-timedout = false;
try {
- // this will consume the only slot in the backlog
- conn.getInputStream();
-} catch (SocketTimeoutException ste) {
- timedout = true;
- assertEquals(Read timed out, ste.getMessage());
+ HttpURLConnection conn = fs.openConnection(/, );
+ timedout = false;
+ try {
+// this will consume the only slot in the backlog
+conn.getInputStream();
+ } catch (SocketTimeoutException ste) {
+timedout = true;
+assertEquals(Read timed out, ste.getMessage());
+ } finally {
+if (conn != null) conn.disconnect();
+ }
+ assertTrue(read timedout, timedout);
+ assertTrue(connect timedout, checkConnectTimeout(fs, false));
} finally {
- if (conn != null) conn.disconnect();
+ fs.close();
}
-assertTrue(read timedout, timedout);
-assertTrue(connect timedout, checkConnectTimeout(fs, false));
}
@Test
@@ -79,20 +83,24 @@ public class TestHftpURLTimeouts {
boolean timedout = false;
HsftpFileSystem fs = (HsftpFileSystem)FileSystem.get(uri, conf);
-HttpURLConnection conn = null;
-timedout = false;
try {
- // this will consume the only slot in the backlog
- conn = fs.openConnection(/, );
-} catch (SocketTimeoutException ste) {
- // SSL expects a negotiation, so it will timeout on read, unlike hftp
- timedout = true;
- assertEquals(Read timed out, ste.getMessage());
+ HttpURLConnection conn = null;
+ timedout = false;
+ try {
+// this will consume the only slot in the backlog
+conn = fs.openConnection(/, );
+ } catch (SocketTimeoutException ste) {
+// SSL expects a negotiation, so it will timeout on read, unlike hftp
+timedout = true;
+assertEquals(Read timed out, ste.getMessage());
+ } finally {
+if (conn != null) conn.disconnect();
+ }
+ assertTrue(ssl read connect timedout, timedout);
+ assertTrue(connect timedout, checkConnectTimeout(fs, true));
} finally {
- if (conn != null) conn.disconnect();
+ fs.close();
}
-assertTrue(ssl read connect timedout, timedout);
-assertTrue(connect timedout, checkConnectTimeout(fs, true));
}
private boolean checkConnectTimeout(HftpFileSystem fs, boolean
ignoreReadTimeout)
svn commit: r1394617 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Author: tgraves
Date: Fri Oct 5 16:01:09 2012
New Revision: 1394617
URL: http://svn.apache.org/viewvc?rev=1394617view=rev
Log:
merge -r 1394609:1394610 from branch-2. FIXES: HDFS-3829
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1394617r1=1394616r2=1394617view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Fri Oct 5 16:01:09 2012
@@ -18,6 +18,9 @@ Release 0.23.5 - UNRELEASED
HDFS-3905. Secure cluster cannot use hftp to an insecure cluster
(Daryn Sharp via tgraves)
+HDFS-3829. TestHftpURLTimeouts fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java?rev=1394617r1=1394616r2=1394617view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestHftpURLTimeouts.java
Fri Oct 5 16:01:09 2012
@@ -54,19 +54,23 @@ public class TestHftpURLTimeouts {
boolean timedout = false;
HftpFileSystem fs = (HftpFileSystem)FileSystem.get(uri, conf);
-HttpURLConnection conn = fs.openConnection(/, );
-timedout = false;
try {
- // this will consume the only slot in the backlog
- conn.getInputStream();
-} catch (SocketTimeoutException ste) {
- timedout = true;
- assertEquals(Read timed out, ste.getMessage());
+ HttpURLConnection conn = fs.openConnection(/, );
+ timedout = false;
+ try {
+// this will consume the only slot in the backlog
+conn.getInputStream();
+ } catch (SocketTimeoutException ste) {
+timedout = true;
+assertEquals(Read timed out, ste.getMessage());
+ } finally {
+if (conn != null) conn.disconnect();
+ }
+ assertTrue(read timedout, timedout);
+ assertTrue(connect timedout, checkConnectTimeout(fs, false));
} finally {
- if (conn != null) conn.disconnect();
+ fs.close();
}
-assertTrue(read timedout, timedout);
-assertTrue(connect timedout, checkConnectTimeout(fs, false));
}
@Test
@@ -80,20 +84,24 @@ public class TestHftpURLTimeouts {
boolean timedout = false;
HsftpFileSystem fs = (HsftpFileSystem)FileSystem.get(uri, conf);
-HttpURLConnection conn = null;
-timedout = false;
try {
- // this will consume the only slot in the backlog
- conn = fs.openConnection(/, );
-} catch (SocketTimeoutException ste) {
- // SSL expects a negotiation, so it will timeout on read, unlike hftp
- timedout = true;
- assertEquals(Read timed out, ste.getMessage());
+ HttpURLConnection conn = null;
+ timedout = false;
+ try {
+// this will consume the only slot in the backlog
+conn = fs.openConnection(/, );
+ } catch (SocketTimeoutException ste) {
+// SSL expects a negotiation, so it will timeout on read, unlike hftp
+timedout = true;
+assertEquals(Read timed out, ste.getMessage());
+ } finally {
+if (conn != null) conn.disconnect();
+ }
+ assertTrue(ssl read connect timedout, timedout);
+ assertTrue(connect timedout, checkConnectTimeout(fs, true));
} finally {
- if (conn != null) conn.disconnect();
+ fs.close();
}
-assertTrue(ssl read connect timedout, timedout);
-assertTrue(connect timedout, checkConnectTimeout(fs, true));
}
private boolean checkConnectTimeout(HftpFileSystem fs, boolean
ignoreReadTimeout)
svn commit: r1393699 - in /hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs: CHANGES.txt src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
Author: tgraves
Date: Wed Oct 3 19:18:12 2012
New Revision: 1393699
URL: http://svn.apache.org/viewvc?rev=1393699view=rev
Log:
HDFS-3905. Secure cluster cannot use hftp to an insecure cluster (Daryn Sharp
via tgraves)
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1393699r1=1393698r2=1393699view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
Wed Oct 3 19:18:12 2012
@@ -15,6 +15,9 @@ Release 0.23.5 - UNRELEASED
HDFS-3919. MiniDFSCluster:waitClusterUp can hang forever.
(Andy Isaacson via eli)
+HDFS-3905. Secure cluster cannot use hftp to an insecure cluster
+(Daryn Sharp via tgraves)
+
Release 0.23.4 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java?rev=1393699r1=1393698r2=1393699view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
Wed Oct 3 19:18:12 2012
@@ -24,6 +24,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.PrintStream;
+import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.InetSocketAddress;
import java.net.URL;
@@ -385,10 +386,24 @@ public class DelegationTokenFetcher {
return ugi.doAs(
new PrivilegedExceptionActionURLConnection() {
public URLConnection run() throws IOException {
- SecurityUtil.fetchServiceTicket(url);
- URLConnection connection = URLUtils.openConnection(url);
- connection.connect();
- return connection;
+ // might not be fatal if secure port doesn't connect because
+ // security may be disabled on remote cluster
+ IOException ioeForTGS = null;
+ try {
+SecurityUtil.fetchServiceTicket(url);
+ } catch (IOException ioe) {
+ioeForTGS = ioe;
+ }
+ try {
+URLConnection connection = URLUtils.openConnection(url);
+connection.connect();
+return connection;
+ } catch (ConnectException e) {
+throw e;
+ } catch (IOException ioe) {
+// throw TGS exception instead if negotiation fails
+throw (ioeForTGS != null) ? ioeForTGS : ioe;
+ }
}
});
} catch (InterruptedException ie) {
svn commit: r1390557 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapred
Author: tgraves
Date: Wed Sep 26 15:22:21 2012
New Revision: 1390557
URL: http://svn.apache.org/viewvc?rev=1390557view=rev
Log:
MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory in it.
(Robert Evans via tgraves)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1390557r1=1390556r2=1390557view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Sep 26
15:22:21 2012
@@ -550,6 +550,9 @@ Release 0.23.4 - UNRELEASED
BUG FIXES
+MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory
+in it. (Robert Evans via tgraves)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1390557r1=1390556r2=1390557view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
Wed Sep 26 15:22:21 2012
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.MapReduceChildJVM;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapred.Task;
@@ -610,10 +611,12 @@ public abstract class TaskAttemptImpl im
if (jobJar != null) {
Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
.getUri(), remoteFS.getWorkingDirectory());
-localResources.put(
-MRJobConfig.JOB_JAR,
-createLocalResource(remoteFS, remoteJobJar,
-LocalResourceType.ARCHIVE,
LocalResourceVisibility.APPLICATION));
+LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
+LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
+String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
+JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
+rc.setPattern(pattern);
+localResources.put(MRJobConfig.JOB_JAR, rc);
LOG.info(The job-jar file on the remote FS is
+ remoteJobJar.toUri().toASCIIString());
} else {
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1390557r1=1390556r2=1390557view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
Wed Sep 26 15:22:21 2012
@@ -152,6 +152,10 @@ class
svn commit: r1390560 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl
Author: tgraves
Date: Wed Sep 26 15:24:03 2012
New Revision: 1390560
URL: http://svn.apache.org/viewvc?rev=1390560view=rev
Log:
MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory in it.
(Robert Evans via tgraves)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1390560r1=1390559r2=1390560view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Wed
Sep 26 15:24:03 2012
@@ -23,6 +23,9 @@ Release 0.23.4 - UNRELEASED
MAPREDUCE-4193. broken doc link for yarn-default.xml in site.xml.
(phunt via tomwhite)
+MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory
+in it. (Robert Evans via tgraves)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1390560r1=1390559r2=1390560view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
Wed Sep 26 15:24:03 2012
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.MapReduceChildJVM;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapred.Task;
@@ -601,10 +602,12 @@ public abstract class TaskAttemptImpl im
if (jobJar != null) {
Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
.getUri(), remoteFS.getWorkingDirectory());
-localResources.put(
-MRJobConfig.JOB_JAR,
-createLocalResource(remoteFS, remoteJobJar,
-LocalResourceType.ARCHIVE,
LocalResourceVisibility.APPLICATION));
+LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
+LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
+String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
+JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
+rc.setPattern(pattern);
+localResources.put(MRJobConfig.JOB_JAR, rc);
LOG.info(The job-jar file on the remote FS is
+ remoteJobJar.toUri().toASCIIString());
} else {
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1390560r1=1390559r2=1390560view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce
svn commit: r1390563 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ h
Author: tgraves
Date: Wed Sep 26 15:25:05 2012
New Revision: 1390563
URL: http://svn.apache.org/viewvc?rev=1390563view=rev
Log:
merge -r 1390556:1390557 from trunk. FIXES: MAPREDUCE-4647
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/YARNRunner.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1390563r1=1390562r2=1390563view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Wed
Sep 26 15:25:05 2012
@@ -422,6 +422,9 @@ Release 0.23.4 - UNRELEASED
BUG FIXES
+MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory
+in it. (Robert Evans via tgraves)
+
Release 0.23.3 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1390563r1=1390562r2=1390563view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
Wed Sep 26 15:25:05 2012
@@ -45,6 +45,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.JobContext;
import org.apache.hadoop.mapred.MapReduceChildJVM;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.mapred.Task;
@@ -610,10 +611,12 @@ public abstract class TaskAttemptImpl im
if (jobJar != null) {
Path remoteJobJar = (new Path(jobJar)).makeQualified(remoteFS
.getUri(), remoteFS.getWorkingDirectory());
-localResources.put(
-MRJobConfig.JOB_JAR,
-createLocalResource(remoteFS, remoteJobJar,
-LocalResourceType.ARCHIVE,
LocalResourceVisibility.APPLICATION));
+LocalResource rc = createLocalResource(remoteFS, remoteJobJar,
+LocalResourceType.PATTERN, LocalResourceVisibility.APPLICATION);
+String pattern = conf.getPattern(JobContext.JAR_UNPACK_PATTERN,
+JobConf.UNPACK_JAR_PATTERN_DEFAULT).pattern();
+rc.setPattern(pattern);
+localResources.put(MRJobConfig.JOB_JAR, rc);
LOG.info(The job-jar file on the remote FS is
+ remoteJobJar.toUri().toASCIIString());
} else {
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1390563r1=1390562r2=1390563view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client
svn commit: r1390557 - in /hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn: hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/a
Author: tgraves
Date: Wed Sep 26 15:22:21 2012
New Revision: 1390557
URL: http://svn.apache.org/viewvc?rev=1390557view=rev
Log:
MAPREDUCE-4647. We should only unjar jobjar if there is a lib directory in it.
(Robert Evans via tgraves)
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResourcesTrackerImpl.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceRetention.java
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java?rev=1390557r1=1390556r2=1390557view=diff
==
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResource.java
Wed Sep 26 15:22:21 2012
@@ -106,4 +106,20 @@ public interface LocalResource {
* localized
*/
public void setVisibility(LocalResourceVisibility visibility);
+
+ /**
+ * Get the empattern/em that should be used to extract entries from the
+ * archive (only used when type is codePATTERN/code).
+ * @return empattern/em that should be used to extract entries from the
+ * archive.
+ */
+ public String getPattern();
+
+ /**
+ * Set the empattern/em that should be used to extract entries from the
+ * archive (only used when type is codePATTERN/code).
+ * @param pattern empattern/em that should be used to extract entries
+ * from the archive.
+ */
+ public void setPattern(String pattern);
}
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java?rev=1390557r1=1390556r2=1390557view=diff
==
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LocalResourceType.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java
svn commit: r1390479 - in /hadoop/common/site/main: author/src/documentation/content/xdocs/who.xml publish/who.html publish/who.pdf
Author: tgraves Date: Wed Sep 26 13:40:34 2012 New Revision: 1390479 URL: http://svn.apache.org/viewvc?rev=1390479view=rev Log: add tgraves to PMC list Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml hadoop/common/site/main/publish/who.html hadoop/common/site/main/publish/who.pdf Modified: hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml URL: http://svn.apache.org/viewvc/hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml?rev=1390479r1=1390478r2=1390479view=diff == --- hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml (original) +++ hadoop/common/site/main/author/src/documentation/content/xdocs/who.xml Wed Sep 26 13:40:34 2012 @@ -257,6 +257,14 @@ /tr tr + tdtgraves/td + tdThomas Graves/td + tdYahoo!/td + td/td + td-6/td +/tr + +tr tdtodd/td tda href=http://people.apache.org/~todd;Todd Lipcon/a/td tdCloudera/td Modified: hadoop/common/site/main/publish/who.html URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/who.html?rev=1390479r1=1390478r2=1390479view=diff == --- hadoop/common/site/main/publish/who.html (original) +++ hadoop/common/site/main/publish/who.html Wed Sep 26 13:40:34 2012 @@ -572,6 +572,17 @@ document.write(Last Published: + docu tr +td colspan=1 rowspan=1tgraves/td + td colspan=1 rowspan=1Thomas Graves/td + td colspan=1 rowspan=1Yahoo!/td + td colspan=1 rowspan=1/td + td colspan=1 rowspan=1-6/td + +/tr + + +tr + td colspan=1 rowspan=1todd/td td colspan=1 rowspan=1a href=http://people.apache.org/~todd;Todd Lipcon/a/td td colspan=1 rowspan=1Cloudera/td @@ -640,7 +651,7 @@ document.write(Last Published: + docu /div -a name=N103C6/aa name=Emeritus+Hadoop+PMC+Members/a +a name=N103E1/aa name=Emeritus+Hadoop+PMC+Members/a h2 class=h3Emeritus Hadoop PMC Members/h2 div class=section ul @@ -655,7 +666,7 @@ document.write(Last Published: + docu /div -a name=N103D9/aa name=Hadoop+Committers/a +a name=N103F4/aa name=Hadoop+Committers/a h2 class=h3Hadoop Committers/h2 div class=section pHadoop's active committers include:/p @@ -1270,7 +1281,7 @@ document.write(Last Published: + docu /div -a name=N10A10/aa name=Emeritus+Hadoop+Committers/a +a name=N10A2B/aa name=Emeritus+Hadoop+Committers/a h2 class=h3Emeritus Hadoop Committers/h2 div class=section pHadoop committers who are no longer active include:/p Modified: hadoop/common/site/main/publish/who.pdf URL: http://svn.apache.org/viewvc/hadoop/common/site/main/publish/who.pdf?rev=1390479r1=1390478r2=1390479view=diff == Binary files - no diff available.
svn commit: r1389888 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobHistory.java
Author: tgraves
Date: Tue Sep 25 14:38:17 2012
New Revision: 1389888
URL: http://svn.apache.org/viewvc?rev=1389888view=rev
Log:
MAPREDUCE-4662. JobHistoryFilesManager thread pool never expands (Kihwal Lee
via tgraves)
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobHistory.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1389888r1=1389887r2=1389888view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Sep 25 14:38:17 2012
@@ -254,6 +254,9 @@ Release 1.2.0 - unreleased
MAPREDUCE-4652. ValueAggregatorJob sets the wrong job jar.
(Ahmed Radwan via tomwhite)
+MAPREDUCE-4662. JobHistoryFilesManager thread pool never expands
+(Kihwal Lee via tgraves)
+
Release 1.1.0 - unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobHistory.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobHistory.java?rev=1389888r1=1389887r2=1389888view=diff
==
---
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobHistory.java
(original)
+++
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobHistory.java
Tue Sep 25 14:38:17 2012
@@ -221,8 +221,11 @@ public class JobHistory {
void start() {
- executor = new ThreadPoolExecutor(1, 3, 1,
+ executor = new ThreadPoolExecutor(5, 5, 1,
TimeUnit.HOURS, new LinkedBlockingQueueRunnable());
+ // make core threads to terminate if there has been no work
+ // for the keppalive period.
+ executor.allowCoreThreadTimeOut(true);
}
private FilesHolder getFileHolder(JobID id) {
svn commit: r1383430 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
Author: tgraves
Date: Tue Sep 11 14:34:27 2012
New Revision: 1383430
URL: http://svn.apache.org/viewvc?rev=1383430view=rev
Log:
MAPREDUCE-4576. Large dist cache can block tasktracker heartbeat (Robert Evans
via tgraves).
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1383430r1=1383429r2=1383430view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Sep 11 14:34:27 2012
@@ -233,6 +233,9 @@ Release 1.2.0 - unreleased
HADOOP-8781. hadoop-config.sh should add JAVA_LIBRARY_PATH to
LD_LIBRARY_PATH. (tucu)
+MAPREDUCE-4576. Large dist cache can block tasktracker heartbeat
+(Robert Evans via tgraves)
+
Release 1.1.0 - unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java?rev=1383430r1=1383429r2=1383430view=diff
==
---
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
(original)
+++
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/filecache/TrackerDistributedCacheManager.java
Tue Sep 11 14:34:27 2012
@@ -35,6 +35,7 @@ import java.util.Random;
import java.util.Set;
import java.util.TreeMap;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.locks.ReentrantLock;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -566,7 +567,8 @@ public class TrackerDistributedCacheMana
//
long size; //the size of this cache.
boolean inited = false; // is it initialized ?
-
+private final ReentrantLock lock = new ReentrantLock();
+
//
// The following five fields are Immutable.
//
@@ -598,14 +600,20 @@ public class TrackerDistributedCacheMana
this.key = key;
}
-public synchronized void incRefCount() {
- refcount.incrementAndGet() ;
- LOG.debug(localizedLoadPath + : refcount= + refcount.get());
+public void incRefCount() {
+ lock.lock();
+ try {
+refcount.incrementAndGet() ;
+LOG.debug(localizedLoadPath + : refcount= + refcount.get());
+ } finally {
+lock.unlock();
+ }
}
public void decRefCount() {
synchronized (cachedArchives) {
-synchronized (this) {
+lock.lock();
+try {
refcount.decrementAndGet() ;
LOG.debug(localizedLoadPath + : refcount= + refcount.get());
if(refcount.get() = 0) {
@@ -613,6 +621,8 @@ public class TrackerDistributedCacheMana
cachedArchives.remove(key);
cachedArchives.put(key, this);
}
+} finally {
+ lock.unlock();
}
}
}
@@ -621,9 +631,14 @@ public class TrackerDistributedCacheMana
return refcount.get();
}
-public synchronized boolean isUsed() {
- LOG.debug(localizedLoadPath + : refcount= + refcount.get());
- return refcount.get() 0;
+public boolean isUsed() {
+ lock.lock();
+ try {
+LOG.debug(localizedLoadPath + : refcount= + refcount.get());
+return refcount.get() 0;
+ } finally {
+lock.unlock();
+ }
}
Path getBaseDir(){
@@ -1027,19 +1042,24 @@ public class TrackerDistributedCacheMana
CacheDir leftToClean =
toBeCleanedBaseDir.get(cacheStatus.getBaseDir());
if (leftToClean != null (leftToClean.size 0 ||
leftToClean.subdirs 0)) {
-synchronized (cacheStatus) {
- // if reference count is zero mark the cache for deletion
- boolean isUsed = cacheStatus.isUsed();
- long cacheSize = cacheStatus.size;
- LOG.debug(cacheStatus.getLocalizedUniqueDir() + : isUsed= +
isUsed +
- size= + cacheSize + leftToClean.size= +
leftToClean.size);
- if (!isUsed) {
-leftToClean.size -= cacheSize;
-leftToClean.subdirs--;
-// delete this cache entry from the global list
-// and mark the localized file for deletion
-toBeDeletedCache.add(cacheStatus);
-it.remove();
+boolean gotLock = cacheStatus.lock.tryLock();
+if (gotLock) {
+ try {
+// if reference count is zero mark the cache for deletion
+boolean isUsed
svn commit: r1381459 - in /hadoop/common/trunk/hadoop-yarn-project: ./ hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containerma
Author: tgraves
Date: Thu Sep 6 02:08:37 2012
New Revision: 1381459
URL: http://svn.apache.org/viewvc?rev=1381459view=rev
Log:
YARN-87. NM ResourceLocalizationService does not set permissions of local cache
directories (Jason Lowe via tgraves)
Modified:
hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
Modified: hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt?rev=1381459r1=1381458r2=1381459view=diff
==
--- hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-yarn-project/CHANGES.txt Thu Sep 6 02:08:37 2012
@@ -114,3 +114,6 @@ Release 0.23.3 - Unreleased
YARN-68. NodeManager will refuse to shutdown indefinitely due to container
log aggregation (daryn via bobby)
+
+YARN-87. NM ResourceLocalizationService does not set permissions of local
+cache directories (Jason Lowe via tgraves)
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java?rev=1381459r1=1381458r2=1381459view=diff
==
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
Thu Sep 6 02:08:37 2012
@@ -181,6 +181,7 @@ public class ResourceLocalizationService
try {
// TODO queue deletions here, rather than NM init?
FileContext lfs = getLocalFileContext(conf);
+ lfs.setUMask(new FsPermission((short)FsPermission.DEFAULT_UMASK));
ListString localDirs = dirsHandler.getLocalDirs();
for (String localDir : localDirs) {
// $local/usercache
Modified:
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java?rev=1381459r1=1381458r2=1381459view=diff
==
---
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
(original)
+++
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
Thu Sep 6 02:08:37 2012
@@ -50,6 +50,7 @@ import junit.framework.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Options.ChecksumOpt;
@@ -116,6 +117,7 @@ public class TestResourceLocalizationSer
@Test
public void testLocalizationInit() throws Exception {
final Configuration conf = new Configuration();
+conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, 077);
AsyncDispatcher dispatcher = new AsyncDispatcher();
dispatcher.init(new Configuration());
@@ -151,14 +153,18 @@ public class TestResourceLocalizationSer
// initialize ResourceLocalizationService
locService.init(conf);
+ final FsPermission defaultPerm = new FsPermission((short)0755);
+
// verify directory
svn commit: r1379599 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ hadoop-mapreduce-c
Author: tgraves
Date: Fri Aug 31 20:43:46 2012
New Revision: 1379599
URL: http://svn.apache.org/viewvc?rev=1379599view=rev
Log:
MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert Evans via
tgraves)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1379599r1=1379598r2=1379599view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Fri Aug 31
20:43:46 2012
@@ -858,6 +858,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
+MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
+Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1379599r1=1379598r2=1379599view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Fri Aug 31 20:43:46 2012
@@ -99,8 +99,8 @@ public class JobHistoryEventHandler exte
protected static final MapJobId, MetaInfo fileMap =
Collections.JobId,MetaInfosynchronizedMap(new HashMapJobId,MetaInfo());
- // Has a signal (SIGTERM etc) been issued?
- protected volatile boolean isSignalled = false;
+ // should job completion be force when the AM shuts down?
+ protected volatile boolean forceJobCompletion = false;
public JobHistoryEventHandler(AppContext context, int startCount) {
super(JobHistoryEventHandler);
@@ -322,7 +322,7 @@ public class JobHistoryEventHandler exte
// Process JobUnsuccessfulCompletionEvent for jobIds which still haven't
// closed their event writers
IteratorJobId jobIt = fileMap.keySet().iterator();
-if(isSignalled) {
+if(forceJobCompletion) {
while (jobIt.hasNext()) {
JobId toClose = jobIt.next();
MetaInfo mi = fileMap.get(toClose);
@@ -911,9 +911,9 @@ public class JobHistoryEventHandler exte
return tmpFileName.substring(0, tmpFileName.length()-4);
}
- public void setSignalled(boolean isSignalled) {
-this.isSignalled = isSignalled;
-LOG.info(JobHistoryEventHandler notified that isSignalled was
- + isSignalled);
+ public void setForcejobCompletion(boolean forceJobCompletion) {
+this.forceJobCompletion = forceJobCompletion;
+LOG.info(JobHistoryEventHandler notified that forceJobCompletion is
+ + forceJobCompletion);
}
}
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1379599r1=1379598r2=1379599view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce
svn commit: r1379602 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ hadoop
Author: tgraves
Date: Fri Aug 31 20:45:34 2012
New Revision: 1379602
URL: http://svn.apache.org/viewvc?rev=1379602view=rev
Log:
merge -r 1379598:1379599 from trunk. FIXES: MAPREDUCE-4611
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1379602r1=1379601r2=1379602view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 31 20:45:34 2012
@@ -734,6 +734,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
+MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
+Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1379602r1=1379601r2=1379602view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Fri Aug 31 20:45:34 2012
@@ -99,8 +99,8 @@ public class JobHistoryEventHandler exte
protected static final MapJobId, MetaInfo fileMap =
Collections.JobId,MetaInfosynchronizedMap(new HashMapJobId,MetaInfo());
- // Has a signal (SIGTERM etc) been issued?
- protected volatile boolean isSignalled = false;
+ // should job completion be force when the AM shuts down?
+ protected volatile boolean forceJobCompletion = false;
public JobHistoryEventHandler(AppContext context, int startCount) {
super(JobHistoryEventHandler);
@@ -322,7 +322,7 @@ public class JobHistoryEventHandler exte
// Process JobUnsuccessfulCompletionEvent for jobIds which still haven't
// closed their event writers
IteratorJobId jobIt = fileMap.keySet().iterator();
-if(isSignalled) {
+if(forceJobCompletion) {
while (jobIt.hasNext()) {
JobId toClose = jobIt.next();
MetaInfo mi = fileMap.get(toClose);
@@ -911,9 +911,9 @@ public class JobHistoryEventHandler exte
return tmpFileName.substring(0, tmpFileName.length()-4);
}
- public void setSignalled(boolean isSignalled) {
-this.isSignalled = isSignalled;
-LOG.info(JobHistoryEventHandler notified that isSignalled was
- + isSignalled);
+ public void setForcejobCompletion(boolean forceJobCompletion) {
+this.forceJobCompletion = forceJobCompletion;
+LOG.info(JobHistoryEventHandler notified that forceJobCompletion is
+ + forceJobCompletion);
}
}
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1379602r1=1379601r2=1379602view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project
svn commit: r1379604 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/ had
Author: tgraves
Date: Fri Aug 31 20:48:22 2012
New Revision: 1379604
URL: http://svn.apache.org/viewvc?rev=1379604view=rev
Log:
merge -r 1379601:1379602 from branch-2. FIXES: MAPREDUCE-4611
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestJobHistoryEventHandler.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestStagingCleanup.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1379604r1=1379603r2=1379604view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 31 20:48:22 2012
@@ -426,6 +426,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4614. Simplify debugging a job's tokens (daryn via bobby)
+MAPREDUCE-4611. MR AM dies badly when Node is decommissioned (Robert
+Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java?rev=1379604r1=1379603r2=1379604view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
Fri Aug 31 20:48:22 2012
@@ -99,8 +99,8 @@ public class JobHistoryEventHandler exte
protected static final MapJobId, MetaInfo fileMap =
Collections.JobId,MetaInfosynchronizedMap(new HashMapJobId,MetaInfo());
- // Has a signal (SIGTERM etc) been issued?
- protected volatile boolean isSignalled = false;
+ // should job completion be force when the AM shuts down?
+ protected volatile boolean forceJobCompletion = false;
public JobHistoryEventHandler(AppContext context, int startCount) {
super(JobHistoryEventHandler);
@@ -322,7 +322,7 @@ public class JobHistoryEventHandler exte
// Process JobUnsuccessfulCompletionEvent for jobIds which still haven't
// closed their event writers
IteratorJobId jobIt = fileMap.keySet().iterator();
-if(isSignalled) {
+if(forceJobCompletion) {
while (jobIt.hasNext()) {
JobId toClose = jobIt.next();
MetaInfo mi = fileMap.get(toClose);
@@ -911,9 +911,9 @@ public class JobHistoryEventHandler exte
return tmpFileName.substring(0, tmpFileName.length()-4);
}
- public void setSignalled(boolean isSignalled) {
-this.isSignalled = isSignalled;
-LOG.info(JobHistoryEventHandler notified that isSignalled was
- + isSignalled);
+ public void setForcejobCompletion(boolean forceJobCompletion) {
+this.forceJobCompletion = forceJobCompletion;
+LOG.info(JobHistoryEventHandler notified that forceJobCompletion is
+ + forceJobCompletion);
}
}
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java?rev=1379604r1=1379603r2=1379604view=diff
svn commit: r1378614 - in /hadoop/common/branches/branch-1: CHANGES.txt src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Author: tgraves
Date: Wed Aug 29 15:38:40 2012
New Revision: 1378614
URL: http://svn.apache.org/viewvc?rev=1378614view=rev
Log:
MAPREDUCE-4499. Looking for speculative tasks is very expensive in 1.x (Koji
Noguchi via tgraves)
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1378614r1=1378613r2=1378614view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Wed Aug 29 15:38:40 2012
@@ -103,6 +103,9 @@ Release 1.2.0 - unreleased
MAPREDUCE-4408. allow jobs to set a JAR that is in the distributed cached
(rkanter via tucu)
+MAPREDUCE-4499. Looking for speculative tasks is very expensive in 1.x
+(Koji Noguchi via tgraves)
+
OPTIMIZATIONS
HDFS-2533. Backport: Remove needless synchronization on some FSDataSet
Modified:
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java?rev=1378614r1=1378613r2=1378614view=diff
==
---
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
(original)
+++
hadoop/common/branches/branch-1/src/mapred/org/apache/hadoop/mapred/JobInProgress.java
Wed Aug 29 15:38:40 2012
@@ -2246,21 +2246,19 @@ public class JobInProgress {
continue;
}
- if (!tip.hasRunOnMachine(ttStatus.getHost(),
+ if (tip.hasSpeculativeTask(currentTime, avgProgress)) {
+// Check if this tip can be removed from the list.
+// If the list is shared then we should not remove.
+if(shouldRemove){
+ iter.remove();
+}
+if (!tip.hasRunOnMachine(ttStatus.getHost(),
ttStatus.getTrackerName())) {
-if (tip.hasSpeculativeTask(currentTime, avgProgress)) {
- // In case of shared list we don't remove it. Since the TIP failed
- // on this tracker can be scheduled on some other tracker.
- if (shouldRemove) {
-iter.remove(); //this tracker is never going to run it again
- }
return tip;
-}
+}
} else {
-// Check if this tip can be removed from the list.
-// If the list is shared then we should not remove.
-if (shouldRemove) {
- // This tracker will never speculate this tip
+if (shouldRemove tip.hasRunOnMachine(ttStatus.getHost(),
+ ttStatus.getTrackerName())) {
iter.remove();
}
}
svn commit: r1378357 - in /hadoop/common/branches/branch-1: CHANGES.txt src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
Author: tgraves
Date: Tue Aug 28 21:28:39 2012
New Revision: 1378357
URL: http://svn.apache.org/viewvc?rev=1378357view=rev
Log:
MAPREDUCE-1684. ClusterStatus can be cached in
CapacityTaskScheduler.assignTasks() (Koji Noguchi via tgraves)
Modified:
hadoop/common/branches/branch-1/CHANGES.txt
hadoop/common/branches/branch-1/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
Modified: hadoop/common/branches/branch-1/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/CHANGES.txt?rev=1378357r1=1378356r2=1378357view=diff
==
--- hadoop/common/branches/branch-1/CHANGES.txt (original)
+++ hadoop/common/branches/branch-1/CHANGES.txt Tue Aug 28 21:28:39 2012
@@ -213,6 +213,9 @@ Release 1.2.0 - unreleased
MAPREDUCE-4595. TestLostTracker failing - possibly due to a race in
JobHistory.JobHistoryFilesManager#run() (kkambatl via tucu)
+MAPREDUCE-1684. ClusterStatus can be cached in
+CapacityTaskScheduler.assignTasks() (Koji Noguchi via tgraves)
+
Release 1.1.0 - unreleased
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-1/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java?rev=1378357r1=1378356r2=1378357view=diff
==
---
hadoop/common/branches/branch-1/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
(original)
+++
hadoop/common/branches/branch-1/src/contrib/capacity-scheduler/src/java/org/apache/hadoop/mapred/CapacityTaskScheduler.java
Tue Aug 28 21:28:39 2012
@@ -154,7 +154,8 @@ class CapacityTaskScheduler extends Task
protected TaskType type = null;
abstract TaskLookupResult obtainNewTask(TaskTrackerStatus taskTracker,
-JobInProgress job, boolean assignOffSwitch) throws IOException;
+JobInProgress job, boolean assignOffSwitch,
+ClusterStatus clusterStatus) throws IOException;
int getSlotsOccupied(JobInProgress job) {
return (getNumReservedTaskTrackers(job) + getRunningTasks(job)) *
@@ -293,7 +294,8 @@ class CapacityTaskScheduler extends Task
private TaskLookupResult getTaskFromQueue(TaskTracker taskTracker,
int availableSlots,
CapacitySchedulerQueue queue,
- boolean assignOffSwitch)
+ boolean assignOffSwitch,
+ ClusterStatus clusterStatus)
throws IOException {
TaskTrackerStatus taskTrackerStatus = taskTracker.getStatus();
// we only look at jobs in the running queues, as these are the ones
@@ -320,7 +322,8 @@ class CapacityTaskScheduler extends Task
availableSlots))
{
// We found a suitable job. Get task from it.
TaskLookupResult tlr =
-obtainNewTask(taskTrackerStatus, j, assignOffSwitch);
+obtainNewTask(taskTrackerStatus, j, assignOffSwitch,
+ clusterStatus);
//if there is a task return it immediately.
if (tlr.getLookUpStatus() ==
TaskLookupResult.LookUpStatus.LOCAL_TASK_FOUND ||
@@ -379,6 +382,11 @@ class CapacityTaskScheduler extends Task
printQueues();
+ //MAPREDUCE-1684: somehow getClusterStatus seems to be expensive. Caching
+ //here to reuse during the scheduling
+ ClusterStatus clusterStatus =
+scheduler.taskTrackerManager.getClusterStatus();
+
// Check if this tasktracker has been reserved for a job...
JobInProgress job = taskTracker.getJobForFallowSlot(type);
if (job != null) {
@@ -397,7 +405,7 @@ class CapacityTaskScheduler extends Task
// Don't care about locality!
job.overrideSchedulingOpportunities();
}
- return obtainNewTask(taskTrackerStatus, job, true);
+ return obtainNewTask(taskTrackerStatus, job, true, clusterStatus);
} else {
// Re-reserve the current tasktracker
taskTracker.reserveSlots(type, job, availableSlots);
@@ -420,7 +428,8 @@ class CapacityTaskScheduler extends Task
}
TaskLookupResult tlr =
- getTaskFromQueue(taskTracker, availableSlots, queue,
assignOffSwitch);
+ getTaskFromQueue(taskTracker, availableSlots, queue, assignOffSwitch,
+ clusterStatus);
TaskLookupResult.LookUpStatus lookUpStatus = tlr.getLookUpStatus();
if (lookUpStatus
svn commit: r1375834 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/ src/main/java/org/apache/hadoop/util/
Author: tgraves
Date: Tue Aug 21 22:38:46 2012
New Revision: 1375834
URL: http://svn.apache.org/viewvc?rev=1375834view=rev
Log:
HADOOP-8239. Add subclasses of MD5MD5CRC32FileChecksum to support file checksum
with CRC32C. (Kihwal Lee via szetszwo)
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32GzipFileChecksum.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/DataChecksum.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1375834r1=1375833r2=1375834view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Tue Aug 21 22:38:46 2012
@@ -36,6 +36,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8240. Add a new API to allow users to specify a checksum type
on FileSystem.create(..). (Kihwal Lee via szetszwo)
+HADOOP-8239. Add subclasses of MD5MD5CRC32FileChecksum to support file
+checksum with CRC32C. (Kihwal Lee via szetszwo)
+
OPTIMIZATIONS
BUG FIXES
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java?rev=1375834view=auto
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java
(added)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32CastagnoliFileChecksum.java
Tue Aug 21 22:38:46 2012
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs;
+
+import org.apache.hadoop.io.MD5Hash;
+import org.apache.hadoop.util.DataChecksum;
+
+/** For CRC32 with the Castagnoli polynomial */
+public class MD5MD5CRC32CastagnoliFileChecksum extends MD5MD5CRC32FileChecksum
{
+ /** Same as this(0, 0, null) */
+ public MD5MD5CRC32CastagnoliFileChecksum() {
+this(0, 0, null);
+ }
+
+ /** Create a MD5FileChecksum */
+ public MD5MD5CRC32CastagnoliFileChecksum(int bytesPerCRC, long crcPerBlock,
MD5Hash md5) {
+super(bytesPerCRC, crcPerBlock, md5);
+ }
+
+ @Override
+ public DataChecksum.Type getCrcType() {
+// default to the one that is understood by all releases.
+return DataChecksum.Type.CRC32C;
+ }
+}
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java?rev=1375834r1=1375833r2=1375834view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/MD5MD5CRC32FileChecksum.java
Tue Aug 21 22:38:46 2012
@@ -23,12 +23,17 @@ import java.io.IOException;
import org.apache.hadoop.classification.InterfaceAudience
svn commit: r1374407 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/ had
Author: tgraves
Date: Fri Aug 17 19:47:35 2012
New Revision: 1374407
URL: http://svn.apache.org/viewvc?rev=1374407view=rev
Log:
MAPREDUCE-4549. Distributed cache conflicts breaks backwards compatability
(Robert Evans via tgraves)
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapreduce/v2/util/TestMRApps.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1374407r1=1374406r2=1374407view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Fri
Aug 17 19:47:35 2012
@@ -392,6 +392,9 @@ Release 0.23.3 - UNRELEASED
for compatibility reasons is creating incorrect counter name.
(Jarek Jarcec Cecho via tomwhite)
+MAPREDUCE-4549. Distributed cache conflicts breaks backwards compatability
+(Robert Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java?rev=1374407r1=1374406r2=1374407view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/util/MRApps.java
Fri Aug 17 19:47:35 2012
@@ -30,6 +30,8 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -48,14 +50,15 @@ import org.apache.hadoop.mapreduce.v2.ap
import org.apache.hadoop.mapreduce.v2.api.records.TaskType;
import org.apache.hadoop.yarn.ContainerLogAppender;
import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.ApplicationConstants.Environment;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.ConverterUtils;
/**
* Helper class for MR applications
@@ -63,6 +66,8 @@ import org.apache.hadoop.yarn.util.Build
@Private
@Unstable
public class MRApps extends Apps {
+ private static final Log LOG = LogFactory.getLog(MRApps.class);
+
public static String toString(JobId jid) {
return jid.toString();
}
@@ -263,6 +268,23 @@ public class MRApps extends Apps {
DistributedCache.getFileClassPaths(conf));
}
+ private static String getResourceDescription(LocalResourceType type) {
+if(type == LocalResourceType.ARCHIVE) {
+ return cache archive ( + MRJobConfig.CACHE_ARCHIVES + ) ;
+}
+return cache file ( + MRJobConfig.CACHE_FILES + ) ;
+ }
+
+ private static String toString(org.apache.hadoop.yarn.api.records.URL url) {
+StringBuffer b = new StringBuffer();
+b.append(url.getScheme()).append(://).append(url.getHost());
+if(url.getPort() = 0) {
+ b.append(:).append(url.getPort());
+}
+b.append(url.getFile());
+return b.toString();
+ }
+
// TODO - Move this to MR!
// Use TaskDistributedCacheManager.CacheFiles.makeCacheFiles(URI[],
// long[], boolean[], Path[], FileType)
@@ -308,6 +330,17 @@ public class MRApps extends Apps {
throw new IllegalArgumentException(Resource name must be relative);
}
String linkName = name.toUri().getPath();
+LocalResource orig
svn commit: r1374424 - /hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Author: tgraves
Date: Fri Aug 17 20:29:38 2012
New Revision: 1374424
URL: http://svn.apache.org/viewvc?rev=1374424view=rev
Log:
YARN-25. remove old aggregated logs (Robert Evans via tgraves)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1374424r1=1374423r2=1374424view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Fri Aug 17 20:29:38 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.YarnExcept
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService;
import org.apache.hadoop.yarn.service.CompositeService;
/**
@@ -53,6 +54,7 @@ public class JobHistoryServer extends Co
private HistoryClientService clientService;
private JobHistory jobHistoryService;
private JHSDelegationTokenSecretManager jhsDTSecretManager;
+ private AggregatedLogDeletionService aggLogDelService;
public JobHistoryServer() {
super(JobHistoryServer.class.getName());
@@ -74,8 +76,10 @@ public class JobHistoryServer extends Co
this.jhsDTSecretManager = createJHSSecretManager(conf);
clientService = new HistoryClientService(historyContext,
this.jhsDTSecretManager);
+aggLogDelService = new AggregatedLogDeletionService();
addService(jobHistoryService);
addService(clientService);
+addService(aggLogDelService);
super.init(config);
}
svn commit: r1374425 - /hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.jav
Author: tgraves
Date: Fri Aug 17 20:31:09 2012
New Revision: 1374425
URL: http://svn.apache.org/viewvc?rev=1374425view=rev
Log:
merge -r 1374423:1374424 from trunk. FIXES: YARN-25
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1374425r1=1374424r2=1374425view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Fri Aug 17 20:31:09 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.YarnExcept
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService;
import org.apache.hadoop.yarn.service.CompositeService;
/**
@@ -53,6 +54,7 @@ public class JobHistoryServer extends Co
private HistoryClientService clientService;
private JobHistory jobHistoryService;
private JHSDelegationTokenSecretManager jhsDTSecretManager;
+ private AggregatedLogDeletionService aggLogDelService;
public JobHistoryServer() {
super(JobHistoryServer.class.getName());
@@ -74,8 +76,10 @@ public class JobHistoryServer extends Co
this.jhsDTSecretManager = createJHSSecretManager(conf);
clientService = new HistoryClientService(historyContext,
this.jhsDTSecretManager);
+aggLogDelService = new AggregatedLogDeletionService();
addService(jobHistoryService);
addService(clientService);
+addService(aggLogDelService);
super.init(config);
}
svn commit: r1374426 - /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.
Author: tgraves
Date: Fri Aug 17 20:32:22 2012
New Revision: 1374426
URL: http://svn.apache.org/viewvc?rev=1374426view=rev
Log:
merge -r 1374424:1374425 from branch-2. FIXES: YARN-25
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1374426r1=1374425r2=1374426view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Fri Aug 17 20:32:22 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.YarnExcept
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService;
import org.apache.hadoop.yarn.service.CompositeService;
/**
@@ -53,6 +54,7 @@ public class JobHistoryServer extends Co
private HistoryClientService clientService;
private JobHistory jobHistoryService;
private JHSDelegationTokenSecretManager jhsDTSecretManager;
+ private AggregatedLogDeletionService aggLogDelService;
public JobHistoryServer() {
super(JobHistoryServer.class.getName());
@@ -74,8 +76,10 @@ public class JobHistoryServer extends Co
this.jhsDTSecretManager = createJHSSecretManager(conf);
clientService = new HistoryClientService(historyContext,
this.jhsDTSecretManager);
+aggLogDelService = new AggregatedLogDeletionService();
addService(jobHistoryService);
addService(clientService);
+addService(aggLogDelService);
super.init(config);
}
svn commit: r1374427 - /hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistory
Author: tgraves
Date: Fri Aug 17 20:32:42 2012
New Revision: 1374427
URL: http://svn.apache.org/viewvc?rev=1374427view=rev
Log:
merge -r 1374423:1374424 from trunk. FIXES: YARN-25
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java?rev=1374427r1=1374426r2=1374427view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/JobHistoryServer.java
Fri Aug 17 20:32:42 2012
@@ -34,6 +34,7 @@ import org.apache.hadoop.yarn.YarnExcept
import org.apache.hadoop.yarn.YarnUncaughtExceptionHandler;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.logaggregation.AggregatedLogDeletionService;
import org.apache.hadoop.yarn.service.CompositeService;
/**
@@ -53,6 +54,7 @@ public class JobHistoryServer extends Co
private HistoryClientService clientService;
private JobHistory jobHistoryService;
private JHSDelegationTokenSecretManager jhsDTSecretManager;
+ private AggregatedLogDeletionService aggLogDelService;
public JobHistoryServer() {
super(JobHistoryServer.class.getName());
@@ -74,8 +76,10 @@ public class JobHistoryServer extends Co
this.jhsDTSecretManager = createJHSSecretManager(conf);
clientService = new HistoryClientService(historyContext,
this.jhsDTSecretManager);
+aggLogDelService = new AggregatedLogDeletionService();
addService(jobHistoryService);
addService(clientService);
+addService(aggLogDelService);
super.init(config);
}
svn commit: r1374314 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Author: tgraves
Date: Fri Aug 17 15:20:03 2012
New Revision: 1374314
URL: http://svn.apache.org/viewvc?rev=1374314view=rev
Log:
merge -r 1374311:1374312 from trunk. FIXES: HADOOP-8692
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374314r1=1374313r2=1374314view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Aug 17 15:20:03 2012
@@ -654,6 +654,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor
Robinson via tgraves)
+HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7
+(Trevor Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java?rev=1374314r1=1374313r2=1374314view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Fri Aug 17 15:20:03 2012
@@ -343,14 +343,18 @@ public class TestLocalDirAllocator {
@Test
public void testRemoveContext() throws IOException {
String dir = buildBufferDir(ROOT, 0);
-String contextCfgItemName =
application_1340842292563_0004.app.cache.dirs;
-conf.set(contextCfgItemName, dir);
-LocalDirAllocator localDirAllocator = new LocalDirAllocator(
-contextCfgItemName);
-localDirAllocator.getLocalPathForWrite(p1/x, SMALL_FILE_SIZE, conf);
-assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName));
-LocalDirAllocator.removeContext(contextCfgItemName);
-assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName));
+try {
+ String contextCfgItemName =
application_1340842292563_0004.app.cache.dirs;
+ conf.set(contextCfgItemName, dir);
+ LocalDirAllocator localDirAllocator = new LocalDirAllocator(
+ contextCfgItemName);
+ localDirAllocator.getLocalPathForWrite(p1/x, SMALL_FILE_SIZE, conf);
+ assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName));
+ LocalDirAllocator.removeContext(contextCfgItemName);
+ assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName));
+} finally {
+ rmBufferDirs();
+}
}
}
svn commit: r1374316 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Author: tgraves
Date: Fri Aug 17 15:21:48 2012
New Revision: 1374316
URL: http://svn.apache.org/viewvc?rev=1374316view=rev
Log:
merge -r 1374312:1374313 from branch-2. FIXES: HADOOP-8692
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374316r1=1374315r2=1374316view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Aug 17 15:21:48 2012
@@ -159,6 +159,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor
Robinson via tgraves)
+HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7
+(Trevor Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java?rev=1374316r1=1374315r2=1374316view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestLocalDirAllocator.java
Fri Aug 17 15:21:48 2012
@@ -340,14 +340,18 @@ public class TestLocalDirAllocator {
@Test
public void testRemoveContext() throws IOException {
String dir = buildBufferDir(ROOT, 0);
-String contextCfgItemName =
application_1340842292563_0004.app.cache.dirs;
-conf.set(contextCfgItemName, dir);
-LocalDirAllocator localDirAllocator = new LocalDirAllocator(
-contextCfgItemName);
-localDirAllocator.getLocalPathForWrite(p1/x, SMALL_FILE_SIZE, conf);
-assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName));
-LocalDirAllocator.removeContext(contextCfgItemName);
-assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName));
+try {
+ String contextCfgItemName =
application_1340842292563_0004.app.cache.dirs;
+ conf.set(contextCfgItemName, dir);
+ LocalDirAllocator localDirAllocator = new LocalDirAllocator(
+ contextCfgItemName);
+ localDirAllocator.getLocalPathForWrite(p1/x, SMALL_FILE_SIZE, conf);
+ assertTrue(LocalDirAllocator.isContextValid(contextCfgItemName));
+ LocalDirAllocator.removeContext(contextCfgItemName);
+ assertFalse(LocalDirAllocator.isContextValid(contextCfgItemName));
+} finally {
+ rmBufferDirs();
+}
}
}
svn commit: r1374322 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Author: tgraves
Date: Fri Aug 17 15:32:50 2012
New Revision: 1374322
URL: http://svn.apache.org/viewvc?rev=1374322view=rev
Log:
HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor Robinson
via tgraves)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374322r1=1374321r2=1374322view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug
17 15:32:50 2012
@@ -919,6 +919,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7
(Trevor Robinson via tgraves)
+HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java?rev=1374322r1=1374321r2=1374322view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Fri Aug 17 15:32:50 2012
@@ -137,6 +137,7 @@ public class TestSecurityUtil {
@Test
public void testBuildDTServiceName() {
+SecurityUtil.setTokenServiceUseIp(true);
assertEquals(127.0.0.1:123,
SecurityUtil.buildDTServiceName(URI.create(test://LocalHost), 123)
);
@@ -153,6 +154,7 @@ public class TestSecurityUtil {
@Test
public void testBuildTokenServiceSockAddr() {
+SecurityUtil.setTokenServiceUseIp(true);
assertEquals(127.0.0.1:123,
SecurityUtil.buildTokenService(new InetSocketAddress(LocalHost,
123)).toString()
);
svn commit: r1374323 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Author: tgraves
Date: Fri Aug 17 15:37:00 2012
New Revision: 1374323
URL: http://svn.apache.org/viewvc?rev=1374323view=rev
Log:
merge -r 1374321:1374322 from trunk. FIXES: HADOOP-8693
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374323r1=1374322r2=1374323view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Aug 17 15:37:00 2012
@@ -739,6 +739,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7
(Trevor Robinson via tgraves)
+HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java?rev=1374323r1=1374322r2=1374323view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Fri Aug 17 15:37:00 2012
@@ -137,6 +137,7 @@ public class TestSecurityUtil {
@Test
public void testBuildDTServiceName() {
+SecurityUtil.setTokenServiceUseIp(true);
assertEquals(127.0.0.1:123,
SecurityUtil.buildDTServiceName(URI.create(test://LocalHost), 123)
);
@@ -153,6 +154,7 @@ public class TestSecurityUtil {
@Test
public void testBuildTokenServiceSockAddr() {
+SecurityUtil.setTokenServiceUseIp(true);
assertEquals(127.0.0.1:123,
SecurityUtil.buildTokenService(new InetSocketAddress(LocalHost,
123)).toString()
);
svn commit: r1374325 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Author: tgraves
Date: Fri Aug 17 15:38:10 2012
New Revision: 1374325
URL: http://svn.apache.org/viewvc?rev=1374325view=rev
Log:
merge -r 1374322:1374323 from branch-2. FIXES: HADOOP-8693
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374325r1=1374324r2=1374325view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Aug 17 15:38:10 2012
@@ -162,6 +162,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8692. TestLocalDirAllocator fails intermittently with JDK7
(Trevor Robinson via tgraves)
+HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java?rev=1374325r1=1374324r2=1374325view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/security/TestSecurityUtil.java
Fri Aug 17 15:38:10 2012
@@ -137,6 +137,7 @@ public class TestSecurityUtil {
@Test
public void testBuildDTServiceName() {
+SecurityUtil.setTokenServiceUseIp(true);
assertEquals(127.0.0.1:123,
SecurityUtil.buildDTServiceName(URI.create(test://LocalHost), 123)
);
@@ -153,6 +154,7 @@ public class TestSecurityUtil {
@Test
public void testBuildTokenServiceSockAddr() {
+SecurityUtil.setTokenServiceUseIp(true);
assertEquals(127.0.0.1:123,
SecurityUtil.buildTokenService(new InetSocketAddress(LocalHost,
123)).toString()
);
svn commit: r1374349 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/io/TestWritableName.java
Author: tgraves Date: Fri Aug 17 16:38:53 2012 New Revision: 1374349 URL: http://svn.apache.org/viewvc?rev=1374349view=rev Log: HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor Robinson via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374349r1=1374348r2=1374349view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 16:38:53 2012 @@ -922,6 +922,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java?rev=1374349r1=1374348r2=1374349view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java Fri Aug 17 16:38:53 2012 @@ -83,6 +83,7 @@ public class TestWritableName extends Te Configuration conf = new Configuration(); String altName = testName + .alt; +WritableName.setName(SimpleWritable.class, testName); WritableName.addName(SimpleWritable.class, altName); Class? test = WritableName.getClass(altName, conf);
svn commit: r1374351 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/io/TestWritableName.java
Author: tgraves Date: Fri Aug 17 16:41:26 2012 New Revision: 1374351 URL: http://svn.apache.org/viewvc?rev=1374351view=rev Log: merge -r 1374348:1374349 from trunk. FIXES: HADOOP-8697 Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374351r1=1374350r2=1374351view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug 17 16:41:26 2012 @@ -660,6 +660,9 @@ Release 0.23.3 - UNRELEASED HADOOP-8693. TestSecurityUtil fails intermittently with JDK7 (Trevor Robinson via tgraves) +HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java?rev=1374351r1=1374350r2=1374351view=diff == --- hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java (original) +++ hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/io/TestWritableName.java Fri Aug 17 16:41:26 2012 @@ -83,6 +83,7 @@ public class TestWritableName extends Te Configuration conf = new Configuration(); String altName = testName + .alt; +WritableName.setName(SimpleWritable.class, testName); WritableName.addName(SimpleWritable.class, altName); Class? test = WritableName.getClass(altName, conf);
svn commit: r1374447 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Author: tgraves
Date: Fri Aug 17 21:43:31 2012
New Revision: 1374447
URL: http://svn.apache.org/viewvc?rev=1374447view=rev
Log:
HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor Robinson via
tgraves)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374447r1=1374446r2=1374447view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Fri Aug
17 21:43:31 2012
@@ -925,6 +925,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor
Robinson via tgraves)
+HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java?rev=1374447r1=1374446r2=1374447view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Fri Aug 17 21:43:31 2012
@@ -26,23 +26,17 @@ import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
public class TestPathData {
- protected static Configuration conf;
- protected static FileSystem fs;
- protected static String dirString;
- protected static Path testDir;
- protected static PathData item;
-
- protected static String[] d1Paths =
-new String[] { d1/f1, d1/f1.1, d1/f2 };
- protected static String[] d2Paths =
-new String[] { d2/f3 };
-
- @BeforeClass
- public static void initialize() throws Exception {
+ protected Configuration conf;
+ protected FileSystem fs;
+ protected Path testDir;
+
+ @Before
+ public void initialize() throws Exception {
conf = new Configuration();
fs = FileSystem.getLocal(conf);
testDir = new Path(
@@ -60,23 +54,28 @@ public class TestPathData {
fs.create(new Path(d2,f3));
}
+ @After
+ public void cleanup() throws Exception {
+fs.close();
+ }
+
@Test
public void testWithDirStringAndConf() throws Exception {
-dirString = d1;
-item = new PathData(dirString, conf);
-checkPathData();
+String dirString = d1;
+PathData item = new PathData(dirString, conf);
+checkPathData(dirString, item);
// properly implementing symlink support in various commands will require
// trailing slashes to be retained
dirString = d1/;
item = new PathData(dirString, conf);
-checkPathData();
+checkPathData(dirString, item);
}
@Test
public void testUnqualifiedUriContents() throws Exception {
-dirString = d1;
-item = new PathData(dirString, conf);
+String dirString = d1;
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(d1/f1, d1/f1.1, d1/f2),
@@ -86,8 +85,8 @@ public class TestPathData {
@Test
public void testQualifiedUriContents() throws Exception {
-dirString = fs.makeQualified(new Path(d1)).toString();
-item = new PathData(dirString, conf);
+String dirString = fs.makeQualified(new Path(d1)).toString();
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(dirString+/f1, dirString+/f1.1, dirString+/f2),
@@ -97,8 +96,8 @@ public class TestPathData {
@Test
public void testCwdContents() throws Exception {
-dirString = Path.CUR_DIR;
-item = new PathData(dirString, conf);
+String dirString = Path.CUR_DIR;
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(d1, d2),
@@ -106,17 +105,16 @@ public class TestPathData {
);
}
-
- @Test
- public void testToFile() throws Exception {
-item = new PathData(., conf);
+ @Test
+ public void testToFile() throws Exception {
+PathData item = new PathData(., conf
svn commit: r1374449 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Author: tgraves
Date: Fri Aug 17 21:44:35 2012
New Revision: 1374449
URL: http://svn.apache.org/viewvc?rev=1374449view=rev
Log:
merge -r 1374446:1374447 from trunk. FIXES: HADOOP-8695
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374449r1=1374448r2=1374449view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Aug 17 21:44:35 2012
@@ -747,6 +747,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor
Robinson via tgraves)
+HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java?rev=1374449r1=1374448r2=1374449view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Fri Aug 17 21:44:35 2012
@@ -26,23 +26,17 @@ import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
public class TestPathData {
- protected static Configuration conf;
- protected static FileSystem fs;
- protected static String dirString;
- protected static Path testDir;
- protected static PathData item;
-
- protected static String[] d1Paths =
-new String[] { d1/f1, d1/f1.1, d1/f2 };
- protected static String[] d2Paths =
-new String[] { d2/f3 };
-
- @BeforeClass
- public static void initialize() throws Exception {
+ protected Configuration conf;
+ protected FileSystem fs;
+ protected Path testDir;
+
+ @Before
+ public void initialize() throws Exception {
conf = new Configuration();
fs = FileSystem.getLocal(conf);
testDir = new Path(
@@ -60,23 +54,28 @@ public class TestPathData {
fs.create(new Path(d2,f3));
}
+ @After
+ public void cleanup() throws Exception {
+fs.close();
+ }
+
@Test
public void testWithDirStringAndConf() throws Exception {
-dirString = d1;
-item = new PathData(dirString, conf);
-checkPathData();
+String dirString = d1;
+PathData item = new PathData(dirString, conf);
+checkPathData(dirString, item);
// properly implementing symlink support in various commands will require
// trailing slashes to be retained
dirString = d1/;
item = new PathData(dirString, conf);
-checkPathData();
+checkPathData(dirString, item);
}
@Test
public void testUnqualifiedUriContents() throws Exception {
-dirString = d1;
-item = new PathData(dirString, conf);
+String dirString = d1;
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(d1/f1, d1/f1.1, d1/f2),
@@ -86,8 +85,8 @@ public class TestPathData {
@Test
public void testQualifiedUriContents() throws Exception {
-dirString = fs.makeQualified(new Path(d1)).toString();
-item = new PathData(dirString, conf);
+String dirString = fs.makeQualified(new Path(d1)).toString();
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(dirString+/f1, dirString+/f1.1, dirString+/f2),
@@ -97,8 +96,8 @@ public class TestPathData {
@Test
public void testCwdContents() throws Exception {
-dirString = Path.CUR_DIR;
-item = new PathData(dirString, conf);
+String dirString = Path.CUR_DIR;
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(d1, d2),
@@ -106,17 +105,16 @@ public class TestPathData {
);
}
-
- @Test
- public void testToFile() throws Exception {
-item = new PathData(., conf);
+ @Test
svn commit: r1374450 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Author: tgraves
Date: Fri Aug 17 21:45:41 2012
New Revision: 1374450
URL: http://svn.apache.org/viewvc?rev=1374450view=rev
Log:
merge -r 1374446:1374447 from trunk. FIXES: HADOOP-8695
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374450r1=1374449r2=1374450view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Aug 17 21:45:41 2012
@@ -665,6 +665,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor
Robinson via tgraves)
+HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java?rev=1374450r1=1374449r2=1374450view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Fri Aug 17 21:45:41 2012
@@ -26,23 +26,17 @@ import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
public class TestPathData {
- protected static Configuration conf;
- protected static FileSystem fs;
- protected static String dirString;
- protected static Path testDir;
- protected static PathData item;
-
- protected static String[] d1Paths =
-new String[] { d1/f1, d1/f1.1, d1/f2 };
- protected static String[] d2Paths =
-new String[] { d2/f3 };
-
- @BeforeClass
- public static void initialize() throws Exception {
+ protected Configuration conf;
+ protected FileSystem fs;
+ protected Path testDir;
+
+ @Before
+ public void initialize() throws Exception {
conf = new Configuration();
fs = FileSystem.getLocal(conf);
testDir = new Path(
@@ -60,23 +54,28 @@ public class TestPathData {
fs.create(new Path(d2,f3));
}
+ @After
+ public void cleanup() throws Exception {
+fs.close();
+ }
+
@Test
public void testWithDirStringAndConf() throws Exception {
-dirString = d1;
-item = new PathData(dirString, conf);
-checkPathData();
+String dirString = d1;
+PathData item = new PathData(dirString, conf);
+checkPathData(dirString, item);
// properly implementing symlink support in various commands will require
// trailing slashes to be retained
dirString = d1/;
item = new PathData(dirString, conf);
-checkPathData();
+checkPathData(dirString, item);
}
@Test
public void testUnqualifiedUriContents() throws Exception {
-dirString = d1;
-item = new PathData(dirString, conf);
+String dirString = d1;
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(d1/f1, d1/f1.1, d1/f2),
@@ -86,8 +85,8 @@ public class TestPathData {
@Test
public void testQualifiedUriContents() throws Exception {
-dirString = fs.makeQualified(new Path(d1)).toString();
-item = new PathData(dirString, conf);
+String dirString = fs.makeQualified(new Path(d1)).toString();
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(dirString+/f1, dirString+/f1.1, dirString+/f2),
@@ -97,8 +96,8 @@ public class TestPathData {
@Test
public void testCwdContents() throws Exception {
-dirString = Path.CUR_DIR;
-item = new PathData(dirString, conf);
+String dirString = Path.CUR_DIR;
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(d1, d2),
@@ -106,17 +105,16 @@ public class TestPathData {
);
}
-
- @Test
svn commit: r1374451 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Author: tgraves
Date: Fri Aug 17 21:46:05 2012
New Revision: 1374451
URL: http://svn.apache.org/viewvc?rev=1374451view=rev
Log:
merge -r 1374448:1374449 from branch-2. FIXES: HADOOP-8695
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374451r1=1374450r2=1374451view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Fri Aug 17 21:46:05 2012
@@ -168,6 +168,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8697. TestWritableName fails intermittently with JDK7 (Trevor
Robinson via tgraves)
+HADOOP-8695. TestPathData fails intermittently with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java?rev=1374451r1=1374450r2=1374451view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestPathData.java
Fri Aug 17 21:46:05 2012
@@ -26,23 +26,17 @@ import java.util.Arrays;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.junit.BeforeClass;
+import org.junit.After;
+import org.junit.Before;
import org.junit.Test;
public class TestPathData {
- protected static Configuration conf;
- protected static FileSystem fs;
- protected static String dirString;
- protected static Path testDir;
- protected static PathData item;
-
- protected static String[] d1Paths =
-new String[] { d1/f1, d1/f1.1, d1/f2 };
- protected static String[] d2Paths =
-new String[] { d2/f3 };
-
- @BeforeClass
- public static void initialize() throws Exception {
+ protected Configuration conf;
+ protected FileSystem fs;
+ protected Path testDir;
+
+ @Before
+ public void initialize() throws Exception {
conf = new Configuration();
fs = FileSystem.getLocal(conf);
testDir = new Path(
@@ -60,23 +54,28 @@ public class TestPathData {
fs.create(new Path(d2,f3));
}
+ @After
+ public void cleanup() throws Exception {
+fs.close();
+ }
+
@Test
public void testWithDirStringAndConf() throws Exception {
-dirString = d1;
-item = new PathData(dirString, conf);
-checkPathData();
+String dirString = d1;
+PathData item = new PathData(dirString, conf);
+checkPathData(dirString, item);
// properly implementing symlink support in various commands will require
// trailing slashes to be retained
dirString = d1/;
item = new PathData(dirString, conf);
-checkPathData();
+checkPathData(dirString, item);
}
@Test
public void testUnqualifiedUriContents() throws Exception {
-dirString = d1;
-item = new PathData(dirString, conf);
+String dirString = d1;
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(d1/f1, d1/f1.1, d1/f2),
@@ -86,8 +85,8 @@ public class TestPathData {
@Test
public void testQualifiedUriContents() throws Exception {
-dirString = fs.makeQualified(new Path(d1)).toString();
-item = new PathData(dirString, conf);
+String dirString = fs.makeQualified(new Path(d1)).toString();
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(dirString+/f1, dirString+/f1.1, dirString+/f2),
@@ -97,8 +96,8 @@ public class TestPathData {
@Test
public void testCwdContents() throws Exception {
-dirString = Path.CUR_DIR;
-item = new PathData(dirString, conf);
+String dirString = Path.CUR_DIR;
+PathData item = new PathData(dirString, conf);
PathData[] items = item.getDirectoryContents();
assertEquals(
sortedString(d1, d2),
@@ -106,17 +105,16 @@ public class TestPathData {
);
}
-
- @Test
- public void testToFile() throws Exception {
-item = new
svn commit: r1374084 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Author: tgraves
Date: Thu Aug 16 22:16:19 2012
New Revision: 1374084
URL: http://svn.apache.org/viewvc?rev=1374084view=rev
Log:
HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor Robinson
via tgraves)
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374084r1=1374083r2=1374084view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Aug
16 22:16:19 2012
@@ -911,6 +911,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave
Thompson via bobby)
+HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java?rev=1374084r1=1374083r2=1374084view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Thu Aug 16 22:16:19 2012
@@ -18,18 +18,20 @@
package org.apache.hadoop.fs;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
import java.io.IOException;
import java.net.URI;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.NetUtilsTestResolver;
import org.apache.hadoop.util.Progressable;
+import org.junit.BeforeClass;
import org.junit.Test;
-public class TestFileSystemCanonicalization extends TestCase {
+public class TestFileSystemCanonicalization {
static String[] authorities = {
myfs://host,
myfs://host.a,
@@ -41,8 +43,8 @@ public class TestFileSystemCanonicalizat
};
- @Test
- public void testSetupResolver() throws Exception {
+ @BeforeClass
+ public static void initialize() throws Exception {
NetUtilsTestResolver.install();
}
svn commit: r1374085 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Author: tgraves
Date: Thu Aug 16 22:19:23 2012
New Revision: 1374085
URL: http://svn.apache.org/viewvc?rev=1374085view=rev
Log:
merge -r 1374083:1374084 from trunk. FIXES: HADOOP-8390
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374085r1=1374084r2=1374085view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 16 22:19:23 2012
@@ -733,6 +733,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave
Thompson via bobby)
+HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java?rev=1374085r1=1374084r2=1374085view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Thu Aug 16 22:19:23 2012
@@ -18,18 +18,20 @@
package org.apache.hadoop.fs;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
import java.io.IOException;
import java.net.URI;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.NetUtilsTestResolver;
import org.apache.hadoop.util.Progressable;
+import org.junit.BeforeClass;
import org.junit.Test;
-public class TestFileSystemCanonicalization extends TestCase {
+public class TestFileSystemCanonicalization {
static String[] authorities = {
myfs://host,
myfs://host.a,
@@ -41,8 +43,8 @@ public class TestFileSystemCanonicalizat
};
- @Test
- public void testSetupResolver() throws Exception {
+ @BeforeClass
+ public static void initialize() throws Exception {
NetUtilsTestResolver.install();
}
svn commit: r1374087 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Author: tgraves
Date: Thu Aug 16 22:21:56 2012
New Revision: 1374087
URL: http://svn.apache.org/viewvc?rev=1374087view=rev
Log:
merge -r 1374083:1374084 from trunk. FIXES: HADOOP-8390
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374087r1=1374086r2=1374087view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 16 22:21:56 2012
@@ -651,6 +651,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave
Thompson via bobby)
+HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java?rev=1374087r1=1374086r2=1374087view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Thu Aug 16 22:21:56 2012
@@ -18,18 +18,20 @@
package org.apache.hadoop.fs;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
import java.io.IOException;
import java.net.URI;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.NetUtilsTestResolver;
import org.apache.hadoop.util.Progressable;
+import org.junit.BeforeClass;
import org.junit.Test;
-public class TestFileSystemCanonicalization extends TestCase {
+public class TestFileSystemCanonicalization {
static String[] authorities = {
myfs://host,
myfs://host.a,
@@ -41,8 +43,8 @@ public class TestFileSystemCanonicalizat
};
- @Test
- public void testSetupResolver() throws Exception {
+ @BeforeClass
+ public static void initialize() throws Exception {
NetUtilsTestResolver.install();
}
svn commit: r1374088 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: CHANGES.txt src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Author: tgraves
Date: Thu Aug 16 22:23:31 2012
New Revision: 1374088
URL: http://svn.apache.org/viewvc?rev=1374088view=rev
Log:
merge -r 1374084:1374085 from branch-2. FIXES: HADOOP-8390
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1374088r1=1374087r2=1374088view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Thu Aug 16 22:23:31 2012
@@ -156,6 +156,9 @@ Release 0.23.3 - UNRELEASED
HADOOP-8703. distcpV2: turn CRC checking off for 0 byte size (Dave
Thompson via bobby)
+HADOOP-8390. TestFileSystemCanonicalization fails with JDK7 (Trevor
+Robinson via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java?rev=1374088r1=1374087r2=1374088view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileSystemCanonicalization.java
Thu Aug 16 22:23:31 2012
@@ -18,18 +18,20 @@
package org.apache.hadoop.fs;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
import java.io.IOException;
import java.net.URI;
-import junit.framework.TestCase;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.security.NetUtilsTestResolver;
import org.apache.hadoop.util.Progressable;
+import org.junit.BeforeClass;
import org.junit.Test;
-public class TestFileSystemCanonicalization extends TestCase {
+public class TestFileSystemCanonicalization {
static String[] authorities = {
myfs://host,
myfs://host.a,
@@ -41,8 +43,8 @@ public class TestFileSystemCanonicalizat
};
- @Test
- public void testSetupResolver() throws Exception {
+ @BeforeClass
+ public static void initialize() throws Exception {
NetUtilsTestResolver.install();
}
svn commit: r1372636 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/ hadoop-mapreduce-cl
Author: tgraves
Date: Mon Aug 13 21:48:33 2012
New Revision: 1372636
URL: http://svn.apache.org/viewvc?rev=1372636view=rev
Log:
MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over group
names deprecated names don't show up (Robert Evans via tgraves)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1372636r1=1372635r2=1372636view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Mon Aug 13
21:48:33 2012
@@ -809,6 +809,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason
Lowe via bobby)
+MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over
+group names deprecated names don't show up (Robert Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1372636r1=1372635r2=1372636view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
Mon Aug 13 21:48:33 2012
@@ -24,6 +24,7 @@ import static org.apache.hadoop.mapreduc
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -185,7 +186,15 @@ public abstract class AbstractCountersC
* @return Set of counter names.
*/
public synchronized IterableString getGroupNames() {
-return Iterables.concat(fgroups.keySet(), groups.keySet());
+HashSetString deprecated = new HashSetString();
+for(Map.EntryString, String entry : legacyMap.entrySet()) {
+ String newGroup = entry.getValue();
+ boolean isFGroup = isFrameworkGroup(newGroup);
+ if(isFGroup ? fgroups.containsKey(newGroup) :
groups.containsKey(newGroup)) {
+deprecated.add(entry.getKey());
+ }
+}
+return Iterables.concat(fgroups.keySet(), groups.keySet(), deprecated);
}
@Override
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java?rev=1372636r1=1372635r2=1372636view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Mon Aug 13 21:48:33 2012
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.text.ParseException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
@@ -225,6 +226,23 @@ public class TestCounters {
}
@Test
+ public void testLegacyGetGroupNames() {
+Counters counters = new Counters();
+// create 2 filesystem counter groups
+counters.findCounter(fs1, FileSystemCounter.BYTES_READ).increment(1);
+counters.findCounter(fs2, FileSystemCounter.BYTES_READ).increment(1);
+counters.incrCounter(group1, counter1, 1);
+
+HashSetString groups = new HashSetString(counters.getGroupNames());
+HashSetString expectedGroups = new HashSetString();
+expectedGroups.add(group1);
+expectedGroups.add(FileSystemCounter); //Legacy Name
svn commit: r1372639 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/ hado
Author: tgraves
Date: Mon Aug 13 21:51:06 2012
New Revision: 1372639
URL: http://svn.apache.org/viewvc?rev=1372639view=rev
Log:
merge -r 1372637:1372638 from branch-2. FIXES: MAPREDUCE-4053
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1372639r1=1372638r2=1372639view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Mon
Aug 13 21:51:06 2012
@@ -385,6 +385,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason
Lowe via bobby)
+MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over
+group names deprecated names don't show up (Robert Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1372639r1=1372638r2=1372639view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
Mon Aug 13 21:51:06 2012
@@ -24,6 +24,7 @@ import static org.apache.hadoop.mapreduc
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -185,7 +186,15 @@ public abstract class AbstractCountersC
* @return Set of counter names.
*/
public synchronized IterableString getGroupNames() {
-return Iterables.concat(fgroups.keySet(), groups.keySet());
+HashSetString deprecated = new HashSetString();
+for(Map.EntryString, String entry : legacyMap.entrySet()) {
+ String newGroup = entry.getValue();
+ boolean isFGroup = isFrameworkGroup(newGroup);
+ if(isFGroup ? fgroups.containsKey(newGroup) :
groups.containsKey(newGroup)) {
+deprecated.add(entry.getKey());
+ }
+}
+return Iterables.concat(fgroups.keySet(), groups.keySet(), deprecated);
}
@Override
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java?rev=1372639r1=1372638r2=1372639view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Mon Aug 13 21:51:06 2012
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.text.ParseException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
@@ -225,6 +226,23 @@ public class TestCounters {
}
@Test
+ public void testLegacyGetGroupNames() {
+Counters counters = new Counters();
+// create 2 filesystem counter groups
+counters.findCounter(fs1, FileSystemCounter.BYTES_READ).increment(1);
+counters.findCounter(fs2, FileSystemCounter.BYTES_READ).increment(1);
+counters.incrCounter(group1, counter1, 1);
+
+HashSetString groups = new HashSetString(counters.getGroupNames
svn commit: r1372640 - in /hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counter
Author: tgraves
Date: Mon Aug 13 21:53:50 2012
New Revision: 1372640
URL: http://svn.apache.org/viewvc?rev=1372640view=rev
Log:
merge -r 1372635:1372636 from trunk. FIXES: MAPREDUCE-4053
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt?rev=1372640r1=1372639r2=1372640view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/CHANGES.txt
Mon Aug 13 21:53:50 2012
@@ -584,6 +584,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-3782. teragen terasort jobs fail when using webhdfs:// (Jason
Lowe via bobby)
+MAPREDUCE-4053. Counters group names deprecation is wrong, iterating over
+group names deprecated names don't show up (Robert Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java?rev=1372640r1=1372639r2=1372640view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/AbstractCounters.java
Mon Aug 13 21:53:50 2012
@@ -24,6 +24,7 @@ import static org.apache.hadoop.mapreduc
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Map;
import java.util.concurrent.ConcurrentSkipListMap;
@@ -185,7 +186,15 @@ public abstract class AbstractCountersC
* @return Set of counter names.
*/
public synchronized IterableString getGroupNames() {
-return Iterables.concat(fgroups.keySet(), groups.keySet());
+HashSetString deprecated = new HashSetString();
+for(Map.EntryString, String entry : legacyMap.entrySet()) {
+ String newGroup = entry.getValue();
+ boolean isFGroup = isFrameworkGroup(newGroup);
+ if(isFGroup ? fgroups.containsKey(newGroup) :
groups.containsKey(newGroup)) {
+deprecated.add(entry.getKey());
+ }
+}
+return Iterables.concat(fgroups.keySet(), groups.keySet(), deprecated);
}
@Override
Modified:
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java?rev=1372640r1=1372639r2=1372640view=diff
==
---
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
(original)
+++
hadoop/common/branches/branch-2.1.0-alpha/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/test/java/org/apache/hadoop/mapred/TestCounters.java
Mon Aug 13 21:53:50 2012
@@ -22,6 +22,7 @@ import static org.junit.Assert.assertTru
import java.io.IOException;
import java.text.ParseException;
+import java.util.HashSet;
import java.util.Iterator;
import java.util.Random;
@@ -225,6 +226,23 @@ public class TestCounters {
}
@Test
+ public void testLegacyGetGroupNames() {
+Counters counters = new Counters();
+// create 2 filesystem counter groups
+counters.findCounter(fs1, FileSystemCounter.BYTES_READ).increment(1);
+counters.findCounter(fs2, FileSystemCounter.BYTES_READ).increment(1);
+counters.incrCounter
svn commit: r1368294 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ src/test/mapred/org/apache
Author: tgraves
Date: Wed Aug 1 22:14:13 2012
New Revision: 1368294
URL: http://svn.apache.org/viewvc?rev=1368294view=rev
Log:
MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans via
tgraves)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1368294r1=1368293r2=1368294view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Wed Aug 1
22:14:13 2012
@@ -789,6 +789,9 @@ Release 0.23.3 - UNRELEASED
parallel use (due to a /tmp file with static name) (Robert Evans via
jeagles)
+MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans
+via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1368294r1=1368293r2=1368294view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
Wed Aug 1 22:14:13 2012
@@ -344,7 +344,7 @@ public class SortValidator extends Confi
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
- Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir,
/tmp),
+ Path outputPath = new Path(new Path(/tmp,
sortvalidate), UUID.randomUUID().toString());
if (defaultfs.exists(outputPath)) {
defaultfs.delete(outputPath, true);
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java?rev=1368294r1=1368293r2=1368294view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
Wed Aug 1 22:14:13 2012
@@ -344,7 +344,7 @@ public class SortValidator extends Confi
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
- Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir,
/tmp),
+ Path outputPath = new Path(new Path(/tmp,
sortvalidate), UUID.randomUUID().toString());
if (defaultfs.exists(outputPath)) {
defaultfs.delete(outputPath, true);
svn commit: r1368296 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ src/test/mapre
Author: tgraves
Date: Wed Aug 1 22:20:59 2012
New Revision: 1368296
URL: http://svn.apache.org/viewvc?rev=1368296view=rev
Log:
merge -r 1368293:1368294 from trunk. FIXES: MAPREDUCE-4504
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1368296r1=1368295r2=1368296view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Wed
Aug 1 22:20:59 2012
@@ -665,6 +665,9 @@ Release 0.23.3 - UNRELEASED
parallel use (due to a /tmp file with static name) (Robert Evans via
jeagles)
+MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans
+via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1368296r1=1368295r2=1368296view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
Wed Aug 1 22:20:59 2012
@@ -344,7 +344,7 @@ public class SortValidator extends Confi
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
- Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir,
/tmp),
+ Path outputPath = new Path(new Path(/tmp,
sortvalidate), UUID.randomUUID().toString());
if (defaultfs.exists(outputPath)) {
defaultfs.delete(outputPath, true);
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java?rev=1368296r1=1368295r2=1368296view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
Wed Aug 1 22:20:59 2012
@@ -344,7 +344,7 @@ public class SortValidator extends Confi
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
- Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir,
/tmp),
+ Path outputPath = new Path(new Path(/tmp,
sortvalidate), UUID.randomUUID().toString());
if (defaultfs.exists(outputPath)) {
defaultfs.delete(outputPath, true);
svn commit: r1368298 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/ src/test/ma
Author: tgraves
Date: Wed Aug 1 22:22:23 2012
New Revision: 1368298
URL: http://svn.apache.org/viewvc?rev=1368298view=rev
Log:
merge -r 1368295:1368296 from branch-2. FIXES: MAPREDUCE-4504
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1368298r1=1368297r2=1368298view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Wed
Aug 1 22:22:23 2012
@@ -379,6 +379,9 @@ Release 0.23.3 - UNRELEASED
parallel use (due to a /tmp file with static name) (Robert Evans via
jeagles)
+MAPREDUCE-4504. SortValidator writes to wrong directory (Robert Evans
+via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java?rev=1368298r1=1368297r2=1368298view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/SortValidator.java
Wed Aug 1 22:22:23 2012
@@ -344,7 +344,7 @@ public class SortValidator extends Confi
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
- Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir,
/tmp),
+ Path outputPath = new Path(new Path(/tmp,
sortvalidate), UUID.randomUUID().toString());
if (defaultfs.exists(outputPath)) {
defaultfs.delete(outputPath, true);
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java?rev=1368298r1=1368297r2=1368298view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/src/test/mapred/org/apache/hadoop/mapred/SortValidator.java
Wed Aug 1 22:22:23 2012
@@ -344,7 +344,7 @@ public class SortValidator extends Confi
FileInputFormat.setInputPaths(jobConf, sortInput);
FileInputFormat.addInputPath(jobConf, sortOutput);
- Path outputPath = new Path(new Path(jobConf.get(hadoop.tmp.dir,
/tmp),
+ Path outputPath = new Path(new Path(/tmp,
sortvalidate), UUID.randomUUID().toString());
if (defaultfs.exists(outputPath)) {
defaultfs.delete(outputPath, true);
svn commit: r1368002 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/shell/ src/test/java/org/apache/hadoop/fs/shell/
Author: tgraves
Date: Wed Aug 1 14:02:22 2012
New Revision: 1368002
URL: http://svn.apache.org/viewvc?rev=1368002view=rev
Log:
HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp via
tgraves)
Added:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1368002r1=1368001r2=1368002view=diff
==
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Wed Aug
1 14:02:22 2012
@@ -873,6 +873,9 @@ Release 0.23.3 - UNRELEASED
org.apache.hadoop.classification.InterfaceAudience not found (Trevor
Robinson via tgraves)
+HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp
+via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1368002r1=1368001r2=1368002view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
Wed Aug 1 14:02:22 2012
@@ -24,6 +24,8 @@ import java.io.InputStream;
import java.util.LinkedList;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException;
import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
@@ -232,31 +234,65 @@ abstract class CommandWithDestination ex
if (target.exists (target.stat.isDirectory() || !overwrite)) {
throw new PathExistsException(target.toString());
}
-target.fs.setWriteChecksum(writeChecksum);
-PathData tempFile = null;
+TargetFileSystem targetFs = new TargetFileSystem(target.fs);
try {
- tempFile = target.createTempFile(target+._COPYING_);
- FSDataOutputStream out = target.fs.create(tempFile.path, true);
- IOUtils.copyBytes(in, out, getConf(), true);
+ PathData tempTarget = target.suffix(._COPYING_);
+ targetFs.setWriteChecksum(writeChecksum);
+ targetFs.writeStreamToFile(in, tempTarget);
+ targetFs.rename(tempTarget, target);
+} finally {
+ targetFs.close(); // last ditch effort to ensure temp file is removed
+}
+ }
+
+ // Helper filter filesystem that registers created files as temp files to
+ // be deleted on exit unless successfully renamed
+ private static class TargetFileSystem extends FilterFileSystem {
+TargetFileSystem(FileSystem fs) {
+ super(fs);
+}
+
+void writeStreamToFile(InputStream in, PathData target) throws IOException
{
+ FSDataOutputStream out = null;
+ try {
+out = create(target);
+IOUtils.copyBytes(in, out, getConf(), true);
+ } finally {
+IOUtils.closeStream(out); // just in case copyBytes didn't
+ }
+}
+
+// tag created files as temp files
+FSDataOutputStream create(PathData item) throws IOException {
+ try {
+return create(item.path, true);
+ } finally { // might have been created but stream was interrupted
+deleteOnExit(item.path);
+ }
+}
+
+void rename(PathData src, PathData target) throws IOException {
// the rename method with an option to delete the target is deprecated
- if (target.exists !target.fs.delete(target.path, false)) {
+ if (target.exists !delete(target.path, false)) {
// too bad we don't know why it failed
PathIOException e = new PathIOException(target.toString());
e.setOperation(delete);
throw e;
}
- if (!tempFile.fs.rename(tempFile.path, target.path)) {
+ if (!rename(src.path, target.path)) {
// too bad we don't know why it failed
-PathIOException e = new PathIOException(tempFile.toString());
+PathIOException e = new
svn commit: r1368003 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/shell/ src/test/java/org/apache/hadoop/fs/shell/
Author: tgraves
Date: Wed Aug 1 14:03:01 2012
New Revision: 1368003
URL: http://svn.apache.org/viewvc?rev=1368003view=rev
Log:
merge -r 1368001:1368002 from trunk. FIXES: HADOOP-8633
Added:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java
- copied unchanged from r1368002,
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1368003r1=1368002r2=1368003view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Aug 1 14:03:01 2012
@@ -692,6 +692,9 @@ Release 0.23.3 - UNRELEASED
org.apache.hadoop.classification.InterfaceAudience not found (Trevor
Robinson via tgraves)
+HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp
+via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1368003r1=1368002r2=1368003view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
Wed Aug 1 14:03:01 2012
@@ -24,6 +24,8 @@ import java.io.InputStream;
import java.util.LinkedList;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException;
import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
@@ -232,31 +234,65 @@ abstract class CommandWithDestination ex
if (target.exists (target.stat.isDirectory() || !overwrite)) {
throw new PathExistsException(target.toString());
}
-target.fs.setWriteChecksum(writeChecksum);
-PathData tempFile = null;
+TargetFileSystem targetFs = new TargetFileSystem(target.fs);
try {
- tempFile = target.createTempFile(target+._COPYING_);
- FSDataOutputStream out = target.fs.create(tempFile.path, true);
- IOUtils.copyBytes(in, out, getConf(), true);
+ PathData tempTarget = target.suffix(._COPYING_);
+ targetFs.setWriteChecksum(writeChecksum);
+ targetFs.writeStreamToFile(in, tempTarget);
+ targetFs.rename(tempTarget, target);
+} finally {
+ targetFs.close(); // last ditch effort to ensure temp file is removed
+}
+ }
+
+ // Helper filter filesystem that registers created files as temp files to
+ // be deleted on exit unless successfully renamed
+ private static class TargetFileSystem extends FilterFileSystem {
+TargetFileSystem(FileSystem fs) {
+ super(fs);
+}
+
+void writeStreamToFile(InputStream in, PathData target) throws IOException
{
+ FSDataOutputStream out = null;
+ try {
+out = create(target);
+IOUtils.copyBytes(in, out, getConf(), true);
+ } finally {
+IOUtils.closeStream(out); // just in case copyBytes didn't
+ }
+}
+
+// tag created files as temp files
+FSDataOutputStream create(PathData item) throws IOException {
+ try {
+return create(item.path, true);
+ } finally { // might have been created but stream was interrupted
+deleteOnExit(item.path);
+ }
+}
+
+void rename(PathData src, PathData target) throws IOException {
// the rename method with an option to delete the target is deprecated
- if (target.exists !target.fs.delete(target.path, false)) {
+ if (target.exists !delete(target.path, false)) {
// too bad we don't know why it failed
PathIOException e = new PathIOException(target.toString());
e.setOperation(delete);
throw e
svn commit: r1368004 - in /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common: ./ src/main/java/org/apache/hadoop/fs/shell/ src/test/java/org/apache/hadoop/fs/shell/
Author: tgraves
Date: Wed Aug 1 14:03:35 2012
New Revision: 1368004
URL: http://svn.apache.org/viewvc?rev=1368004view=rev
Log:
merge -r 1368002:1368003 from branch-2. FIXES: HADOOP-8633
Added:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java
- copied unchanged from r1368003,
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestCopy.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/PathData.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1368004r1=1368003r2=1368004view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/CHANGES.txt
Wed Aug 1 14:03:35 2012
@@ -146,6 +146,9 @@ Release 0.23.3 - UNRELEASED
org.apache.hadoop.classification.InterfaceAudience not found (Trevor
Robinson via tgraves)
+HADOOP-8633. Interrupted FsShell copies may leave tmp files (Daryn Sharp
+via tgraves)
+
Release 0.23.2 - UNRELEASED
NEW FEATURES
Modified:
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java?rev=1368004r1=1368003r2=1368004view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
Wed Aug 1 14:03:35 2012
@@ -24,6 +24,8 @@ import java.io.InputStream;
import java.util.LinkedList;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FilterFileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.shell.PathExceptions.PathExistsException;
import org.apache.hadoop.fs.shell.PathExceptions.PathIOException;
@@ -232,31 +234,65 @@ abstract class CommandWithDestination ex
if (target.exists (target.stat.isDirectory() || !overwrite)) {
throw new PathExistsException(target.toString());
}
-target.fs.setWriteChecksum(writeChecksum);
-PathData tempFile = null;
+TargetFileSystem targetFs = new TargetFileSystem(target.fs);
try {
- tempFile = target.createTempFile(target+._COPYING_);
- FSDataOutputStream out = target.fs.create(tempFile.path, true);
- IOUtils.copyBytes(in, out, getConf(), true);
+ PathData tempTarget = target.suffix(._COPYING_);
+ targetFs.setWriteChecksum(writeChecksum);
+ targetFs.writeStreamToFile(in, tempTarget);
+ targetFs.rename(tempTarget, target);
+} finally {
+ targetFs.close(); // last ditch effort to ensure temp file is removed
+}
+ }
+
+ // Helper filter filesystem that registers created files as temp files to
+ // be deleted on exit unless successfully renamed
+ private static class TargetFileSystem extends FilterFileSystem {
+TargetFileSystem(FileSystem fs) {
+ super(fs);
+}
+
+void writeStreamToFile(InputStream in, PathData target) throws IOException
{
+ FSDataOutputStream out = null;
+ try {
+out = create(target);
+IOUtils.copyBytes(in, out, getConf(), true);
+ } finally {
+IOUtils.closeStream(out); // just in case copyBytes didn't
+ }
+}
+
+// tag created files as temp files
+FSDataOutputStream create(PathData item) throws IOException {
+ try {
+return create(item.path, true);
+ } finally { // might have been created but stream was interrupted
+deleteOnExit(item.path);
+ }
+}
+
+void rename(PathData src, PathData target) throws IOException {
// the rename method with an option to delete the target is deprecated
- if (target.exists !target.fs.delete(target.path, false)) {
+ if (target.exists !delete(target.path, false)) {
// too bad we don't know why it failed
PathIOException e = new PathIOException(target.toString
svn commit: r1367539 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapred
Author: tgraves
Date: Tue Jul 31 13:39:41 2012
New Revision: 1367539
URL: http://svn.apache.org/viewvc?rev=1367539view=rev
Log:
MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby via tgraves)
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367539r1=1367538r2=1367539view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31
13:39:41 2012
@@ -505,6 +505,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4267. mavenize pipes (tgraves via bobby)
+MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby
+via tgraves)
+
OPTIMIZATIONS
MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367539r1=1367538r2=1367539view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
Tue Jul 31 13:39:41 2012
@@ -1561,7 +1561,7 @@ public class JobImpl implements org.apac
Path confPath = getConfFile();
FileContext fc = FileContext.getFileContext(confPath.toUri(), conf);
Configuration jobConf = new Configuration(false);
-jobConf.addResource(fc.open(confPath));
+jobConf.addResource(fc.open(confPath), confPath.toString());
return jobConf;
}
}
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367539r1=1367538r2=1367539view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
Tue Jul 31 13:39:41 2012
@@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock
tr().
th(_TH, key).
th(_TH, value).
+th(_TH, source chain).
_().
_().
tbody();
for (ConfEntryInfo entry : info.getProperties()) {
+StringBuffer buffer = new StringBuffer();
+String[] sources = entry.getSource();
+//Skip the last entry, because it is always the same HDFS file, and
+// output them in reverse order so most recent is output first
+boolean first = true;
+for(int i
svn commit: r1367540 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ h
Author: tgraves
Date: Tue Jul 31 13:40:39 2012
New Revision: 1367540
URL: http://svn.apache.org/viewvc?rev=1367540view=rev
Log:
merge -r 1367538:1367539 from trunk. FIXES: MAPREDUCE-4375
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367540r1=1367539r2=1367540view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue
Jul 31 13:40:39 2012
@@ -381,6 +381,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4267. mavenize pipes (tgraves via bobby)
+MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby
+via tgraves)
+
OPTIMIZATIONS
MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367540r1=1367539r2=1367540view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
Tue Jul 31 13:40:39 2012
@@ -1561,7 +1561,7 @@ public class JobImpl implements org.apac
Path confPath = getConfFile();
FileContext fc = FileContext.getFileContext(confPath.toUri(), conf);
Configuration jobConf = new Configuration(false);
-jobConf.addResource(fc.open(confPath));
+jobConf.addResource(fc.open(confPath), confPath.toString());
return jobConf;
}
}
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367540r1=1367539r2=1367540view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java
Tue Jul 31 13:40:39 2012
@@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock
tr().
th(_TH, key).
th(_TH, value).
+th(_TH, source chain).
_().
_().
tbody();
for (ConfEntryInfo entry : info.getProperties()) {
+StringBuffer buffer = new StringBuffer
svn commit: r1367544 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl
Author: tgraves Date: Tue Jul 31 13:44:50 2012 New Revision: 1367544 URL: http://svn.apache.org/viewvc?rev=1367544view=rev Log: merge -r 1367539:1367540 from branch-2. FIXES: MAPREDUCE-4375 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfEntryInfo.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/dao/ConfInfo.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MockJobs.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/HistoryServerRest.apt.vm hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/MapredAppMasterRest.apt.vm Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367544r1=1367543r2=1367544view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 13:44:50 2012 @@ -56,6 +56,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-2220. Fix new API FileOutputFormat-related typos in mapred-default.xml (Rui Kubo via harsh) +MAPREDUCE-4375. Show Configuration Tracability in MR UI (bobby +via tgraves) + OPTIMIZATIONS MAPREDUCE-3850. Avoid redundant calls for tokens in TokenCache (Daryn Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367544r1=1367543r2=1367544view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 13:44:50 2012 @@ -1481,7 +1481,7 @@ public class JobImpl implements org.apac Path confPath = getConfFile(); FileContext fc = FileContext.getFileContext(confPath.toUri(), conf); Configuration jobConf = new Configuration(false); -jobConf.addResource(fc.open(confPath)); +jobConf.addResource(fc.open(confPath), confPath.toString()); return jobConf; } } Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java?rev=1367544r1=1367543r2=1367544view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/webapp/ConfBlock.java Tue Jul 31 13:44:50 2012 @@ -78,14 +78,29 @@ public class ConfBlock extends HtmlBlock tr(). th(_TH, key). th(_TH, value). +th(_TH, source chain). _(). _(). tbody
svn commit: r1367581 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-client/hadoo
Author: tgraves
Date: Tue Jul 31 14:46:57 2012
New Revision: 1367581
URL: http://svn.apache.org/viewvc?rev=1367581view=rev
Log:
MAPREDUCE-4456. LocalDistributedCacheManager can get an ArrayIndexOutOfBounds
when creating symlinks (Robert Evans via tgraves)
Added:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367581r1=1367580r2=1367581view=diff
==
--- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31
14:46:57 2012
@@ -766,6 +766,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans
via tgraves)
+MAPREDUCE-4456. LocalDistributedCacheManager can get an
+ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367581r1=1367580r2=1367581view=diff
==
---
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
(original)
+++
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
Tue Jul 31 14:46:57 2012
@@ -18,12 +18,9 @@
package org.apache.hadoop.mapred;
-import com.google.common.collect.Maps;
-
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
-import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
@@ -34,6 +31,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.FSDownload;
+import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
@@ -85,6 +84,9 @@ class LocalDistributedCacheManager {
* @throws IOException
*/
public void setup(JobConf conf) throws IOException {
+boolean mkLinks = DistributedCache.getSymlink(conf);
+File workDir = new File(System.getProperty(user.dir));
+
// Generate YARN local resources objects corresponding to the distributed
// cache configuration
MapString, LocalResource localResources =
@@ -132,7 +134,8 @@ class LocalDistributedCacheManager {
FuturePath future = exec.submit(download);
resourcesToPaths.put(resource, future);
}
- for (LocalResource resource : localResources.values()) {
+ for (EntryString, LocalResource entry : localResources.entrySet()) {
+LocalResource resource = entry.getValue();
Path path;
try {
path = resourcesToPaths.get(resource).get();
@@ -142,6 +145,12 @@ class LocalDistributedCacheManager {
throw new IOException(e);
}
String pathString = path.toUri().toString();
+if(mkLinks) {
+ String link = entry.getKey();
+ String target = new File(path.toUri()).getPath();
+ symlink(workDir, target, link);
+}
+
if (resource.getType() == LocalResourceType.ARCHIVE) {
localArchives.add(pathString);
} else if (resource.getType() == LocalResourceType.FILE) {
@@ -175,27 +184,6 @@ class LocalDistributedCacheManager {
.arrayToString(localFiles.toArray(new String[localArchives
.size
svn commit: r1367585 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-
Author: tgraves
Date: Tue Jul 31 14:49:07 2012
New Revision: 1367585
URL: http://svn.apache.org/viewvc?rev=1367585view=rev
Log:
merge -r 1367580:1367581 from trunk. FIXES: MAPREDUCE-4456
Added:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
- copied unchanged from r1367581,
hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367585r1=1367584r2=1367585view=diff
==
--- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue
Jul 31 14:49:07 2012
@@ -642,6 +642,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4423. Potential infinite fetching of map output (Robert Evans
via tgraves)
+MAPREDUCE-4456. LocalDistributedCacheManager can get an
+ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367585r1=1367584r2=1367585view=diff
==
---
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
Tue Jul 31 14:49:07 2012
@@ -18,12 +18,9 @@
package org.apache.hadoop.mapred;
-import com.google.common.collect.Maps;
-
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
-import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
@@ -34,6 +31,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.FSDownload;
+import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
@@ -85,6 +84,9 @@ class LocalDistributedCacheManager {
* @throws IOException
*/
public void setup(JobConf conf) throws IOException {
+boolean mkLinks = DistributedCache.getSymlink(conf);
+File workDir = new File(System.getProperty(user.dir));
+
// Generate YARN local resources objects corresponding to the distributed
// cache configuration
MapString, LocalResource localResources =
@@ -132,7 +134,8 @@ class LocalDistributedCacheManager {
FuturePath future = exec.submit(download);
resourcesToPaths.put(resource, future);
}
- for (LocalResource resource : localResources.values()) {
+ for (EntryString, LocalResource entry : localResources.entrySet()) {
+LocalResource resource = entry.getValue();
Path path;
try {
path = resourcesToPaths.get(resource).get();
@@ -142,6 +145,12 @@ class LocalDistributedCacheManager {
throw new IOException(e);
}
String pathString = path.toUri().toString();
+if(mkLinks) {
+ String link = entry.getKey();
+ String target = new File(path.toUri()).getPath();
+ symlink(workDir, target, link);
+}
+
if (resource.getType() == LocalResourceType.ARCHIVE
svn commit: r1367587 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapredu
Author: tgraves
Date: Tue Jul 31 14:50:27 2012
New Revision: 1367587
URL: http://svn.apache.org/viewvc?rev=1367587view=rev
Log:
merge -r 1367584:1367585 from branch-2. FIXES: MAPREDUCE-4456
Added:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
- copied unchanged from r1367585,
hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367587r1=1367586r2=1367587view=diff
==
--- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt
(original)
+++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue
Jul 31 14:50:27 2012
@@ -358,6 +358,9 @@ Release 0.23.3 - UNRELEASED
MAPREDUCE-4010. TestWritableJobConf fails on trunk (tucu via bobby)
+MAPREDUCE-4456. LocalDistributedCacheManager can get an
+ArrayIndexOutOfBounds when creating symlinks (Robert Evans via tgraves)
+
Release 0.23.2 - UNRELEASED
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367587r1=1367586r2=1367587view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java
Tue Jul 31 14:50:27 2012
@@ -18,12 +18,9 @@
package org.apache.hadoop.mapred;
-import com.google.common.collect.Maps;
-
import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
-import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLClassLoader;
@@ -34,6 +31,7 @@ import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Random;
import java.util.concurrent.Callable;
import java.util.concurrent.ExecutionException;
@@ -60,6 +58,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.yarn.util.FSDownload;
+import com.google.common.collect.Maps;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
/**
@@ -85,6 +84,9 @@ class LocalDistributedCacheManager {
* @throws IOException
*/
public void setup(JobConf conf) throws IOException {
+boolean mkLinks = DistributedCache.getSymlink(conf);
+File workDir = new File(System.getProperty(user.dir));
+
// Generate YARN local resources objects corresponding to the distributed
// cache configuration
MapString, LocalResource localResources =
@@ -132,7 +134,8 @@ class LocalDistributedCacheManager {
FuturePath future = exec.submit(download);
resourcesToPaths.put(resource, future);
}
- for (LocalResource resource : localResources.values()) {
+ for (EntryString, LocalResource entry : localResources.entrySet()) {
+LocalResource resource = entry.getValue();
Path path;
try {
path = resourcesToPaths.get(resource).get();
@@ -142,6 +145,12 @@ class LocalDistributedCacheManager {
throw new IOException(e);
}
String pathString = path.toUri().toString();
+if(mkLinks) {
+ String link = entry.getKey();
+ String target = new File(path.toUri()).getPath();
+ symlink(workDir, target, link);
+}
+
if (resource.getType
svn commit: r1367715 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapreduce-
Author: tgraves Date: Tue Jul 31 19:21:27 2012 New Revision: 1367715 URL: http://svn.apache.org/viewvc?rev=1367715view=rev Log: merge -r 1367712:1367713 from trunk. FIXES: MAPREDUCE-4493 Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367715r1=1367714r2=1367715view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 19:21:27 2012 @@ -647,6 +647,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby) +MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans +via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367715r1=1367714r2=1367715view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java Tue Jul 31 19:21:27
svn commit: r1367717 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/ hadoop-mapredu
Author: tgraves Date: Tue Jul 31 19:22:25 2012 New Revision: 1367717 URL: http://svn.apache.org/viewvc?rev=1367717view=rev Log: merge -r 1367714:1367715 from branch-2. FIXES: MAPREDUCE-4493 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestLocalDistributedCacheManager.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/test/java/org/apache/hadoop/mapred/TestMRWithDistributedCache.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/filecache/DistributedCache.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/pipes/Submitter.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/Job.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobContext.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/DistributedCache.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/util/ConfigUtil.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/MRCaching.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestMiniMRDFSCaching.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestMRJobs.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapreduce/v2/TestSpeculativeExecution.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/Sort.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/terasort/TeraSort.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367717r1=1367716r2=1367717view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 19:22:25 2012 @@ -363,6 +363,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4496. AM logs link is missing user name (Jason Lowe via bobby) +MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans +via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java?rev=1367717r1=1367716r2=1367717view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapred/LocalDistributedCacheManager.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java
svn commit: r1367771 - in /hadoop/common/trunk/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ hadoop-mapred
Author: tgraves Date: Tue Jul 31 20:52:41 2012 New Revision: 1367771 URL: http://svn.apache.org/viewvc?rev=1367771view=rev Log: MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at FAILED (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java Modified: hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt?rev=1367771r1=1367770r2=1367771view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 20:52:41 2012 @@ -779,6 +779,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at perticular level is sucessfull (Mayank Bansal via bobby) +MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at +FAILED (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367771r1=1367770r2=1367771view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 20:52:41 2012 @@ -1370,7 +1370,8 @@ public class JobImpl implements org.apac } } -float failureRate = (float) fetchFailures / runningReduceTasks; +float failureRate = runningReduceTasks == 0 ? 1.0f : + (float) fetchFailures / runningReduceTasks; // declare faulty if fetch-failures = max-allowed-failures boolean isMapFaulty = (failureRate = MAX_ALLOWED_FETCH_FAILURES_FRACTION); Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1367771r1=1367770r2=1367771view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue Jul 31 20:52:41 2012 @@ -435,7 +435,8 @@ public abstract class TaskAttemptImpl im TaskAttemptEventType.TA_CONTAINER_CLEANED, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, - TaskAttemptEventType.TA_FAILMSG)) + TaskAttemptEventType.TA_FAILMSG, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)) // Transitions from KILLED state .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED, Modified: hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java?rev=1367771r1=1367770r2=1367771view=diff == --- hadoop/common/trunk/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop
svn commit: r1367772 - in /hadoop/common/branches/branch-2/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/ h
Author: tgraves Date: Tue Jul 31 20:53:45 2012 New Revision: 1367772 URL: http://svn.apache.org/viewvc?rev=1367772view=rev Log: merge -r 1367770:1367771 from trunk. FIXES: MAPREDUCE-4457 Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt?rev=1367772r1=1367771r2=1367772view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 20:53:45 2012 @@ -655,6 +655,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at perticular level is sucessfull (Mayank Bansal via bobby) +MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at +FAILED (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367772r1=1367771r2=1367772view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 20:53:45 2012 @@ -1370,7 +1370,8 @@ public class JobImpl implements org.apac } } -float failureRate = (float) fetchFailures / runningReduceTasks; +float failureRate = runningReduceTasks == 0 ? 1.0f : + (float) fetchFailures / runningReduceTasks; // declare faulty if fetch-failures = max-allowed-failures boolean isMapFaulty = (failureRate = MAX_ALLOWED_FETCH_FAILURES_FRACTION); Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1367772r1=1367771r2=1367772view=diff == --- hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue Jul 31 20:53:45 2012 @@ -435,7 +435,8 @@ public abstract class TaskAttemptImpl im TaskAttemptEventType.TA_CONTAINER_CLEANED, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, - TaskAttemptEventType.TA_FAILMSG)) + TaskAttemptEventType.TA_FAILMSG, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)) // Transitions from KILLED state .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED, Modified: hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java?rev=1367772r1=1367771r2=1367772view
svn commit: r1367774 - in /hadoop/common/branches/branch-0.23/hadoop-mapreduce-project: ./ hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl
Author: tgraves Date: Tue Jul 31 20:54:45 2012 New Revision: 1367774 URL: http://svn.apache.org/viewvc?rev=1367774view=rev Log: merge -r 1367771:1367772 from branch-2. FIXES: MAPREDUCE-4457 Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt?rev=1367774r1=1367773r2=1367774view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/CHANGES.txt Tue Jul 31 20:54:45 2012 @@ -369,6 +369,9 @@ Release 0.23.3 - UNRELEASED MAPREDUCE-4492. Configuring total queue capacity between 100.5 and 99.5 at perticular level is sucessfull (Mayank Bansal via bobby) +MAPREDUCE-4457. mr job invalid transition TA_TOO_MANY_FETCH_FAILURE at +FAILED (Robert Evans via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java?rev=1367774r1=1367773r2=1367774view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java Tue Jul 31 20:54:45 2012 @@ -1305,7 +1305,8 @@ public class JobImpl implements org.apac } } -float failureRate = (float) fetchFailures / runningReduceTasks; +float failureRate = runningReduceTasks == 0 ? 1.0f : + (float) fetchFailures / runningReduceTasks; // declare faulty if fetch-failures = max-allowed-failures boolean isMapFaulty = (failureRate = MAX_ALLOWED_FETCH_FAILURES_FRACTION); Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java?rev=1367774r1=1367773r2=1367774view=diff == --- hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java (original) +++ hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java Tue Jul 31 20:54:45 2012 @@ -431,7 +431,8 @@ public abstract class TaskAttemptImpl im TaskAttemptEventType.TA_CONTAINER_CLEANED, TaskAttemptEventType.TA_COMMIT_PENDING, TaskAttemptEventType.TA_DONE, - TaskAttemptEventType.TA_FAILMSG)) + TaskAttemptEventType.TA_FAILMSG, + TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE)) // Transitions from KILLED state .addTransition(TaskAttemptState.KILLED, TaskAttemptState.KILLED, Modified: hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl
svn commit: r1367713 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
Author: tgraves Date: Tue Jul 31 19:20:03 2012 New Revision: 1367713 URL: http://svn.apache.org/viewvc?rev=1367713view=rev Log: MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm?rev=1367713r1=1367712r2=1367713view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Tue Jul 31 19:20:03 2012 @@ -196,7 +196,9 @@ Deprecated Properties *---+---+ |mapred.compress.map.output | mapreduce.map.output.compress *---+---+ -|mapred.create.symlink | mapreduce.job.cache.symlink.create +|mapred.create.symlink | NONE - symlinking is always on +*---+---+ +|mapreduce.job.cache.symlink.create | NONE - symlinking is always on *---+---+ |mapred.data.field.separator | mapreduce.fieldsel.data.field.separator *---+---+
svn commit: r1367713 - /hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Author: tgraves
Date: Tue Jul 31 19:20:03 2012
New Revision: 1367713
URL: http://svn.apache.org/viewvc?rev=1367713view=rev
Log:
MAPREDUCE-4493. Distibuted Cache Compatability Issues (Robert Evans via tgraves)
Modified:
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Modified:
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1367713r1=1367712r2=1367713view=diff
==
---
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++
hadoop/common/trunk/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Tue Jul 31 19:20:03 2012
@@ -958,7 +958,6 @@ public class StreamJob implements Tool {
if (!b)
fail(LINK_URI);
}
-DistributedCache.createSymlink(jobConf_);
// set the jobconf for the caching parameters
if (cacheArchives != null)
DistributedCache.setCacheArchives(archiveURIs, jobConf_);
svn commit: r1367715 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
Author: tgraves Date: Tue Jul 31 19:21:27 2012 New Revision: 1367715 URL: http://svn.apache.org/viewvc?rev=1367715view=rev Log: merge -r 1367712:1367713 from trunk. FIXES: MAPREDUCE-4493 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm?rev=1367715r1=1367714r2=1367715view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Tue Jul 31 19:21:27 2012 @@ -198,7 +198,9 @@ Deprecated Properties *---+---+ |mapred.compress.map.output | mapreduce.map.output.compress *---+---+ -|mapred.create.symlink | mapreduce.job.cache.symlink.create +|mapred.create.symlink | NONE - symlinking is always on +*---+---+ +|mapreduce.job.cache.symlink.create | NONE - symlinking is always on *---+---+ |mapred.data.field.separator | mapreduce.fieldsel.data.field.separator *---+---+
svn commit: r1367715 - /hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Author: tgraves
Date: Tue Jul 31 19:21:27 2012
New Revision: 1367715
URL: http://svn.apache.org/viewvc?rev=1367715view=rev
Log:
merge -r 1367712:1367713 from trunk. FIXES: MAPREDUCE-4493
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1367715r1=1367714r2=1367715view=diff
==
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Tue Jul 31 19:21:27 2012
@@ -958,7 +958,6 @@ public class StreamJob implements Tool {
if (!b)
fail(LINK_URI);
}
-DistributedCache.createSymlink(jobConf_);
// set the jobconf for the caching parameters
if (cacheArchives != null)
DistributedCache.setCacheArchives(archiveURIs, jobConf_);
svn commit: r1367717 - /hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm
Author: tgraves Date: Tue Jul 31 19:22:25 2012 New Revision: 1367717 URL: http://svn.apache.org/viewvc?rev=1367717view=rev Log: merge -r 1367714:1367715 from branch-2. FIXES: MAPREDUCE-4493 Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Modified: hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm?rev=1367717r1=1367716r2=1367717view=diff == --- hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm (original) +++ hadoop/common/branches/branch-0.23/hadoop-common-project/hadoop-common/src/site/apt/DeprecatedProperties.apt.vm Tue Jul 31 19:22:25 2012 @@ -198,7 +198,9 @@ Deprecated Properties *---+---+ |mapred.compress.map.output | mapreduce.map.output.compress *---+---+ -|mapred.create.symlink | mapreduce.job.cache.symlink.create +|mapred.create.symlink | NONE - symlinking is always on +*---+---+ +|mapreduce.job.cache.symlink.create | NONE - symlinking is always on *---+---+ |mapred.data.field.separator | mapreduce.fieldsel.data.field.separator *---+---+
svn commit: r1367717 - /hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Author: tgraves
Date: Tue Jul 31 19:22:25 2012
New Revision: 1367717
URL: http://svn.apache.org/viewvc?rev=1367717view=rev
Log:
merge -r 1367714:1367715 from branch-2. FIXES: MAPREDUCE-4493
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java?rev=1367717r1=1367716r2=1367717view=diff
==
---
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
(original)
+++
hadoop/common/branches/branch-0.23/hadoop-tools/hadoop-streaming/src/main/java/org/apache/hadoop/streaming/StreamJob.java
Tue Jul 31 19:22:25 2012
@@ -958,7 +958,6 @@ public class StreamJob implements Tool {
if (!b)
fail(LINK_URI);
}
-DistributedCache.createSymlink(jobConf_);
// set the jobconf for the caching parameters
if (cacheArchives != null)
DistributedCache.setCacheArchives(archiveURIs, jobConf_);
svn commit: r1367764 - in /hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt pom.xml
Author: tgraves Date: Tue Jul 31 20:43:21 2012 New Revision: 1367764 URL: http://svn.apache.org/viewvc?rev=1367764view=rev Log: HADOOP-8370. Native build failure: javah: class file for org.apache.hadoop.classification.InterfaceAudience not found (Trevor Robinson via tgraves) Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1367764r1=1367763r2=1367764view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jul 31 20:43:21 2012 @@ -869,6 +869,10 @@ Release 0.23.3 - UNRELEASED HADOOP-8637. FilterFileSystem#setWriteChecksum is broken (daryn via bobby) +HADOOP-8370. Native build failure: javah: class file for +org.apache.hadoop.classification.InterfaceAudience not found (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED INCOMPATIBLE CHANGES Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml?rev=1367764r1=1367763r2=1367764view=diff == --- hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml (original) +++ hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml Tue Jul 31 20:43:21 2012 @@ -44,7 +44,7 @@ dependency groupIdorg.apache.hadoop/groupId artifactIdhadoop-annotations/artifactId - scopeprovided/scope + scopecompile/scope /dependency dependency groupIdcom.google.guava/groupId
svn commit: r1367765 - in /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common: CHANGES.txt pom.xml
Author: tgraves Date: Tue Jul 31 20:44:12 2012 New Revision: 1367765 URL: http://svn.apache.org/viewvc?rev=1367765view=rev Log: merge -r 1367763:1367764 from trunk. FIXES: HADOOP-8370 Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1367765r1=1367764r2=1367765view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/CHANGES.txt Tue Jul 31 20:44:12 2012 @@ -688,6 +688,10 @@ Release 0.23.3 - UNRELEASED HADOOP-8637. FilterFileSystem#setWriteChecksum is broken (daryn via bobby) +HADOOP-8370. Native build failure: javah: class file for +org.apache.hadoop.classification.InterfaceAudience not found (Trevor +Robinson via tgraves) + Release 0.23.2 - UNRELEASED NEW FEATURES Modified: hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml?rev=1367765r1=1367764r2=1367765view=diff == --- hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml (original) +++ hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/pom.xml Tue Jul 31 20:44:12 2012 @@ -43,7 +43,7 @@ dependency groupIdorg.apache.hadoop/groupId artifactIdhadoop-annotations/artifactId - scopeprovided/scope + scopecompile/scope /dependency dependency groupIdcom.google.guava/groupId
