svn commit: r1556679 - in /hadoop/common/branches: YARN-321-old/ YARN-321/
Author: zjshen Date: Wed Jan 8 23:51:43 2014 New Revision: 1556679 URL: http://svn.apache.org/r1556679 Log: Deprecate current YARN-321 branch Added: hadoop/common/branches/YARN-321-old/ (props changed) - copied from r1556678, hadoop/common/branches/YARN-321/ Removed: hadoop/common/branches/YARN-321/ Propchange: hadoop/common/branches/YARN-321-old/ -- --- svn:ignore (added) +++ svn:ignore Wed Jan 8 23:51:43 2014 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/branches/YARN-321-old/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Wed Jan 8 23:51:43 2014 @@ -0,0 +1,6 @@ +/hadoop/common/branches/HDFS-2802:1360400-1480829 +/hadoop/common/branches/HDFS-3042:1306184-1342109 +/hadoop/common/branches/branch-0.23-PB:1227776-1294021 +/hadoop/common/branches/branch-2:1503799-1540532 +/hadoop/common/branches/branch-2.0.4-alpha:1463804 +/hadoop/common/trunk:1161777,1161781,1162188,1162421,1162491,1162499,1162613,1162928,1162954,1162979,1163050,1163069,1163490,1163768,1163852,1163858,1163981,1164255,1164301,1164339,1166009,1166402,1167001,1167383,1167662,1170085,1170379,1170459,1171297,1172916,1173402,1176550,1177487,1177531,1177859,1177864,1182189,1182205,1182214,1189613,1189932,1189982,1195575,1196113,1196129,1204114,1204117,1204122,1204124,1204129,1204131,1204177,1204370,1204376,1204388,1205260,1205697,1206786,1206830,1207694,1208153,1208313,1212021,1212062,1212073,1212084,1213537,1213586,1213592-1213593,1213954,1214046,1220510,1221348,1225114,1225192,1225456,1225489,1225591,1226211,1226239,1226350,1227091,1227165,1227423,1227964,1229347,1230398,1231569,1231572,1231627,1231640,1233605,1234555,1235135,1235137,1235956,1236456,1239752,1240897,1240928,1243065,1243104,1244766,1245751,1245762,1293419,1295061,1295227,1296556,1298044,1298696,1298700,1299045,1299434,1299963,1301308,1301312,1301820,1301871,1302624,1302704- 1302705,1303474,1304063,1304099,1304112,1304118,1305230,1309625,1310185,1311556,1312029,1329319,1333557,1334216,1342112,1347867,1348207,1349124,1349616,1351445,1351818,1357442,1361813,1373683,1374696,1375450,1375829,1379646,1380921,1400347,1430682,1430688,1440245,1445965,1458741,1465121,1476395,1477849,1480838,1480840,1487093,1507259,1531125
svn commit: r1556680 - /hadoop/common/branches/YARN-321/
Author: zjshen Date: Wed Jan 8 23:56:35 2014 New Revision: 1556680 URL: http://svn.apache.org/r1556680 Log: Creating a new branch for Generic ApplicationHistoryServer via YARN-321 based on trunk Added: hadoop/common/branches/YARN-321/ (props changed) - copied from r1556679, hadoop/common/trunk/ Propchange: hadoop/common/branches/YARN-321/ -- --- svn:ignore (added) +++ svn:ignore Wed Jan 8 23:56:35 2014 @@ -0,0 +1,5 @@ +.classpath +.git +.project +.settings +target Propchange: hadoop/common/branches/YARN-321/ -- --- svn:mergeinfo (added) +++ svn:mergeinfo Wed Jan 8 23:56:35 2014 @@ -0,0 +1,9 @@ +/hadoop/common/branches/HDFS-1623:1152502-1296519 +/hadoop/common/branches/HDFS-2802:1360400-1480829 +/hadoop/common/branches/HDFS-2832:1513717-1550362 +/hadoop/common/branches/HDFS-3042:1306184-1342109 +/hadoop/common/branches/HDFS-3077:1363593-1396941 +/hadoop/common/branches/HDFS-347:1430995-1467533 +/hadoop/common/branches/HDFS-4949:1509426-1536569 +/hadoop/common/branches/branch-2:1526848,1541667,1541680,1541687,1541699,1541861,1544173,1546957,1547755,1551915,1554802 +/hadoop/common/branches/branch-2.0.4-alpha:1463804
svn commit: r1556744 - /hadoop/common/branches/YARN-321/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
Author: zjshen
Date: Thu Jan 9 07:03:50 2014
New Revision: 1556744
URL: http://svn.apache.org/r1556744
Log:
YARN-1242. Changed yarn scripts to be able to start ApplicationHistoryServer as
an individual process. Contributed by Mayank Bansal.
svn merge --ignore-ancestry -c 1544399 ../../branches/YARN-321-old/
Modified:
hadoop/common/branches/YARN-321/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
Modified:
hadoop/common/branches/YARN-321/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-321/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml?rev=1556744r1=1556743r2=1556744view=diff
==
---
hadoop/common/branches/YARN-321/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
(original)
+++
hadoop/common/branches/YARN-321/hadoop-assemblies/src/main/resources/assemblies/hadoop-yarn-dist.xml
Thu Jan 9 07:03:50 2014
@@ -123,6 +123,13 @@
/includes
/fileSet
fileSet
+
directoryhadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/target/directory
+
outputDirectory/share/hadoop/${hadoop.component}/sources/outputDirectory
+ includes
+include*-sources.jar/include
+ /includes
+/fileSet
+fileSet
directoryhadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/target/directory
outputDirectory/share/hadoop/${hadoop.component}/sources/outputDirectory
includes
svn commit: r1565868 - /hadoop/common/trunk/hadoop-project/pom.xml
Author: zjshen Date: Sat Feb 8 02:15:46 2014 New Revision: 1565868 URL: http://svn.apache.org/r1565868 Log: YARN-1635. Implemented a Leveldb based ApplicationTimelineStore. Contributed by Billie Rinaldi. Modified: hadoop/common/trunk/hadoop-project/pom.xml Modified: hadoop/common/trunk/hadoop-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-project/pom.xml?rev=1565868r1=1565867r2=1565868view=diff == --- hadoop/common/trunk/hadoop-project/pom.xml (original) +++ hadoop/common/trunk/hadoop-project/pom.xml Sat Feb 8 02:15:46 2014 @@ -784,6 +784,12 @@ artifactIdgrizzly-http-servlet/artifactId version2.1.2/version /dependency + + dependency +groupIdorg.fusesource.leveldbjni/groupId +artifactIdleveldbjni-all/artifactId +version1.8/version + /dependency /dependencies /dependencyManagement
svn commit: r1565869 - /hadoop/common/branches/branch-2/hadoop-project/pom.xml
Author: zjshen Date: Sat Feb 8 02:18:06 2014 New Revision: 1565869 URL: http://svn.apache.org/r1565869 Log: YARN-1635. Implemented a Leveldb based ApplicationTimelineStore. Contributed by Billie Rinaldi. svn merge --ignore-ancestry -c 1565868 ../../trunk/ Modified: hadoop/common/branches/branch-2/hadoop-project/pom.xml Modified: hadoop/common/branches/branch-2/hadoop-project/pom.xml URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-project/pom.xml?rev=1565869r1=1565868r2=1565869view=diff == --- hadoop/common/branches/branch-2/hadoop-project/pom.xml (original) +++ hadoop/common/branches/branch-2/hadoop-project/pom.xml Sat Feb 8 02:18:06 2014 @@ -778,6 +778,12 @@ artifactIdgrizzly-http-servlet/artifactId version2.1.2/version /dependency + + dependency +groupIdorg.fusesource.leveldbjni/groupId +artifactIdleveldbjni-all/artifactId +version1.8/version + /dependency /dependencies /dependencyManagement
svn commit: r1602071 - /hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
Author: zjshen
Date: Thu Jun 12 06:40:32 2014
New Revision: 1602071
URL: http://svn.apache.org/r1602071
Log:
YARN-2075. Fixed the test failure of TestRMAdminCLI. Contributed by Kenji
Kikushima.
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java?rev=1602071r1=1602070r2=1602071view=diff
==
---
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
(original)
+++
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
Thu Jun 12 06:40:32 2014
@@ -19,6 +19,7 @@ package org.apache.hadoop.ha;
import java.io.IOException;
import java.io.PrintStream;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
@@ -68,7 +69,7 @@ public abstract class HAAdmin extends Co
protected final static MapString, UsageInfo USAGE =
ImmutableMap.String, UsageInfobuilder()
.put(-transitionToActive,
-new UsageInfo( serviceId [--+FORCEACTIVE+], Transitions the
service into Active state))
+new UsageInfo(serviceId [--+FORCEACTIVE+], Transitions the
service into Active state))
.put(-transitionToStandby,
new UsageInfo(serviceId, Transitions the service into Standby
state))
.put(-failover,
@@ -104,7 +105,8 @@ public abstract class HAAdmin extends Co
protected abstract HAServiceTarget resolveTarget(String string);
protected CollectionString getTargetIds(String targetNodeToActivate) {
-return Arrays.asList(new String[]{targetNodeToActivate});
+return new ArrayListString(
+Arrays.asList(new String[]{targetNodeToActivate}));
}
protected String getUsageString() {
svn commit: r1602073 - /hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
Author: zjshen
Date: Thu Jun 12 06:48:49 2014
New Revision: 1602073
URL: http://svn.apache.org/r1602073
Log:
YARN-2075. Fixed the test failure of TestRMAdminCLI. Contributed by Kenji
Kikushima.
svn merge --ignore-ancestry -c 1602071 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
Modified:
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java?rev=1602073r1=1602072r2=1602073view=diff
==
---
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ha/HAAdmin.java
Thu Jun 12 06:48:49 2014
@@ -19,6 +19,7 @@ package org.apache.hadoop.ha;
import java.io.IOException;
import java.io.PrintStream;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Map;
@@ -68,7 +69,7 @@ public abstract class HAAdmin extends Co
protected final static MapString, UsageInfo USAGE =
ImmutableMap.String, UsageInfobuilder()
.put(-transitionToActive,
-new UsageInfo( serviceId [--+FORCEACTIVE+], Transitions the
service into Active state))
+new UsageInfo(serviceId [--+FORCEACTIVE+], Transitions the
service into Active state))
.put(-transitionToStandby,
new UsageInfo(serviceId, Transitions the service into Standby
state))
.put(-failover,
@@ -104,7 +105,8 @@ public abstract class HAAdmin extends Co
protected abstract HAServiceTarget resolveTarget(String string);
protected CollectionString getTargetIds(String targetNodeToActivate) {
-return Arrays.asList(new String[]{targetNodeToActivate});
+return new ArrayListString(
+Arrays.asList(new String[]{targetNodeToActivate}));
}
protected String getUsageString() {
svn commit: r1618972 - /hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Author: zjshen
Date: Tue Aug 19 20:33:49 2014
New Revision: 1618972
URL: http://svn.apache.org/r1618972
Log:
YARN-2249. Avoided AM release requests being lost on work preserving RM
restart. Contributed by Jian He.
Modified:
hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Modified:
hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1618972r1=1618971r2=1618972view=diff
==
---
hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
(original)
+++
hadoop/common/trunk/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Tue Aug 19 20:33:49 2014
@@ -36,8 +36,8 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import
org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
@@ -933,4 +934,10 @@ final public class ResourceSchedulerWrap
return new HashMapApplicationId,
SchedulerApplicationSchedulerApplicationAttempt();
}
+
+ @Override
+ protected void completedContainer(RMContainer rmContainer,
+ ContainerStatus containerStatus, RMContainerEventType event) {
+// do nothing
+ }
}
svn commit: r1618973 - /hadoop/common/branches/branch-2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Author: zjshen
Date: Tue Aug 19 20:40:37 2014
New Revision: 1618973
URL: http://svn.apache.org/r1618973
Log:
YARN-2249. Avoided AM release requests being lost on work preserving RM
restart. Contributed by Jian He.
svn merge --ignore-ancestry -c 1618972 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Modified:
hadoop/common/branches/branch-2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java?rev=1618973r1=1618972r2=1618973view=diff
==
---
hadoop/common/branches/branch-2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
(original)
+++
hadoop/common/branches/branch-2/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/ResourceSchedulerWrapper.java
Tue Aug 19 20:40:37 2014
@@ -36,8 +36,8 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceAudience.LimitedPrivate;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
@@ -61,6 +61,7 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.recovery.RMStateStore;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+import
org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerEventType;
import
org.apache.hadoop.yarn.server.resourcemanager.rmnode.UpdatedContainerInfo;
import
org.apache.hadoop.yarn.server.resourcemanager.scheduler.AbstractYarnScheduler;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Allocation;
@@ -934,5 +935,11 @@ public class ResourceSchedulerWrapper
return new HashMapApplicationId,
SchedulerApplicationSchedulerApplicationAttempt();
}
+
+ @Override
+ protected void completedContainer(RMContainer rmContainer,
+ ContainerStatus containerStatus, RMContainerEventType event) {
+// do nothing
+ }
}
git commit: YARN-2035. FileSystemApplicationHistoryStore should not make working dir when it already exists. Contributed by Jonathan Eagles.
Repository: hadoop
Updated Branches:
refs/heads/trunk d16bfd1d0 - d778abf02
YARN-2035. FileSystemApplicationHistoryStore should not make working dir when
it already exists. Contributed by Jonathan Eagles.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d778abf0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d778abf0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d778abf0
Branch: refs/heads/trunk
Commit: d778abf022b415c64223153814d4188c2b3dd797
Parents: d16bfd1
Author: Zhijie Shen zjs...@apache.org
Authored: Wed Aug 27 02:01:00 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Wed Aug 27 02:01:00 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../FileSystemApplicationHistoryStore.java | 14 -
.../TestFileSystemApplicationHistoryStore.java | 62 +++-
3 files changed, 75 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d778abf0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index eefa547..36d304c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -240,6 +240,9 @@ Release 2.6.0 - UNRELEASED
YARN-2434. RM should not recover containers from previously failed attempt
when AM restart is not enabled (Jian He via jlowe)
+YARN-2035. FileSystemApplicationHistoryStore should not make working dir
+when it already exists. (Jonathan Eagles via zjshen)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d778abf0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
index a5725eb..a2d9140 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
@@ -110,15 +110,23 @@ public class FileSystemApplicationHistoryStore extends
AbstractService
super(FileSystemApplicationHistoryStore.class.getName());
}
+ protected FileSystem getFileSystem(Path path, Configuration conf) throws
Exception {
+return path.getFileSystem(conf);
+ }
+
@Override
public void serviceInit(Configuration conf) throws Exception {
Path fsWorkingPath =
new Path(conf.get(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI));
rootDirPath = new Path(fsWorkingPath, ROOT_DIR_NAME);
try {
- fs = fsWorkingPath.getFileSystem(conf);
- fs.mkdirs(rootDirPath);
- fs.setPermission(rootDirPath, ROOT_DIR_UMASK);
+ fs = getFileSystem(fsWorkingPath, conf);
+
+ if (!fs.isDirectory(rootDirPath)) {
+fs.mkdirs(rootDirPath);
+fs.setPermission(rootDirPath, ROOT_DIR_UMASK);
+ }
+
} catch (IOException e) {
LOG.error(Error when initializing FileSystemHistoryStorage, e);
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d778abf0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
index d31018c..552a5e5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop
git commit: YARN-2035. FileSystemApplicationHistoryStore should not make working dir when it already exists. Contributed by Jonathan Eagles.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 9c43760e5 - 0c785f082
YARN-2035. FileSystemApplicationHistoryStore should not make working dir when
it already exists. Contributed by Jonathan Eagles.
(cherry picked from commit d778abf022b415c64223153814d4188c2b3dd797)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0c785f08
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0c785f08
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0c785f08
Branch: refs/heads/branch-2
Commit: 0c785f0824144acea1bf30c725efd37c1f9bea6c
Parents: 9c43760
Author: Zhijie Shen zjs...@apache.org
Authored: Wed Aug 27 02:01:00 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Wed Aug 27 02:10:06 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../FileSystemApplicationHistoryStore.java | 14 -
.../TestFileSystemApplicationHistoryStore.java | 62 +++-
3 files changed, 75 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c785f08/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b3a35a3..fe42b45 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -220,6 +220,9 @@ Release 2.6.0 - UNRELEASED
YARN-2434. RM should not recover containers from previously failed attempt
when AM restart is not enabled (Jian He via jlowe)
+YARN-2035. FileSystemApplicationHistoryStore should not make working dir
+when it already exists. (Jonathan Eagles via zjshen)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c785f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
index a5725eb..a2d9140 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
@@ -110,15 +110,23 @@ public class FileSystemApplicationHistoryStore extends
AbstractService
super(FileSystemApplicationHistoryStore.class.getName());
}
+ protected FileSystem getFileSystem(Path path, Configuration conf) throws
Exception {
+return path.getFileSystem(conf);
+ }
+
@Override
public void serviceInit(Configuration conf) throws Exception {
Path fsWorkingPath =
new Path(conf.get(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI));
rootDirPath = new Path(fsWorkingPath, ROOT_DIR_NAME);
try {
- fs = fsWorkingPath.getFileSystem(conf);
- fs.mkdirs(rootDirPath);
- fs.setPermission(rootDirPath, ROOT_DIR_UMASK);
+ fs = getFileSystem(fsWorkingPath, conf);
+
+ if (!fs.isDirectory(rootDirPath)) {
+fs.mkdirs(rootDirPath);
+fs.setPermission(rootDirPath, ROOT_DIR_UMASK);
+ }
+
} catch (IOException e) {
LOG.error(Error when initializing FileSystemHistoryStorage, e);
throw e;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0c785f08/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
index d31018c..552a5e5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server
git commit: YARN-2449. Fixed the bug that TimelineAuthenticationFilterInitializer is not automatically added when hadoop.http.filter.initializers is not configured. Contributed by Varun Vasudev.
Repository: hadoop
Updated Branches:
refs/heads/trunk 4ae8178c5 - 4bd0194e6
YARN-2449. Fixed the bug that TimelineAuthenticationFilterInitializer is not
automatically added when hadoop.http.filter.initializers is not configured.
Contributed by Varun Vasudev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4bd0194e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4bd0194e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4bd0194e
Branch: refs/heads/trunk
Commit: 4bd0194e6be68421eb1dc87f9f031626112e4c50
Parents: 4ae8178
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Aug 29 09:40:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Aug 29 09:40:39 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 4 +++
.../ApplicationHistoryServer.java | 5 ++-
.../TestApplicationHistoryServer.java | 35 ++--
3 files changed, 32 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bd0194e/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fa47c8e..1528cba 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -251,6 +251,10 @@ Release 2.6.0 - UNRELEASED
YARN-2405. NPE in FairSchedulerAppsBlock. (Tsuyoshi Ozawa via kasha)
+YARN-2449. Fixed the bug that TimelineAuthenticationFilterInitializer
+is not automatically added when hadoop.http.filter.initializers is not
+configured. (Varun Vasudev via zjshen)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bd0194e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index c61b80e..6ec0d42 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -197,6 +197,7 @@ public class ApplicationHistoryServer extends
CompositeService {
// the customized filter will be loaded by the timeline server to do
Kerberos
// + DT authentication.
String initializers = conf.get(hadoop.http.filter.initializers);
+boolean modifiedInitialiers = false;
initializers =
initializers == null || initializers.length() == 0 ? : initializers;
@@ -206,6 +207,7 @@ public class ApplicationHistoryServer extends
CompositeService {
initializers =
TimelineAuthenticationFilterInitializer.class.getName() + ,
+ initializers;
+ modifiedInitialiers = true;
}
String[] parts = initializers.split(,);
@@ -214,13 +216,14 @@ public class ApplicationHistoryServer extends
CompositeService {
filterInitializer = filterInitializer.trim();
if (filterInitializer.equals(AuthenticationFilterInitializer.class
.getName())) {
+modifiedInitialiers = true;
continue;
}
target.add(filterInitializer);
}
String actualInitializers =
org.apache.commons.lang.StringUtils.join(target, ,);
-if (!actualInitializers.equals(initializers)) {
+if (modifiedInitialiers) {
conf.set(hadoop.http.filter.initializers, actualInitializers);
}
String bindAddress = WebAppUtils.getWebAppBindURL(conf,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4bd0194e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
b
git commit: YARN-2449. Fixed the bug that TimelineAuthenticationFilterInitializer is not automatically added when hadoop.http.filter.initializers is not configured. Contributed by Varun Vasudev.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 f237dee15 - 77a4b014e
YARN-2449. Fixed the bug that TimelineAuthenticationFilterInitializer is not
automatically added when hadoop.http.filter.initializers is not configured.
Contributed by Varun Vasudev.
(cherry picked from commit 4bd0194e6be68421eb1dc87f9f031626112e4c50)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77a4b014
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77a4b014
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77a4b014
Branch: refs/heads/branch-2
Commit: 77a4b014e6407ad7d0f6110a6ac96e9d31b1a4e8
Parents: f237dee
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Aug 29 09:40:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Aug 29 09:46:04 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 4 +++
.../ApplicationHistoryServer.java | 5 ++-
.../TestApplicationHistoryServer.java | 35 ++--
3 files changed, 32 insertions(+), 12 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77a4b014/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ba70867..5fba3bc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -231,6 +231,10 @@ Release 2.6.0 - UNRELEASED
YARN-2405. NPE in FairSchedulerAppsBlock. (Tsuyoshi Ozawa via kasha)
+YARN-2449. Fixed the bug that TimelineAuthenticationFilterInitializer
+is not automatically added when hadoop.http.filter.initializers is not
+configured. (Varun Vasudev via zjshen)
+
Release 2.5.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77a4b014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
index c61b80e..6ec0d42 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
@@ -197,6 +197,7 @@ public class ApplicationHistoryServer extends
CompositeService {
// the customized filter will be loaded by the timeline server to do
Kerberos
// + DT authentication.
String initializers = conf.get(hadoop.http.filter.initializers);
+boolean modifiedInitialiers = false;
initializers =
initializers == null || initializers.length() == 0 ? : initializers;
@@ -206,6 +207,7 @@ public class ApplicationHistoryServer extends
CompositeService {
initializers =
TimelineAuthenticationFilterInitializer.class.getName() + ,
+ initializers;
+ modifiedInitialiers = true;
}
String[] parts = initializers.split(,);
@@ -214,13 +216,14 @@ public class ApplicationHistoryServer extends
CompositeService {
filterInitializer = filterInitializer.trim();
if (filterInitializer.equals(AuthenticationFilterInitializer.class
.getName())) {
+modifiedInitialiers = true;
continue;
}
target.add(filterInitializer);
}
String actualInitializers =
org.apache.commons.lang.StringUtils.join(target, ,);
-if (!actualInitializers.equals(initializers)) {
+if (modifiedInitialiers) {
conf.set(hadoop.http.filter.initializers, actualInitializers);
}
String bindAddress = WebAppUtils.getWebAppBindURL(conf,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77a4b014/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryServer.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop
git commit: YARN-2511. Allowed all origins by default when CrossOriginFilter is enabled. Contributed by Jonathan Eagles.
Repository: hadoop
Updated Branches:
refs/heads/trunk 3fa5f728c - 51a4faf52
YARN-2511. Allowed all origins by default when CrossOriginFilter is enabled.
Contributed by Jonathan Eagles.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/51a4faf5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/51a4faf5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/51a4faf5
Branch: refs/heads/trunk
Commit: 51a4faf52199b181471d591a9fde900fc166fc48
Parents: 3fa5f72
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Sep 4 15:01:01 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Sep 4 15:01:01 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/timeline/webapp/CrossOriginFilter.java| 10 +++---
.../timeline/webapp/TestCrossOriginFilter.java | 15 +++
3 files changed, 25 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51a4faf5/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index f6f5c4b..1a5ea07 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -178,6 +178,9 @@ Release 2.6.0 - UNRELEASED
YARN-2509. Enable Cross Origin Filter for timeline server only and not all
Yarn servers (Mit Desai via jeagles)
+YARN-2511. Allowed all origins by default when CrossOriginFilter is
+enabled. (Jonathan Eagles via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51a4faf5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index a9fb3e8..d71175f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -76,6 +76,7 @@ public class CrossOriginFilter implements Filter {
private ListString allowedMethods = new ArrayListString();
private ListString allowedHeaders = new ArrayListString();
private ListString allowedOrigins = new ArrayListString();
+ private boolean allowAllOrigins = true;
private String maxAge;
@Override
@@ -171,7 +172,9 @@ public class CrossOriginFilter implements Filter {
}
allowedOrigins =
Arrays.asList(allowedOriginsConfig.trim().split(\\s*,\\s*));
+allowAllOrigins = allowedOrigins.contains(*);
LOG.info(Allowed Origins: + StringUtils.join(allowedOrigins, ','));
+LOG.info(Allow All Origins: + allowAllOrigins);
}
private void initializeMaxAge(FilterConfig filterConfig) {
@@ -199,8 +202,9 @@ public class CrossOriginFilter implements Filter {
return origin != null;
}
- private boolean isOriginAllowed(String origin) {
-return allowedOrigins.contains(origin);
+ @VisibleForTesting
+ boolean isOriginAllowed(String origin) {
+return allowAllOrigins || allowedOrigins.contains(origin);
}
private boolean areHeadersAllowed(String accessControlRequestHeaders) {
@@ -213,7 +217,7 @@ public class CrossOriginFilter implements Filter {
private boolean isMethodAllowed(String accessControlRequestMethod) {
if (accessControlRequestMethod == null) {
- return false;
+ return true;
}
return allowedMethods.contains(accessControlRequestMethod);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/51a4faf5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn
git commit: YARN-2511. Allowed all origins by default when CrossOriginFilter is enabled. Contributed by Jonathan Eagles.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 04d325aff - 1e40cbba7
YARN-2511. Allowed all origins by default when CrossOriginFilter is enabled.
Contributed by Jonathan Eagles.
(cherry picked from commit 51a4faf52199b181471d591a9fde900fc166fc48)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e40cbba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e40cbba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e40cbba
Branch: refs/heads/branch-2
Commit: 1e40cbba7f0ac402328819d3fb8f09e11595117f
Parents: 04d325a
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Sep 4 15:01:01 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Sep 4 15:03:50 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../server/timeline/webapp/CrossOriginFilter.java| 10 +++---
.../timeline/webapp/TestCrossOriginFilter.java | 15 +++
3 files changed, 25 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e40cbba/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7f7f367..b7e092a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -155,6 +155,9 @@ Release 2.6.0 - UNRELEASED
YARN-2509. Enable Cross Origin Filter for timeline server only and not all
Yarn servers (Mit Desai via jeagles)
+YARN-2511. Allowed all origins by default when CrossOriginFilter is
+enabled. (Jonathan Eagles via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e40cbba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index a9fb3e8..d71175f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -76,6 +76,7 @@ public class CrossOriginFilter implements Filter {
private ListString allowedMethods = new ArrayListString();
private ListString allowedHeaders = new ArrayListString();
private ListString allowedOrigins = new ArrayListString();
+ private boolean allowAllOrigins = true;
private String maxAge;
@Override
@@ -171,7 +172,9 @@ public class CrossOriginFilter implements Filter {
}
allowedOrigins =
Arrays.asList(allowedOriginsConfig.trim().split(\\s*,\\s*));
+allowAllOrigins = allowedOrigins.contains(*);
LOG.info(Allowed Origins: + StringUtils.join(allowedOrigins, ','));
+LOG.info(Allow All Origins: + allowAllOrigins);
}
private void initializeMaxAge(FilterConfig filterConfig) {
@@ -199,8 +202,9 @@ public class CrossOriginFilter implements Filter {
return origin != null;
}
- private boolean isOriginAllowed(String origin) {
-return allowedOrigins.contains(origin);
+ @VisibleForTesting
+ boolean isOriginAllowed(String origin) {
+return allowAllOrigins || allowedOrigins.contains(origin);
}
private boolean areHeadersAllowed(String accessControlRequestHeaders) {
@@ -213,7 +217,7 @@ public class CrossOriginFilter implements Filter {
private boolean isMethodAllowed(String accessControlRequestMethod) {
if (accessControlRequestMethod == null) {
- return false;
+ return true;
}
return allowedMethods.contains(accessControlRequestMethod);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e40cbba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server
git commit: YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. Contributed by Jonathan Eagles.
Repository: hadoop
Updated Branches:
refs/heads/trunk a23144fd8 - a092cdf32
YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
Contributed by Jonathan Eagles.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a092cdf3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a092cdf3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a092cdf3
Branch: refs/heads/trunk
Commit: a092cdf32de4d752456286a9f4dda533d8a62bca
Parents: a23144f
Author: Zhijie Shen zjs...@apache.org
Authored: Sun Sep 7 17:49:06 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Sun Sep 7 17:49:06 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../timeline/webapp/CrossOriginFilter.java | 20 ++-
.../timeline/webapp/TestCrossOriginFilter.java | 21 +++-
3 files changed, 42 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a092cdf3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index beafc22..ed31479 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -184,6 +184,9 @@ Release 2.6.0 - UNRELEASED
YARN-2508. Cross Origin configuration parameters prefix are not honored
(Mit Desai via jeagles)
+YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
+(Jonathan Eagles via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a092cdf3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index d71175f..5a0703d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -24,6 +24,8 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@@ -204,7 +206,23 @@ public class CrossOriginFilter implements Filter {
@VisibleForTesting
boolean isOriginAllowed(String origin) {
-return allowAllOrigins || allowedOrigins.contains(origin);
+if (allowAllOrigins) {
+ return true;
+}
+
+for (String allowedOrigin : allowedOrigins) {
+ if (allowedOrigin.contains(*)) {
+String regex = allowedOrigin.replace(., \\.).replace(*, .*);
+Pattern p = Pattern.compile(regex);
+Matcher m = p.matcher(origin);
+if (m.matches()) {
+ return true;
+}
+ } else if (allowedOrigin.equals(origin)) {
+return true;
+ }
+}
+return false;
}
private boolean areHeadersAllowed(String accessControlRequestHeaders) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a092cdf3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
index f666c21..ccc9bbf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
@@ -77,7 +77,26 @@ public class
git commit: YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. Contributed by Jonathan Eagles.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b87d1d7d0 - f2a5a5d4a
YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
Contributed by Jonathan Eagles.
(cherry picked from commit a092cdf32de4d752456286a9f4dda533d8a62bca)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f2a5a5d4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f2a5a5d4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f2a5a5d4
Branch: refs/heads/branch-2
Commit: f2a5a5d4aada5b0a3660e5db281f505e2cd3193a
Parents: b87d1d7
Author: Zhijie Shen zjs...@apache.org
Authored: Sun Sep 7 17:49:06 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Sun Sep 7 17:52:38 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../timeline/webapp/CrossOriginFilter.java | 20 ++-
.../timeline/webapp/TestCrossOriginFilter.java | 21 +++-
3 files changed, 42 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2a5a5d4/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 83274b6..a149028 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -161,6 +161,9 @@ Release 2.6.0 - UNRELEASED
YARN-2508. Cross Origin configuration parameters prefix are not honored
(Mit Desai via jeagles)
+YARN-2512. Allowed pattern matching for origins in CrossOriginFilter.
+(Jonathan Eagles via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2a5a5d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index d71175f..5a0703d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -24,6 +24,8 @@ import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import javax.servlet.Filter;
import javax.servlet.FilterChain;
@@ -204,7 +206,23 @@ public class CrossOriginFilter implements Filter {
@VisibleForTesting
boolean isOriginAllowed(String origin) {
-return allowAllOrigins || allowedOrigins.contains(origin);
+if (allowAllOrigins) {
+ return true;
+}
+
+for (String allowedOrigin : allowedOrigins) {
+ if (allowedOrigin.contains(*)) {
+String regex = allowedOrigin.replace(., \\.).replace(*, .*);
+Pattern p = Pattern.compile(regex);
+Matcher m = p.matcher(origin);
+if (m.matches()) {
+ return true;
+}
+ } else if (allowedOrigin.equals(origin)) {
+return true;
+ }
+}
+return false;
}
private boolean areHeadersAllowed(String accessControlRequestHeaders) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f2a5a5d4/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
index f666c21..ccc9bbf 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestCrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server
git commit: YARN-2507. Documented CrossOriginFilter configurations for the timeline server. Contributed by Jonathan Eagles.
Repository: hadoop Updated Branches: refs/heads/trunk a092cdf32 - 56dc496a1 YARN-2507. Documented CrossOriginFilter configurations for the timeline server. Contributed by Jonathan Eagles. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/56dc496a Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/56dc496a Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/56dc496a Branch: refs/heads/trunk Commit: 56dc496a1031621d2b701801de4ec29179d75f2e Parents: a092cdf Author: Zhijie Shen zjs...@apache.org Authored: Sun Sep 7 18:22:40 2014 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Sun Sep 7 18:22:40 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 ++ .../src/site/apt/TimelineServer.apt.vm | 37 2 files changed, 40 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/56dc496a/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index ed31479..ed9de87 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -187,6 +187,9 @@ Release 2.6.0 - UNRELEASED YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. (Jonathan Eagles via zjshen) +YARN-2507. Documented CrossOriginFilter configurations for the timeline +server. (Jonathan Eagles via zjshen) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/56dc496a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm index c704d37..92c7377 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm @@ -102,6 +102,43 @@ YARN Timeline Server nameyarn.timeline-service.handler-thread-count/name value10/value /property + +property + descriptionEnables cross-origin support (CORS) for web services where + cross-origin web response headers are needed. For example, javascript making + a web services request to the timeline server./description + nameyarn.timeline-service.http-cross-origin.enabled/name + valuefalse/value +/property + +property + descriptionComma separated list of origins that are allowed for web + services needing cross-origin (CORS) support. Wildcards (*) and patterns + allowed/description + nameyarn.timeline-service.http-cross-origin.allowed-origins/name + value*/value +/property + +property + descriptionComma separated list of methods that are allowed for web + services needing cross-origin (CORS) support./description + nameyarn.timeline-service.http-cross-origin.allowed-methods/name + valueGET,POST,HEAD/value +/property + +property + descriptionComma separated list of headers that are allowed for web + services needing cross-origin (CORS) support./description + nameyarn.timeline-service.http-cross-origin.allowed-headers/name + valueX-Requested-With,Content-Type,Accept,Origin/value +/property + +property + descriptionThe number of seconds a pre-flighted request can be cached + for web services needing cross-origin (CORS) support./description + nameyarn.timeline-service.http-cross-origin.max-age/name + value1800/value +/property +---+ * Generic-data related Configuration
git commit: YARN-2507. Documented CrossOriginFilter configurations for the timeline server. Contributed by Jonathan Eagles.
Repository: hadoop Updated Branches: refs/heads/branch-2 f2a5a5d4a - 0974400b7 YARN-2507. Documented CrossOriginFilter configurations for the timeline server. Contributed by Jonathan Eagles. (cherry picked from commit 56dc496a1031621d2b701801de4ec29179d75f2e) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0974400b Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0974400b Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0974400b Branch: refs/heads/branch-2 Commit: 0974400b7401cebef0e9212d7400d868271876f9 Parents: f2a5a5d Author: Zhijie Shen zjs...@apache.org Authored: Sun Sep 7 18:22:40 2014 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Sun Sep 7 18:25:13 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 ++ .../src/site/apt/TimelineServer.apt.vm | 37 2 files changed, 40 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/0974400b/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index a149028..8b362a5 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -164,6 +164,9 @@ Release 2.6.0 - UNRELEASED YARN-2512. Allowed pattern matching for origins in CrossOriginFilter. (Jonathan Eagles via zjshen) +YARN-2507. Documented CrossOriginFilter configurations for the timeline +server. (Jonathan Eagles via zjshen) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/0974400b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm index c704d37..92c7377 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/TimelineServer.apt.vm @@ -102,6 +102,43 @@ YARN Timeline Server nameyarn.timeline-service.handler-thread-count/name value10/value /property + +property + descriptionEnables cross-origin support (CORS) for web services where + cross-origin web response headers are needed. For example, javascript making + a web services request to the timeline server./description + nameyarn.timeline-service.http-cross-origin.enabled/name + valuefalse/value +/property + +property + descriptionComma separated list of origins that are allowed for web + services needing cross-origin (CORS) support. Wildcards (*) and patterns + allowed/description + nameyarn.timeline-service.http-cross-origin.allowed-origins/name + value*/value +/property + +property + descriptionComma separated list of methods that are allowed for web + services needing cross-origin (CORS) support./description + nameyarn.timeline-service.http-cross-origin.allowed-methods/name + valueGET,POST,HEAD/value +/property + +property + descriptionComma separated list of headers that are allowed for web + services needing cross-origin (CORS) support./description + nameyarn.timeline-service.http-cross-origin.allowed-headers/name + valueX-Requested-With,Content-Type,Accept,Origin/value +/property + +property + descriptionThe number of seconds a pre-flighted request can be cached + for web services needing cross-origin (CORS) support./description + nameyarn.timeline-service.http-cross-origin.max-age/name + value1800/value +/property +---+ * Generic-data related Configuration
git commit: YARN-2538. Added logs when RM sends roll-overed AMRMToken to AM. Contributed by Xuan Gong.
Repository: hadoop Updated Branches: refs/heads/trunk 5633da2a0 - 469ea3dce YARN-2538. Added logs when RM sends roll-overed AMRMToken to AM. Contributed by Xuan Gong. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/469ea3dc Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/469ea3dc Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/469ea3dc Branch: refs/heads/trunk Commit: 469ea3dcef6e427d02fd08b859b2789cc25189f9 Parents: 5633da2 Author: Zhijie Shen zjs...@apache.org Authored: Thu Sep 11 22:19:00 2014 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Thu Sep 11 22:19:00 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/server/resourcemanager/ApplicationMasterService.java | 2 ++ 2 files changed, 5 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/469ea3dc/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 570a855..c5f5cb0 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -208,6 +208,9 @@ Release 2.6.0 - UNRELEASED YARN-2448. Changed ApplicationMasterProtocol to expose RM-recognized resource types to the AMs. (Varun Vasudev via vinodkv) +YARN-2538. Added logs when RM sends roll-overed AMRMToken to AM. (Xuan Gong +via zjshen) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/469ea3dc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index e6d878a..707cf1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -581,6 +581,8 @@ public class ApplicationMasterService extends AbstractService implements .newInstance(amrmToken.getIdentifier(), amrmToken.getKind() .toString(), amrmToken.getPassword(), amrmToken.getService() .toString())); +LOG.info(The AMRMToken has been rolled-over. Send new AMRMToken back ++ to application: + appAttemptId.getApplicationId()); } /*
git commit: YARN-2538. Added logs when RM sends roll-overed AMRMToken to AM. Contributed by Xuan Gong.
Repository: hadoop Updated Branches: refs/heads/branch-2 f580586ab - 466bdf79c YARN-2538. Added logs when RM sends roll-overed AMRMToken to AM. Contributed by Xuan Gong. (cherry picked from commit 469ea3dcef6e427d02fd08b859b2789cc25189f9) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/466bdf79 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/466bdf79 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/466bdf79 Branch: refs/heads/branch-2 Commit: 466bdf79c531ab70fd8c61aedd28b20867cc07a4 Parents: f580586 Author: Zhijie Shen zjs...@apache.org Authored: Thu Sep 11 22:19:00 2014 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Thu Sep 11 22:21:59 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 +++ .../yarn/server/resourcemanager/ApplicationMasterService.java | 2 ++ 2 files changed, 5 insertions(+) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/466bdf79/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4eaf36a..b40db52 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -182,6 +182,9 @@ Release 2.6.0 - UNRELEASED YARN-2448. Changed ApplicationMasterProtocol to expose RM-recognized resource types to the AMs. (Varun Vasudev via vinodkv) +YARN-2538. Added logs when RM sends roll-overed AMRMToken to AM. (Xuan Gong +via zjshen) + OPTIMIZATIONS BUG FIXES http://git-wip-us.apache.org/repos/asf/hadoop/blob/466bdf79/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java index e6d878a..707cf1b 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ApplicationMasterService.java @@ -581,6 +581,8 @@ public class ApplicationMasterService extends AbstractService implements .newInstance(amrmToken.getIdentifier(), amrmToken.getKind() .toString(), amrmToken.getPassword(), amrmToken.getService() .toString())); +LOG.info(The AMRMToken has been rolled-over. Send new AMRMToken back ++ to application: + appAttemptId.getApplicationId()); } /*
git commit: YARN-2528. Relaxed http response split vulnerability protection for the origins header and made it accept multiple origins in CrossOriginFilter. Contributed by Jonathan Eagles.
Repository: hadoop
Updated Branches:
refs/heads/trunk 3c9181722 - 98588cf04
YARN-2528. Relaxed http response split vulnerability protection for the origins
header and made it accept multiple origins in CrossOriginFilter. Contributed by
Jonathan Eagles.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/98588cf0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/98588cf0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/98588cf0
Branch: refs/heads/trunk
Commit: 98588cf044d9908ecf767257c09a52cf17aa2ec2
Parents: 3c91817
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Sep 12 21:33:01 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Sep 12 21:33:01 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 4 ++
.../timeline/webapp/CrossOriginFilter.java | 47 ++--
.../timeline/webapp/TestCrossOriginFilter.java | 41 ++---
3 files changed, 61 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98588cf0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 06d94ca..065869a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -351,6 +351,10 @@ Release 2.6.0 - UNRELEASED
YARN-2542. Fixed NPE when retrieving ApplicationReport from TimeLineServer.
(Zhijie Shen via jianhe)
+YARN-2528. Relaxed http response split vulnerability protection for the
origins
+header and made it accept multiple origins in CrossOriginFilter. (Jonathan
+Eagles via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/98588cf0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index cceee54..d5fab7a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.timeline.webapp;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -106,12 +104,12 @@ public class CrossOriginFilter implements Filter {
private void doCrossFilter(HttpServletRequest req, HttpServletResponse res) {
-String origin = encodeHeader(req.getHeader(ORIGIN));
-if (!isCrossOrigin(origin)) {
+String originsList = encodeHeader(req.getHeader(ORIGIN));
+if (!isCrossOrigin(originsList)) {
return;
}
-if (!isOriginAllowed(origin)) {
+if (!areOriginsAllowed(originsList)) {
return;
}
@@ -127,7 +125,7 @@ public class CrossOriginFilter implements Filter {
return;
}
-res.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, origin);
+res.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, originsList);
res.setHeader(ACCESS_CONTROL_ALLOW_CREDENTIALS, Boolean.TRUE.toString());
res.setHeader(ACCESS_CONTROL_ALLOW_METHODS, getAllowedMethodsHeader());
res.setHeader(ACCESS_CONTROL_ALLOW_HEADERS, getAllowedHeadersHeader());
@@ -191,35 +189,36 @@ public class CrossOriginFilter implements Filter {
if (header == null) {
return null;
}
-try {
- // Protect against HTTP response splitting vulnerability
- // since value is written as part of the response header
- return URLEncoder.encode(header, ASCII);
-} catch (UnsupportedEncodingException e) {
- return null;
-}
+// Protect against HTTP response splitting vulnerability
+// since value is written as part of the response header
+// Ensure this header only has one header by removing
+// CRs and LFs
+return header.split(\n|\r)[0].trim();
}
- static boolean isCrossOrigin(String origin) {
-return origin != null;
+ static boolean isCrossOrigin(String originsList) {
+return
git commit: YARN-2528. Relaxed http response split vulnerability protection for the origins header and made it accept multiple origins in CrossOriginFilter. Contributed by Jonathan Eagles.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 09ad86d70 - 99ccd842d
YARN-2528. Relaxed http response split vulnerability protection for the origins
header and made it accept multiple origins in CrossOriginFilter. Contributed by
Jonathan Eagles.
(cherry picked from commit 98588cf044d9908ecf767257c09a52cf17aa2ec2)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99ccd842
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99ccd842
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99ccd842
Branch: refs/heads/branch-2
Commit: 99ccd842d8c0e13705f6b8255d80f06775d4b937
Parents: 09ad86d
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Sep 12 21:33:01 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Sep 12 21:35:49 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 4 ++
.../timeline/webapp/CrossOriginFilter.java | 47 ++--
.../timeline/webapp/TestCrossOriginFilter.java | 41 ++---
3 files changed, 61 insertions(+), 31 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99ccd842/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9c264ec..56c8b45 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -323,6 +323,10 @@ Release 2.6.0 - UNRELEASED
YARN-2542. Fixed NPE when retrieving ApplicationReport from TimeLineServer.
(Zhijie Shen via jianhe)
+YARN-2528. Relaxed http response split vulnerability protection for the
origins
+header and made it accept multiple origins in CrossOriginFilter. (Jonathan
+Eagles via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99ccd842/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
index cceee54..d5fab7a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/CrossOriginFilter.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.timeline.webapp;
import java.io.IOException;
-import java.io.UnsupportedEncodingException;
-import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -106,12 +104,12 @@ public class CrossOriginFilter implements Filter {
private void doCrossFilter(HttpServletRequest req, HttpServletResponse res) {
-String origin = encodeHeader(req.getHeader(ORIGIN));
-if (!isCrossOrigin(origin)) {
+String originsList = encodeHeader(req.getHeader(ORIGIN));
+if (!isCrossOrigin(originsList)) {
return;
}
-if (!isOriginAllowed(origin)) {
+if (!areOriginsAllowed(originsList)) {
return;
}
@@ -127,7 +125,7 @@ public class CrossOriginFilter implements Filter {
return;
}
-res.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, origin);
+res.setHeader(ACCESS_CONTROL_ALLOW_ORIGIN, originsList);
res.setHeader(ACCESS_CONTROL_ALLOW_CREDENTIALS, Boolean.TRUE.toString());
res.setHeader(ACCESS_CONTROL_ALLOW_METHODS, getAllowedMethodsHeader());
res.setHeader(ACCESS_CONTROL_ALLOW_HEADERS, getAllowedHeadersHeader());
@@ -191,35 +189,36 @@ public class CrossOriginFilter implements Filter {
if (header == null) {
return null;
}
-try {
- // Protect against HTTP response splitting vulnerability
- // since value is written as part of the response header
- return URLEncoder.encode(header, ASCII);
-} catch (UnsupportedEncodingException e) {
- return null;
-}
+// Protect against HTTP response splitting vulnerability
+// since value is written as part of the response header
+// Ensure this header only has one header by removing
+// CRs and LFs
+return header.split(\n|\r)[0].trim();
}
- static boolean isCrossOrigin(String origin) {
-return origin
git commit: YARN-2568. Fixed the potential test failures due to race conditions when RM work-preserving recovery is enabled. Contributed by Jian He.
Repository: hadoop
Updated Branches:
refs/heads/trunk 643457229 - 6fe5c6b74
YARN-2568. Fixed the potential test failures due to race conditions when RM
work-preserving recovery is enabled. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6fe5c6b7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6fe5c6b7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6fe5c6b7
Branch: refs/heads/trunk
Commit: 6fe5c6b746a40019b9a43676c33efec0f971c4b9
Parents: 6434572
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Sep 18 21:56:56 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Sep 18 21:56:56 2014 -0700
--
.../hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java| 1 +
.../yarn/server/resourcemanager/TestWorkPreservingRMRestart.java | 1 +
4 files changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fe5c6b7/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 9664b8f..4c03b31 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -2095,7 +2095,7 @@ public class TestRMContainerAllocator {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED,
true);
-
+
conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
0);
conf.setBoolean(MRJobConfig.MR_AM_JOB_NODE_BLACKLISTING_ENABLE, true);
conf.setInt(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, 1);
conf.setInt(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fe5c6b7/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 759f2bf..8f717ae 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -397,6 +397,9 @@ Release 2.6.0 - UNRELEASED
YARN-2563. Fixed YarnClient to call getTimeLineDelegationToken only if the
Token is not present. (Zhijie Shen via jianhe)
+YARN-2568. Fixed the potential test failures due to race conditions when RM
+work-preserving recovery is enabled. (Jian He via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fe5c6b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
index dfad6d6..ce3086f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
@@ -84,6 +84,7 @@ public class TestAMRMClientOnRMRestart {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED,
true);
+
conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
0);
}
// Test does major 6 steps verification.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6fe5c6b7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
git commit: YARN-2568. Fixed the potential test failures due to race conditions when RM work-preserving recovery is enabled. Contributed by Jian He.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 58d9cc091 - 9333ee306
YARN-2568. Fixed the potential test failures due to race conditions when RM
work-preserving recovery is enabled. Contributed by Jian He.
(cherry picked from commit 6fe5c6b746a40019b9a43676c33efec0f971c4b9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9333ee30
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9333ee30
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9333ee30
Branch: refs/heads/branch-2
Commit: 9333ee306815bd38e0aa5e94fb7cd057bc183bd8
Parents: 58d9cc0
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Sep 18 21:56:56 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Sep 18 22:02:53 2014 -0700
--
.../hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java| 1 +
.../yarn/server/resourcemanager/TestWorkPreservingRMRestart.java | 1 +
4 files changed, 6 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9333ee30/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
index 9d9ee18..7f88666 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/rm/TestRMContainerAllocator.java
@@ -2090,7 +2090,7 @@ public class TestRMContainerAllocator {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED,
true);
-
+
conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
0);
conf.setBoolean(MRJobConfig.MR_AM_JOB_NODE_BLACKLISTING_ENABLE, true);
conf.setInt(MRJobConfig.MAX_TASK_FAILURES_PER_TRACKER, 1);
conf.setInt(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9333ee30/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3e798ee..e1bc797 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -367,6 +367,9 @@ Release 2.6.0 - UNRELEASED
YARN-2563. Fixed YarnClient to call getTimeLineDelegationToken only if the
Token is not present. (Zhijie Shen via jianhe)
+YARN-2568. Fixed the potential test failures due to race conditions when RM
+work-preserving recovery is enabled. (Jian He via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9333ee30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
index dfad6d6..ce3086f 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClientOnRMRestart.java
@@ -84,6 +84,7 @@ public class TestAMRMClientOnRMRestart {
conf.setInt(YarnConfiguration.RM_AM_MAX_ATTEMPTS,
YarnConfiguration.DEFAULT_RM_AM_MAX_ATTEMPTS);
conf.setBoolean(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_ENABLED,
true);
+
conf.setLong(YarnConfiguration.RM_WORK_PRESERVING_RECOVERY_SCHEDULING_WAIT_MS,
0);
}
// Test does major 6 steps verification.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9333ee30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager
git commit: MAPREDUCE-5279. Made MR headroom calculation honor cpu dimension when YARN scheduler resource type is memory plus cpu. Contributed by Peng Zhang and Varun Vasudev.
Repository: hadoop
Updated Branches:
refs/heads/trunk 26cba7f35 - 376233cdd
MAPREDUCE-5279. Made MR headroom calculation honor cpu dimension when YARN
scheduler resource type is memory plus cpu. Contributed by Peng Zhang and Varun
Vasudev.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/376233cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/376233cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/376233cd
Branch: refs/heads/trunk
Commit: 376233cdd4a4ddbde5a92a0627f78338cb4c38b7
Parents: 26cba7f
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Sep 22 09:28:47 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Sep 22 09:28:47 2014 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 4 +
.../mapreduce/v2/app/rm/RMCommunicator.java | 12 +-
.../v2/app/rm/RMContainerAllocator.java | 272 +++
.../v2/app/rm/ResourceCalculatorUtils.java | 52
.../v2/app/rm/TestRMContainerAllocator.java | 82 --
.../hadoop/yarn/util/resource/Resources.java| 4 +-
6 files changed, 288 insertions(+), 138 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376233cd/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 815b3fb..859c899 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -254,6 +254,10 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-5891. Improved shuffle error handling across NM restarts
(Junping Du via jlowe)
+MAPREDUCE-5279. Made MR headroom calculation honor cpu dimension when YARN
+scheduler resource type is memory plus cpu. (Peng Zhang and Varun Vasudev
+via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/376233cd/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
index 6e9f313..6c58a68 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/rm/RMCommunicator.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.mapreduce.v2.app.rm;
import java.io.IOException;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
+import java.util.EnumSet;
import java.util.Map;
import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.atomic.AtomicBoolean;
@@ -59,6 +60,7 @@ import org.apache.hadoop.yarn.factories.RecordFactory;
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.proto.YarnServiceProtos.SchedulerResourceTypes;
/**
* Registers/unregisters to RM and sends heartbeats to RM.
@@ -90,6 +92,8 @@ public abstract class RMCommunicator extends AbstractService
private volatile boolean shouldUnregister = true;
private boolean isApplicationMasterRegistered = false;
+ private EnumSetSchedulerResourceTypes schedulerResourceTypes;
+
public RMCommunicator(ClientService clientService, AppContext context) {
super(RMCommunicator);
this.clientService = clientService;
@@ -98,6 +102,7 @@ public abstract class RMCommunicator extends AbstractService
this.applicationId = context.getApplicationID();
this.stopped = new AtomicBoolean(false);
this.heartbeatCallbacks = new ConcurrentLinkedQueueRunnable();
+this.schedulerResourceTypes = EnumSet.of(SchedulerResourceTypes.MEMORY);
}
@Override
@@ -163,10 +168,11 @@ public abstract class RMCommunicator extends
AbstractService
setClientToAMToken(response.getClientToAMTokenMasterKey());
}
this.applicationACLs = response.getApplicationACLs();
- LOG.info(maxContainerCapability: +
maxContainerCapability.getMemory());
+ LOG.info(maxContainerCapability: + maxContainerCapability);
String queue = response.getQueue();
LOG.info(queue: + queue);
job.setQueueName(queue);
+ this.schedulerResourceTypes.addAll(response.getSchedulerResourceTypes
git commit: YARN-2569. Added the log handling APIs for the long running services. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk a1fd804a3 - 5338ac416
YARN-2569. Added the log handling APIs for the long running services.
Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5338ac41
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5338ac41
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5338ac41
Branch: refs/heads/trunk
Commit: 5338ac416ab8ab3e7e0a7bfb4a53151fc457f673
Parents: a1fd804
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Sep 23 10:36:57 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Sep 23 10:36:57 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/ApplicationSubmissionContext.java | 36 +
.../yarn/api/records/LogAggregationContext.java | 121 +
.../src/main/proto/yarn_protos.proto| 7 +
.../pb/ApplicationSubmissionContextPBImpl.java | 40 ++
.../impl/pb/LogAggregationContextPBImpl.java| 134 +++
.../hadoop/yarn/api/TestPBImplRecords.java | 1 +
7 files changed, 342 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5338ac41/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a2d0536..0e4909e 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -91,6 +91,9 @@ Release 2.6.0 - UNRELEASED
YARN-1250. Generic history service should support application-acls.
(Zhijie Shen
via junping_du)
+YARN-2569. Added the log handling APIs for the long running services. (Xuan
+Gong via zjshen)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5338ac41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index 723a2e0..2202380 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -54,6 +54,7 @@ import java.util.Set;
* validityInterval into failure count. If failure count reaches to
* maxAppAttempts, the application will be failed.
* /li
+ * liOptional, application-specific {@link LogAggregationContext}/li
* /ul
* /p
*
@@ -128,6 +129,21 @@ public abstract class ApplicationSubmissionContext {
return context;
}
+ @Public
+ @Stable
+ public static ApplicationSubmissionContext newInstance(
+ ApplicationId applicationId, String applicationName, String queue,
+ Priority priority, ContainerLaunchContext amContainer,
+ boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
+ int maxAppAttempts, Resource resource, String applicationType,
+ boolean keepContainers, LogAggregationContext logAggregationContext) {
+ApplicationSubmissionContext context =
+newInstance(applicationId, applicationName, queue, priority,
+ amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts,
+ resource, applicationType, keepContainers);
+context.setLogAggregationContext(logAggregationContext);
+return context;
+ }
/**
* Get the codeApplicationId/code of the submitted application.
* @return codeApplicationId/code of the submitted application
@@ -381,4 +397,24 @@ public abstract class ApplicationSubmissionContext {
@Stable
public abstract void setAttemptFailuresValidityInterval(
long attemptFailuresValidityInterval);
+
+ /**
+ * Get codeLogAggregationContext/code of the application
+ *
+ * @return codeLogAggregationContext/code of the application
+ */
+ @Public
+ @Stable
+ public abstract LogAggregationContext getLogAggregationContext();
+
+ /**
+ * Set codeLogAggregationContext/code for the application
+ *
+ * @param logAggregationContext
+ * for the application
+ */
+ @Public
+ @Stable
+ public abstract void setLogAggregationContext(
+ LogAggregationContext logAggregationContext);
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5338ac41/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src
git commit: YARN-2569. Added the log handling APIs for the long running services. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 ac3b62dae - 19cfe2f80
YARN-2569. Added the log handling APIs for the long running services.
Contributed by Xuan Gong.
(cherry picked from commit 5338ac416ab8ab3e7e0a7bfb4a53151fc457f673)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/19cfe2f8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/19cfe2f8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/19cfe2f8
Branch: refs/heads/branch-2
Commit: 19cfe2f8009a88aa4166638dd19fe2423eb91aef
Parents: ac3b62d
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Sep 23 10:36:57 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Sep 23 10:38:27 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../records/ApplicationSubmissionContext.java | 36 +
.../yarn/api/records/LogAggregationContext.java | 121 +
.../src/main/proto/yarn_protos.proto| 7 +
.../pb/ApplicationSubmissionContextPBImpl.java | 40 ++
.../impl/pb/LogAggregationContextPBImpl.java| 134 +++
.../hadoop/yarn/api/TestPBImplRecords.java | 1 +
7 files changed, 342 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19cfe2f8/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7152697..15df498 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -64,6 +64,9 @@ Release 2.6.0 - UNRELEASED
YARN-1250. Generic history service should support application-acls.
(Zhijie Shen
via junping_du)
+YARN-2569. Added the log handling APIs for the long running services. (Xuan
+Gong via zjshen)
+
IMPROVEMENTS
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
http://git-wip-us.apache.org/repos/asf/hadoop/blob/19cfe2f8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index 723a2e0..2202380 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -54,6 +54,7 @@ import java.util.Set;
* validityInterval into failure count. If failure count reaches to
* maxAppAttempts, the application will be failed.
* /li
+ * liOptional, application-specific {@link LogAggregationContext}/li
* /ul
* /p
*
@@ -128,6 +129,21 @@ public abstract class ApplicationSubmissionContext {
return context;
}
+ @Public
+ @Stable
+ public static ApplicationSubmissionContext newInstance(
+ ApplicationId applicationId, String applicationName, String queue,
+ Priority priority, ContainerLaunchContext amContainer,
+ boolean isUnmanagedAM, boolean cancelTokensWhenComplete,
+ int maxAppAttempts, Resource resource, String applicationType,
+ boolean keepContainers, LogAggregationContext logAggregationContext) {
+ApplicationSubmissionContext context =
+newInstance(applicationId, applicationName, queue, priority,
+ amContainer, isUnmanagedAM, cancelTokensWhenComplete, maxAppAttempts,
+ resource, applicationType, keepContainers);
+context.setLogAggregationContext(logAggregationContext);
+return context;
+ }
/**
* Get the codeApplicationId/code of the submitted application.
* @return codeApplicationId/code of the submitted application
@@ -381,4 +397,24 @@ public abstract class ApplicationSubmissionContext {
@Stable
public abstract void setAttemptFailuresValidityInterval(
long attemptFailuresValidityInterval);
+
+ /**
+ * Get codeLogAggregationContext/code of the application
+ *
+ * @return codeLogAggregationContext/code of the application
+ */
+ @Public
+ @Stable
+ public abstract LogAggregationContext getLogAggregationContext();
+
+ /**
+ * Set codeLogAggregationContext/code for the application
+ *
+ * @param logAggregationContext
+ * for the application
+ */
+ @Public
+ @Stable
+ public abstract void setLogAggregationContext(
+ LogAggregationContext logAggregationContext);
}
\ No newline at end of file
http://git-wip-us.apache.org
git commit: YARN-2581. Passed LogAggregationContext to NM via ContainerTokenIdentifier. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk 116f83157 - c86674a3a
YARN-2581. Passed LogAggregationContext to NM via ContainerTokenIdentifier.
Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c86674a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c86674a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c86674a3
Branch: refs/heads/trunk
Commit: c86674a3a4d99aa56bb8ed3f6df51e3fef215eba
Parents: 116f831
Author: Zhijie Shen zjs...@apache.org
Authored: Wed Sep 24 17:50:26 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Wed Sep 24 17:50:26 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/security/ContainerTokenIdentifier.java | 34 +++
.../containermanager/ContainerManagerImpl.java | 26 +++--
.../application/ApplicationImpl.java| 18 +-
.../application/ApplicationInitEvent.java | 13 +
.../event/LogHandlerAppStartedEvent.java| 13 +
.../yarn_server_nodemanager_recovery.proto | 1 +
.../containermanager/TestContainerManager.java | 12 +++-
.../TestContainerManagerRecovery.java | 25 +++-
.../scheduler/SchedulerApplicationAttempt.java | 6 +-
.../security/RMContainerTokenSecretManager.java | 23 +++-
.../yarn/server/resourcemanager/MockRM.java | 22 +--
.../capacity/TestContainerAllocation.java | 60 +++-
13 files changed, 239 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c86674a3/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c33bf45..4e71c1b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -97,6 +97,9 @@ Release 2.6.0 - UNRELEASED
YARN-2102. Added the concept of a Timeline Domain to handle read/write ACLs
on Timeline service event data. (Zhijie Shen via vinodkv)
+YARN-2581. Passed LogAggregationContext to NM via ContainerTokenIdentifier.
+(Xuan Gong via zjshen)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c86674a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index ca847e0..0bb016a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -34,8 +34,11 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.LogAggregationContextPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationContextProto;
/**
* TokenIdentifier for a container. Encodes {@link ContainerId},
@@ -59,10 +62,19 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
private long rmIdentifier;
private Priority priority;
private long creationTime;
+ private LogAggregationContext logAggregationContext;
public ContainerTokenIdentifier(ContainerId containerID,
String hostName, String appSubmitter, Resource r, long expiryTimeStamp,
int masterKeyId, long rmIdentifier, Priority priority, long
creationTime) {
+this(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId,
+rmIdentifier, priority, creationTime, null);
+ }
+
+ public ContainerTokenIdentifier(ContainerId containerID, String hostName,
+ String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId,
+ long rmIdentifier, Priority priority, long creationTime,
+ LogAggregationContext logAggregationContext) {
this.containerId = containerID;
this.nmHostAddr = hostName;
this.appSubmitter = appSubmitter;
@@ -72,6 +84,7 @@ public class ContainerTokenIdentifier extends TokenIdentifier
git commit: YARN-2581. Passed LogAggregationContext to NM via ContainerTokenIdentifier. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 835ade706 - 3a2e40037
YARN-2581. Passed LogAggregationContext to NM via ContainerTokenIdentifier.
Contributed by Xuan Gong.
(cherry picked from commit c86674a3a4d99aa56bb8ed3f6df51e3fef215eba)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a2e4003
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a2e4003
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a2e4003
Branch: refs/heads/branch-2
Commit: 3a2e400377b53f37ee4a52df9db9ea2f15a82230
Parents: 835ade7
Author: Zhijie Shen zjs...@apache.org
Authored: Wed Sep 24 17:50:26 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Wed Sep 24 17:51:54 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../yarn/security/ContainerTokenIdentifier.java | 34 +++
.../containermanager/ContainerManagerImpl.java | 26 +++--
.../application/ApplicationImpl.java| 18 +-
.../application/ApplicationInitEvent.java | 13 +
.../event/LogHandlerAppStartedEvent.java| 13 +
.../yarn_server_nodemanager_recovery.proto | 1 +
.../containermanager/TestContainerManager.java | 12 +++-
.../TestContainerManagerRecovery.java | 25 +++-
.../scheduler/SchedulerApplicationAttempt.java | 6 +-
.../security/RMContainerTokenSecretManager.java | 23 +++-
.../yarn/server/resourcemanager/MockRM.java | 22 +--
.../capacity/TestContainerAllocation.java | 60 +++-
13 files changed, 239 insertions(+), 17 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a2e4003/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ef70865..491222b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -70,6 +70,9 @@ Release 2.6.0 - UNRELEASED
YARN-2102. Added the concept of a Timeline Domain to handle read/write ACLs
on Timeline service event data. (Zhijie Shen via vinodkv)
+YARN-2581. Passed LogAggregationContext to NM via ContainerTokenIdentifier.
+(Xuan Gong via zjshen)
+
IMPROVEMENTS
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a2e4003/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
index ca847e0..0bb016a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
@@ -34,8 +34,11 @@ import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.impl.pb.LogAggregationContextPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.LogAggregationContextProto;
/**
* TokenIdentifier for a container. Encodes {@link ContainerId},
@@ -59,10 +62,19 @@ public class ContainerTokenIdentifier extends
TokenIdentifier {
private long rmIdentifier;
private Priority priority;
private long creationTime;
+ private LogAggregationContext logAggregationContext;
public ContainerTokenIdentifier(ContainerId containerID,
String hostName, String appSubmitter, Resource r, long expiryTimeStamp,
int masterKeyId, long rmIdentifier, Priority priority, long
creationTime) {
+this(containerID, hostName, appSubmitter, r, expiryTimeStamp, masterKeyId,
+rmIdentifier, priority, creationTime, null);
+ }
+
+ public ContainerTokenIdentifier(ContainerId containerID, String hostName,
+ String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId,
+ long rmIdentifier, Priority priority, long creationTime,
+ LogAggregationContext logAggregationContext) {
this.containerId = containerID;
this.nmHostAddr = hostName;
this.appSubmitter = appSubmitter;
@@ -72,6
git commit: YARN-2546. Made REST API for application creation/submission use numeric and boolean types instead of the string of them. Contributed by Varun Vasudev.
Repository: hadoop Updated Branches: refs/heads/trunk c86674a3a - 72b0881ca YARN-2546. Made REST API for application creation/submission use numeric and boolean types instead of the string of them. Contributed by Varun Vasudev. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/72b0881c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/72b0881c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/72b0881c Branch: refs/heads/trunk Commit: 72b0881ca641fa830c907823f674a5c5e39aa15a Parents: c86674a Author: Zhijie Shen zjs...@apache.org Authored: Wed Sep 24 17:57:32 2014 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Wed Sep 24 17:57:32 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 + .../webapp/JAXBContextResolver.java | 77 ++-- .../TestRMWebServicesAppsModification.java | 9 ++- .../src/site/apt/ResourceManagerRest.apt.vm | 30 4 files changed, 62 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/72b0881c/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 4e71c1b..aaac7b7 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -444,6 +444,9 @@ Release 2.6.0 - UNRELEASED YARN-2596. TestWorkPreservingRMRestart fails with FairScheduler. (kasha) +YARN-2546. Made REST API for application creation/submission use numeric and +boolean types instead of the string of them. (Varun Vasudev via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/72b0881c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index c577629..ca7edb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -22,62 +22,61 @@ import com.google.inject.Singleton; import com.sun.jersey.api.json.JSONConfiguration; import com.sun.jersey.api.json.JSONJAXBContext; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import java.util.*; import javax.ws.rs.ext.ContextResolver; import javax.ws.rs.ext.Provider; import javax.xml.bind.JAXBContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.UserMetricsInfo; -import
git commit: YARN-2546. Made REST API for application creation/submission use numeric and boolean types instead of the string of them. Contributed by Varun Vasudev.
Repository: hadoop Updated Branches: refs/heads/branch-2 3a2e40037 - 83926b3c5 YARN-2546. Made REST API for application creation/submission use numeric and boolean types instead of the string of them. Contributed by Varun Vasudev. (cherry picked from commit 72b0881ca641fa830c907823f674a5c5e39aa15a) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/83926b3c Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/83926b3c Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/83926b3c Branch: refs/heads/branch-2 Commit: 83926b3c57bc8b48adcf19b52567f8bd7268bd48 Parents: 3a2e400 Author: Zhijie Shen zjs...@apache.org Authored: Wed Sep 24 17:57:32 2014 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Wed Sep 24 17:58:48 2014 -0700 -- hadoop-yarn-project/CHANGES.txt | 3 + .../webapp/JAXBContextResolver.java | 77 ++-- .../TestRMWebServicesAppsModification.java | 9 ++- .../src/site/apt/ResourceManagerRest.apt.vm | 30 4 files changed, 62 insertions(+), 57 deletions(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/83926b3c/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 491222b..6608015 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -414,6 +414,9 @@ Release 2.6.0 - UNRELEASED YARN-2596. TestWorkPreservingRMRestart fails with FairScheduler. (kasha) +YARN-2546. Made REST API for application creation/submission use numeric and +boolean types instead of the string of them. (Varun Vasudev via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/83926b3c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java index c577629..ca7edb7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/JAXBContextResolver.java @@ -22,62 +22,61 @@ import com.google.inject.Singleton; import com.sun.jersey.api.json.JSONConfiguration; import com.sun.jersey.api.json.JSONJAXBContext; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; +import java.util.*; import javax.ws.rs.ext.ContextResolver; import javax.ws.rs.ext.Provider; import javax.xml.bind.JAXBContext; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.UserInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.CapacitySchedulerQueueInfoList; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.StatisticsItemInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.FifoSchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ResourceInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerInfo; -import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo; -import
git commit: MAPREDUCE-5831. Make MR client ignore unknown counters received from AM. Contributed by Junping Du.
Repository: hadoop
Updated Branches:
refs/heads/trunk 4ea77efa3 - 662fc11ae
MAPREDUCE-5831. Make MR client ignore unknown counters received from AM.
Contributed by Junping Du.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/662fc11a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/662fc11a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/662fc11a
Branch: refs/heads/trunk
Commit: 662fc11ae7ccea9d05dfe54b2a794bb08cfd70a4
Parents: 4ea77ef
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Sep 25 23:45:07 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Sep 25 23:45:07 2014 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../apache/hadoop/mapreduce/TypeConverter.java | 5 +-
.../counters/FileSystemCounterGroup.java| 15 --
.../counters/FrameworkCounterGroup.java | 28 +++---
.../org/apache/hadoop/mapred/TestCounters.java | 54
5 files changed, 95 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/662fc11a/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index d53e748..5b57756 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -377,6 +377,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6093. minor distcp doc edits (Charles Lamb via aw)
+MAPREDUCE-5831. Make MR client ignore unknown counters received from AM.
+(Junping Du via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/662fc11a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
index 6b4aa4e..553ba70 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
@@ -241,7 +241,10 @@ public class TypeConverter {
org.apache.hadoop.mapreduce.Counter c =
counters.findCounter(yGrp.getName(),
yCntr.getName());
-c.setValue(yCntr.getValue());
+// if c can be found, or it will be skipped.
+if (c != null) {
+ c.setValue(yCntr.getValue());
+}
}
}
return counters;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/662fc11a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
index b1b6796..a53b76a 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
@@ -34,6 +34,8 @@ import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
import com.google.common.collect.Maps;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.io.WritableUtils;
@@ -53,6 +55,8 @@ public abstract class FileSystemCounterGroupC extends
Counter
static final int MAX_NUM_SCHEMES = 100; // intern/sanity check
static final ConcurrentMapString, String schemes = Maps.newConcurrentMap();
+
+ private static final Log LOG =
LogFactory.getLog(FileSystemCounterGroup.class);
// C[] would need Array.newInstance which requires a ClassC reference.
// Just a few local casts probably worth not having
git commit: MAPREDUCE-5831. Make MR client ignore unknown counters received from AM. Contributed by Junping Du.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 0eaa4ba02 - 7b3ecc547
MAPREDUCE-5831. Make MR client ignore unknown counters received from AM.
Contributed by Junping Du.
(cherry picked from commit 662fc11ae7ccea9d05dfe54b2a794bb08cfd70a4)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7b3ecc54
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7b3ecc54
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7b3ecc54
Branch: refs/heads/branch-2
Commit: 7b3ecc547f352b17de4655be5e0d81dc23fd6b7c
Parents: 0eaa4ba
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Sep 25 23:45:07 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Sep 25 23:46:57 2014 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 3 ++
.../apache/hadoop/mapreduce/TypeConverter.java | 5 +-
.../counters/FileSystemCounterGroup.java| 15 --
.../counters/FrameworkCounterGroup.java | 28 +++---
.../org/apache/hadoop/mapred/TestCounters.java | 54
5 files changed, 95 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b3ecc54/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 7d60919..84eae0c 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -158,6 +158,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-6093. minor distcp doc edits (Charles Lamb via aw)
+MAPREDUCE-5831. Make MR client ignore unknown counters received from AM.
+(Junping Du via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b3ecc54/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
index 6b4aa4e..553ba70 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/TypeConverter.java
@@ -241,7 +241,10 @@ public class TypeConverter {
org.apache.hadoop.mapreduce.Counter c =
counters.findCounter(yGrp.getName(),
yCntr.getName());
-c.setValue(yCntr.getValue());
+// if c can be found, or it will be skipped.
+if (c != null) {
+ c.setValue(yCntr.getValue());
+}
}
}
return counters;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7b3ecc54/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
index b1b6796..a53b76a 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/counters/FileSystemCounterGroup.java
@@ -34,6 +34,8 @@ import com.google.common.collect.AbstractIterator;
import com.google.common.collect.Iterators;
import com.google.common.collect.Maps;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.io.WritableUtils;
@@ -53,6 +55,8 @@ public abstract class FileSystemCounterGroupC extends
Counter
static final int MAX_NUM_SCHEMES = 100; // intern/sanity check
static final ConcurrentMapString, String schemes = Maps.newConcurrentMap();
+
+ private static final Log LOG =
LogFactory.getLog(FileSystemCounterGroup.class);
// C[] would need Array.newInstance which requires
git commit: YARN-2627. Added the info logs of attemptFailuresValidityInterval and number of previous failed attempts. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk f7743dd07 - 9582a5017
YARN-2627. Added the info logs of attemptFailuresValidityInterval and number of
previous failed attempts. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9582a501
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9582a501
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9582a501
Branch: refs/heads/trunk
Commit: 9582a50176800433ad3fa8829a50c28b859812a3
Parents: f7743dd
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Sep 30 12:57:45 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Sep 30 12:57:45 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/rmapp/RMAppImpl.java | 11 +++
2 files changed, 14 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9582a501/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index c1172de..6771839 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -280,6 +280,9 @@ Release 2.6.0 - UNRELEASED
YARN-1769. CapacityScheduler: Improve reservations (Thomas Graves via
jlowe)
+YARN-2627. Added the info logs of attemptFailuresValidityInterval and
number
+of previous failed attempts. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9582a501/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 0b8f321..eaef7d2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -376,6 +376,11 @@ public class RMAppImpl implements RMApp, Recoverable {
this.attemptFailuresValidityInterval =
submissionContext.getAttemptFailuresValidityInterval();
+if (this.attemptFailuresValidityInterval 0) {
+ LOG.info(The attemptFailuresValidityInterval for the application:
+ + this.applicationId + is + this.attemptFailuresValidityInterval
+ + .);
+}
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
@@ -1179,6 +1184,11 @@ public class RMAppImpl implements RMApp, Recoverable {
@Override
public RMAppState transition(RMAppImpl app, RMAppEvent event) {
int numberOfFailure = app.getNumFailedAppAttempts();
+ LOG.info(The number of failed attempts
+ + (app.attemptFailuresValidityInterval 0 ? in previous
+ + app.attemptFailuresValidityInterval + milliseconds : )
+ + is + numberOfFailure + . The max attempts is
+ + app.maxAppAttempts);
if (!app.submissionContext.getUnmanagedAM()
numberOfFailure app.maxAppAttempts) {
boolean transferStateFromPreviousAttempt;
@@ -1293,4 +1303,5 @@ public class RMAppImpl implements RMApp, Recoverable {
public void setSystemClock(Clock clock) {
this.systemClock = clock;
}
+
}
git commit: YARN-2627. Added the info logs of attemptFailuresValidityInterval and number of previous failed attempts. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 21f920ab8 - d977205b9
YARN-2627. Added the info logs of attemptFailuresValidityInterval and number of
previous failed attempts. Contributed by Xuan Gong.
(cherry picked from commit 9582a50176800433ad3fa8829a50c28b859812a3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d977205b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d977205b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d977205b
Branch: refs/heads/branch-2
Commit: d977205b9b1538f3e5ca250b9ce61f799ec4374c
Parents: 21f920a
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Sep 30 12:57:45 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Sep 30 12:59:34 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/rmapp/RMAppImpl.java | 11 +++
2 files changed, 14 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d977205b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ce1d357..c773145 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -250,6 +250,9 @@ Release 2.6.0 - UNRELEASED
YARN-1769. CapacityScheduler: Improve reservations (Thomas Graves via
jlowe)
+YARN-2627. Added the info logs of attemptFailuresValidityInterval and
number
+of previous failed attempts. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d977205b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 0b8f321..eaef7d2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -376,6 +376,11 @@ public class RMAppImpl implements RMApp, Recoverable {
this.attemptFailuresValidityInterval =
submissionContext.getAttemptFailuresValidityInterval();
+if (this.attemptFailuresValidityInterval 0) {
+ LOG.info(The attemptFailuresValidityInterval for the application:
+ + this.applicationId + is + this.attemptFailuresValidityInterval
+ + .);
+}
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
@@ -1179,6 +1184,11 @@ public class RMAppImpl implements RMApp, Recoverable {
@Override
public RMAppState transition(RMAppImpl app, RMAppEvent event) {
int numberOfFailure = app.getNumFailedAppAttempts();
+ LOG.info(The number of failed attempts
+ + (app.attemptFailuresValidityInterval 0 ? in previous
+ + app.attemptFailuresValidityInterval + milliseconds : )
+ + is + numberOfFailure + . The max attempts is
+ + app.maxAppAttempts);
if (!app.submissionContext.getUnmanagedAM()
numberOfFailure app.maxAppAttempts) {
boolean transferStateFromPreviousAttempt;
@@ -1293,4 +1303,5 @@ public class RMAppImpl implements RMApp, Recoverable {
public void setSystemClock(Clock clock) {
this.systemClock = clock;
}
+
}
git commit: YARN-2627. Added the info logs of attemptFailuresValidityInterval and number of previous failed attempts. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 62d47e396 - b19f668f5
YARN-2627. Added the info logs of attemptFailuresValidityInterval and number of
previous failed attempts. Contributed by Xuan Gong.
(cherry picked from commit 9582a50176800433ad3fa8829a50c28b859812a3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b19f668f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b19f668f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b19f668f
Branch: refs/heads/branch-2.6
Commit: b19f668f5df12aa0b6d7fc42e481ba5d12f5e396
Parents: 62d47e3
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Sep 30 12:57:45 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Sep 30 13:01:13 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/rmapp/RMAppImpl.java | 11 +++
2 files changed, 14 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b19f668f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 4e62631..628f64f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -238,6 +238,9 @@ Release 2.6.0 - UNRELEASED
YARN-1769. CapacityScheduler: Improve reservations (Thomas Graves via
jlowe)
+YARN-2627. Added the info logs of attemptFailuresValidityInterval and
number
+of previous failed attempts. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b19f668f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
index 0b8f321..eaef7d2 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
@@ -376,6 +376,11 @@ public class RMAppImpl implements RMApp, Recoverable {
this.attemptFailuresValidityInterval =
submissionContext.getAttemptFailuresValidityInterval();
+if (this.attemptFailuresValidityInterval 0) {
+ LOG.info(The attemptFailuresValidityInterval for the application:
+ + this.applicationId + is + this.attemptFailuresValidityInterval
+ + .);
+}
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
@@ -1179,6 +1184,11 @@ public class RMAppImpl implements RMApp, Recoverable {
@Override
public RMAppState transition(RMAppImpl app, RMAppEvent event) {
int numberOfFailure = app.getNumFailedAppAttempts();
+ LOG.info(The number of failed attempts
+ + (app.attemptFailuresValidityInterval 0 ? in previous
+ + app.attemptFailuresValidityInterval + milliseconds : )
+ + is + numberOfFailure + . The max attempts is
+ + app.maxAppAttempts);
if (!app.submissionContext.getUnmanagedAM()
numberOfFailure app.maxAppAttempts) {
boolean transferStateFromPreviousAttempt;
@@ -1293,4 +1303,5 @@ public class RMAppImpl implements RMApp, Recoverable {
public void setSystemClock(Clock clock) {
this.systemClock = clock;
}
+
}
git commit: YARN-2630. Prevented previous AM container status from being acquired by the current restarted AM. Contributed by Jian He.
Repository: hadoop
Updated Branches:
refs/heads/trunk dd1b8f2ed - 52bbe0f11
YARN-2630. Prevented previous AM container status from being acquired by the
current restarted AM. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52bbe0f1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52bbe0f1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52bbe0f1
Branch: refs/heads/trunk
Commit: 52bbe0f11bc8e97df78a1ab9b63f4eff65fd7a76
Parents: dd1b8f2
Author: Zhijie Shen zjs...@apache.org
Authored: Wed Oct 1 15:38:11 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Wed Oct 1 15:38:11 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/ApplicationMaster.java | 1 +
.../protocolrecords/NodeHeartbeatResponse.java | 9 +--
.../impl/pb/NodeHeartbeatResponsePBImpl.java| 45 ++---
.../yarn_server_common_service_protos.proto | 2 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 2 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 69 +---
.../resourcemanager/rmnode/RMNodeImpl.java | 25 +++
.../applicationsmanager/TestAMRestart.java | 24 ++-
.../attempt/TestRMAppAttemptTransitions.java| 12 ++--
11 files changed, 108 insertions(+), 86 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52bbe0f1/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3878ca1..4d05757 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -503,6 +503,9 @@ Release 2.6.0 - UNRELEASED
YARN-2602. Fixed possible NPE in ApplicationHistoryManagerOnTimelineStore.
(Zhijie Shen via jianhe)
+YARN-2630. Prevented previous AM container status from being acquired by
the
+current restarted AM. (Jian He via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52bbe0f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 2451030..df9f34b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -668,6 +668,7 @@ public class ApplicationMaster {
+ , completed= + numCompletedContainers.get() + , allocated=
+ numAllocatedContainers.get() + , failed=
+ numFailedContainers.get();
+ LOG.info(appMessage);
success = false;
}
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52bbe0f1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
index 9887acc..12e1f54 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
@@ -30,7 +30,7 @@ public interface NodeHeartbeatResponse {
NodeAction getNodeAction();
ListContainerId getContainersToCleanup();
- ListContainerId getFinishedContainersPulledByAM();
+ ListContainerId getContainersToBeRemovedFromNM
git commit: YARN-2630. Prevented previous AM container status from being acquired by the current restarted AM. Contributed by Jian He.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 1725d5613 - 4b50e2327
YARN-2630. Prevented previous AM container status from being acquired by the
current restarted AM. Contributed by Jian He.
(cherry picked from commit 52bbe0f11bc8e97df78a1ab9b63f4eff65fd7a76)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b50e232
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b50e232
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b50e232
Branch: refs/heads/branch-2
Commit: 4b50e23271f3de5b18f0305cb82acbe0a5a97328
Parents: 1725d56
Author: Zhijie Shen zjs...@apache.org
Authored: Wed Oct 1 15:38:11 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Wed Oct 1 15:39:36 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/ApplicationMaster.java | 1 +
.../protocolrecords/NodeHeartbeatResponse.java | 9 +--
.../impl/pb/NodeHeartbeatResponsePBImpl.java| 45 ++---
.../yarn_server_common_service_protos.proto | 2 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 2 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 69 +---
.../resourcemanager/rmnode/RMNodeImpl.java | 25 +++
.../applicationsmanager/TestAMRestart.java | 24 ++-
.../attempt/TestRMAppAttemptTransitions.java| 12 ++--
11 files changed, 108 insertions(+), 86 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b50e232/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e8db12d..bbd6013 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -470,6 +470,9 @@ Release 2.6.0 - UNRELEASED
YARN-2602. Fixed possible NPE in ApplicationHistoryManagerOnTimelineStore.
(Zhijie Shen via jianhe)
+YARN-2630. Prevented previous AM container status from being acquired by
the
+current restarted AM. (Jian He via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b50e232/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 2451030..df9f34b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -668,6 +668,7 @@ public class ApplicationMaster {
+ , completed= + numCompletedContainers.get() + , allocated=
+ numAllocatedContainers.get() + , failed=
+ numFailedContainers.get();
+ LOG.info(appMessage);
success = false;
}
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b50e232/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
index 9887acc..12e1f54 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
@@ -30,7 +30,7 @@ public interface NodeHeartbeatResponse {
NodeAction getNodeAction();
ListContainerId getContainersToCleanup();
- ListContainerId
git commit: YARN-2630. Prevented previous AM container status from being acquired by the current restarted AM. Contributed by Jian He.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 5d2f32548 - 8531f93d2
YARN-2630. Prevented previous AM container status from being acquired by the
current restarted AM. Contributed by Jian He.
(cherry picked from commit 52bbe0f11bc8e97df78a1ab9b63f4eff65fd7a76)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8531f93d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8531f93d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8531f93d
Branch: refs/heads/branch-2.6
Commit: 8531f93d2d7a59fb78880c35f8b70f8c93c85389
Parents: 5d2f325
Author: Zhijie Shen zjs...@apache.org
Authored: Wed Oct 1 15:38:11 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Wed Oct 1 15:41:12 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/ApplicationMaster.java | 1 +
.../protocolrecords/NodeHeartbeatResponse.java | 9 +--
.../impl/pb/NodeHeartbeatResponsePBImpl.java| 45 ++---
.../yarn_server_common_service_protos.proto | 2 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../nodemanager/TestNodeStatusUpdater.java | 2 +-
.../rmapp/attempt/RMAppAttemptImpl.java | 69 +---
.../resourcemanager/rmnode/RMNodeImpl.java | 25 +++
.../applicationsmanager/TestAMRestart.java | 24 ++-
.../attempt/TestRMAppAttemptTransitions.java| 12 ++--
11 files changed, 108 insertions(+), 86 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8531f93d/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 25fbc48..fad966b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -452,6 +452,9 @@ Release 2.6.0 - UNRELEASED
YARN-2602. Fixed possible NPE in ApplicationHistoryManagerOnTimelineStore.
(Zhijie Shen via jianhe)
+YARN-2630. Prevented previous AM container status from being acquired by
the
+current restarted AM. (Jian He via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8531f93d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index 2451030..df9f34b 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -668,6 +668,7 @@ public class ApplicationMaster {
+ , completed= + numCompletedContainers.get() + , allocated=
+ numAllocatedContainers.get() + , failed=
+ numFailedContainers.get();
+ LOG.info(appMessage);
success = false;
}
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8531f93d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
index 9887acc..12e1f54 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
@@ -30,7 +30,7 @@ public interface NodeHeartbeatResponse {
NodeAction getNodeAction();
ListContainerId getContainersToCleanup();
- ListContainerId
git commit: YARN-2527. Fixed the potential NPE in ApplicationACLsManager and added test cases for it. Contributed by Benoy Antony.
Repository: hadoop
Updated Branches:
refs/heads/trunk f679ca38c - 1c93025a1
YARN-2527. Fixed the potential NPE in ApplicationACLsManager and added test
cases for it. Contributed by Benoy Antony.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1c93025a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1c93025a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1c93025a
Branch: refs/heads/trunk
Commit: 1c93025a1b370db46e345161dbc15e03f829823f
Parents: f679ca3
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Oct 2 14:55:37 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Oct 2 14:56:13 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/security/ApplicationACLsManager.java | 22 ++-
.../security/TestApplicationACLsManager.java| 180 +++
3 files changed, 199 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c93025a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index deedb83..2471673 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -524,6 +524,9 @@ Release 2.6.0 - UNRELEASED
YARN-2624. Resource Localization fails on a cluster due to existing cache
directories (Anubhav Dhoot via jlowe)
+YARN-2527. Fixed the potential NPE in ApplicationACLsManager and added test
+cases for it. (Benoy Antony via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c93025a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
index 75c8478..e8e3cb5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
@@ -41,6 +41,8 @@ public class ApplicationACLsManager {
private static final Log LOG = LogFactory
.getLog(ApplicationACLsManager.class);
+ private static AccessControlList DEFAULT_YARN_APP_ACL
+= new AccessControlList(YarnConfiguration.DEFAULT_YARN_APP_ACL);
private final Configuration conf;
private final AdminACLsManager adminAclsManager;
private final ConcurrentMapApplicationId, MapApplicationAccessType,
AccessControlList applicationACLS
@@ -100,18 +102,26 @@ public class ApplicationACLsManager {
if (!areACLsEnabled()) {
return true;
}
-
-AccessControlList applicationACL = this.applicationACLS
-.get(applicationId).get(applicationAccessType);
-if (applicationACL == null) {
+AccessControlList applicationACL = DEFAULT_YARN_APP_ACL;
+MapApplicationAccessType, AccessControlList acls = this.applicationACLS
+.get(applicationId);
+if (acls == null) {
if (LOG.isDebugEnabled()) {
+LOG.debug(ACL not found for application
++ applicationId + owned by
++ applicationOwner + . Using default [
++ YarnConfiguration.DEFAULT_YARN_APP_ACL + ]);
+ }
+} else {
+ AccessControlList applicationACLInMap = acls.get(applicationAccessType);
+ if (applicationACLInMap != null) {
+applicationACL = applicationACLInMap;
+ } else if (LOG.isDebugEnabled()) {
LOG.debug(ACL not found for access-type + applicationAccessType
+ for application + applicationId + owned by
+ applicationOwner + . Using default [
+ YarnConfiguration.DEFAULT_YARN_APP_ACL + ]);
}
- applicationACL =
- new AccessControlList(YarnConfiguration.DEFAULT_YARN_APP_ACL);
}
// Allow application-owner for any type of access on the application
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1c93025a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/server/security/TestApplicationACLsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/server/security/TestApplicationACLsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org
git commit: YARN-2527. Fixed the potential NPE in ApplicationACLsManager and added test cases for it. Contributed by Benoy Antony.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 ebeb9da80 - ecaae67ab
YARN-2527. Fixed the potential NPE in ApplicationACLsManager and added test
cases for it. Contributed by Benoy Antony.
(cherry picked from commit 1c93025a1b370db46e345161dbc15e03f829823f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ecaae67a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ecaae67a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ecaae67a
Branch: refs/heads/branch-2
Commit: ecaae67ab32f842627c0bee06bba7b4773972a67
Parents: ebeb9da
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Oct 2 14:55:37 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Oct 2 14:57:27 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/security/ApplicationACLsManager.java | 22 ++-
.../security/TestApplicationACLsManager.java| 180 +++
3 files changed, 199 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecaae67a/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b4b0b75..6aeb961 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -494,6 +494,9 @@ Release 2.6.0 - UNRELEASED
YARN-2624. Resource Localization fails on a cluster due to existing cache
directories (Anubhav Dhoot via jlowe)
+YARN-2527. Fixed the potential NPE in ApplicationACLsManager and added test
+cases for it. (Benoy Antony via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecaae67a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
index 75c8478..e8e3cb5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
@@ -41,6 +41,8 @@ public class ApplicationACLsManager {
private static final Log LOG = LogFactory
.getLog(ApplicationACLsManager.class);
+ private static AccessControlList DEFAULT_YARN_APP_ACL
+= new AccessControlList(YarnConfiguration.DEFAULT_YARN_APP_ACL);
private final Configuration conf;
private final AdminACLsManager adminAclsManager;
private final ConcurrentMapApplicationId, MapApplicationAccessType,
AccessControlList applicationACLS
@@ -100,18 +102,26 @@ public class ApplicationACLsManager {
if (!areACLsEnabled()) {
return true;
}
-
-AccessControlList applicationACL = this.applicationACLS
-.get(applicationId).get(applicationAccessType);
-if (applicationACL == null) {
+AccessControlList applicationACL = DEFAULT_YARN_APP_ACL;
+MapApplicationAccessType, AccessControlList acls = this.applicationACLS
+.get(applicationId);
+if (acls == null) {
if (LOG.isDebugEnabled()) {
+LOG.debug(ACL not found for application
++ applicationId + owned by
++ applicationOwner + . Using default [
++ YarnConfiguration.DEFAULT_YARN_APP_ACL + ]);
+ }
+} else {
+ AccessControlList applicationACLInMap = acls.get(applicationAccessType);
+ if (applicationACLInMap != null) {
+applicationACL = applicationACLInMap;
+ } else if (LOG.isDebugEnabled()) {
LOG.debug(ACL not found for access-type + applicationAccessType
+ for application + applicationId + owned by
+ applicationOwner + . Using default [
+ YarnConfiguration.DEFAULT_YARN_APP_ACL + ]);
}
- applicationACL =
- new AccessControlList(YarnConfiguration.DEFAULT_YARN_APP_ACL);
}
// Allow application-owner for any type of access on the application
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ecaae67a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/server/security/TestApplicationACLsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/server/security
git commit: YARN-2527. Fixed the potential NPE in ApplicationACLsManager and added test cases for it. Contributed by Benoy Antony.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 cfc5f8f68 - 20fb25cde
YARN-2527. Fixed the potential NPE in ApplicationACLsManager and added test
cases for it. Contributed by Benoy Antony.
(cherry picked from commit 1c93025a1b370db46e345161dbc15e03f829823f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/20fb25cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/20fb25cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/20fb25cd
Branch: refs/heads/branch-2.6
Commit: 20fb25cde204d0d49126a89e5a6412de9d68091b
Parents: cfc5f8f
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Oct 2 14:55:37 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Oct 2 14:58:32 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../server/security/ApplicationACLsManager.java | 22 ++-
.../security/TestApplicationACLsManager.java| 180 +++
3 files changed, 199 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20fb25cd/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ffc9d79..11c0c7f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -467,6 +467,9 @@ Release 2.6.0 - UNRELEASED
YARN-2624. Resource Localization fails on a cluster due to existing cache
directories (Anubhav Dhoot via jlowe)
+YARN-2527. Fixed the potential NPE in ApplicationACLsManager and added test
+cases for it. (Benoy Antony via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20fb25cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
index 75c8478..e8e3cb5 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/security/ApplicationACLsManager.java
@@ -41,6 +41,8 @@ public class ApplicationACLsManager {
private static final Log LOG = LogFactory
.getLog(ApplicationACLsManager.class);
+ private static AccessControlList DEFAULT_YARN_APP_ACL
+= new AccessControlList(YarnConfiguration.DEFAULT_YARN_APP_ACL);
private final Configuration conf;
private final AdminACLsManager adminAclsManager;
private final ConcurrentMapApplicationId, MapApplicationAccessType,
AccessControlList applicationACLS
@@ -100,18 +102,26 @@ public class ApplicationACLsManager {
if (!areACLsEnabled()) {
return true;
}
-
-AccessControlList applicationACL = this.applicationACLS
-.get(applicationId).get(applicationAccessType);
-if (applicationACL == null) {
+AccessControlList applicationACL = DEFAULT_YARN_APP_ACL;
+MapApplicationAccessType, AccessControlList acls = this.applicationACLS
+.get(applicationId);
+if (acls == null) {
if (LOG.isDebugEnabled()) {
+LOG.debug(ACL not found for application
++ applicationId + owned by
++ applicationOwner + . Using default [
++ YarnConfiguration.DEFAULT_YARN_APP_ACL + ]);
+ }
+} else {
+ AccessControlList applicationACLInMap = acls.get(applicationAccessType);
+ if (applicationACLInMap != null) {
+applicationACL = applicationACLInMap;
+ } else if (LOG.isDebugEnabled()) {
LOG.debug(ACL not found for access-type + applicationAccessType
+ for application + applicationId + owned by
+ applicationOwner + . Using default [
+ YarnConfiguration.DEFAULT_YARN_APP_ACL + ]);
}
- applicationACL =
- new AccessControlList(YarnConfiguration.DEFAULT_YARN_APP_ACL);
}
// Allow application-owner for any type of access on the application
http://git-wip-us.apache.org/repos/asf/hadoop/blob/20fb25cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/server/security/TestApplicationACLsManager.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/server/security
git commit: HADOOP-11179. Java untar should handle the case that the file entry comes without its parent directory entry. Contributed by Craig Welch.
Repository: hadoop
Updated Branches:
refs/heads/trunk 2217e2f8f - a16905193
HADOOP-11179. Java untar should handle the case that the file entry comes
without its parent directory entry. Contributed by Craig Welch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a1690519
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a1690519
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a1690519
Branch: refs/heads/trunk
Commit: a1690519317068d9855174752d22ff45f0e4c962
Parents: 2217e2f
Author: Zhijie Shen zjs...@apache.org
Authored: Wed Oct 8 15:35:24 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Wed Oct 8 15:36:16 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../src/main/java/org/apache/hadoop/fs/FileUtil.java | 6 +++---
.../src/test/java/org/apache/hadoop/fs/TestFileUtil.java | 6 +++---
3 files changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1690519/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 5cd71ab..404d978 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -952,6 +952,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11163. MetricsSystemImpl may miss a registered source.
(Chuan Liu via cnauroth)
+HADOOP-11179. Java untar should handle the case that the file entry comes
+without its parent directory entry. (Craig Welch via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1690519/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 8aa8423..2b05293 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -707,7 +707,7 @@ public class FileUtil {
TarArchiveEntry entry, File outputDir) throws IOException {
if (entry.isDirectory()) {
File subDir = new File(outputDir, entry.getName());
- if (!subDir.mkdir() !subDir.isDirectory()) {
+ if (!subDir.mkdirs() !subDir.isDirectory()) {
throw new IOException(Mkdirs failed to create tar internal dir
+ outputDir);
}
@@ -720,8 +720,8 @@ public class FileUtil {
}
File outputFile = new File(outputDir, entry.getName());
-if (!outputDir.exists()) {
- if (!outputDir.mkdirs()) {
+if (!outputFile.getParentFile().exists()) {
+ if (!outputFile.getParentFile().mkdirs()) {
throw new IOException(Mkdirs failed to create tar internal dir
+ outputDir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a1690519/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 2234f2f..2a03988 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -619,7 +619,7 @@ public class TestFileUtil {
OutputStream os = new FileOutputStream(simpleTar);
TarOutputStream tos = new TarOutputStream(os);
try {
- TarEntry te = new TarEntry(foo);
+ TarEntry te = new TarEntry(/bar/foo);
byte[] data = some-content.getBytes(UTF-8);
te.setSize(data.length);
tos.putNextEntry(te);
@@ -634,8 +634,8 @@ public class TestFileUtil {
// successfully untar it into an existing dir:
FileUtil.unTar(simpleTar, tmp);
// check result:
-assertTrue(new File(tmp, foo).exists());
-assertEquals(12, new File(tmp, foo).length());
+assertTrue(new File(tmp, /bar/foo).exists());
+assertEquals(12, new File(tmp, /bar/foo).length());
final File regularFile = new File(tmp, QuickBrownFoxJumpsOverTheLazyDog);
regularFile.createNewFile();
git commit: HADOOP-11179. Java untar should handle the case that the file entry comes without its parent directory entry. Contributed by Craig Welch.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 19c307f36 - f27a88562
HADOOP-11179. Java untar should handle the case that the file entry comes
without its parent directory entry. Contributed by Craig Welch.
(cherry picked from commit a1690519317068d9855174752d22ff45f0e4c962)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f27a8856
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f27a8856
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f27a8856
Branch: refs/heads/branch-2
Commit: f27a88562edfbd0e6e12acb3fb5d29559960aaf4
Parents: 19c307f
Author: Zhijie Shen zjs...@apache.org
Authored: Wed Oct 8 15:35:24 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Wed Oct 8 15:37:57 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../src/main/java/org/apache/hadoop/fs/FileUtil.java | 6 +++---
.../src/test/java/org/apache/hadoop/fs/TestFileUtil.java | 6 +++---
3 files changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f27a8856/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index e0883aa..fe15cf5 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -611,6 +611,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11163. MetricsSystemImpl may miss a registered source.
(Chuan Liu via cnauroth)
+HADOOP-11179. Java untar should handle the case that the file entry comes
+without its parent directory entry. (Craig Welch via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f27a8856/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 8aa8423..2b05293 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -707,7 +707,7 @@ public class FileUtil {
TarArchiveEntry entry, File outputDir) throws IOException {
if (entry.isDirectory()) {
File subDir = new File(outputDir, entry.getName());
- if (!subDir.mkdir() !subDir.isDirectory()) {
+ if (!subDir.mkdirs() !subDir.isDirectory()) {
throw new IOException(Mkdirs failed to create tar internal dir
+ outputDir);
}
@@ -720,8 +720,8 @@ public class FileUtil {
}
File outputFile = new File(outputDir, entry.getName());
-if (!outputDir.exists()) {
- if (!outputDir.mkdirs()) {
+if (!outputFile.getParentFile().exists()) {
+ if (!outputFile.getParentFile().mkdirs()) {
throw new IOException(Mkdirs failed to create tar internal dir
+ outputDir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f27a8856/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 2234f2f..2a03988 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -619,7 +619,7 @@ public class TestFileUtil {
OutputStream os = new FileOutputStream(simpleTar);
TarOutputStream tos = new TarOutputStream(os);
try {
- TarEntry te = new TarEntry(foo);
+ TarEntry te = new TarEntry(/bar/foo);
byte[] data = some-content.getBytes(UTF-8);
te.setSize(data.length);
tos.putNextEntry(te);
@@ -634,8 +634,8 @@ public class TestFileUtil {
// successfully untar it into an existing dir:
FileUtil.unTar(simpleTar, tmp);
// check result:
-assertTrue(new File(tmp, foo).exists());
-assertEquals(12, new File(tmp, foo).length());
+assertTrue(new File(tmp, /bar/foo).exists());
+assertEquals(12, new File(tmp, /bar/foo).length());
final File regularFile = new File(tmp, QuickBrownFoxJumpsOverTheLazyDog);
regularFile.createNewFile();
git commit: HADOOP-11179. Java untar should handle the case that the file entry comes without its parent directory entry. Contributed by Craig Welch.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 204148f0d - ab448565c
HADOOP-11179. Java untar should handle the case that the file entry comes
without its parent directory entry. Contributed by Craig Welch.
(cherry picked from commit a1690519317068d9855174752d22ff45f0e4c962)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab448565
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab448565
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab448565
Branch: refs/heads/branch-2.6
Commit: ab448565c90b6e713fee079b71dbd334863e8687
Parents: 204148f
Author: Zhijie Shen zjs...@apache.org
Authored: Wed Oct 8 15:35:24 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Wed Oct 8 15:39:30 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt| 3 +++
.../src/main/java/org/apache/hadoop/fs/FileUtil.java | 6 +++---
.../src/test/java/org/apache/hadoop/fs/TestFileUtil.java | 6 +++---
3 files changed, 9 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab448565/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index c63051e..e15a185 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -591,6 +591,9 @@ Release 2.6.0 - UNRELEASED
HADOOP-11163. MetricsSystemImpl may miss a registered source.
(Chuan Liu via cnauroth)
+HADOOP-11179. Java untar should handle the case that the file entry comes
+without its parent directory entry. (Craig Welch via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab448565/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 8aa8423..2b05293 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -707,7 +707,7 @@ public class FileUtil {
TarArchiveEntry entry, File outputDir) throws IOException {
if (entry.isDirectory()) {
File subDir = new File(outputDir, entry.getName());
- if (!subDir.mkdir() !subDir.isDirectory()) {
+ if (!subDir.mkdirs() !subDir.isDirectory()) {
throw new IOException(Mkdirs failed to create tar internal dir
+ outputDir);
}
@@ -720,8 +720,8 @@ public class FileUtil {
}
File outputFile = new File(outputDir, entry.getName());
-if (!outputDir.exists()) {
- if (!outputDir.mkdirs()) {
+if (!outputFile.getParentFile().exists()) {
+ if (!outputFile.getParentFile().mkdirs()) {
throw new IOException(Mkdirs failed to create tar internal dir
+ outputDir);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab448565/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 2234f2f..2a03988 100644
---
a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++
b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -619,7 +619,7 @@ public class TestFileUtil {
OutputStream os = new FileOutputStream(simpleTar);
TarOutputStream tos = new TarOutputStream(os);
try {
- TarEntry te = new TarEntry(foo);
+ TarEntry te = new TarEntry(/bar/foo);
byte[] data = some-content.getBytes(UTF-8);
te.setSize(data.length);
tos.putNextEntry(te);
@@ -634,8 +634,8 @@ public class TestFileUtil {
// successfully untar it into an existing dir:
FileUtil.unTar(simpleTar, tmp);
// check result:
-assertTrue(new File(tmp, foo).exists());
-assertEquals(12, new File(tmp, foo).length());
+assertTrue(new File(tmp, /bar/foo).exists());
+assertEquals(12, new File(tmp, /bar/foo).length());
final File regularFile = new File(tmp, QuickBrownFoxJumpsOverTheLazyDog);
regularFile.createNewFile();
git commit: YARN-2629. Made the distributed shell use the domain-based timeline ACLs. Contributed by Zhijie Shen.
Repository: hadoop
Updated Branches:
refs/heads/trunk d7b647f0e - 1d4612f5a
YARN-2629. Made the distributed shell use the domain-based timeline ACLs.
Contributed by Zhijie Shen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1d4612f5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1d4612f5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1d4612f5
Branch: refs/heads/trunk
Commit: 1d4612f5ad9678c952b416e798dccd20c88f96ef
Parents: d7b647f
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Oct 9 12:59:47 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Oct 9 12:59:47 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/ApplicationMaster.java | 120 ---
.../applications/distributedshell/Client.java | 75 +++-
.../distributedshell/DSConstants.java | 5 +
.../distributedshell/TestDistributedShell.java | 52 +++-
.../client/api/impl/TimelineClientImpl.java | 3 -
6 files changed, 205 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d4612f5/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5186224..1e016e7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -324,6 +324,9 @@ Release 2.6.0 - UNRELEASED
YARN-2615. Changed
ClientToAMTokenIdentifier/RM(Timeline)DelegationTokenIdentifier
to use protobuf as payload. (Junping Du via jianhe)
+YARN-2629. Made the distributed shell use the domain-based timeline ACLs.
+(zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1d4612f5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index df9f34b..e6ded00 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.StringReader;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
@@ -86,6 +87,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
@@ -246,6 +248,9 @@ public class ApplicationMaster {
// File length needed for local resource
private long shellScriptPathLen = 0;
+ // Timeline domain ID
+ private String domainId = null;
+
// Hardcoded path to shell script in launch container's local env
private static final String ExecShellStringPath = Client.SCRIPT_PATH + .sh;
private static final String ExecBatScripStringtPath = Client.SCRIPT_PATH
@@ -465,7 +470,9 @@ public class ApplicationMaster {
shellScriptPathLen = Long.valueOf(envs
.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN));
}
-
+ if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN)) {
+domainId = envs.get(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN);
+ }
if (!scriptPath.isEmpty()
(shellScriptPathTimestamp = 0 || shellScriptPathLen = 0)) {
LOG.error(Illegal values in env for shell script path + , path=
@@ -515,13 +522,6 @@ public class ApplicationMaster {
@SuppressWarnings({ unchecked })
public void run() throws YarnException, IOException {
LOG.info(Starting
git commit: YARN-2629. Made the distributed shell use the domain-based timeline ACLs. Contributed by Zhijie Shen.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 971eb31d2 - 4d170d3db
YARN-2629. Made the distributed shell use the domain-based timeline ACLs.
Contributed by Zhijie Shen.
(cherry picked from commit 1d4612f5ad9678c952b416e798dccd20c88f96ef)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4d170d3d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4d170d3d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4d170d3d
Branch: refs/heads/branch-2
Commit: 4d170d3db6bd0b07e5e49b6b41a5a60d59466e66
Parents: 971eb31
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Oct 9 12:59:47 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Oct 9 13:01:44 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/ApplicationMaster.java | 120 ---
.../applications/distributedshell/Client.java | 75 +++-
.../distributedshell/DSConstants.java | 5 +
.../distributedshell/TestDistributedShell.java | 52 +++-
.../client/api/impl/TimelineClientImpl.java | 3 -
6 files changed, 205 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d170d3d/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0cba8fb..4109efe 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -294,6 +294,9 @@ Release 2.6.0 - UNRELEASED
YARN-2615. Changed
ClientToAMTokenIdentifier/RM(Timeline)DelegationTokenIdentifier
to use protobuf as payload. (Junping Du via jianhe)
+YARN-2629. Made the distributed shell use the domain-based timeline ACLs.
+(zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4d170d3d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index df9f34b..e6ded00 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.StringReader;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
@@ -86,6 +87,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
@@ -246,6 +248,9 @@ public class ApplicationMaster {
// File length needed for local resource
private long shellScriptPathLen = 0;
+ // Timeline domain ID
+ private String domainId = null;
+
// Hardcoded path to shell script in launch container's local env
private static final String ExecShellStringPath = Client.SCRIPT_PATH + .sh;
private static final String ExecBatScripStringtPath = Client.SCRIPT_PATH
@@ -465,7 +470,9 @@ public class ApplicationMaster {
shellScriptPathLen = Long.valueOf(envs
.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN));
}
-
+ if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN)) {
+domainId = envs.get(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN);
+ }
if (!scriptPath.isEmpty()
(shellScriptPathTimestamp = 0 || shellScriptPathLen = 0)) {
LOG.error(Illegal values in env for shell script path + , path=
@@ -515,13 +522,6 @@ public class ApplicationMaster {
@SuppressWarnings({ unchecked })
public void
git commit: YARN-2629. Made the distributed shell use the domain-based timeline ACLs. Contributed by Zhijie Shen.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 e7b215dfa - a4522e90f
YARN-2629. Made the distributed shell use the domain-based timeline ACLs.
Contributed by Zhijie Shen.
(cherry picked from commit 1d4612f5ad9678c952b416e798dccd20c88f96ef)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a4522e90
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a4522e90
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a4522e90
Branch: refs/heads/branch-2.6
Commit: a4522e90f6e42921183b5eed539cb04761c7a35b
Parents: e7b215d
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Oct 9 12:59:47 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Oct 9 13:03:03 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../distributedshell/ApplicationMaster.java | 120 ---
.../applications/distributedshell/Client.java | 75 +++-
.../distributedshell/DSConstants.java | 5 +
.../distributedshell/TestDistributedShell.java | 52 +++-
.../client/api/impl/TimelineClientImpl.java | 3 -
6 files changed, 205 insertions(+), 53 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4522e90/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cddeba2..b6b56e7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -264,6 +264,9 @@ Release 2.6.0 - UNRELEASED
YARN-2615. Changed
ClientToAMTokenIdentifier/RM(Timeline)DelegationTokenIdentifier
to use protobuf as payload. (Junping Du via jianhe)
+YARN-2629. Made the distributed shell use the domain-based timeline ACLs.
+(zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a4522e90/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
index df9f34b..e6ded00 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
@@ -24,6 +24,7 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.StringReader;
+import java.lang.reflect.UndeclaredThrowableException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.ByteBuffer;
@@ -86,6 +87,7 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
+import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync;
@@ -246,6 +248,9 @@ public class ApplicationMaster {
// File length needed for local resource
private long shellScriptPathLen = 0;
+ // Timeline domain ID
+ private String domainId = null;
+
// Hardcoded path to shell script in launch container's local env
private static final String ExecShellStringPath = Client.SCRIPT_PATH + .sh;
private static final String ExecBatScripStringtPath = Client.SCRIPT_PATH
@@ -465,7 +470,9 @@ public class ApplicationMaster {
shellScriptPathLen = Long.valueOf(envs
.get(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN));
}
-
+ if (envs.containsKey(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN)) {
+domainId = envs.get(DSConstants.DISTRIBUTEDSHELLTIMELINEDOMAIN);
+ }
if (!scriptPath.isEmpty()
(shellScriptPathTimestamp = 0 || shellScriptPathLen = 0)) {
LOG.error(Illegal values in env for shell script path + , path=
@@ -515,13 +522,6 @@ public class ApplicationMaster {
@SuppressWarnings({ unchecked })
public
git commit: YARN-2617. Fixed ApplicationSubmissionContext to still set resource for backward compatibility. Contributed by Wangda Tan.
Repository: hadoop
Updated Branches:
refs/heads/trunk 4f426fe22 - e532ed8fa
YARN-2617. Fixed ApplicationSubmissionContext to still set resource for
backward compatibility. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e532ed8f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e532ed8f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e532ed8f
Branch: refs/heads/trunk
Commit: e532ed8faa8db4b008a5b8d3f82b48a1b314fa6c
Parents: 4f426fe
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Oct 9 17:57:50 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Oct 9 17:57:50 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/api/records/ApplicationSubmissionContext.java | 1 +
.../java/org/apache/hadoop/yarn/api/TestPBImplRecords.java| 7 +++
3 files changed, 11 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e532ed8f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1633cf7..2fd13d3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -611,6 +611,9 @@ Release 2.6.0 - UNRELEASED
YARN-2598 GHS should show N/A instead of null for the inaccessible
information
(Zhijie Shen via mayank)
+YARN-2617. Fixed ApplicationSubmissionContext to still set resource for
+backward compatibility. (Wangda Tan via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e532ed8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index f186650..f1ebbfe 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -89,6 +89,7 @@ public abstract class ApplicationSubmissionContext {
context.setApplicationType(applicationType);
context.setKeepContainersAcrossApplicationAttempts(keepContainers);
context.setNodeLabelExpression(appLabelExpression);
+context.setResource(resource);
ResourceRequest amReq = Records.newRecord(ResourceRequest.class);
amReq.setResourceName(ResourceRequest.ANY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e532ed8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 451d39b..b8f6e9c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -45,6 +45,7 @@ import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.*;
import org.apache.hadoop.yarn.proto.YarnProtos.*;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.*;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.*;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -654,6 +655,12 @@ public class TestPBImplRecords {
public void testApplicationSubmissionContextPBImpl() throws Exception {
validatePBImplRecord(ApplicationSubmissionContextPBImpl.class,
ApplicationSubmissionContextProto.class);
+
+ApplicationSubmissionContext ctx =
+ApplicationSubmissionContext.newInstance(null, null, null, null, null,
+false, false, 0, Resources.none(), null, false, null, null);
+
+Assert.assertNotNull(ctx.getResource());
}
@Test
git commit: YARN-2617. Fixed ApplicationSubmissionContext to still set resource for backward compatibility. Contributed by Wangda Tan.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 7e20187f9 - ad47a27db
YARN-2617. Fixed ApplicationSubmissionContext to still set resource for
backward compatibility. Contributed by Wangda Tan.
(cherry picked from commit e532ed8faa8db4b008a5b8d3f82b48a1b314fa6c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad47a27d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad47a27d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad47a27d
Branch: refs/heads/branch-2
Commit: ad47a27dbfd80abb83c09f6b7724a989416d36d7
Parents: 7e20187
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Oct 9 17:57:50 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Oct 9 18:00:01 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/api/records/ApplicationSubmissionContext.java | 1 +
.../java/org/apache/hadoop/yarn/api/TestPBImplRecords.java| 7 +++
3 files changed, 11 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad47a27d/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 47cf718..dd0ee56 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -581,6 +581,9 @@ Release 2.6.0 - UNRELEASED
YARN-2598 GHS should show N/A instead of null for the inaccessible
information
(Zhijie Shen via mayank)
+YARN-2617. Fixed ApplicationSubmissionContext to still set resource for
+backward compatibility. (Wangda Tan via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad47a27d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index f186650..f1ebbfe 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -89,6 +89,7 @@ public abstract class ApplicationSubmissionContext {
context.setApplicationType(applicationType);
context.setKeepContainersAcrossApplicationAttempts(keepContainers);
context.setNodeLabelExpression(appLabelExpression);
+context.setResource(resource);
ResourceRequest amReq = Records.newRecord(ResourceRequest.class);
amReq.setResourceName(ResourceRequest.ANY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad47a27d/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 451d39b..b8f6e9c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -45,6 +45,7 @@ import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.*;
import org.apache.hadoop.yarn.proto.YarnProtos.*;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.*;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.*;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -654,6 +655,12 @@ public class TestPBImplRecords {
public void testApplicationSubmissionContextPBImpl() throws Exception {
validatePBImplRecord(ApplicationSubmissionContextPBImpl.class,
ApplicationSubmissionContextProto.class);
+
+ApplicationSubmissionContext ctx =
+ApplicationSubmissionContext.newInstance(null, null, null, null, null,
+false, false, 0, Resources.none(), null, false, null, null);
+
+Assert.assertNotNull(ctx.getResource());
}
@Test
git commit: YARN-2617. Fixed ApplicationSubmissionContext to still set resource for backward compatibility. Contributed by Wangda Tan.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 14080ea3d - 647874206
YARN-2617. Fixed ApplicationSubmissionContext to still set resource for
backward compatibility. Contributed by Wangda Tan.
(cherry picked from commit e532ed8faa8db4b008a5b8d3f82b48a1b314fa6c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/64787420
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/64787420
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/64787420
Branch: refs/heads/branch-2.6
Commit: 647874206c683ade6ea43a390670913b7e0d6467
Parents: 14080ea
Author: Zhijie Shen zjs...@apache.org
Authored: Thu Oct 9 17:57:50 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Thu Oct 9 18:01:13 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/api/records/ApplicationSubmissionContext.java | 1 +
.../java/org/apache/hadoop/yarn/api/TestPBImplRecords.java| 7 +++
3 files changed, 11 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64787420/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 18d08c7..a3961a4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -548,6 +548,9 @@ Release 2.6.0 - UNRELEASED
YARN-2598 GHS should show N/A instead of null for the inaccessible
information
(Zhijie Shen via mayank)
+YARN-2617. Fixed ApplicationSubmissionContext to still set resource for
+backward compatibility. (Wangda Tan via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64787420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
index f186650..f1ebbfe 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationSubmissionContext.java
@@ -89,6 +89,7 @@ public abstract class ApplicationSubmissionContext {
context.setApplicationType(applicationType);
context.setKeepContainersAcrossApplicationAttempts(keepContainers);
context.setNodeLabelExpression(appLabelExpression);
+context.setResource(resource);
ResourceRequest amReq = Records.newRecord(ResourceRequest.class);
amReq.setResourceName(ResourceRequest.ANY);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/64787420/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
index 451d39b..b8f6e9c 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestPBImplRecords.java
@@ -45,6 +45,7 @@ import
org.apache.hadoop.yarn.proto.YarnServerResourceManagerServiceProtos.*;
import org.apache.hadoop.yarn.proto.YarnProtos.*;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.*;
import org.apache.hadoop.yarn.server.api.protocolrecords.impl.pb.*;
+import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
@@ -654,6 +655,12 @@ public class TestPBImplRecords {
public void testApplicationSubmissionContextPBImpl() throws Exception {
validatePBImplRecord(ApplicationSubmissionContextPBImpl.class,
ApplicationSubmissionContextProto.class);
+
+ApplicationSubmissionContext ctx =
+ApplicationSubmissionContext.newInstance(null, null, null, null, null,
+false, false, 0, Resources.none(), null, false, null, null);
+
+Assert.assertNotNull(ctx.getResource());
}
@Test
git commit: YARN-2583. Modified AggregatedLogDeletionService to be able to delete rolling aggregated logs. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk d3afd730a - cb81bac00
YARN-2583. Modified AggregatedLogDeletionService to be able to delete rolling
aggregated logs. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb81bac0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb81bac0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb81bac0
Branch: refs/heads/trunk
Commit: cb81bac0029fce3a9726df3523f0b692cd3375b8
Parents: d3afd73
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Oct 10 00:10:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Oct 10 00:11:30 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../AggregatedLogDeletionService.java | 93 -
.../TestAggregatedLogDeletionService.java | 192 ---
.../logaggregation/AppLogAggregatorImpl.java| 151 ++-
.../logaggregation/LogAggregationService.java | 2 +-
.../TestLogAggregationService.java | 94 ++---
6 files changed, 467 insertions(+), 68 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb81bac0/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 578ab6e..210297a 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -333,6 +333,9 @@ Release 2.6.0 - UNRELEASED
YARN-2629. Made the distributed shell use the domain-based timeline ACLs.
(zjshen)
+YARN-2583. Modified AggregatedLogDeletionService to be able to delete
rolling
+aggregated logs. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb81bac0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
index 590cfe2..4c1d152 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
@@ -24,38 +24,53 @@ import java.util.TimerTask;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+import com.google.common.annotations.VisibleForTesting;
/**
* A service that periodically deletes aggregated logs.
*/
-@Private
+@InterfaceAudience.LimitedPrivate({yarn, mapreduce})
public class AggregatedLogDeletionService extends AbstractService {
private static final Log LOG =
LogFactory.getLog(AggregatedLogDeletionService.class);
private Timer timer = null;
private long checkIntervalMsecs;
+ private LogDeletionTask task;
static class LogDeletionTask extends TimerTask {
private Configuration conf;
private long retentionMillis;
private String suffix = null;
private Path remoteRootLogDir = null;
+private ApplicationClientProtocol rmClient = null;
-public LogDeletionTask(Configuration conf, long retentionSecs) {
+public LogDeletionTask(Configuration conf, long retentionSecs,
ApplicationClientProtocol rmClient) {
this.conf = conf;
this.retentionMillis = retentionSecs
git commit: YARN-2583. Modified AggregatedLogDeletionService to be able to delete rolling aggregated logs. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b81641a31 - 1e6d81a88
YARN-2583. Modified AggregatedLogDeletionService to be able to delete rolling
aggregated logs. Contributed by Xuan Gong.
(cherry picked from commit cb81bac0029fce3a9726df3523f0b692cd3375b8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1e6d81a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1e6d81a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1e6d81a8
Branch: refs/heads/branch-2
Commit: 1e6d81a8869ceeb2f0f81f2ee4b89833f2b22cd4
Parents: b81641a
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Oct 10 00:10:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Oct 10 00:16:34 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../AggregatedLogDeletionService.java | 93 -
.../TestAggregatedLogDeletionService.java | 192 ---
.../logaggregation/AppLogAggregatorImpl.java| 151 ++-
.../logaggregation/LogAggregationService.java | 2 +-
.../TestLogAggregationService.java | 94 ++---
6 files changed, 467 insertions(+), 68 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e6d81a8/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 77f710f..47a16f2 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -303,6 +303,9 @@ Release 2.6.0 - UNRELEASED
YARN-2629. Made the distributed shell use the domain-based timeline ACLs.
(zjshen)
+YARN-2583. Modified AggregatedLogDeletionService to be able to delete
rolling
+aggregated logs. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1e6d81a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
index 590cfe2..4c1d152 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
@@ -24,38 +24,53 @@ import java.util.TimerTask;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+import com.google.common.annotations.VisibleForTesting;
/**
* A service that periodically deletes aggregated logs.
*/
-@Private
+@InterfaceAudience.LimitedPrivate({yarn, mapreduce})
public class AggregatedLogDeletionService extends AbstractService {
private static final Log LOG =
LogFactory.getLog(AggregatedLogDeletionService.class);
private Timer timer = null;
private long checkIntervalMsecs;
+ private LogDeletionTask task;
static class LogDeletionTask extends TimerTask {
private Configuration conf;
private long retentionMillis;
private String suffix = null;
private Path remoteRootLogDir = null;
+private ApplicationClientProtocol rmClient = null;
-public LogDeletionTask(Configuration conf, long retentionSecs) {
+public LogDeletionTask(Configuration conf, long retentionSecs,
ApplicationClientProtocol rmClient
git commit: YARN-2583. Modified AggregatedLogDeletionService to be able to delete rolling aggregated logs. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 281041a57 - 16d714433
YARN-2583. Modified AggregatedLogDeletionService to be able to delete rolling
aggregated logs. Contributed by Xuan Gong.
(cherry picked from commit cb81bac0029fce3a9726df3523f0b692cd3375b8)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/16d71443
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/16d71443
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/16d71443
Branch: refs/heads/branch-2.6
Commit: 16d71443373e8eda8644bee54f3f9a6979d66a56
Parents: 281041a
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Oct 10 00:10:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Oct 10 00:18:24 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../AggregatedLogDeletionService.java | 93 -
.../TestAggregatedLogDeletionService.java | 192 ---
.../logaggregation/AppLogAggregatorImpl.java| 151 ++-
.../logaggregation/LogAggregationService.java | 2 +-
.../TestLogAggregationService.java | 94 ++---
6 files changed, 467 insertions(+), 68 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16d71443/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 674909c..ec31f8f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -270,6 +270,9 @@ Release 2.6.0 - UNRELEASED
YARN-2629. Made the distributed shell use the domain-based timeline ACLs.
(zjshen)
+YARN-2583. Modified AggregatedLogDeletionService to be able to delete
rolling
+aggregated logs. (Xuan Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/16d71443/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
index 590cfe2..4c1d152 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogDeletionService.java
@@ -24,38 +24,53 @@ import java.util.TimerTask;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.security.AccessControlException;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
+import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationReportRequest;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.ClientRMProxy;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.ApplicationNotFoundException;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+
+import com.google.common.annotations.VisibleForTesting;
/**
* A service that periodically deletes aggregated logs.
*/
-@Private
+@InterfaceAudience.LimitedPrivate({yarn, mapreduce})
public class AggregatedLogDeletionService extends AbstractService {
private static final Log LOG =
LogFactory.getLog(AggregatedLogDeletionService.class);
private Timer timer = null;
private long checkIntervalMsecs;
+ private LogDeletionTask task;
static class LogDeletionTask extends TimerTask {
private Configuration conf;
private long retentionMillis;
private String suffix = null;
private Path remoteRootLogDir = null;
+private ApplicationClientProtocol rmClient = null;
-public LogDeletionTask(Configuration conf, long retentionSecs) {
+public LogDeletionTask(Configuration conf, long retentionSecs,
ApplicationClientProtocol
git commit: YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk 53100318e - 4aed2d8e9
YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed
by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4aed2d8e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4aed2d8e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4aed2d8e
Branch: refs/heads/trunk
Commit: 4aed2d8e91c7dccc78fbaffc409d3076c3316289
Parents: 5310031
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 13 10:53:37 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 13 10:54:09 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/api/records/LogAggregationContext.java | 28 +---
.../hadoop/yarn/conf/YarnConfiguration.java | 7 +
.../src/main/proto/yarn_protos.proto| 1 -
.../impl/pb/LogAggregationContextPBImpl.java| 15 ---
.../src/main/resources/yarn-default.xml | 10 +++
.../logaggregation/AppLogAggregatorImpl.java| 8 +++---
.../TestContainerManagerRecovery.java | 5 +---
.../TestLogAggregationService.java | 4 ++-
.../capacity/TestContainerAllocation.java | 4 +--
10 files changed, 31 insertions(+), 54 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aed2d8e/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9e61b2d..782cd13 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -630,6 +630,9 @@ Release 2.6.0 - UNRELEASED
YARN-2667. Fix the release audit warning caused by hadoop-yarn-registry
(Yi Liu via jlowe)
+YARN-2651. Spun off LogRollingInterval from LogAggregationContext. (Xuan
Gong
+via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aed2d8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
index 9a0a157..46c1809 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
@@ -37,13 +37,6 @@ import org.apache.hadoop.yarn.util.Records;
* which match the defined exclude pattern and those log files
* will not be uploaded. If the log file name matches both the
* include and the exclude pattern, this file will be excluded
eventually/li
- * lirollingIntervalSeconds. The default value is -1. By default,
- * the logAggregationService only uploads container logs when
- * the application is finished. This configure defines
- * how often the logAggregationSerivce uploads container logs in seconds.
- * By setting this configure, the logAggregationSerivce can upload
container
- * logs periodically when the application is running.
- * /li
* /ul
* /p
*
@@ -57,11 +50,10 @@ public abstract class LogAggregationContext {
@Public
@Unstable
public static LogAggregationContext newInstance(String includePattern,
- String excludePattern, long rollingIntervalSeconds) {
+ String excludePattern) {
LogAggregationContext context =
Records.newRecord(LogAggregationContext.class);
context.setIncludePattern(includePattern);
context.setExcludePattern(excludePattern);
-context.setRollingIntervalSeconds(rollingIntervalSeconds);
return context;
}
@@ -100,22 +92,4 @@ public abstract class LogAggregationContext {
@Public
@Unstable
public abstract void setExcludePattern(String excludePattern);
-
- /**
- * Get rollingIntervalSeconds
- *
- * @return the rollingIntervalSeconds
- */
- @Public
- @Unstable
- public abstract long getRollingIntervalSeconds();
-
- /**
- * Set rollingIntervalSeconds
- *
- * @param rollingIntervalSeconds
- */
- @Public
- @Unstable
- public abstract void setRollingIntervalSeconds(long rollingIntervalSeconds);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4aed2d8e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
git commit: YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 c3dfc7119 - e51ae6476
YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed
by Xuan Gong.
(cherry picked from commit 4aed2d8e91c7dccc78fbaffc409d3076c3316289)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e51ae647
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e51ae647
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e51ae647
Branch: refs/heads/branch-2
Commit: e51ae6476181c0675546854da31966c7d021c48d
Parents: c3dfc71
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 13 10:53:37 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 13 10:55:09 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/api/records/LogAggregationContext.java | 28 +---
.../hadoop/yarn/conf/YarnConfiguration.java | 7 +
.../src/main/proto/yarn_protos.proto| 1 -
.../impl/pb/LogAggregationContextPBImpl.java| 15 ---
.../src/main/resources/yarn-default.xml | 10 +++
.../logaggregation/AppLogAggregatorImpl.java| 8 +++---
.../TestContainerManagerRecovery.java | 5 +---
.../TestLogAggregationService.java | 4 ++-
.../capacity/TestContainerAllocation.java | 4 +--
10 files changed, 31 insertions(+), 54 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2cf5a93..81de1d3 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -600,6 +600,9 @@ Release 2.6.0 - UNRELEASED
YARN-2667. Fix the release audit warning caused by hadoop-yarn-registry
(Yi Liu via jlowe)
+YARN-2651. Spun off LogRollingInterval from LogAggregationContext. (Xuan
Gong
+via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
index 9a0a157..46c1809 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
@@ -37,13 +37,6 @@ import org.apache.hadoop.yarn.util.Records;
* which match the defined exclude pattern and those log files
* will not be uploaded. If the log file name matches both the
* include and the exclude pattern, this file will be excluded
eventually/li
- * lirollingIntervalSeconds. The default value is -1. By default,
- * the logAggregationService only uploads container logs when
- * the application is finished. This configure defines
- * how often the logAggregationSerivce uploads container logs in seconds.
- * By setting this configure, the logAggregationSerivce can upload
container
- * logs periodically when the application is running.
- * /li
* /ul
* /p
*
@@ -57,11 +50,10 @@ public abstract class LogAggregationContext {
@Public
@Unstable
public static LogAggregationContext newInstance(String includePattern,
- String excludePattern, long rollingIntervalSeconds) {
+ String excludePattern) {
LogAggregationContext context =
Records.newRecord(LogAggregationContext.class);
context.setIncludePattern(includePattern);
context.setExcludePattern(excludePattern);
-context.setRollingIntervalSeconds(rollingIntervalSeconds);
return context;
}
@@ -100,22 +92,4 @@ public abstract class LogAggregationContext {
@Public
@Unstable
public abstract void setExcludePattern(String excludePattern);
-
- /**
- * Get rollingIntervalSeconds
- *
- * @return the rollingIntervalSeconds
- */
- @Public
- @Unstable
- public abstract long getRollingIntervalSeconds();
-
- /**
- * Set rollingIntervalSeconds
- *
- * @param rollingIntervalSeconds
- */
- @Public
- @Unstable
- public abstract void setRollingIntervalSeconds(long rollingIntervalSeconds);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e51ae647/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf
git commit: YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 4fb157655 - 6da637ec1
YARN-2651. Spun off LogRollingInterval from LogAggregationContext. Contributed
by Xuan Gong.
(cherry picked from commit 4aed2d8e91c7dccc78fbaffc409d3076c3316289)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6da637ec
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6da637ec
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6da637ec
Branch: refs/heads/branch-2.6
Commit: 6da637ec1282fcb9b54bbd95ace842c0e00ecbb6
Parents: 4fb1576
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 13 10:53:37 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 13 10:56:22 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/api/records/LogAggregationContext.java | 28 +---
.../hadoop/yarn/conf/YarnConfiguration.java | 7 +
.../src/main/proto/yarn_protos.proto| 1 -
.../impl/pb/LogAggregationContextPBImpl.java| 15 ---
.../src/main/resources/yarn-default.xml | 10 +++
.../logaggregation/AppLogAggregatorImpl.java| 8 +++---
.../TestContainerManagerRecovery.java | 5 +---
.../TestLogAggregationService.java | 4 ++-
.../capacity/TestContainerAllocation.java | 4 +--
10 files changed, 31 insertions(+), 54 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 13a09c0..2ebae00 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -567,6 +567,9 @@ Release 2.6.0 - UNRELEASED
YARN-2667. Fix the release audit warning caused by hadoop-yarn-registry
(Yi Liu via jlowe)
+YARN-2651. Spun off LogRollingInterval from LogAggregationContext. (Xuan
Gong
+via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
index 9a0a157..46c1809 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/LogAggregationContext.java
@@ -37,13 +37,6 @@ import org.apache.hadoop.yarn.util.Records;
* which match the defined exclude pattern and those log files
* will not be uploaded. If the log file name matches both the
* include and the exclude pattern, this file will be excluded
eventually/li
- * lirollingIntervalSeconds. The default value is -1. By default,
- * the logAggregationService only uploads container logs when
- * the application is finished. This configure defines
- * how often the logAggregationSerivce uploads container logs in seconds.
- * By setting this configure, the logAggregationSerivce can upload
container
- * logs periodically when the application is running.
- * /li
* /ul
* /p
*
@@ -57,11 +50,10 @@ public abstract class LogAggregationContext {
@Public
@Unstable
public static LogAggregationContext newInstance(String includePattern,
- String excludePattern, long rollingIntervalSeconds) {
+ String excludePattern) {
LogAggregationContext context =
Records.newRecord(LogAggregationContext.class);
context.setIncludePattern(includePattern);
context.setExcludePattern(excludePattern);
-context.setRollingIntervalSeconds(rollingIntervalSeconds);
return context;
}
@@ -100,22 +92,4 @@ public abstract class LogAggregationContext {
@Public
@Unstable
public abstract void setExcludePattern(String excludePattern);
-
- /**
- * Get rollingIntervalSeconds
- *
- * @return the rollingIntervalSeconds
- */
- @Public
- @Unstable
- public abstract long getRollingIntervalSeconds();
-
- /**
- * Set rollingIntervalSeconds
- *
- * @param rollingIntervalSeconds
- */
- @Public
- @Unstable
- public abstract void setRollingIntervalSeconds(long rollingIntervalSeconds);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6da637ec/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf
git commit: HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all sub-classes of AbstractDelegationTokenIdentifier. Contributed by Zhijie Shen.
Repository: hadoop
Updated Branches:
refs/heads/trunk 7dcad8414 - cdce88376
HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all
sub-classes of AbstractDelegationTokenIdentifier. Contributed by Zhijie Shen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cdce8837
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cdce8837
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cdce8837
Branch: refs/heads/trunk
Commit: cdce88376a60918dfe2f3bcd82a7666d74992a19
Parents: 7dcad84
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 14 11:35:38 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 14 11:35:38 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++
.../AbstractDelegationTokenIdentifier.java | 56 +++---
.../AbstractDelegationTokenSecretManager.java | 13 +
.../DelegationTokenAuthenticationHandler.java | 11 ++--
.../delegation/web/DelegationTokenManager.java | 60 ++--
.../TestZKDelegationTokenSecretManager.java | 7 ++-
...tionTokenAuthenticationHandlerWithMocks.java | 12 ++--
.../web/TestDelegationTokenManager.java | 21 ++-
.../TestRMWebServicesDelegationTokens.java | 6 +-
9 files changed, 135 insertions(+), 55 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdce8837/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index 57402e1..9f43937 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -594,6 +594,10 @@ Release 2.6.0 - UNRELEASED
HADOOP-11184. Update Hadoop's lz4 to version r123. (cmccabe)
+HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all
+sub-classes of AbstractDelegationTokenIdentifier. (zjshen)
+
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cdce8837/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
index 089cd99..168a77b 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
@@ -53,26 +53,9 @@ extends TokenIdentifier {
}
public AbstractDelegationTokenIdentifier(Text owner, Text renewer, Text
realUser) {
-if (owner == null) {
- this.owner = new Text();
-} else {
- this.owner = owner;
-}
-if (renewer == null) {
- this.renewer = new Text();
-} else {
- HadoopKerberosName renewerKrbName = new
HadoopKerberosName(renewer.toString());
- try {
-this.renewer = new Text(renewerKrbName.getShortName());
- } catch (IOException e) {
-throw new RuntimeException(e);
- }
-}
-if (realUser == null) {
- this.realUser = new Text();
-} else {
- this.realUser = realUser;
-}
+setOwner(owner);
+setRenewer(renewer);
+setRealUser(realUser);
issueDate = 0;
maxDate = 0;
}
@@ -107,14 +90,43 @@ extends TokenIdentifier {
return owner;
}
+ public void setOwner(Text owner) {
+if (owner == null) {
+ this.owner = new Text();
+} else {
+ this.owner = owner;
+}
+ }
+
public Text getRenewer() {
return renewer;
}
-
+
+ public void setRenewer(Text renewer) {
+if (renewer == null) {
+ this.renewer = new Text();
+} else {
+ HadoopKerberosName renewerKrbName = new
HadoopKerberosName(renewer.toString());
+ try {
+this.renewer = new Text(renewerKrbName.getShortName());
+ } catch (IOException e) {
+throw new RuntimeException(e);
+ }
+}
+ }
+
public Text getRealUser() {
return realUser;
}
-
+
+ public void setRealUser(Text realUser) {
+if (realUser == null) {
+ this.realUser = new Text();
+} else {
+ this.realUser = realUser;
+}
+ }
+
public void setIssueDate(long issueDate) {
this.issueDate = issueDate;
}
http://git-wip-us.apache.org/repos/asf
git commit: HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all sub-classes of AbstractDelegationTokenIdentifier. Contributed by Zhijie Shen.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 6c3ac6a48 - 8aeda20b2
HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all
sub-classes of AbstractDelegationTokenIdentifier. Contributed by Zhijie Shen.
(cherry picked from commit cdce88376a60918dfe2f3bcd82a7666d74992a19)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8aeda20b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8aeda20b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8aeda20b
Branch: refs/heads/branch-2
Commit: 8aeda20b25fd483d6055dda0b473656a2278d8f1
Parents: 6c3ac6a
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 14 11:35:38 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 14 11:37:28 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++
.../AbstractDelegationTokenIdentifier.java | 56 +++---
.../AbstractDelegationTokenSecretManager.java | 13 +
.../DelegationTokenAuthenticationHandler.java | 11 ++--
.../delegation/web/DelegationTokenManager.java | 60 ++--
.../TestZKDelegationTokenSecretManager.java | 7 ++-
...tionTokenAuthenticationHandlerWithMocks.java | 12 ++--
.../web/TestDelegationTokenManager.java | 21 ++-
.../TestRMWebServicesDelegationTokens.java | 6 +-
9 files changed, 135 insertions(+), 55 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeda20b/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index f93eb7f..c42d4fa 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -249,6 +249,10 @@ Release 2.6.0 - UNRELEASED
HADOOP-11184. Update Hadoop's lz4 to version r123. (cmccabe)
+HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all
+sub-classes of AbstractDelegationTokenIdentifier. (zjshen)
+
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeda20b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
index 089cd99..168a77b 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
@@ -53,26 +53,9 @@ extends TokenIdentifier {
}
public AbstractDelegationTokenIdentifier(Text owner, Text renewer, Text
realUser) {
-if (owner == null) {
- this.owner = new Text();
-} else {
- this.owner = owner;
-}
-if (renewer == null) {
- this.renewer = new Text();
-} else {
- HadoopKerberosName renewerKrbName = new
HadoopKerberosName(renewer.toString());
- try {
-this.renewer = new Text(renewerKrbName.getShortName());
- } catch (IOException e) {
-throw new RuntimeException(e);
- }
-}
-if (realUser == null) {
- this.realUser = new Text();
-} else {
- this.realUser = realUser;
-}
+setOwner(owner);
+setRenewer(renewer);
+setRealUser(realUser);
issueDate = 0;
maxDate = 0;
}
@@ -107,14 +90,43 @@ extends TokenIdentifier {
return owner;
}
+ public void setOwner(Text owner) {
+if (owner == null) {
+ this.owner = new Text();
+} else {
+ this.owner = owner;
+}
+ }
+
public Text getRenewer() {
return renewer;
}
-
+
+ public void setRenewer(Text renewer) {
+if (renewer == null) {
+ this.renewer = new Text();
+} else {
+ HadoopKerberosName renewerKrbName = new
HadoopKerberosName(renewer.toString());
+ try {
+this.renewer = new Text(renewerKrbName.getShortName());
+ } catch (IOException e) {
+throw new RuntimeException(e);
+ }
+}
+ }
+
public Text getRealUser() {
return realUser;
}
-
+
+ public void setRealUser(Text realUser) {
+if (realUser == null) {
+ this.realUser = new Text();
+} else {
+ this.realUser = realUser;
+}
+ }
+
public void setIssueDate(long issueDate
git commit: HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all sub-classes of AbstractDelegationTokenIdentifier. Contributed by Zhijie Shen.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 3d2d501ab - e71fa82ee
HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all
sub-classes of AbstractDelegationTokenIdentifier. Contributed by Zhijie Shen.
(cherry picked from commit cdce88376a60918dfe2f3bcd82a7666d74992a19)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e71fa82e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e71fa82e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e71fa82e
Branch: refs/heads/branch-2.6
Commit: e71fa82ee51cce2141b10fe842f28bec18b93f22
Parents: 3d2d501
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 14 11:35:38 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 14 11:39:06 2014 -0700
--
hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++
.../AbstractDelegationTokenIdentifier.java | 56 +++---
.../AbstractDelegationTokenSecretManager.java | 13 +
.../DelegationTokenAuthenticationHandler.java | 11 ++--
.../delegation/web/DelegationTokenManager.java | 60 ++--
.../TestZKDelegationTokenSecretManager.java | 7 ++-
...tionTokenAuthenticationHandlerWithMocks.java | 12 ++--
.../web/TestDelegationTokenManager.java | 21 ++-
.../TestRMWebServicesDelegationTokens.java | 6 +-
9 files changed, 135 insertions(+), 55 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e71fa82e/hadoop-common-project/hadoop-common/CHANGES.txt
--
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt
b/hadoop-common-project/hadoop-common/CHANGES.txt
index bfaa9b3..77a2677 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -229,6 +229,10 @@ Release 2.6.0 - UNRELEASED
HADOOP-11184. Update Hadoop's lz4 to version r123. (cmccabe)
+HADOOP-11181. Generalized o.a.h.s.t.d.DelegationTokenManager to handle all
+sub-classes of AbstractDelegationTokenIdentifier. (zjshen)
+
+
OPTIMIZATIONS
HADOOP-10838. Byte array native checksumming. (James Thomas via todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e71fa82e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
index 089cd99..168a77b 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/AbstractDelegationTokenIdentifier.java
@@ -53,26 +53,9 @@ extends TokenIdentifier {
}
public AbstractDelegationTokenIdentifier(Text owner, Text renewer, Text
realUser) {
-if (owner == null) {
- this.owner = new Text();
-} else {
- this.owner = owner;
-}
-if (renewer == null) {
- this.renewer = new Text();
-} else {
- HadoopKerberosName renewerKrbName = new
HadoopKerberosName(renewer.toString());
- try {
-this.renewer = new Text(renewerKrbName.getShortName());
- } catch (IOException e) {
-throw new RuntimeException(e);
- }
-}
-if (realUser == null) {
- this.realUser = new Text();
-} else {
- this.realUser = realUser;
-}
+setOwner(owner);
+setRenewer(renewer);
+setRealUser(realUser);
issueDate = 0;
maxDate = 0;
}
@@ -107,14 +90,43 @@ extends TokenIdentifier {
return owner;
}
+ public void setOwner(Text owner) {
+if (owner == null) {
+ this.owner = new Text();
+} else {
+ this.owner = owner;
+}
+ }
+
public Text getRenewer() {
return renewer;
}
-
+
+ public void setRenewer(Text renewer) {
+if (renewer == null) {
+ this.renewer = new Text();
+} else {
+ HadoopKerberosName renewerKrbName = new
HadoopKerberosName(renewer.toString());
+ try {
+this.renewer = new Text(renewerKrbName.getShortName());
+ } catch (IOException e) {
+throw new RuntimeException(e);
+ }
+}
+ }
+
public Text getRealUser() {
return realUser;
}
-
+
+ public void setRealUser(Text realUser) {
+if (realUser == null) {
+ this.realUser = new Text();
+} else {
+ this.realUser = realUser;
+}
+ }
+
public void setIssueDate(long issueDate
git commit: YARN-2656. Made RM web services authentication filter support proxy user. Contributed by Varun Vasudev and Zhijie Shen.
Repository: hadoop
Updated Branches:
refs/heads/trunk 026023166 - 1220bb72d
YARN-2656. Made RM web services authentication filter support proxy user.
Contributed by Varun Vasudev and Zhijie Shen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1220bb72
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1220bb72
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1220bb72
Branch: refs/heads/trunk
Commit: 1220bb72d452521c6f09cebe1dd77341054ee9dd
Parents: 0260231
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 14 21:50:46 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 14 21:50:46 2014 -0700
--
.../DelegationTokenAuthenticationHandler.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../security/http/RMAuthenticationFilter.java | 70 ++---
.../http/RMAuthenticationFilterInitializer.java | 20 ++-
.../server/resourcemanager/ResourceManager.java | 2 +
.../resourcemanager/webapp/RMWebServices.java | 13 +-
.../resourcemanager/TestRMAdminService.java | 1 +
...ebServicesDelegationTokenAuthentication.java | 151 ---
.../src/site/apt/ResourceManagerRest.apt.vm | 2 +-
9 files changed, 218 insertions(+), 46 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1220bb72/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index b73b062..8321c5e 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -81,7 +81,7 @@ public abstract class DelegationTokenAuthenticationHandler
private static final SetString DELEGATION_TOKEN_OPS = new
HashSetString();
- static final String DELEGATION_TOKEN_UGI_ATTRIBUTE =
+ public static final String DELEGATION_TOKEN_UGI_ATTRIBUTE =
hadoop.security.delegation-token.ugi;
static {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1220bb72/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5cadc49..4c5d2e7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -162,6 +162,9 @@ Release 2.6.0 - UNRELEASED
YARN-2501. Enhanced AMRMClient library to support requests against node
labels. (Wangda Tan via vinodkv)
+YARN-2656. Made RM web services authentication filter support proxy user.
+(Varun Vasudev and Zhijie Shen via zjshen)
+
IMPROVEMENTS
YARN-2197. Add a link to YARN CHANGES.txt in the left side of doc
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1220bb72/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
index 651b5b0..3eeb620 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
@@ -18,46 +18,74 @@
package org.apache.hadoop.yarn.server.security.http;
-import java.util.Properties;
+import java.io.IOException;
+import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable
git commit: YARN-2656. Made RM web services authentication filter support proxy user. Contributed by Varun Vasudev and Zhijie Shen.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 174138ec2 - bec165e16
YARN-2656. Made RM web services authentication filter support proxy user.
Contributed by Varun Vasudev and Zhijie Shen.
(cherry picked from commit 1220bb72d452521c6f09cebe1dd77341054ee9dd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/bec165e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/bec165e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/bec165e1
Branch: refs/heads/branch-2
Commit: bec165e163e827f1a1a8f39fe5491c26faa9409e
Parents: 174138e
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 14 21:50:46 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 14 21:56:44 2014 -0700
--
.../DelegationTokenAuthenticationHandler.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../security/http/RMAuthenticationFilter.java | 70 ++---
.../http/RMAuthenticationFilterInitializer.java | 20 ++-
.../server/resourcemanager/ResourceManager.java | 2 +
.../resourcemanager/webapp/RMWebServices.java | 13 +-
.../resourcemanager/TestRMAdminService.java | 1 +
...ebServicesDelegationTokenAuthentication.java | 151 ---
.../src/site/apt/ResourceManagerRest.apt.vm | 2 +-
9 files changed, 218 insertions(+), 46 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bec165e1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index b73b062..8321c5e 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -81,7 +81,7 @@ public abstract class DelegationTokenAuthenticationHandler
private static final SetString DELEGATION_TOKEN_OPS = new
HashSetString();
- static final String DELEGATION_TOKEN_UGI_ATTRIBUTE =
+ public static final String DELEGATION_TOKEN_UGI_ATTRIBUTE =
hadoop.security.delegation-token.ugi;
static {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bec165e1/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b58d5f1..ceeebd6 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -135,6 +135,9 @@ Release 2.6.0 - UNRELEASED
YARN-2501. Enhanced AMRMClient library to support requests against node
labels. (Wangda Tan via vinodkv)
+YARN-2656. Made RM web services authentication filter support proxy user.
+(Varun Vasudev and Zhijie Shen via zjshen)
+
IMPROVEMENTS
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bec165e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
index 651b5b0..3eeb620 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
@@ -18,46 +18,74 @@
package org.apache.hadoop.yarn.server.security.http;
-import java.util.Properties;
+import java.io.IOException;
+import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
import
git commit: YARN-2656. Made RM web services authentication filter support proxy user. Contributed by Varun Vasudev and Zhijie Shen.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 e71fa82ee - eb055eadd
YARN-2656. Made RM web services authentication filter support proxy user.
Contributed by Varun Vasudev and Zhijie Shen.
(cherry picked from commit 1220bb72d452521c6f09cebe1dd77341054ee9dd)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eb055ead
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eb055ead
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eb055ead
Branch: refs/heads/branch-2.6
Commit: eb055eadd74d83755c4def6781388785bb264a6f
Parents: e71fa82
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 14 21:50:46 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 14 21:58:12 2014 -0700
--
.../DelegationTokenAuthenticationHandler.java | 2 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../security/http/RMAuthenticationFilter.java | 70 ++---
.../http/RMAuthenticationFilterInitializer.java | 20 ++-
.../server/resourcemanager/ResourceManager.java | 2 +
.../resourcemanager/webapp/RMWebServices.java | 13 +-
.../resourcemanager/TestRMAdminService.java | 1 +
...ebServicesDelegationTokenAuthentication.java | 151 ---
.../src/site/apt/ResourceManagerRest.apt.vm | 2 +-
9 files changed, 218 insertions(+), 46 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb055ead/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
--
diff --git
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
index b73b062..8321c5e 100644
---
a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
+++
b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/security/token/delegation/web/DelegationTokenAuthenticationHandler.java
@@ -81,7 +81,7 @@ public abstract class DelegationTokenAuthenticationHandler
private static final SetString DELEGATION_TOKEN_OPS = new
HashSetString();
- static final String DELEGATION_TOKEN_UGI_ATTRIBUTE =
+ public static final String DELEGATION_TOKEN_UGI_ATTRIBUTE =
hadoop.security.delegation-token.ugi;
static {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb055ead/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9d5dbd1..e0d7cb4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -99,6 +99,9 @@ Release 2.6.0 - UNRELEASED
YARN-2501. Enhanced AMRMClient library to support requests against node
labels. (Wangda Tan via vinodkv)
+YARN-2656. Made RM web services authentication filter support proxy user.
+(Varun Vasudev and Zhijie Shen via zjshen)
+
IMPROVEMENTS
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eb055ead/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
index 651b5b0..3eeb620 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/http/RMAuthenticationFilter.java
@@ -18,46 +18,74 @@
package org.apache.hadoop.yarn.server.security.http;
-import java.util.Properties;
+import java.io.IOException;
+import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
+import javax.servlet.ServletRequest;
+import javax.servlet.ServletResponse;
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletRequestWrapper;
import
git commit: YARN-2673. Made timeline client put APIs retry if ConnectException happens. Contributed by Li Lu.
Repository: hadoop
Updated Branches:
refs/heads/trunk e4d6a8785 - 89427419a
YARN-2673. Made timeline client put APIs retry if ConnectException happens.
Contributed by Li Lu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/89427419
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/89427419
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/89427419
Branch: refs/heads/trunk
Commit: 89427419a3c5eaab0f73bae98d675979b9efab5f
Parents: e4d6a87
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 20 12:20:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 20 12:22:17 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 17
.../client/api/impl/TimelineClientImpl.java | 86
.../src/main/resources/yarn-default.xml | 16
.../client/api/impl/TestTimelineClient.java | 26 ++
5 files changed, 148 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89427419/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 52485c8..b32d58b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -373,6 +373,9 @@ Release 2.6.0 - UNRELEASED
YARN-2676. Enhanced Timeline auth-filter to support proxy users. (Zhijie
Shen
via vinodkv)
+YARN-2673. Made timeline client put APIs retry if ConnectException happens.
+(Li Lu via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89427419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index a81c1a6..992d3ea 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1317,6 +1317,23 @@ public class YarnConfiguration extends Configuration {
public static final boolean
TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT = false;
+ /** Timeline client settings */
+ public static final String TIMELINE_SERVICE_CLIENT_PREFIX =
+ TIMELINE_SERVICE_PREFIX + client.;
+
+ /** Timeline client call, max retries (-1 means no limit) */
+ public static final String TIMELINE_SERVICE_CLIENT_MAX_RETRIES =
+ TIMELINE_SERVICE_CLIENT_PREFIX + max-retries;
+
+ public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES = 30;
+
+ /** Timeline client call, retry interval */
+ public static final String TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS =
+ TIMELINE_SERVICE_CLIENT_PREFIX + retry-interval-ms;
+
+ public static final long
+ DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000;
+
// ///
// Shared Cache Configs
// ///
http://git-wip-us.apache.org/repos/asf/hadoop/blob/89427419/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 1b863d5..a2efbc6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.client.api.impl;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
+import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
@@ -67,7 +68,10 @@ import org.codehaus.jackson.map.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.sun.jersey.api.client.Client;
+import com.sun.jersey.api.client.filter.ClientFilter;
import com.sun.jersey.api.client.ClientResponse;
+import
git commit: YARN-2673. Made timeline client put APIs retry if ConnectException happens. Contributed by Li Lu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 f8e4cec08 - 42ceed8f2
YARN-2673. Made timeline client put APIs retry if ConnectException happens.
Contributed by Li Lu.
(cherry picked from commit 89427419a3c5eaab0f73bae98d675979b9efab5f)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/42ceed8f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/42ceed8f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/42ceed8f
Branch: refs/heads/branch-2
Commit: 42ceed8f200a5a04db36fe481079d1597b75bc50
Parents: f8e4cec
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 20 12:20:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 20 12:23:10 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 17
.../client/api/impl/TimelineClientImpl.java | 86
.../src/main/resources/yarn-default.xml | 16
.../client/api/impl/TestTimelineClient.java | 26 ++
5 files changed, 148 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ceed8f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 009e643..9251235 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -343,6 +343,9 @@ Release 2.6.0 - UNRELEASED
YARN-2676. Enhanced Timeline auth-filter to support proxy users. (Zhijie
Shen
via vinodkv)
+YARN-2673. Made timeline client put APIs retry if ConnectException happens.
+(Li Lu via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ceed8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 03a1f60..9ca2a66 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1317,6 +1317,23 @@ public class YarnConfiguration extends Configuration {
public static final boolean
TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT = false;
+ /** Timeline client settings */
+ public static final String TIMELINE_SERVICE_CLIENT_PREFIX =
+ TIMELINE_SERVICE_PREFIX + client.;
+
+ /** Timeline client call, max retries (-1 means no limit) */
+ public static final String TIMELINE_SERVICE_CLIENT_MAX_RETRIES =
+ TIMELINE_SERVICE_CLIENT_PREFIX + max-retries;
+
+ public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES = 30;
+
+ /** Timeline client call, retry interval */
+ public static final String TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS =
+ TIMELINE_SERVICE_CLIENT_PREFIX + retry-interval-ms;
+
+ public static final long
+ DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000;
+
// ///
// Shared Cache Configs
// ///
http://git-wip-us.apache.org/repos/asf/hadoop/blob/42ceed8f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 1b863d5..a2efbc6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.client.api.impl;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
+import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
@@ -67,7 +68,10 @@ import org.codehaus.jackson.map.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
import com.sun.jersey.api.client.Client;
+import
git commit: YARN-2673. Made timeline client put APIs retry if ConnectException happens. Contributed by Li Lu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 cdf265de4 - 6d78dc3ee
YARN-2673. Made timeline client put APIs retry if ConnectException happens.
Contributed by Li Lu.
(cherry picked from commit 89427419a3c5eaab0f73bae98d675979b9efab5f)
Conflicts:
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d78dc3e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d78dc3e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d78dc3e
Branch: refs/heads/branch-2.6
Commit: 6d78dc3ee465c0fbc62da13db3f127bd97133b1d
Parents: cdf265d
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 20 12:20:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 20 12:31:33 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../hadoop/yarn/conf/YarnConfiguration.java | 17
.../client/api/impl/TimelineClientImpl.java | 86
.../src/main/resources/yarn-default.xml | 16
.../client/api/impl/TestTimelineClient.java | 26 ++
5 files changed, 148 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d78dc3e/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index aacac34..1f78e26 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -303,6 +303,9 @@ Release 2.6.0 - UNRELEASED
YARN-2676. Enhanced Timeline auth-filter to support proxy users. (Zhijie
Shen
via vinodkv)
+YARN-2673. Made timeline client put APIs retry if ConnectException happens.
+(Li Lu via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d78dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index cd5ff5e..57c8da1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1311,6 +1311,23 @@ public class YarnConfiguration extends Configuration {
public static final boolean
TIMELINE_SERVICE_HTTP_CROSS_ORIGIN_ENABLED_DEFAULT = false;
+ /** Timeline client settings */
+ public static final String TIMELINE_SERVICE_CLIENT_PREFIX =
+ TIMELINE_SERVICE_PREFIX + client.;
+
+ /** Timeline client call, max retries (-1 means no limit) */
+ public static final String TIMELINE_SERVICE_CLIENT_MAX_RETRIES =
+ TIMELINE_SERVICE_CLIENT_PREFIX + max-retries;
+
+ public static final int DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES = 30;
+
+ /** Timeline client call, retry interval */
+ public static final String TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS =
+ TIMELINE_SERVICE_CLIENT_PREFIX + retry-interval-ms;
+
+ public static final long
+ DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS = 1000;
+
// Other Configs
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d78dc3e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index 1b863d5..a2efbc6 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.client.api.impl;
import java.io.File;
import java.io.IOException;
import java.lang.reflect.UndeclaredThrowableException;
+import java.net.ConnectException;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
@@ -67,7 +68,10 @@ import
git commit: YARN-2582. Fixed Log CLI and Web UI for showing aggregated logs of LRS. Contributed Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 42ceed8f2 - f75570734
YARN-2582. Fixed Log CLI and Web UI for showing aggregated logs of LRS.
Contributed Xuan Gong.
(cherry picked from commit e90718fa5a0e7c18592af61534668acebb9db51b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f7557073
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f7557073
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f7557073
Branch: refs/heads/branch-2
Commit: f75570734bb41a61d66b10922b6fc1ed48119067
Parents: 42ceed8
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 20 13:38:59 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 20 13:42:22 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 31 +--
.../hadoop/yarn/client/cli/TestLogsCLI.java | 129 +++-
.../logaggregation/LogAggregationUtils.java | 5 +
.../yarn/logaggregation/LogCLIHelpers.java | 122 +++
.../yarn/webapp/log/AggregatedLogsBlock.java| 207 ++-
.../logaggregation/TestAggregatedLogsBlock.java | 3 -
7 files changed, 342 insertions(+), 158 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7557073/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 9251235..ccfc1db 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -346,6 +346,9 @@ Release 2.6.0 - UNRELEASED
YARN-2673. Made timeline client put APIs retry if ConnectException happens.
(Li Lu via zjshen)
+YARN-2582. Fixed Log CLI and Web UI for showing aggregated logs of LRS.
(Xuan
+Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f7557073/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index eb6169c..0b34a46 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -31,7 +31,6 @@ import
org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -39,8 +38,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
-import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -113,17 +110,16 @@ public class LogsCLI extends Configured implements Tool {
System.err.println(Invalid ApplicationId specified);
return -1;
}
-
+
try {
int resultCode = verifyApplicationState(appId);
if (resultCode != 0) {
-System.out.println(Application has not completed. +
-Logs are only available after an application
completes);
+System.out.println(Logs are not avaiable right now.);
return resultCode;
}
} catch (Exception e) {
- System.err.println(Unable to get ApplicationState. +
-Attempting to fetch logs directly from the filesystem.);
+ System.err.println(Unable to get ApplicationState.
+ + Attempting to fetch logs directly from the filesystem.);
}
LogCLIHelpers logCliHelper = new LogCLIHelpers();
@@ -141,18 +137,9 @@ public class LogsCLI extends Configured implements Tool {
printHelpMessage(printOpts);
resultCode = -1;
} else {
- Path remoteRootLogDir =
-new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
-YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
- AggregatedLogFormat.LogReader reader =
- new
git commit: YARN-2582. Fixed Log CLI and Web UI for showing aggregated logs of LRS. Contributed Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 6d78dc3ee - 2af7f4862
YARN-2582. Fixed Log CLI and Web UI for showing aggregated logs of LRS.
Contributed Xuan Gong.
(cherry picked from commit e90718fa5a0e7c18592af61534668acebb9db51b)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2af7f486
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2af7f486
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2af7f486
Branch: refs/heads/branch-2.6
Commit: 2af7f48625e3a357d1822035fdfb1f7fa6c70f6d
Parents: 6d78dc3
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 20 13:38:59 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 20 13:43:03 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 31 +--
.../hadoop/yarn/client/cli/TestLogsCLI.java | 129 +++-
.../logaggregation/LogAggregationUtils.java | 5 +
.../yarn/logaggregation/LogCLIHelpers.java | 122 +++
.../yarn/webapp/log/AggregatedLogsBlock.java| 207 ++-
.../logaggregation/TestAggregatedLogsBlock.java | 3 -
7 files changed, 342 insertions(+), 158 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2af7f486/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 1f78e26..ea525e1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -306,6 +306,9 @@ Release 2.6.0 - UNRELEASED
YARN-2673. Made timeline client put APIs retry if ConnectException happens.
(Li Lu via zjshen)
+YARN-2582. Fixed Log CLI and Web UI for showing aggregated logs of LRS.
(Xuan
+Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2af7f486/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index eb6169c..0b34a46 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -31,7 +31,6 @@ import
org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -39,8 +38,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
-import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -113,17 +110,16 @@ public class LogsCLI extends Configured implements Tool {
System.err.println(Invalid ApplicationId specified);
return -1;
}
-
+
try {
int resultCode = verifyApplicationState(appId);
if (resultCode != 0) {
-System.out.println(Application has not completed. +
-Logs are only available after an application
completes);
+System.out.println(Logs are not avaiable right now.);
return resultCode;
}
} catch (Exception e) {
- System.err.println(Unable to get ApplicationState. +
-Attempting to fetch logs directly from the filesystem.);
+ System.err.println(Unable to get ApplicationState.
+ + Attempting to fetch logs directly from the filesystem.);
}
LogCLIHelpers logCliHelper = new LogCLIHelpers();
@@ -141,18 +137,9 @@ public class LogsCLI extends Configured implements Tool {
printHelpMessage(printOpts);
resultCode = -1;
} else {
- Path remoteRootLogDir =
-new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
-YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
- AggregatedLogFormat.LogReader reader =
- new
git commit: YARN-2582. Fixed Log CLI and Web UI for showing aggregated logs of LRS. Contributed Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk 89427419a - e90718fa5
YARN-2582. Fixed Log CLI and Web UI for showing aggregated logs of LRS.
Contributed Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e90718fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e90718fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e90718fa
Branch: refs/heads/trunk
Commit: e90718fa5a0e7c18592af61534668acebb9db51b
Parents: 8942741
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 20 13:38:59 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 20 13:38:59 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../apache/hadoop/yarn/client/cli/LogsCLI.java | 31 +--
.../hadoop/yarn/client/cli/TestLogsCLI.java | 129 +++-
.../logaggregation/LogAggregationUtils.java | 5 +
.../yarn/logaggregation/LogCLIHelpers.java | 122 +++
.../yarn/webapp/log/AggregatedLogsBlock.java| 207 ++-
.../logaggregation/TestAggregatedLogsBlock.java | 3 -
7 files changed, 342 insertions(+), 158 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e90718fa/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b32d58b..8fd5aa4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -376,6 +376,9 @@ Release 2.6.0 - UNRELEASED
YARN-2673. Made timeline client put APIs retry if ConnectException happens.
(Li Lu via zjshen)
+YARN-2582. Fixed Log CLI and Web UI for showing aggregated logs of LRS.
(Xuan
+Gong via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e90718fa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
index eb6169c..0b34a46 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/LogsCLI.java
@@ -31,7 +31,6 @@ import
org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -39,8 +38,6 @@ import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
-import org.apache.hadoop.yarn.logaggregation.AggregatedLogFormat;
-import org.apache.hadoop.yarn.logaggregation.LogAggregationUtils;
import org.apache.hadoop.yarn.logaggregation.LogCLIHelpers;
import org.apache.hadoop.yarn.util.ConverterUtils;
@@ -113,17 +110,16 @@ public class LogsCLI extends Configured implements Tool {
System.err.println(Invalid ApplicationId specified);
return -1;
}
-
+
try {
int resultCode = verifyApplicationState(appId);
if (resultCode != 0) {
-System.out.println(Application has not completed. +
-Logs are only available after an application
completes);
+System.out.println(Logs are not avaiable right now.);
return resultCode;
}
} catch (Exception e) {
- System.err.println(Unable to get ApplicationState. +
-Attempting to fetch logs directly from the filesystem.);
+ System.err.println(Unable to get ApplicationState.
+ + Attempting to fetch logs directly from the filesystem.);
}
LogCLIHelpers logCliHelper = new LogCLIHelpers();
@@ -141,18 +137,9 @@ public class LogsCLI extends Configured implements Tool {
printHelpMessage(printOpts);
resultCode = -1;
} else {
- Path remoteRootLogDir =
-new Path(getConf().get(YarnConfiguration.NM_REMOTE_APP_LOG_DIR,
-YarnConfiguration.DEFAULT_NM_REMOTE_APP_LOG_DIR));
- AggregatedLogFormat.LogReader reader =
- new AggregatedLogFormat.LogReader(getConf
git commit: YARN-2717. Avoided duplicate logging when container logs are not found. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk 4a78a7522 - 171f2376d
YARN-2717. Avoided duplicate logging when container logs are not found.
Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/171f2376
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/171f2376
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/171f2376
Branch: refs/heads/trunk
Commit: 171f2376d23d51b61b9c9b3804ee86dbd4de033a
Parents: 4a78a75
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 20 22:21:37 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 20 22:21:37 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java | 1 -
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/171f2376/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 08b7342..b130ecf 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -702,6 +702,9 @@ Release 2.6.0 - UNRELEASED
YARN-2701. Potential race condition in startLocalizer when using
LinuxContainerExecutor. (Xuan Gong via jianhe)
+YARN-2717. Avoided duplicate logging when container logs are not found.
(Xuan
+Gong via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/171f2376/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 9efdef8..de06d48 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -109,7 +109,6 @@ public class LogCLIHelpers implements Configurable {
}
if (valueStream == null) {
- containerLogNotFound(containerIdStr);
return -1;
}
git commit: YARN-2717. Avoided duplicate logging when container logs are not found. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b52928a6e - fd7ba56f6
YARN-2717. Avoided duplicate logging when container logs are not found.
Contributed by Xuan Gong.
(cherry picked from commit 171f2376d23d51b61b9c9b3804ee86dbd4de033a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fd7ba56f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fd7ba56f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fd7ba56f
Branch: refs/heads/branch-2
Commit: fd7ba56f6adec6c6ee7472b687f8c32f59109f84
Parents: b52928a
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 20 22:21:37 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 20 22:23:12 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java | 1 -
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd7ba56f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 04b7e1e..271444c 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -672,6 +672,9 @@ Release 2.6.0 - UNRELEASED
YARN-2701. Potential race condition in startLocalizer when using
LinuxContainerExecutor. (Xuan Gong via jianhe)
+YARN-2717. Avoided duplicate logging when container logs are not found.
(Xuan
+Gong via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fd7ba56f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 9efdef8..de06d48 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -109,7 +109,6 @@ public class LogCLIHelpers implements Configurable {
}
if (valueStream == null) {
- containerLogNotFound(containerIdStr);
return -1;
}
git commit: YARN-2717. Avoided duplicate logging when container logs are not found. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 852ffea84 - fcdeba204
YARN-2717. Avoided duplicate logging when container logs are not found.
Contributed by Xuan Gong.
(cherry picked from commit 171f2376d23d51b61b9c9b3804ee86dbd4de033a)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fcdeba20
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fcdeba20
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fcdeba20
Branch: refs/heads/branch-2.6
Commit: fcdeba204ca956be62f25c036cccbab9d7884b99
Parents: 852ffea
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 20 22:21:37 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 20 22:24:19 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java | 1 -
2 files changed, 3 insertions(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcdeba20/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index fff076d..ef1e8ad 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -635,6 +635,9 @@ Release 2.6.0 - UNRELEASED
YARN-2701. Potential race condition in startLocalizer when using
LinuxContainerExecutor. (Xuan Gong via jianhe)
+YARN-2717. Avoided duplicate logging when container logs are not found.
(Xuan
+Gong via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fcdeba20/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
index 9efdef8..de06d48 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
@@ -109,7 +109,6 @@ public class LogCLIHelpers implements Configurable {
}
if (valueStream == null) {
- containerLogNotFound(containerIdStr);
return -1;
}
git commit: YARN-2709. Made timeline client getDelegationToken API retry if ConnectException happens. Contributed by Li Lu.
Repository: hadoop
Updated Branches:
refs/heads/trunk 4baca311f - b2942762d
YARN-2709. Made timeline client getDelegationToken API retry if
ConnectException happens. Contributed by Li Lu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2942762
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2942762
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2942762
Branch: refs/heads/trunk
Commit: b2942762d7f76d510ece5621c71116346a6b12f6
Parents: 4baca31
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 21 16:06:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 21 16:06:39 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../client/api/impl/TimelineClientImpl.java | 177 +--
.../client/api/impl/TestTimelineClient.java | 41 -
3 files changed, 164 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2942762/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 06c0eab..6a0d38f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -382,6 +382,9 @@ Release 2.6.0 - UNRELEASED
YARN-90. NodeManager should identify failed disks becoming good again
(Varun Vasudev via jlowe)
+YARN-2709. Made timeline client getDelegationToken API retry if
ConnectException
+happens. (Li Lu via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2942762/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index a2efbc6..26e6d33 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -107,9 +107,23 @@ public class TimelineClientImpl extends TimelineClient {
private URI resURI;
private boolean isEnabled;
- private TimelineJerseyRetryFilter retryFilter;
+ @Private
+ @VisibleForTesting
+ TimelineClientConnectionRetry connectionRetry;
+
+ // Abstract class for an operation that should be retried by timeline client
+ private static abstract class TimelineClientRetryOp {
+// The operation that should be retried
+public abstract Object run() throws IOException;
+// The method to indicate if we should retry given the incoming exception
+public abstract boolean shouldRetryOn(Exception e);
+ }
- static class TimelineJerseyRetryFilter extends ClientFilter {
+ // Class to handle retry
+ // Outside this class, only visible to tests
+ @Private
+ @VisibleForTesting
+ static class TimelineClientConnectionRetry {
// maxRetries 0 means keep trying
@Private
@VisibleForTesting
@@ -119,14 +133,14 @@ public class TimelineClientImpl extends TimelineClient {
@VisibleForTesting
public long retryInterval;
-// Indicates if retries happened last time
+// Indicates if retries happened last time. Only tests should read it.
+// In unit tests, retryOn() calls should _not_ be concurrent.
@Private
@VisibleForTesting
public boolean retried = false;
// Constructor with default retry settings
-public TimelineJerseyRetryFilter(Configuration conf) {
- super();
+public TimelineClientConnectionRetry(Configuration conf) {
maxRetries = conf.getInt(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES);
@@ -135,32 +149,36 @@ public class TimelineClientImpl extends TimelineClient {
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS);
}
-@Override
-public ClientResponse handle(ClientRequest cr)
-throws ClientHandlerException {
+public Object retryOn(TimelineClientRetryOp op)
+throws RuntimeException, IOException {
int leftRetries = maxRetries;
retried = false;
+
// keep trying
while (true) {
try {
- // try pass the request on, if fail, keep retrying
- return getNext().handle(cr);
-} catch (ClientHandlerException e) {
+ // try perform the op, if fail, keep
git commit: YARN-2709. Made timeline client getDelegationToken API retry if ConnectException happens. Contributed by Li Lu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2d0996ae6 - 296bbd897
YARN-2709. Made timeline client getDelegationToken API retry if
ConnectException happens. Contributed by Li Lu.
(cherry picked from commit b2942762d7f76d510ece5621c71116346a6b12f6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/296bbd89
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/296bbd89
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/296bbd89
Branch: refs/heads/branch-2
Commit: 296bbd897149372e764a4bd28218f962758a3661
Parents: 2d0996a
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 21 16:06:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 21 16:08:18 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../client/api/impl/TimelineClientImpl.java | 177 +--
.../client/api/impl/TestTimelineClient.java | 41 -
3 files changed, 164 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/296bbd89/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index efab925..aca62b7 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -352,6 +352,9 @@ Release 2.6.0 - UNRELEASED
YARN-90. NodeManager should identify failed disks becoming good again
(Varun Vasudev via jlowe)
+YARN-2709. Made timeline client getDelegationToken API retry if
ConnectException
+happens. (Li Lu via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/296bbd89/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index a2efbc6..26e6d33 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -107,9 +107,23 @@ public class TimelineClientImpl extends TimelineClient {
private URI resURI;
private boolean isEnabled;
- private TimelineJerseyRetryFilter retryFilter;
+ @Private
+ @VisibleForTesting
+ TimelineClientConnectionRetry connectionRetry;
+
+ // Abstract class for an operation that should be retried by timeline client
+ private static abstract class TimelineClientRetryOp {
+// The operation that should be retried
+public abstract Object run() throws IOException;
+// The method to indicate if we should retry given the incoming exception
+public abstract boolean shouldRetryOn(Exception e);
+ }
- static class TimelineJerseyRetryFilter extends ClientFilter {
+ // Class to handle retry
+ // Outside this class, only visible to tests
+ @Private
+ @VisibleForTesting
+ static class TimelineClientConnectionRetry {
// maxRetries 0 means keep trying
@Private
@VisibleForTesting
@@ -119,14 +133,14 @@ public class TimelineClientImpl extends TimelineClient {
@VisibleForTesting
public long retryInterval;
-// Indicates if retries happened last time
+// Indicates if retries happened last time. Only tests should read it.
+// In unit tests, retryOn() calls should _not_ be concurrent.
@Private
@VisibleForTesting
public boolean retried = false;
// Constructor with default retry settings
-public TimelineJerseyRetryFilter(Configuration conf) {
- super();
+public TimelineClientConnectionRetry(Configuration conf) {
maxRetries = conf.getInt(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES);
@@ -135,32 +149,36 @@ public class TimelineClientImpl extends TimelineClient {
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS);
}
-@Override
-public ClientResponse handle(ClientRequest cr)
-throws ClientHandlerException {
+public Object retryOn(TimelineClientRetryOp op)
+throws RuntimeException, IOException {
int leftRetries = maxRetries;
retried = false;
+
// keep trying
while (true) {
try {
- // try pass the request on, if fail, keep retrying
- return getNext().handle(cr);
-} catch
git commit: YARN-2709. Made timeline client getDelegationToken API retry if ConnectException happens. Contributed by Li Lu.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 1ec2dbf09 - 7df8cbdd0
YARN-2709. Made timeline client getDelegationToken API retry if
ConnectException happens. Contributed by Li Lu.
(cherry picked from commit b2942762d7f76d510ece5621c71116346a6b12f6)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7df8cbdd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7df8cbdd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7df8cbdd
Branch: refs/heads/branch-2.6
Commit: 7df8cbdd0f13ade74c6e1cff13b49b0d3dc41f1b
Parents: 1ec2dbf
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 21 16:06:39 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 21 16:09:23 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../client/api/impl/TimelineClientImpl.java | 177 +--
.../client/api/impl/TestTimelineClient.java | 41 -
3 files changed, 164 insertions(+), 57 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7df8cbdd/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index a47b0c1..6ca05b4 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -315,6 +315,9 @@ Release 2.6.0 - UNRELEASED
YARN-90. NodeManager should identify failed disks becoming good again
(Varun Vasudev via jlowe)
+YARN-2709. Made timeline client getDelegationToken API retry if
ConnectException
+happens. (Li Lu via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7df8cbdd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
index a2efbc6..26e6d33 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
@@ -107,9 +107,23 @@ public class TimelineClientImpl extends TimelineClient {
private URI resURI;
private boolean isEnabled;
- private TimelineJerseyRetryFilter retryFilter;
+ @Private
+ @VisibleForTesting
+ TimelineClientConnectionRetry connectionRetry;
+
+ // Abstract class for an operation that should be retried by timeline client
+ private static abstract class TimelineClientRetryOp {
+// The operation that should be retried
+public abstract Object run() throws IOException;
+// The method to indicate if we should retry given the incoming exception
+public abstract boolean shouldRetryOn(Exception e);
+ }
- static class TimelineJerseyRetryFilter extends ClientFilter {
+ // Class to handle retry
+ // Outside this class, only visible to tests
+ @Private
+ @VisibleForTesting
+ static class TimelineClientConnectionRetry {
// maxRetries 0 means keep trying
@Private
@VisibleForTesting
@@ -119,14 +133,14 @@ public class TimelineClientImpl extends TimelineClient {
@VisibleForTesting
public long retryInterval;
-// Indicates if retries happened last time
+// Indicates if retries happened last time. Only tests should read it.
+// In unit tests, retryOn() calls should _not_ be concurrent.
@Private
@VisibleForTesting
public boolean retried = false;
// Constructor with default retry settings
-public TimelineJerseyRetryFilter(Configuration conf) {
- super();
+public TimelineClientConnectionRetry(Configuration conf) {
maxRetries = conf.getInt(
YarnConfiguration.TIMELINE_SERVICE_CLIENT_MAX_RETRIES,
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_MAX_RETRIES);
@@ -135,32 +149,36 @@ public class TimelineClientImpl extends TimelineClient {
YarnConfiguration.DEFAULT_TIMELINE_SERVICE_CLIENT_RETRY_INTERVAL_MS);
}
-@Override
-public ClientResponse handle(ClientRequest cr)
-throws ClientHandlerException {
+public Object retryOn(TimelineClientRetryOp op)
+throws RuntimeException, IOException {
int leftRetries = maxRetries;
retried = false;
+
// keep trying
while (true) {
try {
- // try pass the request on, if fail, keep retrying
- return getNext().handle(cr
git commit: YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it retries creating znode. Contributed by Jian He.
Repository: hadoop
Updated Branches:
refs/heads/trunk c0e034336 - 7e3b5e6f5
YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it
retries creating znode. Contributed by Jian He.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e3b5e6f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e3b5e6f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e3b5e6f
Branch: refs/heads/trunk
Commit: 7e3b5e6f5cb4945b4fab27e8a83d04280df50e17
Parents: c0e0343
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 21 21:49:17 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 21 21:49:17 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/recovery/ZKRMStateStore.java | 4
2 files changed, 7 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e3b5e6f/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index b638ab0..4a71b6b 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -717,6 +717,9 @@ Release 2.6.0 - UNRELEASED
YARN-2720. Windows: Wildcard classpath variables not expanded against
resources contained in archives. (Craig Welch via cnauroth)
+YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it
+retries creating znode. (Jian He via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e3b5e6f/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 87c8cbe..fdbf125 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -1094,6 +1094,10 @@ public class ZKRMStateStore extends RMStateStore {
throw new StoreFencedException();
}
} catch (KeeperException ke) {
+ if (ke.code() == Code.NODEEXISTS) {
+LOG.info(znode already exists!);
+return null;
+ }
LOG.info(Exception while executing a ZK operation., ke);
if (shouldRetry(ke.code()) ++retry numRetries) {
LOG.info(Retrying operation on ZK. Retry no. + retry);
git commit: YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it retries creating znode. Contributed by Jian He.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 77dc9afe8 - 4b9de2cdf
YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it
retries creating znode. Contributed by Jian He.
(cherry picked from commit 7e3b5e6f5cb4945b4fab27e8a83d04280df50e17)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4b9de2cd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4b9de2cd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4b9de2cd
Branch: refs/heads/branch-2
Commit: 4b9de2cdfe4f9b4861fb67e0c0c72a4029fc7476
Parents: 77dc9af
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 21 21:49:17 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 21 21:52:01 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/recovery/ZKRMStateStore.java | 4
2 files changed, 7 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b9de2cd/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index be3ca2b..01e1483 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -687,6 +687,9 @@ Release 2.6.0 - UNRELEASED
YARN-2720. Windows: Wildcard classpath variables not expanded against
resources contained in archives. (Craig Welch via cnauroth)
+YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it
+retries creating znode. (Jian He via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4b9de2cd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 87c8cbe..fdbf125 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -1094,6 +1094,10 @@ public class ZKRMStateStore extends RMStateStore {
throw new StoreFencedException();
}
} catch (KeeperException ke) {
+ if (ke.code() == Code.NODEEXISTS) {
+LOG.info(znode already exists!);
+return null;
+ }
LOG.info(Exception while executing a ZK operation., ke);
if (shouldRetry(ke.code()) ++retry numRetries) {
LOG.info(Retrying operation on ZK. Retry no. + retry);
git commit: YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it retries creating znode. Contributed by Jian He.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 a1c5d4c51 - b1ca12bc0
YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it
retries creating znode. Contributed by Jian He.
(cherry picked from commit 7e3b5e6f5cb4945b4fab27e8a83d04280df50e17)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b1ca12bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b1ca12bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b1ca12bc
Branch: refs/heads/branch-2.6
Commit: b1ca12bc0bc5079b4c8bbc7de22ee43a26ec3fec
Parents: a1c5d4c
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 21 21:49:17 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 21 21:53:27 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/recovery/ZKRMStateStore.java | 4
2 files changed, 7 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1ca12bc/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7eaced9..f1442c8 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -650,6 +650,9 @@ Release 2.6.0 - UNRELEASED
YARN-2720. Windows: Wildcard classpath variables not expanded against
resources contained in archives. (Craig Welch via cnauroth)
+YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it
+retries creating znode. (Jian He via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b1ca12bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 87c8cbe..fdbf125 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -1094,6 +1094,10 @@ public class ZKRMStateStore extends RMStateStore {
throw new StoreFencedException();
}
} catch (KeeperException ke) {
+ if (ke.code() == Code.NODEEXISTS) {
+LOG.info(znode already exists!);
+return null;
+ }
LOG.info(Exception while executing a ZK operation., ke);
if (shouldRetry(ke.code()) ++retry numRetries) {
LOG.info(Retrying operation on ZK. Retry no. + retry);
git commit: YARN-2732. Fixed syntax error in SecureContainer.apt.vm. Contributed by Jian He.
Repository: hadoop Updated Branches: refs/heads/trunk f729ecf9d - b94b8b30f YARN-2732. Fixed syntax error in SecureContainer.apt.vm. Contributed by Jian He. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b94b8b30 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b94b8b30 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b94b8b30 Branch: refs/heads/trunk Commit: b94b8b30f282563ee2ecdd25761b2345aaf06c9b Parents: f729ecf Author: Zhijie Shen zjs...@apache.org Authored: Wed Oct 22 18:39:29 2014 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Wed Oct 22 18:39:29 2014 -0700 -- hadoop-yarn-project/CHANGES.txt| 2 ++ .../hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/b94b8b30/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index 422b033..ba6de1e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -728,6 +728,8 @@ Release 2.6.0 - UNRELEASED YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it retries creating znode. (Jian He via zjshen) +YARN-2732. Fixed syntax error in SecureContainer.apt.vm. (Jian He via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/b94b8b30/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm index 722870c..0365bf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm @@ -111,7 +111,7 @@ min.user.id=1000#Prevent other super-users +---+ *** wsce-site.xml - The hadoopwinutilsvc uses %HADOOP_HOME%\etc\hadoop\wsce_site.xml to configure access to the privileged operations. + The hadoopwinutilsvc uses %HADOOP_HOME%\etc\hadoop\wsce_site.xml to configure access to the privileged operations. +---+ property
git commit: YARN-2732. Fixed syntax error in SecureContainer.apt.vm. Contributed by Jian He.
Repository: hadoop Updated Branches: refs/heads/branch-2 59e0ea877 - 36515435f YARN-2732. Fixed syntax error in SecureContainer.apt.vm. Contributed by Jian He. (cherry picked from commit b94b8b30f282563ee2ecdd25761b2345aaf06c9b) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/36515435 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/36515435 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/36515435 Branch: refs/heads/branch-2 Commit: 36515435ff3c34feef7c4771960e7c01c4bcfdde Parents: 59e0ea8 Author: Zhijie Shen zjs...@apache.org Authored: Wed Oct 22 18:39:29 2014 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Wed Oct 22 18:41:05 2014 -0700 -- hadoop-yarn-project/CHANGES.txt| 2 ++ .../hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/36515435/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index f323ff9..143b08e 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -698,6 +698,8 @@ Release 2.6.0 - UNRELEASED YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it retries creating znode. (Jian He via zjshen) +YARN-2732. Fixed syntax error in SecureContainer.apt.vm. (Jian He via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/36515435/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm index 722870c..0365bf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm @@ -111,7 +111,7 @@ min.user.id=1000#Prevent other super-users +---+ *** wsce-site.xml - The hadoopwinutilsvc uses %HADOOP_HOME%\etc\hadoop\wsce_site.xml to configure access to the privileged operations. + The hadoopwinutilsvc uses %HADOOP_HOME%\etc\hadoop\wsce_site.xml to configure access to the privileged operations. +---+ property
git commit: YARN-2732. Fixed syntax error in SecureContainer.apt.vm. Contributed by Jian He.
Repository: hadoop Updated Branches: refs/heads/branch-2.6 29d0164ee - 78efd4fd5 YARN-2732. Fixed syntax error in SecureContainer.apt.vm. Contributed by Jian He. (cherry picked from commit b94b8b30f282563ee2ecdd25761b2345aaf06c9b) Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/78efd4fd Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/78efd4fd Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/78efd4fd Branch: refs/heads/branch-2.6 Commit: 78efd4fd5c9ebb0427cd712bf3d6e3cdf0144717 Parents: 29d0164 Author: Zhijie Shen zjs...@apache.org Authored: Wed Oct 22 18:39:29 2014 -0700 Committer: Zhijie Shen zjs...@apache.org Committed: Wed Oct 22 18:42:27 2014 -0700 -- hadoop-yarn-project/CHANGES.txt| 2 ++ .../hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/hadoop/blob/78efd4fd/hadoop-yarn-project/CHANGES.txt -- diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt index d44e1ca..184fe50 100644 --- a/hadoop-yarn-project/CHANGES.txt +++ b/hadoop-yarn-project/CHANGES.txt @@ -668,6 +668,8 @@ Release 2.6.0 - UNRELEASED YARN-2721. Suppress NodeExist exception thrown by ZKRMStateStore when it retries creating znode. (Jian He via zjshen) +YARN-2732. Fixed syntax error in SecureContainer.apt.vm. (Jian He via zjshen) + Release 2.5.1 - 2014-09-05 INCOMPATIBLE CHANGES http://git-wip-us.apache.org/repos/asf/hadoop/blob/78efd4fd/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm -- diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm index 722870c..0365bf7 100644 --- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm +++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/SecureContainer.apt.vm @@ -111,7 +111,7 @@ min.user.id=1000#Prevent other super-users +---+ *** wsce-site.xml - The hadoopwinutilsvc uses %HADOOP_HOME%\etc\hadoop\wsce_site.xml to configure access to the privileged operations. + The hadoopwinutilsvc uses %HADOOP_HOME%\etc\hadoop\wsce_site.xml to configure access to the privileged operations. +---+ property
git commit: YARN-2209. Replaced AM resync/shutdown command with corresponding exceptions and made related MR changes. Contributed by Jian He.
request to 2nd RM and get resync command
allocator.schedule();
dispatcher.await();
-Assert.assertTrue(Last allocate response is not RESYNC,
-allocator.isResyncCommand());
// Step-5 : On Resync,AM sends all outstanding
// asks,release,blacklistAaddition
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3b6900/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index ba6de1e..bc93be1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -388,6 +388,9 @@ Release 2.6.0 - UNRELEASED
YARN-2709. Made timeline client getDelegationToken API retry if
ConnectException
happens. (Li Lu via zjshen)
+YARN-2209. Replaced AM resync/shutdown command with corresponding
exceptions and
+made related MR changes. (Jian He via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3b6900/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
index e56ba61..46ac642 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
@@ -86,7 +86,7 @@ public abstract class AllocateResponse {
response.setNMTokens(nmTokens);
return response;
}
-
+
@Public
@Stable
public static AllocateResponse newInstance(int responseId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3b6900/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
index 5c9a985..7e8ba2a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
@@ -20,7 +20,11 @@ package org.apache.hadoop.yarn.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import
org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
/**
* Command sent by the Resource Manager to the Application Master in the
@@ -30,16 +34,26 @@ import
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@Public
@Unstable
public enum AMCommand {
+
/**
- * Sent by Resource Manager when it is out of sync with the AM and wants the
- * AM get back in sync.
+ * @deprecated Sent by Resource Manager when it is out of sync with the AM
and
+ * wants the AM get back in sync.
+ *
+ * Note: Instead of sending this command,
+ * {@link ApplicationMasterNotRegisteredException} will be thrown
+ * when ApplicationMaster is out of sync with ResourceManager and
+ * ApplicationMaster is expected to re-register with RM by
calling
+ * {@link
ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}
*/
AM_RESYNC,
-
+
/**
- * Sent by Resource Manager when it wants the AM to shutdown. Eg. when the
- * node is going down for maintenance. The AM should save any state and
- * prepare to be restarted at a later time.
+ * @deprecated Sent by Resource Manager when it wants the AM to shutdown.
+ * Note: This command was earlier sent by ResourceManager to
+ * instruct AM to shutdown if RM had restarted. Now
+ * {@link ApplicationAttemptNotFoundException} will be thrown in
case
+ * that RM has restarted and AM is supposed to handle this
+ * exception by shutting down itself.
*/
AM_SHUTDOWN
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0f3b6900
git commit: YARN-2209. Replaced AM resync/shutdown command with corresponding exceptions and made related MR changes. Contributed by Jian He.
+2256,6 @@ public class TestRMContainerAllocator {
// send allocate request to 2nd RM and get resync command
allocator.schedule();
dispatcher.await();
-Assert.assertTrue(Last allocate response is not RESYNC,
-allocator.isResyncCommand());
// Step-5 : On Resync,AM sends all outstanding
// asks,release,blacklistAaddition
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b03ea6b/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 143b08e..dfcdd31 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -358,6 +358,9 @@ Release 2.6.0 - UNRELEASED
YARN-2709. Made timeline client getDelegationToken API retry if
ConnectException
happens. (Li Lu via zjshen)
+YARN-2209. Replaced AM resync/shutdown command with corresponding
exceptions and
+made related MR changes. (Jian He via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b03ea6b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
index e56ba61..46ac642 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
@@ -86,7 +86,7 @@ public abstract class AllocateResponse {
response.setNMTokens(nmTokens);
return response;
}
-
+
@Public
@Stable
public static AllocateResponse newInstance(int responseId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3b03ea6b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
index 5c9a985..7e8ba2a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
@@ -20,7 +20,11 @@ package org.apache.hadoop.yarn.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import
org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
/**
* Command sent by the Resource Manager to the Application Master in the
@@ -30,16 +34,26 @@ import
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@Public
@Unstable
public enum AMCommand {
+
/**
- * Sent by Resource Manager when it is out of sync with the AM and wants the
- * AM get back in sync.
+ * @deprecated Sent by Resource Manager when it is out of sync with the AM
and
+ * wants the AM get back in sync.
+ *
+ * Note: Instead of sending this command,
+ * {@link ApplicationMasterNotRegisteredException} will be thrown
+ * when ApplicationMaster is out of sync with ResourceManager and
+ * ApplicationMaster is expected to re-register with RM by
calling
+ * {@link
ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}
*/
AM_RESYNC,
-
+
/**
- * Sent by Resource Manager when it wants the AM to shutdown. Eg. when the
- * node is going down for maintenance. The AM should save any state and
- * prepare to be restarted at a later time.
+ * @deprecated Sent by Resource Manager when it wants the AM to shutdown.
+ * Note: This command was earlier sent by ResourceManager to
+ * instruct AM to shutdown if RM had restarted. Now
+ * {@link ApplicationAttemptNotFoundException} will be thrown in
case
+ * that RM has restarted and AM is supposed to handle this
+ * exception by shutting down itself
git commit: YARN-2209. Replaced AM resync/shutdown command with corresponding exceptions and made related MR changes. Contributed by Jian He.
super.isResyncCommand(allocateResponse);
-}
}
@Test
@@ -2250,8 +2256,6 @@ public class TestRMContainerAllocator {
// send allocate request to 2nd RM and get resync command
allocator.schedule();
dispatcher.await();
-Assert.assertTrue(Last allocate response is not RESYNC,
-allocator.isResyncCommand());
// Step-5 : On Resync,AM sends all outstanding
// asks,release,blacklistAaddition
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09e42230/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 184fe50..6df461d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -328,6 +328,9 @@ Release 2.6.0 - UNRELEASED
DefaultContainerExecutor#getFirstApplicationDir and use getWorkingDir()
instead. (Zhihai Xu via jianhe)
+YARN-2209. Replaced AM resync/shutdown command with corresponding
exceptions and
+made related MR changes. (Jian He via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09e42230/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
index e56ba61..46ac642 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/AllocateResponse.java
@@ -86,7 +86,7 @@ public abstract class AllocateResponse {
response.setNMTokens(nmTokens);
return response;
}
-
+
@Public
@Stable
public static AllocateResponse newInstance(int responseId,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/09e42230/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
index 5c9a985..7e8ba2a 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/AMCommand.java
@@ -20,7 +20,11 @@ package org.apache.hadoop.yarn.api.records;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
+import
org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterRequest;
+import org.apache.hadoop.yarn.exceptions.ApplicationAttemptNotFoundException;
+import
org.apache.hadoop.yarn.exceptions.ApplicationMasterNotRegisteredException;
/**
* Command sent by the Resource Manager to the Application Master in the
@@ -30,16 +34,26 @@ import
org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@Public
@Unstable
public enum AMCommand {
+
/**
- * Sent by Resource Manager when it is out of sync with the AM and wants the
- * AM get back in sync.
+ * @deprecated Sent by Resource Manager when it is out of sync with the AM
and
+ * wants the AM get back in sync.
+ *
+ * Note: Instead of sending this command,
+ * {@link ApplicationMasterNotRegisteredException} will be thrown
+ * when ApplicationMaster is out of sync with ResourceManager and
+ * ApplicationMaster is expected to re-register with RM by
calling
+ * {@link
ApplicationMasterProtocol#registerApplicationMaster(RegisterApplicationMasterRequest)}
*/
AM_RESYNC,
-
+
/**
- * Sent by Resource Manager when it wants the AM to shutdown. Eg. when the
- * node is going down for maintenance. The AM should save any state and
- * prepare to be restarted at a later time.
+ * @deprecated Sent by Resource Manager when it wants the AM to shutdown.
+ * Note: This command was earlier sent by ResourceManager to
+ * instruct AM to shutdown if RM had restarted. Now
+ * {@link ApplicationAttemptNotFoundException} will be thrown in
case
+ * that RM has restarted and AM is supposed to handle
git commit: YARN-2724. Skipped uploading a local log file to HDFS if exception is raised when opening it. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk b3d8a642a - e31f0a655
YARN-2724. Skipped uploading a local log file to HDFS if exception is raised
when opening it. Contributed by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e31f0a65
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e31f0a65
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e31f0a65
Branch: refs/heads/trunk
Commit: e31f0a6558b106662c83e1f797216e412b6689a9
Parents: b3d8a64
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Oct 24 11:13:44 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Oct 24 11:13:44 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../logaggregation/AggregatedLogFormat.java | 33 +++-
.../logaggregation/TestAggregatedLogFormat.java | 17 --
3 files changed, 43 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e31f0a65/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 7c76688..cfa0727 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -733,6 +733,9 @@ Release 2.6.0 - UNRELEASED
YARN-2732. Fixed syntax error in SecureContainer.apt.vm. (Jian He via
zjshen)
+YARN-2724. Skipped uploading a local log file to HDFS if exception is
raised
+when opening it. (Xuan Gong via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e31f0a65/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index e1d1e00..46e39cc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@@ -211,6 +212,16 @@ public class AggregatedLogFormat {
Collections.sort(fileList);
for (File logFile : fileList) {
+
+FileInputStream in = null;
+try {
+ in = secureOpenFile(logFile);
+} catch (IOException e) {
+ logErrorMessage(logFile, e);
+ IOUtils.cleanup(LOG, in);
+ continue;
+}
+
final long fileLength = logFile.length();
// Write the logFile Type
out.writeUTF(logFile.getName());
@@ -219,9 +230,7 @@ public class AggregatedLogFormat {
out.writeUTF(String.valueOf(fileLength));
// Write the log itself
-FileInputStream in = null;
try {
- in = SecureIOUtils.openForRead(logFile, getUser(), null);
byte[] buf = new byte[65535];
int len = 0;
long bytesLeft = fileLength;
@@ -244,18 +253,26 @@ public class AggregatedLogFormat {
}
this.uploadedFiles.add(logFile);
} catch (IOException e) {
- String message = Error aggregating log file. Log file :
- + logFile.getAbsolutePath() + e.getMessage();
- LOG.error(message, e);
+ String message = logErrorMessage(logFile, e);
out.write(message.getBytes());
} finally {
- if (in != null) {
-in.close();
- }
+ IOUtils.cleanup(LOG, in);
}
}
}
+@VisibleForTesting
+public FileInputStream secureOpenFile(File logFile) throws IOException {
+ return SecureIOUtils.openForRead(logFile, getUser(), null);
+}
+
+private static String logErrorMessage(File logFile, Exception e) {
+ String message = Error aggregating log file. Log file :
+ + logFile.getAbsolutePath() + . + e.getMessage();
+ LOG.error(message, e);
+ return message;
+}
+
// Added for testing purpose.
public String getUser() {
return user;
http://git-wip-us.apache.org/repos/asf/hadoop/blob
git commit: YARN-2724. Skipped uploading a local log file to HDFS if exception is raised when opening it. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 1f6258f53 - 13c60bac5
YARN-2724. Skipped uploading a local log file to HDFS if exception is raised
when opening it. Contributed by Xuan Gong.
(cherry picked from commit e31f0a6558b106662c83e1f797216e412b6689a9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/13c60bac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/13c60bac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/13c60bac
Branch: refs/heads/branch-2
Commit: 13c60bac57c579526a33107cdb2deb7b093d96b1
Parents: 1f6258f
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Oct 24 11:13:44 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Oct 24 11:15:19 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../logaggregation/AggregatedLogFormat.java | 33 +++-
.../logaggregation/TestAggregatedLogFormat.java | 17 --
3 files changed, 43 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13c60bac/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2e191be..23466a5 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -703,6 +703,9 @@ Release 2.6.0 - UNRELEASED
YARN-2732. Fixed syntax error in SecureContainer.apt.vm. (Jian He via
zjshen)
+YARN-2724. Skipped uploading a local log file to HDFS if exception is
raised
+when opening it. (Xuan Gong via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/13c60bac/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index e1d1e00..46e39cc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@@ -211,6 +212,16 @@ public class AggregatedLogFormat {
Collections.sort(fileList);
for (File logFile : fileList) {
+
+FileInputStream in = null;
+try {
+ in = secureOpenFile(logFile);
+} catch (IOException e) {
+ logErrorMessage(logFile, e);
+ IOUtils.cleanup(LOG, in);
+ continue;
+}
+
final long fileLength = logFile.length();
// Write the logFile Type
out.writeUTF(logFile.getName());
@@ -219,9 +230,7 @@ public class AggregatedLogFormat {
out.writeUTF(String.valueOf(fileLength));
// Write the log itself
-FileInputStream in = null;
try {
- in = SecureIOUtils.openForRead(logFile, getUser(), null);
byte[] buf = new byte[65535];
int len = 0;
long bytesLeft = fileLength;
@@ -244,18 +253,26 @@ public class AggregatedLogFormat {
}
this.uploadedFiles.add(logFile);
} catch (IOException e) {
- String message = Error aggregating log file. Log file :
- + logFile.getAbsolutePath() + e.getMessage();
- LOG.error(message, e);
+ String message = logErrorMessage(logFile, e);
out.write(message.getBytes());
} finally {
- if (in != null) {
-in.close();
- }
+ IOUtils.cleanup(LOG, in);
}
}
}
+@VisibleForTesting
+public FileInputStream secureOpenFile(File logFile) throws IOException {
+ return SecureIOUtils.openForRead(logFile, getUser(), null);
+}
+
+private static String logErrorMessage(File logFile, Exception e) {
+ String message = Error aggregating log file. Log file :
+ + logFile.getAbsolutePath() + . + e.getMessage();
+ LOG.error(message, e);
+ return message;
+}
+
// Added for testing purpose.
public String getUser
git commit: YARN-2724. Skipped uploading a local log file to HDFS if exception is raised when opening it. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 09e422302 - 38fa39aa1
YARN-2724. Skipped uploading a local log file to HDFS if exception is raised
when opening it. Contributed by Xuan Gong.
(cherry picked from commit e31f0a6558b106662c83e1f797216e412b6689a9)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/38fa39aa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/38fa39aa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/38fa39aa
Branch: refs/heads/branch-2.6
Commit: 38fa39aa17fec13db6ae84de7ad19db5be5f0c6a
Parents: 09e4223
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Oct 24 11:13:44 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Oct 24 11:16:44 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 ++
.../logaggregation/AggregatedLogFormat.java | 33 +++-
.../logaggregation/TestAggregatedLogFormat.java | 17 --
3 files changed, 43 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38fa39aa/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 6df461d..26f9e25 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -673,6 +673,9 @@ Release 2.6.0 - UNRELEASED
YARN-2732. Fixed syntax error in SecureContainer.apt.vm. (Jian He via
zjshen)
+YARN-2724. Skipped uploading a local log file to HDFS if exception is
raised
+when opening it. (Xuan Gong via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/38fa39aa/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index e1d1e00..46e39cc 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -69,6 +69,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
import com.google.common.collect.Iterables;
import com.google.common.collect.Sets;
@@ -211,6 +212,16 @@ public class AggregatedLogFormat {
Collections.sort(fileList);
for (File logFile : fileList) {
+
+FileInputStream in = null;
+try {
+ in = secureOpenFile(logFile);
+} catch (IOException e) {
+ logErrorMessage(logFile, e);
+ IOUtils.cleanup(LOG, in);
+ continue;
+}
+
final long fileLength = logFile.length();
// Write the logFile Type
out.writeUTF(logFile.getName());
@@ -219,9 +230,7 @@ public class AggregatedLogFormat {
out.writeUTF(String.valueOf(fileLength));
// Write the log itself
-FileInputStream in = null;
try {
- in = SecureIOUtils.openForRead(logFile, getUser(), null);
byte[] buf = new byte[65535];
int len = 0;
long bytesLeft = fileLength;
@@ -244,18 +253,26 @@ public class AggregatedLogFormat {
}
this.uploadedFiles.add(logFile);
} catch (IOException e) {
- String message = Error aggregating log file. Log file :
- + logFile.getAbsolutePath() + e.getMessage();
- LOG.error(message, e);
+ String message = logErrorMessage(logFile, e);
out.write(message.getBytes());
} finally {
- if (in != null) {
-in.close();
- }
+ IOUtils.cleanup(LOG, in);
}
}
}
+@VisibleForTesting
+public FileInputStream secureOpenFile(File logFile) throws IOException {
+ return SecureIOUtils.openForRead(logFile, getUser(), null);
+}
+
+private static String logErrorMessage(File logFile, Exception e) {
+ String message = Error aggregating log file. Log file :
+ + logFile.getAbsolutePath() + . + e.getMessage();
+ LOG.error(message, e);
+ return message;
+}
+
// Added for testing purpose.
public String getUser
git commit: YARN-2703. Added logUploadedTime into LogValue for better display. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/trunk a52eb4bc5 - f81dc3f99
YARN-2703. Added logUploadedTime into LogValue for better display. Contributed
by Xuan Gong.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f81dc3f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f81dc3f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f81dc3f9
Branch: refs/heads/trunk
Commit: f81dc3f995579c1b94b11d60e9fc6da56c8a9496
Parents: a52eb4b
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Oct 24 14:10:46 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Oct 24 14:10:46 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/client/cli/TestLogsCLI.java | 1 +
.../yarn/logaggregation/AggregatedLogFormat.java | 19 +++
.../yarn/webapp/log/AggregatedLogsBlock.java | 4 +++-
.../logaggregation/TestAggregatedLogFormat.java | 4 +++-
.../TestLogAggregationService.java | 15 +--
6 files changed, 38 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cfa0727..c30499f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -391,6 +391,9 @@ Release 2.6.0 - UNRELEASED
YARN-2209. Replaced AM resync/shutdown command with corresponding
exceptions and
made related MR changes. (Jian He via zjshen)
+YARN-2703. Added logUploadedTime into LogValue for better display. (Xuan
Gong
+via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 132dca2..2e9e92d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -227,6 +227,7 @@ public class TestLogsCLI {
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
Hello container_0_0001_01_01!));
+assertTrue(sysOutStream.toString().contains(LogUploadTime));
fs.delete(new Path(remoteLogRootDir), true);
fs.delete(new Path(rootLogDir), true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f81dc3f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 46e39cc..035d61d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -68,6 +68,7 @@ import
org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
@@ -226,6 +227,9 @@ public class AggregatedLogFormat {
// Write the logFile Type
out.writeUTF(logFile.getName());
+// Write the uploaded TimeStamp
+out.writeLong(System.currentTimeMillis());
+
// Write the log length as UTF so that it is printable
out.writeUTF(String.valueOf(fileLength));
@@ -636,6 +640,7 @@ public class AggregatedLogFormat {
int bufferSize = 65536;
char[] cbuf = new char[bufferSize];
String fileType;
+ long uploadTime;
String fileLengthStr;
long fileLength;
@@ -646,10 +651,13 @@ public class AggregatedLogFormat {
// EndOfFile
return;
}
+uploadTime
git commit: YARN-2703. Added logUploadedTime into LogValue for better display. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 2bcda17ee - 1b8110514
YARN-2703. Added logUploadedTime into LogValue for better display. Contributed
by Xuan Gong.
(cherry picked from commit f81dc3f995579c1b94b11d60e9fc6da56c8a9496)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1b811051
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1b811051
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1b811051
Branch: refs/heads/branch-2
Commit: 1b811051438b942f2f5d9c99a4ad6fecf82759cf
Parents: 2bcda17
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Oct 24 14:10:46 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Oct 24 14:12:17 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/client/cli/TestLogsCLI.java | 1 +
.../yarn/logaggregation/AggregatedLogFormat.java | 19 +++
.../yarn/webapp/log/AggregatedLogsBlock.java | 4 +++-
.../logaggregation/TestAggregatedLogFormat.java | 4 +++-
.../TestLogAggregationService.java | 15 +--
6 files changed, 38 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b811051/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 23466a5..36b70b1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -361,6 +361,9 @@ Release 2.6.0 - UNRELEASED
YARN-2209. Replaced AM resync/shutdown command with corresponding
exceptions and
made related MR changes. (Jian He via zjshen)
+YARN-2703. Added logUploadedTime into LogValue for better display. (Xuan
Gong
+via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b811051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 132dca2..2e9e92d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -227,6 +227,7 @@ public class TestLogsCLI {
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
Hello container_0_0001_01_01!));
+assertTrue(sysOutStream.toString().contains(LogUploadTime));
fs.delete(new Path(remoteLogRootDir), true);
fs.delete(new Path(rootLogDir), true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1b811051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 46e39cc..035d61d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -68,6 +68,7 @@ import
org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
@@ -226,6 +227,9 @@ public class AggregatedLogFormat {
// Write the logFile Type
out.writeUTF(logFile.getName());
+// Write the uploaded TimeStamp
+out.writeLong(System.currentTimeMillis());
+
// Write the log length as UTF so that it is printable
out.writeUTF(String.valueOf(fileLength));
@@ -636,6 +640,7 @@ public class AggregatedLogFormat {
int bufferSize = 65536;
char[] cbuf = new char[bufferSize];
String fileType;
+ long uploadTime;
String fileLengthStr;
long fileLength;
@@ -646,10 +651,13 @@ public class AggregatedLogFormat
git commit: YARN-2703. Added logUploadedTime into LogValue for better display. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 62f23d0b7 - a41371e17
YARN-2703. Added logUploadedTime into LogValue for better display. Contributed
by Xuan Gong.
(cherry picked from commit f81dc3f995579c1b94b11d60e9fc6da56c8a9496)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a41371e1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a41371e1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a41371e1
Branch: refs/heads/branch-2.6
Commit: a41371e1738b813cd86a0ecd6eb501d281903d9b
Parents: 62f23d0
Author: Zhijie Shen zjs...@apache.org
Authored: Fri Oct 24 14:10:46 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Fri Oct 24 14:13:28 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/client/cli/TestLogsCLI.java | 1 +
.../yarn/logaggregation/AggregatedLogFormat.java | 19 +++
.../yarn/webapp/log/AggregatedLogsBlock.java | 4 +++-
.../logaggregation/TestAggregatedLogFormat.java | 4 +++-
.../TestLogAggregationService.java | 15 +--
6 files changed, 38 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41371e1/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 26f9e25..e8a38aa 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -331,6 +331,9 @@ Release 2.6.0 - UNRELEASED
YARN-2209. Replaced AM resync/shutdown command with corresponding
exceptions and
made related MR changes. (Jian He via zjshen)
+YARN-2703. Added logUploadedTime into LogValue for better display. (Xuan
Gong
+via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41371e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
index 132dca2..2e9e92d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestLogsCLI.java
@@ -227,6 +227,7 @@ public class TestLogsCLI {
assertTrue(exitCode == 0);
assertTrue(sysOutStream.toString().contains(
Hello container_0_0001_01_01!));
+assertTrue(sysOutStream.toString().contains(LogUploadTime));
fs.delete(new Path(remoteLogRootDir), true);
fs.delete(new Path(rootLogDir), true);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a41371e1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 46e39cc..035d61d 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -68,6 +68,7 @@ import
org.apache.hadoop.yarn.api.records.LogAggregationContext;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.hadoop.yarn.util.Times;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Predicate;
@@ -226,6 +227,9 @@ public class AggregatedLogFormat {
// Write the logFile Type
out.writeUTF(logFile.getName());
+// Write the uploaded TimeStamp
+out.writeLong(System.currentTimeMillis());
+
// Write the log length as UTF so that it is printable
out.writeUTF(String.valueOf(fileLength));
@@ -636,6 +640,7 @@ public class AggregatedLogFormat {
int bufferSize = 65536;
char[] cbuf = new char[bufferSize];
String fileType;
+ long uploadTime;
String fileLengthStr;
long fileLength;
@@ -646,10 +651,13 @@ public class AggregatedLogFormat
git commit: YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 b1e17d2da - cb3974524
YARN-2734. Skipped sub-folders in the local log dir when aggregating logs.
Contributed by Xuan Gong.
(cherry picked from commit caecd9fffe7c6216be31f3ab65349182045451fa)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb397452
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb397452
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb397452
Branch: refs/heads/branch-2
Commit: cb3974524c5b265e70d94d0d40b24e2e9224f050
Parents: b1e17d2
Author: Zhijie Shen zjs...@apache.org
Authored: Sun Oct 26 12:57:31 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Sun Oct 26 12:59:29 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/logaggregation/AggregatedLogFormat.java | 6 ++
.../hadoop/yarn/logaggregation/TestAggregatedLogFormat.java | 9 +
3 files changed, 18 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb397452/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0c9e525..74d1dff 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -730,6 +730,9 @@ Release 2.6.0 - UNRELEASED
YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port.
(Naganarasimha G R via xgong)
+YARN-2734. Skipped sub-folders in the local log dir when aggregating logs.
+(Xuan Gong via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb397452/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 035d61d..22219be 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -213,6 +213,12 @@ public class AggregatedLogFormat {
Collections.sort(fileList);
for (File logFile : fileList) {
+// We only aggregate top level files.
+// Ignore anything inside sub-folders.
+if (logFile.isDirectory()) {
+ LOG.warn(logFile.getAbsolutePath() + is a directory. Ignore it.);
+ continue;
+}
FileInputStream in = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb397452/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
index fa9de74..1d607b1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
@@ -194,6 +194,14 @@ public class TestAggregatedLogFormat {
int numChars = 8;
+// create a sub-folder under srcFilePath
+// and create file logs in this sub-folder.
+// We only aggregate top level files.
+// So, this log file should be ignored.
+Path subDir = new Path(srcFilePath, subDir);
+fs.mkdirs(subDir);
+writeSrcFile(subDir, logs, numChars);
+
// create file stderr and stdout in containerLogDir
writeSrcFile(srcFilePath, stderr, numChars);
writeSrcFile(srcFilePath, stdout, numChars);
@@ -238,6 +246,7 @@ public class TestAggregatedLogFormat {
+ \nLog Contents:\n.length() + numChars;
Assert.assertTrue(LogType not matched, s.contains(LogType:stdout));
Assert.assertTrue(log file:stderr should not be aggregated.,
!s.contains(LogType:stderr));
+Assert.assertTrue(log file:logs should not be aggregated.,
!s.contains(LogType:logs));
Assert.assertTrue(LogLength not matched, s.contains(LogLength: +
numChars));
Assert.assertTrue
git commit: YARN-2734. Skipped sub-folders in the local log dir when aggregating logs. Contributed by Xuan Gong.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 259db7a72 - ab7a21569
YARN-2734. Skipped sub-folders in the local log dir when aggregating logs.
Contributed by Xuan Gong.
(cherry picked from commit caecd9fffe7c6216be31f3ab65349182045451fa)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab7a2156
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab7a2156
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab7a2156
Branch: refs/heads/branch-2.6
Commit: ab7a21569cbceee482aa8b009e4a2d8a2f766434
Parents: 259db7a
Author: Zhijie Shen zjs...@apache.org
Authored: Sun Oct 26 12:57:31 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Sun Oct 26 13:00:26 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/logaggregation/AggregatedLogFormat.java | 6 ++
.../hadoop/yarn/logaggregation/TestAggregatedLogFormat.java | 9 +
3 files changed, 18 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab7a2156/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 30f87e2..a613c80 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -694,6 +694,9 @@ Release 2.6.0 - UNRELEASED
YARN-2723. Fix rmadmin -replaceLabelsOnNode does not correctly parse port.
(Naganarasimha G R via xgong)
+YARN-2734. Skipped sub-folders in the local log dir when aggregating logs.
+(Xuan Gong via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab7a2156/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
index 035d61d..22219be 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/AggregatedLogFormat.java
@@ -213,6 +213,12 @@ public class AggregatedLogFormat {
Collections.sort(fileList);
for (File logFile : fileList) {
+// We only aggregate top level files.
+// Ignore anything inside sub-folders.
+if (logFile.isDirectory()) {
+ LOG.warn(logFile.getAbsolutePath() + is a directory. Ignore it.);
+ continue;
+}
FileInputStream in = null;
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab7a2156/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
index fa9de74..1d607b1 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/logaggregation/TestAggregatedLogFormat.java
@@ -194,6 +194,14 @@ public class TestAggregatedLogFormat {
int numChars = 8;
+// create a sub-folder under srcFilePath
+// and create file logs in this sub-folder.
+// We only aggregate top level files.
+// So, this log file should be ignored.
+Path subDir = new Path(srcFilePath, subDir);
+fs.mkdirs(subDir);
+writeSrcFile(subDir, logs, numChars);
+
// create file stderr and stdout in containerLogDir
writeSrcFile(srcFilePath, stderr, numChars);
writeSrcFile(srcFilePath, stdout, numChars);
@@ -238,6 +246,7 @@ public class TestAggregatedLogFormat {
+ \nLog Contents:\n.length() + numChars;
Assert.assertTrue(LogType not matched, s.contains(LogType:stdout));
Assert.assertTrue(log file:stderr should not be aggregated.,
!s.contains(LogType:stderr));
+Assert.assertTrue(log file:logs should not be aggregated.,
!s.contains(LogType:logs));
Assert.assertTrue(LogLength not matched, s.contains(LogLength: +
numChars
git commit: MAPREDUCE-5933. Enabled MR AM to post history events to the timeline server. Contributed by Robert Kanter.
Repository: hadoop
Updated Branches:
refs/heads/trunk f6b963fdf - 6b2f11b54
MAPREDUCE-5933. Enabled MR AM to post history events to the timeline server.
Contributed by Robert Kanter.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6b2f11b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6b2f11b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6b2f11b5
Branch: refs/heads/trunk
Commit: 6b2f11b54bc679b0715fe66bd129e340e8c61c5c
Parents: f6b963f
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 27 20:35:40 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 27 20:35:40 2014 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../hadoop-mapreduce-client-app/pom.xml | 6 +
.../mapreduce/jobhistory/JobHistoryEvent.java | 7 +-
.../jobhistory/JobHistoryEventHandler.java | 340 ++-
.../jobhistory/TestJobHistoryEventHandler.java | 248 +-
.../mapred/TestMRTimelineEventHandling.java | 88 +
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 7 +-
7 files changed, 691 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b2f11b5/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 84016b9..fd82d49 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -242,6 +242,9 @@ Release 2.6.0 - UNRELEASED
NEW FEATURES
+MAPREDUCE-5933. Enabled MR AM to post history events to the timeline
server.
+(Robert Kanter via zjshen)
+
IMPROVEMENTS
MAPREDUCE-5971. Move the default options for distcp -p to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b2f11b5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index 2e597d1..4f6f837 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -79,6 +79,12 @@
typetest-jar/type
scopetest/scope
/dependency
+dependency
+ groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-yarn-server-tests/artifactId
+ typetest-jar/type
+ scopetest/scope
+ /dependency
/dependencies
build
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b2f11b5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
index e853b1c..4fe35b2 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
@@ -27,7 +27,12 @@ public class JobHistoryEvent extends
AbstractEventEventType{
private final HistoryEvent historyEvent;
public JobHistoryEvent(JobId jobID, HistoryEvent historyEvent) {
-super(historyEvent.getEventType());
+this(jobID, historyEvent, System.currentTimeMillis());
+ }
+
+ public JobHistoryEvent(JobId jobID, HistoryEvent historyEvent,
+ long timestamp) {
+super(historyEvent.getEventType(), timestamp);
this.jobID = jobID;
this.historyEvent = historyEvent;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6b2f11b5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index c566740..ebedc1b
git commit: MAPREDUCE-5933. Enabled MR AM to post history events to the timeline server. Contributed by Robert Kanter.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 a1c7a707c - 733380fa7
MAPREDUCE-5933. Enabled MR AM to post history events to the timeline server.
Contributed by Robert Kanter.
(cherry picked from commit 6b2f11b54bc679b0715fe66bd129e340e8c61c5c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/733380fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/733380fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/733380fa
Branch: refs/heads/branch-2
Commit: 733380fa7055fe0296626ea50036bc533bf0092a
Parents: a1c7a70
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 27 20:35:40 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 27 20:41:02 2014 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../hadoop-mapreduce-client-app/pom.xml | 6 +
.../mapreduce/jobhistory/JobHistoryEvent.java | 7 +-
.../jobhistory/JobHistoryEventHandler.java | 340 ++-
.../jobhistory/TestJobHistoryEventHandler.java | 248 +-
.../mapred/TestMRTimelineEventHandling.java | 88 +
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 7 +-
7 files changed, 691 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/733380fa/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 76a7130..8738bc5 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -20,6 +20,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-5910. Make MR AM resync with RM in case of work-preserving
RM-restart. Contributed by Rohith
+MAPREDUCE-5933. Enabled MR AM to post history events to the timeline
server.
+(Robert Kanter via zjshen)
+
IMPROVEMENTS
MAPREDUCE-5971. Move the default options for distcp -p to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/733380fa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index 2251252..4a7d843 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -79,6 +79,12 @@
typetest-jar/type
scopetest/scope
/dependency
+dependency
+ groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-yarn-server-tests/artifactId
+ typetest-jar/type
+ scopetest/scope
+ /dependency
/dependencies
build
http://git-wip-us.apache.org/repos/asf/hadoop/blob/733380fa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
index e853b1c..4fe35b2 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
@@ -27,7 +27,12 @@ public class JobHistoryEvent extends
AbstractEventEventType{
private final HistoryEvent historyEvent;
public JobHistoryEvent(JobId jobID, HistoryEvent historyEvent) {
-super(historyEvent.getEventType());
+this(jobID, historyEvent, System.currentTimeMillis());
+ }
+
+ public JobHistoryEvent(JobId jobID, HistoryEvent historyEvent,
+ long timestamp) {
+super(historyEvent.getEventType(), timestamp);
this.jobID = jobID;
this.historyEvent = historyEvent;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/733380fa/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop
git commit: MAPREDUCE-5933. Enabled MR AM to post history events to the timeline server. Contributed by Robert Kanter.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 8d4c93aab - 71ab776b2
MAPREDUCE-5933. Enabled MR AM to post history events to the timeline server.
Contributed by Robert Kanter.
(cherry picked from commit 6b2f11b54bc679b0715fe66bd129e340e8c61c5c)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/71ab776b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/71ab776b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/71ab776b
Branch: refs/heads/branch-2.6
Commit: 71ab776b21b48d332f9b6e427ed675eca0c60390
Parents: 8d4c93a
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 27 20:35:40 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 27 20:45:22 2014 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../hadoop-mapreduce-client-app/pom.xml | 6 +
.../mapreduce/jobhistory/JobHistoryEvent.java | 7 +-
.../jobhistory/JobHistoryEventHandler.java | 340 ++-
.../jobhistory/TestJobHistoryEventHandler.java | 248 +-
.../mapred/TestMRTimelineEventHandling.java | 88 +
.../hadoop/mapreduce/v2/MiniMRYarnCluster.java | 7 +-
7 files changed, 691 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71ab776b/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 4f7d1d8..1f67326 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -8,6 +8,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-5910. Make MR AM resync with RM in case of work-preserving
RM-restart. Contributed by Rohith
+MAPREDUCE-5933. Enabled MR AM to post history events to the timeline
server.
+(Robert Kanter via zjshen)
+
IMPROVEMENTS
MAPREDUCE-5971. Move the default options for distcp -p to
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71ab776b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
index 2251252..4a7d843 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/pom.xml
@@ -79,6 +79,12 @@
typetest-jar/type
scopetest/scope
/dependency
+dependency
+ groupIdorg.apache.hadoop/groupId
+ artifactIdhadoop-yarn-server-tests/artifactId
+ typetest-jar/type
+ scopetest/scope
+ /dependency
/dependencies
build
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71ab776b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
index e853b1c..4fe35b2 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEvent.java
@@ -27,7 +27,12 @@ public class JobHistoryEvent extends
AbstractEventEventType{
private final HistoryEvent historyEvent;
public JobHistoryEvent(JobId jobID, HistoryEvent historyEvent) {
-super(historyEvent.getEventType());
+this(jobID, historyEvent, System.currentTimeMillis());
+ }
+
+ public JobHistoryEvent(JobId jobID, HistoryEvent historyEvent,
+ long timestamp) {
+super(historyEvent.getEventType(), timestamp);
this.jobID = jobID;
this.historyEvent = historyEvent;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/71ab776b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop
git commit: MAPREDUCE-6018. Added an MR specific config to enable emitting job history data to the timeline server. Contributed by Robert Kanter.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 733380fa7 - 52695d38f
MAPREDUCE-6018. Added an MR specific config to enable emitting job history data
to the timeline server. Contributed by Robert Kanter.
(cherry picked from commit 971e91c8c03a23e4613ed3f071b4f982ee5a1b63)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/52695d38
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/52695d38
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/52695d38
Branch: refs/heads/branch-2
Commit: 52695d38fcf40b8ee52077aaa15534676351d6e0
Parents: 733380f
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 27 21:03:53 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 27 21:08:38 2014 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../jobhistory/JobHistoryEventHandler.java | 26 +--
.../apache/hadoop/mapreduce/MRJobConfig.java| 5 ++
.../src/main/resources/mapred-default.xml | 8 ++
.../mapred/TestMRTimelineEventHandling.java | 80
5 files changed, 114 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52695d38/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 8738bc5..a181bb1 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -61,6 +61,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-5970. Provide a boolean switch to enable MR-AM profiling (Gera
Shegalov via jlowe)
+MAPREDUCE-6018. Added an MR specific config to enable emitting job history
+data to the timeline server. (Robert Kanter via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/52695d38/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index ebedc1b..184baaa 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -241,8 +241,14 @@ public class JobHistoryEventHandler extends AbstractService
MRJobConfig.MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD,
MRJobConfig.DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD);
-timelineClient = TimelineClient.createTimelineClient();
-timelineClient.init(conf);
+if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA,
+MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)) {
+ timelineClient = TimelineClient.createTimelineClient();
+ timelineClient.init(conf);
+ LOG.info(Emitting job history data to the timeline server is enabled);
+} else {
+ LOG.info(Emitting job history data to the timeline server is not
enabled);
+}
super.serviceInit(conf);
}
@@ -268,7 +274,9 @@ public class JobHistoryEventHandler extends AbstractService
@Override
protected void serviceStart() throws Exception {
-timelineClient.start();
+if (timelineClient != null) {
+ timelineClient.start();
+}
eventHandlingThread = new Thread(new Runnable() {
@Override
public void run() {
@@ -537,7 +545,7 @@ public class JobHistoryEventHandler extends AbstractService
// For all events
// (1) Write it out
// (2) Process it for JobSummary
- // (3) Process it for ATS
+ // (3) Process it for ATS (if enabled)
MetaInfo mi = fileMap.get(event.getJobID());
try {
HistoryEvent historyEvent = event.getHistoryEvent();
@@ -546,8 +554,10 @@ public class JobHistoryEventHandler extends AbstractService
}
processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(),
event.getJobID());
-processEventForTimelineServer(historyEvent, event.getJobID(),
-event.getTimestamp());
+if (timelineClient != null) {
+ processEventForTimelineServer(historyEvent, event.getJobID(),
+ event.getTimestamp
git commit: MAPREDUCE-6018. Added an MR specific config to enable emitting job history data to the timeline server. Contributed by Robert Kanter.
Repository: hadoop
Updated Branches:
refs/heads/branch-2.6 71ab776b2 - d92d4e32f
MAPREDUCE-6018. Added an MR specific config to enable emitting job history data
to the timeline server. Contributed by Robert Kanter.
(cherry picked from commit 971e91c8c03a23e4613ed3f071b4f982ee5a1b63)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d92d4e32
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d92d4e32
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d92d4e32
Branch: refs/heads/branch-2.6
Commit: d92d4e32f2477c38edcd49a23326103aa064f2c3
Parents: 71ab776
Author: Zhijie Shen zjs...@apache.org
Authored: Mon Oct 27 21:03:53 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Mon Oct 27 21:09:20 2014 -0700
--
hadoop-mapreduce-project/CHANGES.txt| 3 +
.../jobhistory/JobHistoryEventHandler.java | 26 +--
.../apache/hadoop/mapreduce/MRJobConfig.java| 5 ++
.../src/main/resources/mapred-default.xml | 8 ++
.../mapred/TestMRTimelineEventHandling.java | 80
5 files changed, 114 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d92d4e32/hadoop-mapreduce-project/CHANGES.txt
--
diff --git a/hadoop-mapreduce-project/CHANGES.txt
b/hadoop-mapreduce-project/CHANGES.txt
index 1f67326..290641b 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -49,6 +49,9 @@ Release 2.6.0 - UNRELEASED
MAPREDUCE-5970. Provide a boolean switch to enable MR-AM profiling (Gera
Shegalov via jlowe)
+MAPREDUCE-6018. Added an MR specific config to enable emitting job history
+data to the timeline server. (Robert Kanter via zjshen)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d92d4e32/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
--
diff --git
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index ebedc1b..184baaa 100644
---
a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++
b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -241,8 +241,14 @@ public class JobHistoryEventHandler extends AbstractService
MRJobConfig.MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD,
MRJobConfig.DEFAULT_MR_AM_HISTORY_USE_BATCHED_FLUSH_QUEUE_SIZE_THRESHOLD);
-timelineClient = TimelineClient.createTimelineClient();
-timelineClient.init(conf);
+if (conf.getBoolean(MRJobConfig.MAPREDUCE_JOB_EMIT_TIMELINE_DATA,
+MRJobConfig.DEFAULT_MAPREDUCE_JOB_EMIT_TIMELINE_DATA)) {
+ timelineClient = TimelineClient.createTimelineClient();
+ timelineClient.init(conf);
+ LOG.info(Emitting job history data to the timeline server is enabled);
+} else {
+ LOG.info(Emitting job history data to the timeline server is not
enabled);
+}
super.serviceInit(conf);
}
@@ -268,7 +274,9 @@ public class JobHistoryEventHandler extends AbstractService
@Override
protected void serviceStart() throws Exception {
-timelineClient.start();
+if (timelineClient != null) {
+ timelineClient.start();
+}
eventHandlingThread = new Thread(new Runnable() {
@Override
public void run() {
@@ -537,7 +545,7 @@ public class JobHistoryEventHandler extends AbstractService
// For all events
// (1) Write it out
// (2) Process it for JobSummary
- // (3) Process it for ATS
+ // (3) Process it for ATS (if enabled)
MetaInfo mi = fileMap.get(event.getJobID());
try {
HistoryEvent historyEvent = event.getHistoryEvent();
@@ -546,8 +554,10 @@ public class JobHistoryEventHandler extends AbstractService
}
processEventForJobSummary(event.getHistoryEvent(), mi.getJobSummary(),
event.getJobID());
-processEventForTimelineServer(historyEvent, event.getJobID(),
-event.getTimestamp());
+if (timelineClient != null) {
+ processEventForTimelineServer(historyEvent, event.getJobID(),
+ event.getTimestamp
git commit: YARN-2741. Made NM web UI serve logs on the drive other than C: on Windows. Contributed by Craig Welch.
Repository: hadoop
Updated Branches:
refs/heads/trunk c6f04f391 - 8984e9b17
YARN-2741. Made NM web UI serve logs on the drive other than C: on Windows.
Contributed by Craig Welch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8984e9b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8984e9b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8984e9b1
Branch: refs/heads/trunk
Commit: 8984e9b1774033e379b57da1bd30a5c81888c7a3
Parents: c6f04f3
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 28 14:10:16 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 28 14:11:19 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodemanager/webapp/ContainerLogsUtils.java | 10 +--
.../webapp/TestContainerLogsPage.java | 82
3 files changed, 87 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8984e9b1/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 0bff6d4..01fcc8f 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -784,6 +784,9 @@ Release 2.6.0 - UNRELEASED
YARN-2758. Update TestApplicationHistoryClientService to use the new
generic
history store. (Zhijie Shen via xgong)
+YARN-2741. Made NM web UI serve logs on the drive other than C: on
Windows. (Craig
+Welch via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8984e9b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
index 3d0f19d..c588a89 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
@@ -77,11 +77,7 @@ public class ContainerLogsUtils {
ListString logDirs = dirsHandler.getLogDirs();
ListFile containerLogDirs = new ArrayListFile(logDirs.size());
for (String logDir : logDirs) {
- try {
-logDir = new URI(logDir).getPath();
- } catch (URISyntaxException e) {
-throw new YarnException(Internal error, e);
- }
+ logDir = new File(logDir).toURI().getPath();
String appIdStr = ConverterUtils.toString(containerId
.getApplicationAttemptId().getApplicationId());
File appLogDir = new File(logDir, appIdStr);
@@ -109,11 +105,9 @@ public class ContainerLogsUtils {
application.getAppId().toString(), containerId.toString());
Path logPath = dirsHandler.getLogPathToRead(
relativeContainerLogDir + Path.SEPARATOR + fileName);
- URI logPathURI = new URI(logPath.toString());
+ URI logPathURI = new File(logPath.toString()).toURI();
File logFile = new File(logPathURI.getPath());
return logFile;
-} catch (URISyntaxException e) {
- throw new YarnException(Internal error, e);
} catch (IOException e) {
LOG.warn(Failed to find log file, e);
throw new NotFoundException(Cannot find this log on the local disk.);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8984e9b1/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
index 9305b84..b1d4397 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
+++
b/hadoop-yarn-project/hadoop
git commit: YARN-2741. Made NM web UI serve logs on the drive other than C: on Windows. Contributed by Craig Welch.
Repository: hadoop
Updated Branches:
refs/heads/branch-2 43b1878f8 - f40389ae0
YARN-2741. Made NM web UI serve logs on the drive other than C: on Windows.
Contributed by Craig Welch.
(cherry picked from commit 8984e9b1774033e379b57da1bd30a5c81888c7a3)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f40389ae
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f40389ae
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f40389ae
Branch: refs/heads/branch-2
Commit: f40389ae08572c60fc36c7a6d71c6b9c47f5b902
Parents: 43b1878
Author: Zhijie Shen zjs...@apache.org
Authored: Tue Oct 28 14:10:16 2014 -0700
Committer: Zhijie Shen zjs...@apache.org
Committed: Tue Oct 28 14:12:09 2014 -0700
--
hadoop-yarn-project/CHANGES.txt | 3 +
.../nodemanager/webapp/ContainerLogsUtils.java | 10 +--
.../webapp/TestContainerLogsPage.java | 82
3 files changed, 87 insertions(+), 8 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f40389ae/hadoop-yarn-project/CHANGES.txt
--
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d58fdb8..46c8993 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -754,6 +754,9 @@ Release 2.6.0 - UNRELEASED
YARN-2758. Update TestApplicationHistoryClientService to use the new
generic
history store. (Zhijie Shen via xgong)
+YARN-2741. Made NM web UI serve logs on the drive other than C: on
Windows. (Craig
+Welch via zjshen)
+
Release 2.5.1 - 2014-09-05
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f40389ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
index 3d0f19d..c588a89 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
+++
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsUtils.java
@@ -77,11 +77,7 @@ public class ContainerLogsUtils {
ListString logDirs = dirsHandler.getLogDirs();
ListFile containerLogDirs = new ArrayListFile(logDirs.size());
for (String logDir : logDirs) {
- try {
-logDir = new URI(logDir).getPath();
- } catch (URISyntaxException e) {
-throw new YarnException(Internal error, e);
- }
+ logDir = new File(logDir).toURI().getPath();
String appIdStr = ConverterUtils.toString(containerId
.getApplicationAttemptId().getApplicationId());
File appLogDir = new File(logDir, appIdStr);
@@ -109,11 +105,9 @@ public class ContainerLogsUtils {
application.getAppId().toString(), containerId.toString());
Path logPath = dirsHandler.getLogPathToRead(
relativeContainerLogDir + Path.SEPARATOR + fileName);
- URI logPathURI = new URI(logPath.toString());
+ URI logPathURI = new File(logPath.toString()).toURI();
File logFile = new File(logPathURI.getPath());
return logFile;
-} catch (URISyntaxException e) {
- throw new YarnException(Internal error, e);
} catch (IOException e) {
LOG.warn(Failed to find log file, e);
throw new NotFoundException(Cannot find this log on the local disk.);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f40389ae/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
--
diff --git
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestContainerLogsPage.java
index 9305b84..b1d4397 100644
---
a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server
