Author: subru
Date: Fri Jul 25 20:33:09 2014
New Revision: 1613514
URL: http://svn.apache.org/r1613514
Log:
syncing YARN-1051 branch with trunk
Added:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/ForbiddenException.java
- copied unchanged from r1613507,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/ForbiddenException.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/records/
- copied from r1613507,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/records/
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/
- copied from r1613507,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/security/
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/DelegationToken.java
- copied unchanged from r1613507,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/DelegationToken.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
- copied unchanged from r1613507,
hadoop/common/trunk/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesDelegationTokens.java
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMNullStateStoreService.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMStateStoreService.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NodePage.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_recovery.proto
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMMemoryStateStoreService.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/RMAppImpl.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainer.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AppSchedulingInfo.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/common/fica/FiCaSchedulerApp.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAddedSchedulerEvent.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/AppAttemptAddedSchedulerEvent.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSLeafQueue.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSParentQueue.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSQueue.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FSSchedulerApp.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerConfiguration.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/policies/FairSharePolicy.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/AMRMTokenSecretManager.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/DelegationTokenRenewer.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMDelegationTokenSecretManager.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestApplicationCleanup.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestWorkPreservingRMRestart.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/RMStateStoreTestBase.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/TestRMContainerImpl.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairSchedulerTestBase.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairScheduler.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/TestFairSchedulerPreemption.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestAMRMTokens.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestDelegationTokenRenewer.java
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/FairScheduler.apt.vm
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/ResourceManagerRest.apt.vm
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/apt/YarnCommands.apt.vm
Modified: hadoop/common/branches/YARN-1051/hadoop-yarn-project/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/CHANGES.txt?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/CHANGES.txt Fri Jul 25
20:33:09 2014
@@ -40,6 +40,34 @@ Release 2.6.0 - UNRELEASED
YARN-2242. Improve exception information on AM launch crashes. (Li Lu
via junping_du)
+ YARN-2274. FairScheduler: Add debug information about cluster capacity,
+ availability and reservations. (kasha)
+
+ YARN-2228. Augmented TimelineServer to load pseudo authentication filter
when
+ authentication = simple. (Zhijie Shen via vinodkv)
+
+ YARN-1341. Recover NMTokens upon nodemanager restart. (Jason Lowe via
+ junping_du)
+
+ YARN-2208. AMRMTokenManager need to have a way to roll over AMRMToken.
(xgong)
+
+ YARN-2323. FairShareComparator creates too many Resource objects (Hong
Zhiguo
+ via Sandy Ryza)
+
+ YARN-2045. Data persisted in NM should be versioned (Junping Du via jlowe)
+
+ YARN-2013. The diagnostics is always the ExitCodeException stack when the
container
+ crashes. (Tsuyoshi OZAWA via junping_du)
+
+ YARN-2295. Refactored DistributedShell to use public APIs of protocol
records.
+ (Li Lu via jianhe)
+
+ YARN-1342. Recover container tokens upon nodemanager restart. (Jason Lowe
via
+ devaraj)
+
+ YARN-2214. FairScheduler: preemptContainerPreCheck() in FSParentQueue
delays
+ convergence towards fairness. (Ashwin Shankar via kasha)
+
OPTIMIZATIONS
BUG FIXES
@@ -50,6 +78,31 @@ Release 2.6.0 - UNRELEASED
YARN-2088. Fixed a bug in GetApplicationsRequestPBImpl#mergeLocalToBuilder.
(Binglin Chang via jianhe)
+ YARN-2260. Fixed ResourceManager's RMNode to correctly remember containers
+ when nodes resync during work-preserving RM restart. (Jian He via vinodkv)
+
+ YARN-2264. Fixed a race condition in DrainDispatcher which may cause random
+ test failures. (Li Lu via jianhe)
+
+ YARN-2219. Changed ResourceManager to avoid AMs and NMs getting exceptions
+ after RM recovery but before scheduler learns about apps and app-attempts.
+ (Jian He via vinodkv)
+
+ YARN-2244. FairScheduler missing handling of containers for unknown
+ application attempts. (Anubhav Dhoot via kasha)
+
+ YARN-2321. NodeManager web UI can incorrectly report Pmem enforcement
+ (Leitao Guo via jlowe)
+
+ YARN-2273. NPE in ContinuousScheduling thread when we lose a node.
+ (Wei Yan via kasha)
+
+ YARN-2313. Livelock can occur in FairScheduler when there are lots of
+ running apps (Tsuyoshi Ozawa via Sandy Ryza)
+
+ YARN-2147. client lacks delegation token exception details when
+ application submit fails (Chen He via jlowe)
+
Release 2.5.0 - UNRELEASED
INCOMPATIBLE CHANGES
@@ -86,6 +139,9 @@ Release 2.5.0 - UNRELEASED
YARN-1713. Added get-new-app and submit-app functionality to RM web
services.
(Varun Vasudev via vinodkv)
+ YARN-2233. Implemented ResourceManager web-services to create, renew and
+ cancel delegation tokens. (Varun Vasudev via vinodkv)
+
IMPROVEMENTS
YARN-1479. Invalid NaN values in Hadoop REST API JSON response (Chen He via
@@ -250,6 +306,12 @@ Release 2.5.0 - UNRELEASED
YARN-2241. ZKRMStateStore: On startup, show nicer messages if znodes
already
exist. (Robert Kanter via kasha)
+ YARN-1408 Preemption caused Invalid State Event: ACQUIRED at KILLED and
+ caused a task timeout for 30mins. (Sunil G via mayank)
+
+ YARN-2300. Improved the documentation of the sample requests for RM REST
API -
+ submitting an app. (Varun Vasudev via zjshen)
+
OPTIMIZATIONS
BUG FIXES
@@ -371,6 +433,14 @@ Release 2.5.0 - UNRELEASED
YARN-2269. Remove external links from YARN UI. (Craig Welch via xgong)
+ YARN-2270. Made TestFSDownload#testDownloadPublicWithStatCache be skipped
+ when thereâs no ancestor permissions. (Akira Ajisaka via zjshen)
+
+ YARN-2319. Made the MiniKdc instance start/close before/after the class of
+ TestRMWebServicesDelegationTokens. (Wenwu Peng via zjshen)
+
+ YARN-2335. Annotate all hadoop-sls APIs as @Private. (Wei Yan via kasha)
+
Release 2.4.1 - 2014-06-23
INCOMPATIBLE CHANGES
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
--- hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn
(original)
+++ hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn
Fri Jul 25 20:33:09 2014
@@ -61,23 +61,27 @@ HADOOP_LIBEXEC_DIR=${HADOOP_LIBEXEC_DIR:
function print_usage(){
echo "Usage: yarn [--config confdir] COMMAND"
echo "where COMMAND is one of:"
- echo " resourcemanager -format deletes the RMStateStore"
- echo " resourcemanager run the ResourceManager"
- echo " nodemanager run a nodemanager on each slave"
- echo " timelineserver run the timeline server"
- echo " rmadmin admin tools"
- echo " version print the version"
- echo " jar <jar> run a jar file"
- echo " application prints application(s) report/kill
application"
- echo " applicationattempt prints applicationattempt(s) report"
- echo " container prints container(s) report"
- echo " node prints node report(s)"
- echo " logs dump container logs"
- echo " classpath prints the class path needed to get the"
- echo " Hadoop jar and the required libraries"
- echo " daemonlog get/set the log level for each daemon"
+ echo " resourcemanager -format-state-store deletes the RMStateStore"
+ echo " resourcemanager run the ResourceManager"
+ echo " nodemanager run a nodemanager on each
slave"
+ echo " timelineserver run the timeline server"
+ echo " rmadmin admin tools"
+ echo " version print the version"
+ echo " jar <jar> run a jar file"
+ echo " application prints application(s)"
+ echo " report/kill application"
+ echo " applicationattempt prints applicationattempt(s)"
+ echo " report"
+ echo " container prints container(s) report"
+ echo " node prints node report(s)"
+ echo " logs dump container logs"
+ echo " classpath prints the class path needed
to"
+ echo " get the Hadoop jar and the"
+ echo " required libraries"
+ echo " daemonlog get/set the log level for each"
+ echo " daemon"
echo " or"
- echo " CLASSNAME run the class named CLASSNAME"
+ echo " CLASSNAME run the class named CLASSNAME"
echo "Most commands print help when invoked w/o parameters."
}
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/bin/yarn-daemon.sh
Fri Jul 25 20:33:09 2014
@@ -145,6 +145,7 @@ case $startStop in
else
echo no $command to stop
fi
+ rm -f $pid
else
echo no $command to stop
fi
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
Fri Jul 25 20:33:09 2014
@@ -194,6 +194,12 @@
<Field name="scheduleAsynchronously" />
<Bug pattern="IS2_INCONSISTENT_SYNC" />
</Match>
+ <!-- Inconsistent sync warning - updateInterval is only initialized once and
never changed -->
+ <Match>
+ <Class
name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.fair.FairScheduler"
/>
+ <Field name="updateInterval" />
+ <Bug pattern="IS2_INCONSISTENT_SYNC" />
+ </Match>
<!-- Inconsistent sync warning - numRetries is only initialized once and
never changed -->
<Match>
<Class
name="org.apache.hadoop.yarn.server.resourcemanager.recovery.ZKRMStateStore" />
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/ApplicationMaster.java
Fri Jul 25 20:33:09 2014
@@ -83,6 +83,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
+import org.apache.hadoop.yarn.api.records.URL;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
import org.apache.hadoop.yarn.client.api.AMRMClient.ContainerRequest;
@@ -94,7 +95,6 @@ import org.apache.hadoop.yarn.conf.YarnC
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.util.ConverterUtils;
-import org.apache.hadoop.yarn.util.Records;
import org.apache.log4j.LogManager;
import com.google.common.annotations.VisibleForTesting;
@@ -522,6 +522,8 @@ public class ApplicationMaster {
+ appAttemptID.toString(), e);
}
+ // Note: Credentials, Token, UserGroupInformation, DataOutputBuffer class
+ // are marked as LimitedPrivate
Credentials credentials =
UserGroupInformation.getCurrentUser().getCredentials();
DataOutputBuffer dob = new DataOutputBuffer();
@@ -900,11 +902,6 @@ public class ApplicationMaster {
public void run() {
LOG.info("Setting up container launch container for containerid="
+ container.getId());
- ContainerLaunchContext ctx = Records
- .newRecord(ContainerLaunchContext.class);
-
- // Set the environment
- ctx.setEnvironment(shellEnv);
// Set the local resources
Map<String, LocalResource> localResources = new HashMap<String,
LocalResource>();
@@ -935,16 +932,13 @@ public class ApplicationMaster {
return;
}
- LocalResource shellRsrc = Records.newRecord(LocalResource.class);
- shellRsrc.setType(LocalResourceType.FILE);
- shellRsrc.setVisibility(LocalResourceVisibility.APPLICATION);
+ URL yarnUrl = null;
try {
- shellRsrc.setResource(ConverterUtils.getYarnUrlFromURI(new URI(
- renamedScriptPath.toString())));
+ yarnUrl = ConverterUtils.getYarnUrlFromURI(
+ new URI(renamedScriptPath.toString()));
} catch (URISyntaxException e) {
LOG.error("Error when trying to use shell script path specified"
+ " in env, path=" + renamedScriptPath, e);
-
// A failure scenario on bad input such as invalid shell script path
// We know we cannot continue launching the container
// so we should release it.
@@ -953,13 +947,13 @@ public class ApplicationMaster {
numFailedContainers.incrementAndGet();
return;
}
- shellRsrc.setTimestamp(shellScriptPathTimestamp);
- shellRsrc.setSize(shellScriptPathLen);
+ LocalResource shellRsrc = LocalResource.newInstance(yarnUrl,
+ LocalResourceType.FILE, LocalResourceVisibility.APPLICATION,
+ shellScriptPathLen, shellScriptPathTimestamp);
localResources.put(Shell.WINDOWS ? ExecBatScripStringtPath :
ExecShellStringPath, shellRsrc);
shellCommand = Shell.WINDOWS ? windows_command : linux_bash_command;
}
- ctx.setLocalResources(localResources);
// Set the necessary command to execute on the allocated container
Vector<CharSequence> vargs = new Vector<CharSequence>(5);
@@ -986,16 +980,18 @@ public class ApplicationMaster {
List<String> commands = new ArrayList<String>();
commands.add(command.toString());
- ctx.setCommands(commands);
- // Set up tokens for the container too. Today, for normal shell commands,
- // the container in distribute-shell doesn't need any tokens. We are
- // populating them mainly for NodeManagers to be able to download any
- // files in the distributed file-system. The tokens are otherwise also
- // useful in cases, for e.g., when one is running a "hadoop dfs" command
- // inside the distributed shell.
- ctx.setTokens(allTokens.duplicate());
+ // Set up ContainerLaunchContext, setting local resource, environment,
+ // command and token for constructor.
+ // Note for tokens: Set up tokens for the container too. Today, for
normal
+ // shell commands, the container in distribute-shell doesn't need any
+ // tokens. We are populating them mainly for NodeManagers to be able to
+ // download anyfiles in the distributed file-system. The tokens are
+ // otherwise also useful in cases, for e.g., when one is running a
+ // "hadoop dfs" command inside the distributed shell.
+ ContainerLaunchContext ctx = ContainerLaunchContext.newInstance(
+ localResources, shellEnv, commands, null, allTokens.duplicate(), null);
containerListener.addContainer(container.getId(), container);
nmClientAsync.startContainerAsync(container, ctx);
}
@@ -1024,15 +1020,13 @@ public class ApplicationMaster {
// setup requirements for hosts
// using * as any host will do for the distributed shell app
// set the priority for the request
- Priority pri = Records.newRecord(Priority.class);
// TODO - what is the range for priority? how to decide?
- pri.setPriority(requestPriority);
+ Priority pri = Priority.newInstance(requestPriority);
// Set up resource type requirements
// For now, memory and CPU are supported so we set memory and cpu
requirements
- Resource capability = Records.newRecord(Resource.class);
- capability.setMemory(containerMemory);
- capability.setVirtualCores(containerVirtualCores);
+ Resource capability = Resource.newInstance(containerMemory,
+ containerVirtualCores);
ContainerRequest request = new ContainerRequest(capability, null, null,
pri);
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-distributedshell/src/main/java/org/apache/hadoop/yarn/applications/distributedshell/Client.java
Fri Jul 25 20:33:09 2014
@@ -456,9 +456,6 @@ public class Client {
appContext.setKeepContainersAcrossApplicationAttempts(keepContainers);
appContext.setApplicationName(appName);
- // Set up the container launch context for the application master
- ContainerLaunchContext amContainer =
Records.newRecord(ContainerLaunchContext.class);
-
// set local resources for the application master
// local files or archives as needed
// In this scenario, the jar file for the application master is part of
the local resources
@@ -508,8 +505,6 @@ public class Client {
addToLocalResources(fs, null, shellArgsPath, appId.toString(),
localResources, StringUtils.join(shellArgs, " "));
}
- // Set local resource info into app master container launch context
- amContainer.setLocalResources(localResources);
// Set the necessary security tokens as needed
//amContainer.setContainerTokens(containerToken);
@@ -550,8 +545,6 @@ public class Client {
env.put("CLASSPATH", classPathEnv.toString());
- amContainer.setEnvironment(env);
-
// Set the necessary command to execute the application master
Vector<CharSequence> vargs = new Vector<CharSequence>(30);
@@ -587,14 +580,15 @@ public class Client {
LOG.info("Completed setting up app master command " + command.toString());
List<String> commands = new ArrayList<String>();
commands.add(command.toString());
- amContainer.setCommands(commands);
+
+ // Set up the container launch context for the application master
+ ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance(
+ localResources, env, commands, null, null, null);
// Set up resource type requirements
// For now, both memory and vcores are supported, so we set memory and
// vcores requirements
- Resource capability = Records.newRecord(Resource.class);
- capability.setMemory(amMemory);
- capability.setVirtualCores(amVCores);
+ Resource capability = Resource.newInstance(amMemory, amVCores);
appContext.setResource(capability);
// Service data is a binary blob that can be passed to the application
@@ -603,6 +597,7 @@ public class Client {
// Setup security tokens
if (UserGroupInformation.isSecurityEnabled()) {
+ // Note: Credentials class is marked as LimitedPrivate for HDFS and
MapReduce
Credentials credentials = new Credentials();
String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL);
if (tokenRenewer == null || tokenRenewer.length() == 0) {
@@ -627,9 +622,8 @@ public class Client {
appContext.setAMContainerSpec(amContainer);
// Set the priority for the application master
- Priority pri = Records.newRecord(Priority.class);
// TODO - what is the range for priority? how to decide?
- pri.setPriority(amPriority);
+ Priority pri = Priority.newInstance(amPriority);
appContext.setPriority(pri);
// Set the queue to which this application is to be submitted in the RM
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/api/impl/TimelineClientImpl.java
Fri Jul 25 20:33:09 2014
@@ -72,6 +72,7 @@ public class TimelineClientImpl extends
private static final Log LOG = LogFactory.getLog(TimelineClientImpl.class);
private static final String RESOURCE_URI_STR = "/ws/v1/timeline/";
+ private static final String URL_PARAM_USER_NAME = "user.name";
private static final Joiner JOINER = Joiner.on("");
private static Options opts;
static {
@@ -84,17 +85,18 @@ public class TimelineClientImpl extends
private Client client;
private URI resURI;
private boolean isEnabled;
- private TimelineAuthenticatedURLConnectionFactory urlFactory;
+ private KerberosAuthenticatedURLConnectionFactory urlFactory;
public TimelineClientImpl() {
super(TimelineClientImpl.class.getName());
ClientConfig cc = new DefaultClientConfig();
cc.getClasses().add(YarnJacksonJaxbJsonProvider.class);
if (UserGroupInformation.isSecurityEnabled()) {
- urlFactory = new TimelineAuthenticatedURLConnectionFactory();
+ urlFactory = new KerberosAuthenticatedURLConnectionFactory();
client = new Client(new URLConnectionClientHandler(urlFactory), cc);
} else {
- client = Client.create(cc);
+ client = new Client(new URLConnectionClientHandler(
+ new PseudoAuthenticatedURLConnectionFactory()), cc);
}
}
@@ -177,7 +179,23 @@ public class TimelineClientImpl extends
.post(ClientResponse.class, entities);
}
- private static class TimelineAuthenticatedURLConnectionFactory
+ private static class PseudoAuthenticatedURLConnectionFactory
+ implements HttpURLConnectionFactory {
+
+ @Override
+ public HttpURLConnection getHttpURLConnection(URL url) throws IOException {
+ Map<String, String> params = new HashMap<String, String>();
+ params.put(URL_PARAM_USER_NAME,
+ UserGroupInformation.getCurrentUser().getShortUserName());
+ url = TimelineAuthenticator.appendParams(url, params);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("URL with delegation token: " + url);
+ }
+ return (HttpURLConnection) url.openConnection();
+ }
+
+ }
+ private static class KerberosAuthenticatedURLConnectionFactory
implements HttpURLConnectionFactory {
private AuthenticatedURL.Token token;
@@ -185,7 +203,7 @@ public class TimelineClientImpl extends
private Token<TimelineDelegationTokenIdentifier> dToken;
private Text service;
- public TimelineAuthenticatedURLConnectionFactory() {
+ public KerberosAuthenticatedURLConnectionFactory() {
token = new AuthenticatedURL.Token();
authenticator = new TimelineAuthenticator();
}
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java
Fri Jul 25 20:33:09 2014
@@ -44,6 +44,7 @@ public class AMRMTokenIdentifier extends
public static final Text KIND_NAME = new Text("YARN_AM_RM_TOKEN");
private ApplicationAttemptId applicationAttemptId;
+ private int keyId = Integer.MIN_VALUE;
public AMRMTokenIdentifier() {
}
@@ -53,6 +54,13 @@ public class AMRMTokenIdentifier extends
this.applicationAttemptId = appAttemptId;
}
+ public AMRMTokenIdentifier(ApplicationAttemptId appAttemptId,
+ int masterKeyId) {
+ this();
+ this.applicationAttemptId = appAttemptId;
+ this.keyId = masterKeyId;
+ }
+
@Private
public ApplicationAttemptId getApplicationAttemptId() {
return this.applicationAttemptId;
@@ -64,6 +72,7 @@ public class AMRMTokenIdentifier extends
out.writeLong(appId.getClusterTimestamp());
out.writeInt(appId.getId());
out.writeInt(this.applicationAttemptId.getAttemptId());
+ out.writeInt(this.keyId);
}
@Override
@@ -75,6 +84,7 @@ public class AMRMTokenIdentifier extends
ApplicationId.newInstance(clusterTimeStamp, appId);
this.applicationAttemptId =
ApplicationAttemptId.newInstance(applicationId, attemptId);
+ this.keyId = in.readInt();
}
@Override
@@ -92,6 +102,10 @@ public class AMRMTokenIdentifier extends
.toString());
}
+ public int getKeyId() {
+ return this.keyId;
+ }
+
// TODO: Needed?
@InterfaceAudience.Private
public static class Renewer extends Token.TrivialRenewer {
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
Fri Jul 25 20:33:09 2014
@@ -177,9 +177,10 @@ public class FSDownload implements Calla
/**
* Returns true if all ancestors of the specified path have the 'execute'
* permission set for all users (i.e. that other users can traverse
- * the directory heirarchy to the given path)
+ * the directory hierarchy to the given path)
*/
- private static boolean ancestorsHaveExecutePermissions(FileSystem fs,
+ @VisibleForTesting
+ static boolean ancestorsHaveExecutePermissions(FileSystem fs,
Path path, LoadingCache<Path,Future<FileStatus>> statCache)
throws IOException {
Path current = path;
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
Fri Jul 25 20:33:09 2014
@@ -81,6 +81,8 @@ public class GenericExceptionHandler imp
s = Response.Status.NOT_FOUND;
} else if (e instanceof IOException) {
s = Response.Status.NOT_FOUND;
+ } else if (e instanceof ForbiddenException) {
+ s = Response.Status.FORBIDDEN;
} else if (e instanceof UnsupportedOperationException) {
s = Response.Status.BAD_REQUEST;
} else if (e instanceof IllegalArgumentException) {
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
Fri Jul 25 20:33:09 2014
@@ -1218,6 +1218,24 @@
</property>
<property>
+ <name>yarn.timeline-service.http-authentication.type</name>
+ <value>simple</value>
+ <description>
+ Defines authentication used for the timeline server HTTP endpoint.
+ Supported values are: simple | kerberos |
#AUTHENTICATION_HANDLER_CLASSNAME#
+ </description>
+ </property>
+
+ <property>
+
<name>yarn.timeline-service.http-authentication.simple.anonymous.allowed</name>
+ <value>true</value>
+ <description>
+ Indicates if anonymous requests are allowed by the timeline server when
using
+ 'simple' authentication.
+ </description>
+ </property>
+
+ <property>
<description>The Kerberos principal for the timeline server.</description>
<name>yarn.timeline-service.principal</name>
<value></value>
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
Fri Jul 25 20:33:09 2014
@@ -28,6 +28,7 @@ public class DrainDispatcher extends Asy
// and similar grotesqueries
private volatile boolean drained = false;
private final BlockingQueue<Event> queue;
+ final Object mutex;
public DrainDispatcher() {
this(new LinkedBlockingQueue<Event>());
@@ -36,6 +37,7 @@ public class DrainDispatcher extends Asy
private DrainDispatcher(BlockingQueue<Event> eventQueue) {
super(eventQueue);
this.queue = eventQueue;
+ this.mutex = this;
}
/**
@@ -53,8 +55,10 @@ public class DrainDispatcher extends Asy
@Override
public void run() {
while (!Thread.currentThread().isInterrupted()) {
- // !drained if dispatch queued new events on this dispatcher
- drained = queue.isEmpty();
+ synchronized (mutex) {
+ // !drained if dispatch queued new events on this dispatcher
+ drained = queue.isEmpty();
+ }
Event event;
try {
event = queue.take();
@@ -75,8 +79,10 @@ public class DrainDispatcher extends Asy
return new EventHandler() {
@Override
public void handle(Event event) {
- drained = false;
- actual.handle(event);
+ synchronized (mutex) {
+ actual.handle(event);
+ drained = false;
+ }
}
};
}
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestFSDownload.java
Fri Jul 25 20:33:09 2014
@@ -23,6 +23,7 @@ import static org.apache.hadoop.fs.Creat
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
+import static org.junit.Assume.assumeTrue;
import java.io.File;
import java.io.FileOutputStream;
@@ -66,6 +67,7 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocalDirAllocator;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.util.Shell;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceType;
@@ -308,6 +310,11 @@ public class TestFSDownload {
FileContext files = FileContext.getLocalFSFileContext(conf);
Path basedir = files.makeQualified(new Path("target",
TestFSDownload.class.getSimpleName()));
+
+ // if test directory doesn't have ancestor permission, skip this test
+ FileSystem f = basedir.getFileSystem(conf);
+ assumeTrue(FSDownload.ancestorsHaveExecutePermissions(f, basedir, null));
+
files.mkdir(basedir, null, true);
conf.setStrings(TestFSDownload.class.getName(), basedir.toString());
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryServer.java
Fri Jul 25 20:33:09 2014
@@ -28,7 +28,6 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.metrics2.source.JvmMetrics;
import org.apache.hadoop.security.SecurityUtil;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.service.Service;
import org.apache.hadoop.util.ExitUtil;
@@ -178,23 +177,20 @@ public class ApplicationHistoryServer ex
protected void startWebApp() {
Configuration conf = getConfig();
- // Play trick to make the customized filter will only be loaded by the
- // timeline server when security is enabled and Kerberos authentication
- // is used.
- if (UserGroupInformation.isSecurityEnabled()
- && conf
- .get(TimelineAuthenticationFilterInitializer.PREFIX + "type", "")
- .equals("kerberos")) {
- String initializers = conf.get("hadoop.http.filter.initializers");
- initializers =
- initializers == null || initializers.length() == 0 ? "" : ","
- + initializers;
- if (!initializers.contains(
- TimelineAuthenticationFilterInitializer.class.getName())) {
- conf.set("hadoop.http.filter.initializers",
- TimelineAuthenticationFilterInitializer.class.getName()
- + initializers);
- }
+ // Always load pseudo authentication filter to parse "user.name" in an URL
+ // to identify a HTTP request's user in insecure mode.
+ // When Kerberos authentication type is set (i.e., secure mode is turned
on),
+ // the customized filter will be loaded by the timeline server to do
Kerberos
+ // + DT authentication.
+ String initializers = conf.get("hadoop.http.filter.initializers");
+ initializers =
+ initializers == null || initializers.length() == 0 ? "" : ","
+ + initializers;
+ if (!initializers.contains(
+ TimelineAuthenticationFilterInitializer.class.getName())) {
+ conf.set("hadoop.http.filter.initializers",
+ TimelineAuthenticationFilterInitializer.class.getName()
+ + initializers);
}
String bindAddress = WebAppUtils.getAHSWebAppURLWithoutScheme(conf);
LOG.info("Instantiating AHSWebApp at " + bindAddress);
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineACLsManager.java
Fri Jul 25 20:33:09 2014
@@ -51,7 +51,8 @@ public class TimelineACLsManager {
public boolean checkAccess(UserGroupInformation callerUGI,
TimelineEntity entity) throws YarnException, IOException {
if (LOG.isDebugEnabled()) {
- LOG.debug("Verifying the access of " + callerUGI.getShortUserName()
+ LOG.debug("Verifying the access of "
+ + (callerUGI == null ? null : callerUGI.getShortUserName())
+ " on the timeline entity "
+ new EntityIdentifier(entity.getEntityId(),
entity.getEntityType()));
}
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilter.java
Fri Jul 25 20:33:09 2014
@@ -38,7 +38,8 @@ public class TimelineAuthenticationFilte
// to replace the name here to use the customized Kerberos + DT service
// instead of the standard Kerberos handler.
Properties properties = super.getConfiguration(configPrefix, filterConfig);
- if (properties.getProperty(AUTH_TYPE).equals("kerberos")) {
+ String authType = properties.getProperty(AUTH_TYPE);
+ if (authType != null && authType.equals("kerberos")) {
properties.setProperty(
AUTH_TYPE, TimelineClientAuthenticationService.class.getName());
}
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/security/TimelineAuthenticationFilterInitializer.java
Fri Jul 25 20:33:09 2014
@@ -47,9 +47,9 @@ import org.apache.hadoop.security.Securi
public class TimelineAuthenticationFilterInitializer extends FilterInitializer
{
/**
- * The configuration prefix of timeline Kerberos + DT authentication
+ * The configuration prefix of timeline HTTP authentication
*/
- public static final String PREFIX =
"yarn.timeline-service.http.authentication.";
+ public static final String PREFIX =
"yarn.timeline-service.http-authentication.";
private static final String SIGNATURE_SECRET_FILE =
TimelineAuthenticationFilter.SIGNATURE_SECRET + ".file";
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/timeline/webapp/TimelineWebServices.java
Fri Jul 25 20:33:09 2014
@@ -62,11 +62,12 @@ import org.apache.hadoop.yarn.exceptions
import org.apache.hadoop.yarn.server.timeline.EntityIdentifier;
import org.apache.hadoop.yarn.server.timeline.GenericObjectMapper;
import org.apache.hadoop.yarn.server.timeline.NameValuePair;
-import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineReader.Field;
+import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.hadoop.yarn.webapp.BadRequestException;
+import org.apache.hadoop.yarn.webapp.ForbiddenException;
import org.apache.hadoop.yarn.webapp.NotFoundException;
import com.google.inject.Inject;
@@ -336,6 +337,11 @@ public class TimelineWebServices {
return new TimelinePutResponse();
}
UserGroupInformation callerUGI = getUser(req);
+ if (callerUGI == null) {
+ String msg = "The owner of the posted timeline entities is not set";
+ LOG.error(msg);
+ throw new ForbiddenException(msg);
+ }
try {
List<EntityIdentifier> entityIDs = new ArrayList<EntityIdentifier>();
TimelineEntities entitiesToPut = new TimelineEntities();
@@ -375,8 +381,7 @@ public class TimelineWebServices {
// the timeline data.
try {
if (existingEntity == null) {
- injectOwnerInfo(entity,
- callerUGI == null ? "" : callerUGI.getShortUserName());
+ injectOwnerInfo(entity, callerUGI.getShortUserName());
}
} catch (YarnException e) {
// Skip the entity which messes up the primary filter and record the
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestMemoryApplicationHistoryStore.java
Fri Jul 25 20:33:09 2014
@@ -198,7 +198,7 @@ public class TestMemoryApplicationHistor
writeContainerFinishData(containerId);
}
long usedMemoryAfter = (runtime.totalMemory() - runtime.freeMemory()) / mb;
- Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 200);
+ Assert.assertTrue((usedMemoryAfter - usedMemoryBefore) < 400);
}
}
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/timeline/webapp/TestTimelineWebServices.java
Fri Jul 25 20:33:09 2014
@@ -19,26 +19,26 @@
package org.apache.hadoop.yarn.server.timeline.webapp;
import static org.junit.Assert.assertEquals;
+import static org.mockito.Matchers.any;
+import static org.mockito.Mockito.doNothing;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.when;
-import java.io.IOException;
+import java.util.Enumeration;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import javax.inject.Singleton;
-import javax.servlet.Filter;
-import javax.servlet.FilterChain;
import javax.servlet.FilterConfig;
import javax.servlet.ServletException;
-import javax.servlet.ServletRequest;
-import javax.servlet.ServletResponse;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletRequestWrapper;
import javax.ws.rs.core.MediaType;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
+import
org.apache.hadoop.security.authentication.server.PseudoAuthenticationHandler;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntities;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEntity;
import org.apache.hadoop.yarn.api.records.timeline.TimelineEvent;
@@ -46,12 +46,11 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse;
import
org.apache.hadoop.yarn.api.records.timeline.TimelinePutResponse.TimelinePutError;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
-import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.security.AdminACLsManager;
import org.apache.hadoop.yarn.server.timeline.TestMemoryTimelineStore;
import org.apache.hadoop.yarn.server.timeline.TimelineStore;
import org.apache.hadoop.yarn.server.timeline.security.TimelineACLsManager;
-import
org.apache.hadoop.yarn.server.timeline.webapp.TimelineWebServices.AboutInfo;
+import
org.apache.hadoop.yarn.server.timeline.security.TimelineAuthenticationFilter;
import org.apache.hadoop.yarn.webapp.GenericExceptionHandler;
import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.junit.Assert;
@@ -74,11 +73,11 @@ public class TestTimelineWebServices ext
private static TimelineStore store;
private static TimelineACLsManager timelineACLsManager;
private static AdminACLsManager adminACLsManager;
- private static String remoteUser;
private long beforeTime;
private Injector injector = Guice.createInjector(new ServletModule() {
+ @SuppressWarnings("unchecked")
@Override
protected void configureServlets() {
bind(YarnJacksonJaxbJsonProvider.class);
@@ -98,7 +97,35 @@ public class TestTimelineWebServices ext
adminACLsManager = new AdminACLsManager(conf);
bind(TimelineACLsManager.class).toInstance(timelineACLsManager);
serve("/*").with(GuiceContainer.class);
- filter("/*").through(TestFilter.class);
+ TimelineAuthenticationFilter taFilter = new
TimelineAuthenticationFilter();
+ FilterConfig filterConfig = mock(FilterConfig.class);
+ when(filterConfig.getInitParameter(AuthenticationFilter.CONFIG_PREFIX))
+ .thenReturn(null);
+ when(filterConfig.getInitParameter(AuthenticationFilter.AUTH_TYPE))
+ .thenReturn("simple");
+ when(filterConfig.getInitParameter(
+ PseudoAuthenticationHandler.ANONYMOUS_ALLOWED)).thenReturn("true");
+ Enumeration<Object> names = mock(Enumeration.class);
+ when(names.hasMoreElements()).thenReturn(true, true, false);
+ when(names.nextElement()).thenReturn(
+ AuthenticationFilter.AUTH_TYPE,
+ PseudoAuthenticationHandler.ANONYMOUS_ALLOWED);
+ when(filterConfig.getInitParameterNames()).thenReturn(names);
+ try {
+ taFilter.init(filterConfig);
+ } catch (ServletException e) {
+ Assert.fail("Unable to initialize TimelineAuthenticationFilter: " +
+ e.getMessage());
+ }
+
+ taFilter = spy(taFilter);
+ try {
+ doNothing().when(taFilter).init(any(FilterConfig.class));
+ } catch (ServletException e) {
+ Assert.fail("Unable to initialize TimelineAuthenticationFilter: " +
+ e.getMessage());
+ }
+ filter("/*").through(taFilter);
}
});
@@ -382,6 +409,7 @@ public class TestTimelineWebServices ext
entities.addEntity(entity);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .queryParam("user.name", "tester")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
@@ -401,11 +429,21 @@ public class TestTimelineWebServices ext
entity.setStartTime(System.currentTimeMillis());
entities.addEntity(entity);
WebResource r = resource();
+ // No owner, will be rejected
ClientResponse response = r.path("ws").path("v1").path("timeline")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
+ assertEquals(ClientResponse.Status.FORBIDDEN,
+ response.getClientResponseStatus());
+
+ response = r.path("ws").path("v1").path("timeline")
+ .queryParam("user.name", "tester")
+ .accept(MediaType.APPLICATION_JSON)
+ .type(MediaType.APPLICATION_JSON)
+ .post(ClientResponse.class, entities);
+ assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
TimelinePutResponse putResposne =
response.getEntity(TimelinePutResponse.class);
Assert.assertNotNull(putResposne);
Assert.assertEquals(0, putResposne.getErrors().size());
@@ -425,7 +463,6 @@ public class TestTimelineWebServices ext
public void testPostEntitiesWithYarnACLsEnabled() throws Exception {
AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager);
- remoteUser = "tester";
try {
TimelineEntities entities = new TimelineEntities();
TimelineEntity entity = new TimelineEntity();
@@ -435,6 +472,7 @@ public class TestTimelineWebServices ext
entities.addEntity(entity);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .queryParam("user.name", "tester")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
@@ -444,8 +482,8 @@ public class TestTimelineWebServices ext
Assert.assertEquals(0, putResponse.getErrors().size());
// override/append timeline data in the same entity with different user
- remoteUser = "other";
response = r.path("ws").path("v1").path("timeline")
+ .queryParam("user.name", "other")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
@@ -457,7 +495,6 @@ public class TestTimelineWebServices ext
putResponse.getErrors().get(0).getErrorCode());
} finally {
timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
- remoteUser = null;
}
}
@@ -465,7 +502,6 @@ public class TestTimelineWebServices ext
public void testGetEntityWithYarnACLsEnabled() throws Exception {
AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager);
- remoteUser = "tester";
try {
TimelineEntities entities = new TimelineEntities();
TimelineEntity entity = new TimelineEntity();
@@ -475,6 +511,7 @@ public class TestTimelineWebServices ext
entities.addEntity(entity);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .queryParam("user.name", "tester")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
@@ -482,6 +519,7 @@ public class TestTimelineWebServices ext
// 1. No field specification
response = r.path("ws").path("v1").path("timeline")
.path("test type 3").path("test id 3")
+ .queryParam("user.name", "tester")
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
@@ -492,6 +530,7 @@ public class TestTimelineWebServices ext
response = r.path("ws").path("v1").path("timeline")
.path("test type 3").path("test id 3")
.queryParam("fields", "relatedentities")
+ .queryParam("user.name", "tester")
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
@@ -502,6 +541,7 @@ public class TestTimelineWebServices ext
response = r.path("ws").path("v1").path("timeline")
.path("test type 3").path("test id 3")
.queryParam("fields", "primaryfilters")
+ .queryParam("user.name", "tester")
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
@@ -510,9 +550,9 @@ public class TestTimelineWebServices ext
TimelineStore.SystemFilter.ENTITY_OWNER.toString()));
// get entity with other user
- remoteUser = "other";
response = r.path("ws").path("v1").path("timeline")
.path("test type 3").path("test id 3")
+ .queryParam("user.name", "other")
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
assertEquals(MediaType.APPLICATION_JSON_TYPE, response.getType());
@@ -520,7 +560,6 @@ public class TestTimelineWebServices ext
response.getClientResponseStatus());
} finally {
timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
- remoteUser = null;
}
}
@@ -528,7 +567,6 @@ public class TestTimelineWebServices ext
public void testGetEntitiesWithYarnACLsEnabled() {
AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager);
- remoteUser = "tester";
try {
TimelineEntities entities = new TimelineEntities();
TimelineEntity entity = new TimelineEntity();
@@ -538,11 +576,11 @@ public class TestTimelineWebServices ext
entities.addEntity(entity);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .queryParam("user.name", "tester")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
- remoteUser = "other";
entities = new TimelineEntities();
entity = new TimelineEntity();
entity.setEntityId("test id 5");
@@ -551,11 +589,13 @@ public class TestTimelineWebServices ext
entities.addEntity(entity);
r = resource();
response = r.path("ws").path("v1").path("timeline")
+ .queryParam("user.name", "other")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
response = r.path("ws").path("v1").path("timeline")
+ .queryParam("user.name", "other")
.path("test type 4")
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
@@ -566,7 +606,6 @@ public class TestTimelineWebServices ext
assertEquals("test id 5", entities.getEntities().get(0).getEntityId());
} finally {
timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
- remoteUser = null;
}
}
@@ -574,7 +613,6 @@ public class TestTimelineWebServices ext
public void testGetEventsWithYarnACLsEnabled() {
AdminACLsManager oldAdminACLsManager =
timelineACLsManager.setAdminACLsManager(adminACLsManager);
- remoteUser = "tester";
try {
TimelineEntities entities = new TimelineEntities();
TimelineEntity entity = new TimelineEntity();
@@ -588,11 +626,11 @@ public class TestTimelineWebServices ext
entities.addEntity(entity);
WebResource r = resource();
ClientResponse response = r.path("ws").path("v1").path("timeline")
+ .queryParam("user.name", "tester")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
- remoteUser = "other";
entities = new TimelineEntities();
entity = new TimelineEntity();
entity.setEntityId("test id 6");
@@ -605,12 +643,14 @@ public class TestTimelineWebServices ext
entities.addEntity(entity);
r = resource();
response = r.path("ws").path("v1").path("timeline")
+ .queryParam("user.name", "other")
.accept(MediaType.APPLICATION_JSON)
.type(MediaType.APPLICATION_JSON)
.post(ClientResponse.class, entities);
response = r.path("ws").path("v1").path("timeline")
.path("test type 5").path("events")
+ .queryParam("user.name", "other")
.queryParam("entityId", "test id 5,test id 6")
.accept(MediaType.APPLICATION_JSON)
.get(ClientResponse.class);
@@ -620,43 +660,7 @@ public class TestTimelineWebServices ext
assertEquals("test id 6", events.getAllEvents().get(0).getEntityId());
} finally {
timelineACLsManager.setAdminACLsManager(oldAdminACLsManager);
- remoteUser = null;
}
}
- @Singleton
- private static class TestFilter implements Filter {
-
- @Override
- public void init(FilterConfig filterConfig) throws ServletException {
- }
-
- @Override
- public void doFilter(ServletRequest request, ServletResponse response,
- FilterChain chain) throws IOException, ServletException {
- if (request instanceof HttpServletRequest) {
- request =
- new TestHttpServletRequestWrapper((HttpServletRequest) request);
- }
- chain.doFilter(request, response);
- }
-
- @Override
- public void destroy() {
- }
-
- }
-
- private static class TestHttpServletRequestWrapper extends
HttpServletRequestWrapper {
-
- public TestHttpServletRequestWrapper(HttpServletRequest request) {
- super(request);
- }
-
- @Override
- public String getRemoteUser() {
- return TestTimelineWebServices.remoteUser;
- }
-
- }
}
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
Fri Jul 25 20:33:09 2014
@@ -43,7 +43,7 @@ public class BaseContainerTokenSecretMan
private static Log LOG = LogFactory
.getLog(BaseContainerTokenSecretManager.class);
- private int serialNo = new SecureRandom().nextInt();
+ protected int serialNo = new SecureRandom().nextInt();
protected final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
protected final Lock readLock = readWriteLock.readLock();
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseNMTokenSecretManager.java
Fri Jul 25 20:33:09 2014
@@ -42,7 +42,7 @@ public class BaseNMTokenSecretManager ex
private static Log LOG = LogFactory
.getLog(BaseNMTokenSecretManager.class);
- private int serialNo = new SecureRandom().nextInt();
+ protected int serialNo = new SecureRandom().nextInt();
protected final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
protected final Lock readLock = readWriteLock.readLock();
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
Fri Jul 25 20:33:09 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import com.google.common.base.Optional;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
@@ -212,10 +213,21 @@ public class DefaultContainerExecutor ex
&& exitCode != ExitCode.TERMINATED.getExitCode()) {
LOG.warn("Exception from container-launch with container ID: "
+ containerId + " and exit code: " + exitCode , e);
- logOutput(shExec.getOutput());
- String diagnostics = "Exception from container-launch: "
- + e + "\n"
- + StringUtils.stringifyException(e) + "\n" + shExec.getOutput();
+
+ StringBuilder builder = new StringBuilder();
+ builder.append("Exception from container-launch.\n");
+ builder.append("Container id: " + containerId + "\n");
+ builder.append("Exit code: " + exitCode + "\n");
+ if (!Optional.fromNullable(e.getMessage()).or("").isEmpty()) {
+ builder.append("Exception message: " + e.getMessage() + "\n");
+ }
+ builder.append("Stack trace: "
+ + StringUtils.stringifyException(e) + "\n");
+ if (!shExec.getOutput().isEmpty()) {
+ builder.append("Shell output: " + shExec.getOutput() + "\n");
+ }
+ String diagnostics = builder.toString();
+ logOutput(diagnostics);
container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
diagnostics));
} else {
Modified:
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL:
http://svn.apache.org/viewvc/hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1613514&r1=1613513&r2=1613514&view=diff
==============================================================================
---
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
(original)
+++
hadoop/common/branches/YARN-1051/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
Fri Jul 25 20:33:09 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.server.nodemanager;
+import com.google.common.base.Optional;
import java.io.File;
import java.io.IOException;
import java.net.InetSocketAddress;
@@ -296,9 +297,21 @@ public class LinuxContainerExecutor exte
&& exitCode != ExitCode.TERMINATED.getExitCode()) {
LOG.warn("Exception from container-launch with container ID: "
+ containerId + " and exit code: " + exitCode , e);
- logOutput(shExec.getOutput());
- String diagnostics = "Exception from container-launch: \n"
- + StringUtils.stringifyException(e) + "\n" + shExec.getOutput();
+
+ StringBuilder builder = new StringBuilder();
+ builder.append("Exception from container-launch.\n");
+ builder.append("Container id: " + containerId + "\n");
+ builder.append("Exit code: " + exitCode + "\n");
+ if (!Optional.fromNullable(e.getMessage()).or("").isEmpty()) {
+ builder.append("Exception message: " + e.getMessage() + "\n");
+ }
+ builder.append("Stack trace: "
+ + StringUtils.stringifyException(e) + "\n");
+ if (!shExec.getOutput().isEmpty()) {
+ builder.append("Shell output: " + shExec.getOutput() + "\n");
+ }
+ String diagnostics = builder.toString();
+ logOutput(diagnostics);
container.handle(new ContainerDiagnosticsUpdateEvent(containerId,
diagnostics));
} else {
