Merge r1609845 through r1618416 from trunk.
git-svn-id: https://svn.apache.org/repos/asf/hadoop/common/branches/HDFS-6584@1618417 13f79535-47bb-0310-9956-ffa450edef68 Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/cb75b6b0 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/cb75b6b0 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/cb75b6b0 Branch: refs/heads/trunk Commit: cb75b6b07d30a63eefc278badafce96e849267b9 Parents: 9b250d7 c3084d6 Author: Tsz-wo Sze <[email protected]> Authored: Sat Aug 16 21:02:21 2014 +0000 Committer: Tsz-wo Sze <[email protected]> Committed: Sat Aug 16 21:02:21 2014 +0000 ---------------------------------------------------------------------- hadoop-common-project/hadoop-common/CHANGES.txt | 36 + .../crypto/key/KeyProviderCryptoExtension.java | 13 +- .../KeyProviderDelegationTokenExtension.java | 8 +- .../crypto/key/kms/KMSClientProvider.java | 110 +- .../org/apache/hadoop/fs/ContentSummary.java | 46 +- .../java/org/apache/hadoop/fs/shell/Count.java | 36 +- .../apache/hadoop/ipc/DecayRpcScheduler.java | 522 +++++++ .../hadoop/ipc/DecayRpcSchedulerMXBean.java | 30 + .../org/apache/hadoop/ipc/RpcScheduler.java | 29 + .../net/NetworkTopologyWithNodeGroup.java | 2 +- .../apache/hadoop/security/NetgroupCache.java | 17 +- .../DelegationTokenAuthenticationHandler.java | 2 +- .../org/apache/hadoop/util/DataChecksum.java | 6 + .../org/apache/hadoop/util/NativeCrc32.java | 16 + .../org/apache/hadoop/io/nativeio/NativeIO.c | 35 +- .../src/org/apache/hadoop/util/NativeCrc32.c | 82 ++ .../src/site/apt/CommandsManual.apt.vm | 166 +-- .../src/site/apt/FileSystemShell.apt.vm | 6 +- .../src/site/apt/NativeLibraries.apt.vm | 14 +- .../apache/hadoop/fs/TestContentSummary.java | 248 ++++ .../org/apache/hadoop/fs/shell/TestCount.java | 270 ++++ .../hadoop/ipc/TestDecayRpcScheduler.java | 225 +++ .../hadoop/security/TestNetgroupCache.java | 127 ++ .../src/test/resources/testConf.xml | 6 +- .../hadoop/crypto/key/kms/server/KMS.java | 272 ++-- .../hadoop/crypto/key/kms/server/KMSACLs.java | 3 +- .../hadoop/crypto/key/kms/server/KMSAudit.java | 18 +- .../key/kms/server/KMSAuthenticationFilter.java | 32 +- .../key/kms/server/KMSExceptionsProvider.java | 8 +- .../crypto/key/kms/server/KMSMDCFilter.java | 17 +- .../hadoop-kms/src/site/apt/index.apt.vm | 80 + .../hadoop/crypto/key/kms/server/TestKMS.java | 261 +++- .../crypto/key/kms/server/TestKMSACLs.java | 10 +- .../crypto/key/kms/server/TestKMSAudit.java | 10 +- .../http/server/HttpFSAuthenticationFilter.java | 10 + .../http/server/HttpFSParametersProvider.java | 108 +- .../hadoop/fs/http/server/HttpFSServer.java | 144 +- .../fs/http/server/HttpFSServerWebApp.java | 4 - .../apache/hadoop/lib/service/ProxyUser.java | 31 - .../lib/service/security/ProxyUserService.java | 179 --- .../apache/hadoop/lib/wsrs/UserProvider.java | 109 -- .../src/main/resources/httpfs-default.xml | 6 +- .../service/security/TestProxyUserService.java | 226 --- .../hadoop/lib/wsrs/TestUserProvider.java | 142 -- .../apache/hadoop/hdfs/nfs/nfs3/TestWrites.java | 77 + hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 26 +- .../hadoop/hdfs/protocol/HdfsFileStatus.java | 38 +- .../hdfs/protocol/HdfsLocatedFileStatus.java | 2 +- .../hadoop/hdfs/server/balancer/Dispatcher.java | 17 +- .../hdfs/server/blockmanagement/BlockInfo.java | 26 +- .../server/blockmanagement/BlockManager.java | 4 +- .../CacheReplicationMonitor.java | 47 +- .../blockmanagement/DatanodeStorageInfo.java | 22 +- .../hdfs/server/datanode/BlockReceiver.java | 32 +- .../hdfs/server/datanode/BlockSender.java | 6 +- .../hdfs/server/datanode/DataXceiver.java | 9 +- .../src/main/proto/datatransfer.proto | 1 + .../server/blockmanagement/TestBlockInfo.java | 20 +- .../server/datanode/TestBlockReplacement.java | 6 +- .../src/test/resources/testHDFSConf.xml | 44 + hadoop-mapreduce-project/CHANGES.txt | 40 + .../hadoop-mapreduce-client-app/pom.xml | 6 + .../jobhistory/JobHistoryEventHandler.java | 12 +- .../jobhistory/TestJobHistoryEventHandler.java | 97 +- .../v2/jobhistory/JobHistoryUtils.java | 75 +- .../org/apache/hadoop/mapred/InputFormat.java | 2 +- .../hadoop/mapred/TaskCompletionEvent.java | 4 +- .../mapred/join/CompositeInputFormat.java | 2 +- .../apache/hadoop/mapreduce/InputFormat.java | 2 +- .../java/org/apache/hadoop/mapreduce/Job.java | 11 +- .../apache/hadoop/mapreduce/MRJobConfig.java | 1 + .../hadoop/mapreduce/TaskCompletionEvent.java | 4 +- .../lib/input/KeyValueTextInputFormat.java | 3 + .../lib/join/CompositeInputFormat.java | 2 +- .../mapreduce/task/reduce/MergeManagerImpl.java | 5 +- .../src/main/resources/mapred-default.xml | 6 +- .../src/site/apt/MapredCommands.apt.vm | 224 +++ .../src/site/markdown/DistCp.md.vm | 8 +- .../src/site/markdown/HadoopArchives.md.vm | 15 + ...istoryServerFileSystemStateStoreService.java | 101 +- .../mapreduce/v2/hs/TestHistoryFileManager.java | 70 +- ...istoryServerFileSystemStateStoreService.java | 85 +- hadoop-project/src/site/site.xml | 2 + .../org/apache/hadoop/tools/HadoopArchives.java | 2 +- .../mapred/gridmix/TestGridmixRecord.java | 10 +- .../sls/scheduler/FairSchedulerMetrics.java | 8 +- hadoop-yarn-project/CHANGES.txt | 30 + .../hadoop/yarn/conf/YarnConfiguration.java | 2 +- .../distributedshell/ApplicationMaster.java | 12 +- .../src/main/resources/yarn-default.xml | 2 +- .../ApplicationHistoryServer.java | 33 +- .../timeline/webapp/CrossOriginFilter.java | 220 +++ .../webapp/CrossOriginFilterInitializer.java | 42 + .../TestApplicationHistoryServer.java | 28 + .../timeline/webapp/TestCrossOriginFilter.java | 214 +++ .../TestCrossOriginFilterInitializer.java | 60 + .../http/RMAuthenticationFilterInitializer.java | 2 +- .../server/resourcemanager/ResourceManager.java | 80 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 19 +- .../scheduler/AbstractYarnScheduler.java | 52 + .../scheduler/AppSchedulingInfo.java | 3 +- .../scheduler/YarnScheduler.java | 18 + .../scheduler/capacity/CSQueue.java | 18 + .../scheduler/capacity/CapacityScheduler.java | 58 + .../scheduler/capacity/LeafQueue.java | 41 +- .../scheduler/capacity/ParentQueue.java | 33 + .../scheduler/fair/AppSchedulable.java | 463 ------ .../scheduler/fair/FSAppAttempt.java | 768 ++++++++++ .../scheduler/fair/FSLeafQueue.java | 66 +- .../resourcemanager/scheduler/fair/FSQueue.java | 24 +- .../scheduler/fair/FSSchedulerApp.java | 360 ----- .../scheduler/fair/FSSchedulerNode.java | 6 +- .../scheduler/fair/FairScheduler.java | 70 +- .../scheduler/fair/FifoAppComparator.java | 8 +- .../scheduler/fair/MaxRunningAppsEnforcer.java | 52 +- .../scheduler/fair/NewAppWeightBooster.java | 2 +- .../scheduler/fair/Schedulable.java | 70 +- .../scheduler/fair/WeightAdjuster.java | 2 +- .../webapp/dao/FairSchedulerInfo.java | 3 +- .../webapp/dao/FairSchedulerLeafQueueInfo.java | 9 +- .../resourcemanager/TestResourceManager.java | 76 + .../TestWorkPreservingRMRestart.java | 9 +- .../capacity/TestCapacityScheduler.java | 876 +++++++++++ .../scheduler/fair/FakeSchedulable.java | 18 +- .../scheduler/fair/TestFSAppAttempt.java | 188 +++ .../scheduler/fair/TestFSLeafQueue.java | 2 +- .../scheduler/fair/TestFSSchedulerApp.java | 191 --- .../scheduler/fair/TestFairScheduler.java | 109 +- .../fair/TestMaxRunningAppsEnforcer.java | 42 +- ...ebServicesDelegationTokenAuthentication.java | 3 + .../src/site/apt/WritingYarnApplications.apt.vm | 1369 +++++++++--------- 131 files changed, 7224 insertions(+), 3425 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb75b6b0/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb75b6b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java index 9033451,0652de1..9a9e1b6 --- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsFileStatus.java @@@ -244,12 -242,7 +244,12 @@@ public class HdfsFileStatus return childrenNum; } + /** @return the storage policy id */ + public final byte getStoragePolicy() { + return storagePolicy; + } + - final public FileStatus makeQualified(URI defaultUri, Path path) { + public final FileStatus makeQualified(URI defaultUri, Path path) { return new FileStatus(getLen(), isDir(), getReplication(), getBlockSize(), getModificationTime(), getAccessTime(), http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb75b6b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsLocatedFileStatus.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb75b6b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/cb75b6b0/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java ----------------------------------------------------------------------
