Merge branch 'trunk' into HDFS-1312
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a490d6f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a490d6f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a490d6f Branch: refs/heads/HDFS-1312 Commit: 3a490d6fc76196c41d921196d5039af0bf89e38c Parents: 480e5ee 44bbc50 Author: Anu Engineer <aengin...@apache.org> Authored: Mon Apr 11 16:14:45 2016 -0700 Committer: Anu Engineer <aengin...@apache.org> Committed: Mon Apr 11 16:14:45 2016 -0700 ---------------------------------------------------------------------- .../server/AuthenticationFilter.java | 63 +- .../util/ZKSignerSecretProvider.java | 29 +- .../src/site/markdown/Configuration.md | 50 +- .../apache/hadoop/crypto/CryptoInputStream.java | 6 +- .../apache/hadoop/fs/ChecksumFileSystem.java | 20 +- .../java/org/apache/hadoop/fs/ChecksumFs.java | 24 +- .../org/apache/hadoop/fs/FSDataInputStream.java | 10 +- .../apache/hadoop/fs/FSExceptionMessages.java | 6 + .../org/apache/hadoop/fs/FSInputStream.java | 61 +- .../org/apache/hadoop/fs/HarFileSystem.java | 12 +- .../apache/hadoop/fs/PositionedReadable.java | 47 +- .../apache/hadoop/fs/RawLocalFileSystem.java | 8 + .../apache/hadoop/io/AbstractMapWritable.java | 16 +- .../main/java/org/apache/hadoop/ipc/Client.java | 73 +- .../security/http/RestCsrfPreventionFilter.java | 2 +- .../markdown/filesystem/fsdatainputstream.md | 39 +- .../apache/hadoop/conf/TestConfiguration.java | 8 +- .../hadoop/crypto/CryptoStreamsTestBase.java | 10 +- .../crypto/TestCryptoStreamsForLocalFS.java | 5 +- .../apache/hadoop/crypto/key/TestKeyShell.java | 5 +- .../org/apache/hadoop/fs/FSTestWrapper.java | 3 +- .../fs/FileContextMainOperationsBaseTest.java | 7 +- .../apache/hadoop/fs/FileContextTestHelper.java | 6 +- .../apache/hadoop/fs/FileContextURIBase.java | 6 +- .../apache/hadoop/fs/FileSystemTestHelper.java | 4 +- .../org/apache/hadoop/fs/TestAvroFSInput.java | 9 +- .../hadoop/fs/TestChecksumFileSystem.java | 5 +- .../org/apache/hadoop/fs/TestDFVariations.java | 2 +- .../test/java/org/apache/hadoop/fs/TestDU.java | 4 +- .../hadoop/fs/TestFileContextResolveAfs.java | 8 +- .../java/org/apache/hadoop/fs/TestFileUtil.java | 9 +- .../org/apache/hadoop/fs/TestFsShellCopy.java | 6 +- .../apache/hadoop/fs/TestFsShellReturnCode.java | 8 +- .../org/apache/hadoop/fs/TestFsShellTouch.java | 6 +- .../hadoop/fs/TestGetFileBlockLocations.java | 5 +- .../hadoop/fs/TestHarFileSystemBasics.java | 5 +- .../java/org/apache/hadoop/fs/TestHardLink.java | 5 +- .../org/apache/hadoop/fs/TestListFiles.java | 14 +- .../apache/hadoop/fs/TestLocalFileSystem.java | 7 +- .../fs/TestLocalFileSystemPermission.java | 18 +- .../java/org/apache/hadoop/fs/TestPath.java | 6 +- .../java/org/apache/hadoop/fs/TestTrash.java | 6 +- .../apache/hadoop/fs/TestTruncatedInputBug.java | 4 +- .../fs/contract/AbstractContractAppendTest.java | 5 +- .../fs/contract/AbstractContractConcatTest.java | 2 +- .../fs/contract/AbstractContractCreateTest.java | 10 +- .../fs/contract/AbstractContractDeleteTest.java | 18 +- .../fs/contract/AbstractContractMkdirTest.java | 9 +- .../fs/contract/AbstractContractOpenTest.java | 4 +- .../fs/contract/AbstractContractRenameTest.java | 35 +- .../AbstractContractRootDirectoryTest.java | 14 +- .../fs/contract/AbstractContractSeekTest.java | 259 ++++- .../fs/contract/AbstractFSContractTestBase.java | 2 +- .../hadoop/fs/contract/ContractOptions.java | 49 +- .../hadoop/fs/contract/ContractTestUtils.java | 25 +- .../hadoop/fs/sftp/TestSFTPFileSystem.java | 3 +- .../apache/hadoop/fs/shell/TestPathData.java | 5 +- .../apache/hadoop/fs/shell/TestTextCommand.java | 4 +- .../hadoop/fs/viewfs/TestViewfsFileStatus.java | 7 +- .../apache/hadoop/ha/ClientBaseWithFixes.java | 4 +- .../http/TestAuthenticationSessionCookie.java | 5 +- .../apache/hadoop/http/TestHttpCookieFlag.java | 5 +- .../hadoop/http/TestHttpServerLifecycle.java | 5 +- .../apache/hadoop/http/TestSSLHttpServer.java | 5 +- .../org/apache/hadoop/io/TestArrayFile.java | 6 +- .../org/apache/hadoop/io/TestBloomMapFile.java | 6 +- .../java/org/apache/hadoop/io/TestMapFile.java | 6 +- .../org/apache/hadoop/io/TestSequenceFile.java | 48 +- .../hadoop/io/TestSequenceFileAppend.java | 4 +- .../io/TestSequenceFileSerialization.java | 4 +- .../apache/hadoop/io/TestSequenceFileSync.java | 5 +- .../java/org/apache/hadoop/io/TestSetFile.java | 5 +- .../apache/hadoop/io/compress/TestCodec.java | 23 +- .../apache/hadoop/io/file/tfile/TestTFile.java | 4 +- .../io/file/tfile/TestTFileByteArrays.java | 4 +- .../io/file/tfile/TestTFileComparator2.java | 4 +- .../io/file/tfile/TestTFileComparators.java | 5 +- .../hadoop/io/file/tfile/TestTFileSeek.java | 4 +- .../file/tfile/TestTFileSeqFileComparison.java | 5 +- .../hadoop/io/file/tfile/TestTFileSplit.java | 4 +- .../hadoop/io/file/tfile/TestTFileStreams.java | 4 +- .../file/tfile/TestTFileUnsortedByteArrays.java | 5 +- .../apache/hadoop/io/file/tfile/TestVLong.java | 4 +- .../apache/hadoop/io/nativeio/TestNativeIO.java | 10 +- .../TestSharedFileDescriptorFactory.java | 4 +- .../org/apache/hadoop/ipc/TestAsyncIPC.java | 346 ++++++ .../java/org/apache/hadoop/ipc/TestIPC.java | 29 +- .../sink/RollingFileSystemSinkTestBase.java | 6 +- .../apache/hadoop/security/TestCredentials.java | 4 +- .../hadoop/security/TestLdapGroupsMapping.java | 7 +- .../hadoop/security/alias/TestCredShell.java | 4 +- .../alias/TestCredentialProviderFactory.java | 5 +- .../hadoop/security/ssl/KeyStoreTestUtil.java | 4 +- .../ssl/TestReloadingX509TrustManager.java | 6 +- .../hadoop/security/ssl/TestSSLFactory.java | 6 +- .../apache/hadoop/test/GenericTestUtils.java | 84 +- .../java/org/apache/hadoop/util/JarFinder.java | 4 +- .../hadoop/util/TestApplicationClassLoader.java | 4 +- .../org/apache/hadoop/util/TestClasspath.java | 5 +- .../hadoop/util/TestGenericOptionsParser.java | 2 +- .../apache/hadoop/util/TestHostsFileReader.java | 4 +- .../org/apache/hadoop/util/TestJarFinder.java | 9 +- .../java/org/apache/hadoop/util/TestRunJar.java | 8 +- .../java/org/apache/hadoop/util/TestShell.java | 4 +- .../apache/hadoop/util/TestSysInfoLinux.java | 5 +- .../org/apache/hadoop/util/TestWinUtils.java | 5 +- .../java/org/apache/hadoop/util/TestZKUtil.java | 5 +- .../src/test/resources/contract/localfs.xml | 4 +- .../hadoop-kms/src/site/markdown/index.md.vm | 39 +- .../apache/hadoop/hdfs/BlockReaderFactory.java | 4 +- .../org/apache/hadoop/hdfs/DFSInputStream.java | 4 + .../apache/hadoop/hdfs/HdfsConfiguration.java | 3 - .../hdfs/client/HdfsClientConfigKeys.java | 6 - .../hdfs/protocol/ClientDatanodeProtocol.java | 7 + .../ClientDatanodeProtocolTranslatorPB.java | 12 + .../hadoop/hdfs/util/StripedBlockUtil.java | 22 +- .../hadoop/hdfs/web/ByteRangeInputStream.java | 22 +- .../src/main/proto/ClientDatanodeProtocol.proto | 10 + .../bkjournal/TestBookKeeperHACheckpoints.java | 46 +- .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 18 - ...tDatanodeProtocolServerSideTranslatorPB.java | 15 + .../server/blockmanagement/BlockManager.java | 2 +- .../hdfs/server/datanode/BlockReceiver.java | 3 + .../hadoop/hdfs/server/datanode/DataNode.java | 7 + .../hdfs/server/datanode/DataXceiver.java | 48 +- .../hdfs/server/datanode/DataXceiverServer.java | 6 + .../hdfs/server/datanode/DirectoryScanner.java | 12 +- .../erasurecode/ErasureCodingWorker.java | 1016 +----------------- .../erasurecode/StripedBlockReader.java | 202 ++++ .../erasurecode/StripedBlockWriter.java | 196 ++++ .../datanode/erasurecode/StripedReader.java | 466 ++++++++ .../erasurecode/StripedReconstructor.java | 273 +++++ .../datanode/erasurecode/StripedWriter.java | 313 ++++++ .../datanode/erasurecode/package-info.java | 26 + .../datanode/fsdataset/impl/FsDatasetImpl.java | 4 +- .../hdfs/server/namenode/FSNamesystem.java | 1 + .../hdfs/server/namenode/NamenodeFsck.java | 2 +- .../org/apache/hadoop/hdfs/tools/DFSAdmin.java | 21 + .../src/main/webapps/hdfs/dfshealth.html | 2 +- .../TestClientProtocolForPipelineRecovery.java | 49 + .../hadoop/hdfs/TestReconstructStripedFile.java | 11 +- .../TestBlockManagerSafeMode.java | 14 +- .../blockmanagement/TestBlockTokenWithDFS.java | 2 +- .../hdfs/server/datanode/TestBlockRecovery.java | 234 +++- .../datanode/TestDataNodeVolumeFailure.java | 2 +- .../hadoop/hdfs/server/namenode/TestFsck.java | 122 ++- .../hdfs/server/namenode/ha/TestHASafeMode.java | 35 + .../v2/app/job/impl/TaskAttemptImpl.java | 41 +- .../apache/hadoop/mapreduce/v2/app/MRApp.java | 10 +- .../hadoop/mapreduce/v2/app/TestRecovery.java | 29 +- .../v2/app/job/impl/TestTaskAttempt.java | 37 +- .../hadoop/mapreduce/task/reduce/Fetcher.java | 2 +- .../mapreduce/task/reduce/TestFetcher.java | 37 + .../mapreduce/v2/hs/TestJobListCache.java | 2 +- .../apache/hadoop/fs/s3a/S3AInputStream.java | 271 ++++- .../hadoop/fs/s3a/TestS3AConfiguration.java | 2 + .../fs/s3a/scale/TestS3ADeleteManyFiles.java | 9 + .../hadoop/fs/azure/NativeAzureFileSystem.java | 4 +- .../src/main/data/2jobs2min-rumen-jh.json | 6 - .../hadoop/yarn/sls/nodemanager/NodeInfo.java | 9 - .../yarn/sls/scheduler/RMNodeWrapper.java | 9 - .../hadoop/yarn/api/records/ContainerId.java | 11 +- .../hadoop/yarn/conf/YarnConfiguration.java | 9 - .../apache/hadoop/yarn/client/cli/TopCLI.java | 10 +- .../org/apache/hadoop/yarn/webapp/WebApps.java | 2 +- .../src/main/resources/yarn-default.xml | 13 - .../hadoop/yarn/conf/TestYarnConfiguration.java | 8 +- .../yarn/nodelabels/NodeLabelTestBase.java | 2 +- .../apache/hadoop/yarn/util/TestFSDownload.java | 2 +- .../client/impl/zk/RegistrySecurity.java | 4 +- .../hadoop/yarn/server/webapp/AppBlock.java | 20 + .../nodemanager/NodeStatusUpdaterImpl.java | 1 + .../container/ContainerImpl.java | 22 + .../launcher/ContainerLaunch.java | 2 +- .../monitor/ContainerMetrics.java | 18 + .../nodemanager/util/ProcessIdFileReader.java | 2 +- .../nodemanager/TestNodeManagerResync.java | 33 +- .../containermanager/TestAuxServices.java | 2 +- .../container/TestContainer.java | 11 + .../TestLogAggregationService.java | 19 +- .../server/resourcemanager/AdminService.java | 6 +- .../resourcemanager/NodesListManager.java | 104 +- .../server/resourcemanager/RMServerUtils.java | 2 +- .../resourcemanager/ResourceTrackerService.java | 8 +- .../rmapp/attempt/RMAppAttemptImpl.java | 24 +- .../server/resourcemanager/rmnode/RMNode.java | 4 - .../resourcemanager/rmnode/RMNodeImpl.java | 22 +- .../scheduler/AppSchedulingInfo.java | 12 +- .../scheduler/capacity/AbstractCSQueue.java | 20 +- .../scheduler/capacity/CSQueueUtils.java | 2 +- .../capacity/CapacityHeadroomProvider.java | 28 +- .../scheduler/capacity/CapacityScheduler.java | 6 + .../capacity/CapacitySchedulerContext.java | 12 +- .../scheduler/capacity/LeafQueue.java | 51 +- .../scheduler/policy/FairOrderingPolicy.java | 3 +- .../webapp/CapacitySchedulerPage.java | 2 + .../yarn/server/resourcemanager/MockNodes.java | 9 - .../yarn/server/resourcemanager/TestRMHA.java | 42 +- .../TestResourceTrackerService.java | 216 +--- .../applicationsmanager/TestAMRestart.java | 107 ++ ...estProportionalCapacityPreemptionPolicy.java | 6 +- ...pacityPreemptionPolicyForNodePartitions.java | 14 +- .../capacity/TestApplicationLimits.java | 22 +- .../TestApplicationLimitsByPartition.java | 202 +++- .../TestCapacitySchedulerNodeLabelUpdate.java | 40 +- .../scheduler/capacity/TestLeafQueue.java | 10 +- .../scheduler/capacity/TestReservations.java | 5 + .../scheduler/capacity/TestUtils.java | 12 +- .../webapp/TestRMWebServicesNodes.java | 12 +- .../hadoop/yarn/server/MiniYARNCluster.java | 2 +- .../TestEntityGroupFSTimelineStore.java | 55 +- .../yarn/server/timeline/TestLogInfo.java | 28 +- .../server/webproxy/WebAppProxyServlet.java | 4 +- .../src/site/markdown/CapacityScheduler.md | 28 + 214 files changed, 4605 insertions(+), 2297 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a490d6f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientDatanodeProtocol.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a490d6f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolTranslatorPB.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a490d6f/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientDatanodeProtocol.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a490d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a490d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientDatanodeProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a490d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a490d6f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java ----------------------------------------------------------------------
