Merge branch 'trunk' into HADOOP-13345
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e93093e Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e93093e Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e93093e Branch: refs/heads/HADOOP-13345 Commit: 5e93093e6fbd2493f020bba770385764ef593114 Parents: 86a67ff 8f6e143 Author: Mingliang Liu <lium...@apache.org> Authored: Tue Nov 29 16:45:38 2016 -0800 Committer: Mingliang Liu <lium...@apache.org> Committed: Tue Nov 29 16:45:38 2016 -0800 ---------------------------------------------------------------------- hadoop-assemblies/pom.xml | 1 - hadoop-build-tools/pom.xml | 2 + hadoop-client/pom.xml | 1 - .../hadoop-cloud-storage/pom.xml | 1 - hadoop-cloud-storage-project/pom.xml | 1 - .../hadoop-annotations/pom.xml | 1 - .../hadoop-auth-examples/pom.xml | 1 - hadoop-common-project/hadoop-auth/pom.xml | 1 - hadoop-common-project/hadoop-common/pom.xml | 15 +- .../org/apache/hadoop/conf/Configuration.java | 8 +- .../crypto/key/kms/KMSClientProvider.java | 43 +- .../java/org/apache/hadoop/fs/FileSystem.java | 1455 +++++++---- .../org/apache/hadoop/fs/shell/Display.java | 11 +- .../org/apache/hadoop/fs/shell/FsUsage.java | 124 +- .../hadoop/fs/viewfs/ChRootedFileSystem.java | 6 + .../apache/hadoop/fs/viewfs/ViewFileSystem.java | 73 +- .../hadoop/fs/viewfs/ViewFileSystemUtil.java | 164 ++ .../org/apache/hadoop/fs/viewfs/ViewFs.java | 12 +- .../java/org/apache/hadoop/io/SequenceFile.java | 74 +- .../io/erasurecode/ErasureCodeConstants.java | 3 + .../apache/hadoop/ipc/DecayRpcScheduler.java | 2 +- .../apache/hadoop/ipc/ProtobufRpcEngine.java | 75 +- .../hadoop/ipc/ProtobufRpcEngineCallback.java | 29 + .../main/java/org/apache/hadoop/ipc/RPC.java | 5 +- .../main/java/org/apache/hadoop/ipc/Server.java | 199 +- .../apache/hadoop/ipc/WritableRpcEngine.java | 3 +- .../hadoop/ipc/metrics/RpcDetailedMetrics.java | 6 + .../apache/hadoop/ipc/metrics/RpcMetrics.java | 33 + .../org/apache/hadoop/jmx/JMXJsonServlet.java | 6 +- .../java/org/apache/hadoop/log/Log4Json.java | 14 +- .../hadoop/metrics2/impl/MetricsSystemImpl.java | 4 +- .../hadoop/security/AccessControlException.java | 2 +- .../org/apache/hadoop/security/Credentials.java | 2 +- .../java/org/apache/hadoop/security/KDiag.java | 67 + .../apache/hadoop/security/SecurityUtil.java | 10 +- .../hadoop/security/UserGroupInformation.java | 6 +- .../authorize/AuthorizationException.java | 2 +- .../org/apache/hadoop/security/token/Token.java | 2 +- .../AbstractDelegationTokenIdentifier.java | 2 +- .../AbstractDelegationTokenSecretManager.java | 2 +- .../DelegationTokenAuthenticationHandler.java | 6 +- .../web/DelegationTokenAuthenticator.java | 2 +- .../apache/hadoop/util/HttpExceptionUtils.java | 2 +- .../src/site/markdown/DeprecatedProperties.md | 7 +- .../src/site/markdown/filesystem/filesystem.md | 2 +- .../apache/hadoop/conf/TestConfiguration.java | 2 +- .../org/apache/hadoop/fs/SymlinkBaseTest.java | 3 +- .../org/apache/hadoop/fs/TestDefaultUri.java | 40 +- .../apache/hadoop/fs/TestFileSystemCaching.java | 8 +- .../hadoop/fs/viewfs/TestViewfsFileStatus.java | 9 +- .../fs/viewfs/ViewFileSystemBaseTest.java | 94 +- .../apache/hadoop/fs/viewfs/ViewFsBaseTest.java | 22 + .../apache/hadoop/io/TestSequenceFileSync.java | 113 +- .../ipc/TestProtoBufRpcServerHandoff.java | 167 ++ .../apache/hadoop/ipc/TestRpcServerHandoff.java | 218 ++ .../org/apache/hadoop/log/TestLog4Json.java | 13 +- .../apache/hadoop/security/TestKDiagNoKDC.java | 8 + ...tionTokenAuthenticationHandlerWithMocks.java | 3 +- .../delegation/web/TestWebDelegationToken.java | 4 +- .../hadoop/util/TestHttpExceptionUtils.java | 2 +- .../hadoop-common/src/test/proto/test.proto | 11 +- .../src/test/proto/test_rpc_service.proto | 6 +- hadoop-common-project/hadoop-kms/pom.xml | 5 +- .../crypto/key/kms/server/KMSJSONReader.java | 3 +- .../crypto/key/kms/server/KMSJSONWriter.java | 3 +- .../hadoop-kms/src/site/markdown/index.md.vm | 2 + .../hadoop/crypto/key/kms/server/TestKMS.java | 97 +- hadoop-common-project/hadoop-minikdc/pom.xml | 1 - hadoop-common-project/hadoop-nfs/pom.xml | 1 - hadoop-common-project/pom.xml | 1 - hadoop-dist/pom.xml | 1 - hadoop-hdfs-project/hadoop-hdfs-client/pom.xml | 1 - .../apache/hadoop/hdfs/HdfsConfiguration.java | 10 +- .../hdfs/client/HdfsClientConfigKeys.java | 10 +- .../hadoop/hdfs/protocol/HdfsConstants.java | 1 + hadoop-hdfs-project/hadoop-hdfs-httpfs/pom.xml | 1 - .../src/site/markdown/ServerSetup.md.vm | 10 +- .../hadoop-hdfs-native-client/pom.xml | 1 - hadoop-hdfs-project/hadoop-hdfs-nfs/pom.xml | 1 - hadoop-hdfs-project/hadoop-hdfs/pom.xml | 1 - .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 19 +- .../hadoop/hdfs/server/balancer/Balancer.java | 4 +- .../server/blockmanagement/BlockManager.java | 62 +- .../BlockPlacementPolicyDefault.java | 8 +- .../apache/hadoop/hdfs/server/mover/Mover.java | 4 +- .../hadoop/hdfs/server/namenode/BackupNode.java | 4 +- .../namenode/ErasureCodingPolicyManager.java | 23 +- .../hdfs/server/namenode/FSNamesystem.java | 7 + .../hadoop/hdfs/server/namenode/INodeFile.java | 8 +- .../src/main/resources/hdfs-default.xml | 8 +- .../src/site/markdown/TransparentEncryption.md | 2 +- .../fs/viewfs/TestViewFileSystemHdfs.java | 53 + .../apache/hadoop/hdfs/AdminStatesBaseTest.java | 4 +- .../org/apache/hadoop/hdfs/DFSTestUtil.java | 28 +- .../hadoop/hdfs/TestBlockStoragePolicy.java | 2 +- .../hadoop/hdfs/TestDFSStripedInputStream.java | 50 +- .../hadoop/hdfs/TestDFSStripedOutputStream.java | 29 +- .../TestDFSStripedOutputStreamWithFailure.java | 39 +- .../hdfs/TestDFSXORStripedInputStream.java | 33 + .../hdfs/TestDFSXORStripedOutputStream.java | 35 + ...estDFSXORStripedOutputStreamWithFailure.java | 36 + .../hdfs/TestDecommissionWithStriped.java | 10 +- .../apache/hadoop/hdfs/TestDeprecatedKeys.java | 22 +- .../hadoop/hdfs/TestEncryptedTransfer.java | 2 +- .../org/apache/hadoop/hdfs/TestFileAppend4.java | 2 +- .../apache/hadoop/hdfs/TestFileChecksum.java | 2 +- .../hadoop/hdfs/TestLeaseRecoveryStriped.java | 2 +- .../hadoop/hdfs/TestMissingBlocksAlert.java | 3 +- .../hdfs/TestReadStripedFileWithDecoding.java | 3 +- .../hadoop/hdfs/TestReconstructStripedFile.java | 4 +- .../hdfs/TestReplaceDatanodeOnFailure.java | 3 +- .../org/apache/hadoop/hdfs/TestReplication.java | 3 +- .../hadoop/hdfs/TestWriteReadStripedFile.java | 2 +- .../hdfs/server/balancer/TestBalancer.java | 26 +- .../blockmanagement/BlockManagerTestUtil.java | 21 +- .../TestBlocksWithNotEnoughRacks.java | 4 +- .../server/blockmanagement/TestNodeCount.java | 2 +- .../TestPendingInvalidateBlock.java | 2 +- .../TestPendingReconstruction.java | 2 +- ...constructStripedBlocksWithRackAwareness.java | 8 +- .../blockmanagement/TestReplicationPolicy.java | 2 +- .../TestReplicationPolicyConsiderLoad.java | 2 +- .../TestDataNodeErasureCodingMetrics.java | 2 +- .../fsdataset/impl/TestLazyPersistFiles.java | 12 +- .../hadoop/hdfs/server/mover/TestMover.java | 9 +- .../hdfs/server/mover/TestStorageMover.java | 4 +- .../server/namenode/NNThroughputBenchmark.java | 10 +- .../TestAddOverReplicatedStripedBlocks.java | 2 +- .../namenode/TestDecommissioningStatus.java | 5 +- .../server/namenode/TestFSEditLogLoader.java | 2 +- .../hdfs/server/namenode/TestHostsFiles.java | 4 +- .../hdfs/server/namenode/TestMetaSave.java | 5 +- .../namenode/TestReconstructStripedBlocks.java | 9 +- .../server/namenode/TestStripedINodeFile.java | 5 +- .../hdfs/server/namenode/ha/TestDNFencing.java | 11 +- .../ha/TestDNFencingWithReplication.java | 2 +- .../namenode/ha/TestPipelinesFailover.java | 5 +- .../TestNNMetricFilesInGetListingOps.java | 2 +- .../namenode/metrics/TestNameNodeMetrics.java | 12 +- hadoop-hdfs-project/pom.xml | 1 - .../hadoop-mapreduce-client-app/pom.xml | 1 - .../org/apache/hadoop/mapred/YarnChild.java | 2 + .../hadoop/mapreduce/v2/app/MRAppMaster.java | 3 + .../hadoop-mapreduce-client-common/pom.xml | 1 - .../hadoop-mapreduce-client-core/pom.xml | 1 - .../hadoop-mapreduce-client-hs-plugins/pom.xml | 1 - .../hadoop-mapreduce-client-hs/pom.xml | 1 - .../hadoop-mapreduce-client-jobclient/pom.xml | 1 - .../hadoop/mapred/ClientServiceDelegate.java | 4 +- .../hadoop-mapreduce-client-nativetask/pom.xml | 11 +- .../src/CMakeLists.txt | 2 + .../src/main/native/gtest/gtest-all.cc | 1483 ++++++++--- .../src/main/native/gtest/include/gtest/gtest.h | 2341 +++++++++++++----- .../hadoop-mapreduce-client-shuffle/pom.xml | 1 - .../hadoop-mapreduce-client/pom.xml | 1 - .../hadoop-mapreduce-examples/pom.xml | 1 - hadoop-mapreduce-project/pom.xml | 1 - hadoop-maven-plugins/pom.xml | 1 - hadoop-minicluster/pom.xml | 1 - hadoop-project-dist/pom.xml | 1 - hadoop-project/pom.xml | 3 +- hadoop-tools/hadoop-archive-logs/pom.xml | 1 - hadoop-tools/hadoop-archives/pom.xml | 1 - .../hadoop/fs/s3a/RenameFailedException.java | 70 + .../org/apache/hadoop/fs/s3a/S3AFileSystem.java | 89 +- .../fs/s3a/ITestS3AFileSystemContract.java | 15 - .../fs/s3a/yarn/ITestS3AMiniYarnCluster.java | 11 +- .../src/test/resources/contract/s3a.xml | 2 +- hadoop-tools/hadoop-azure-datalake/pom.xml | 1 - hadoop-tools/hadoop-datajoin/pom.xml | 1 - hadoop-tools/hadoop-distcp/pom.xml | 1 - hadoop-tools/hadoop-extras/pom.xml | 1 - hadoop-tools/hadoop-gridmix/pom.xml | 1 - hadoop-tools/hadoop-pipes/pom.xml | 1 - hadoop-tools/hadoop-rumen/pom.xml | 6 - hadoop-tools/hadoop-sls/pom.xml | 1 - hadoop-tools/hadoop-streaming/pom.xml | 1 - hadoop-tools/hadoop-tools-dist/pom.xml | 1 - hadoop-tools/pom.xml | 1 - .../hadoop-yarn/hadoop-yarn-api/pom.xml | 1 - .../api/records/AbstractResourceRequest.java | 55 + .../yarn/api/records/ApplicationReport.java | 10 + .../yarn/api/records/ApplicationTimeout.java | 99 + .../yarn/api/records/ResourceRequest.java | 19 +- .../api/records/UpdateContainerRequest.java | 18 +- .../src/main/proto/yarn_protos.proto | 7 + .../pom.xml | 1 - .../pom.xml | 1 - .../hadoop-yarn-applications/pom.xml | 1 - .../hadoop-yarn/hadoop-yarn-client/pom.xml | 1 - .../hadoop/yarn/client/api/YarnClient.java | 11 + .../yarn/client/api/impl/YarnClientImpl.java | 9 + .../hadoop/yarn/client/cli/ApplicationCLI.java | 45 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 40 + .../hadoop-yarn/hadoop-yarn-common/pom.xml | 1 - .../impl/pb/ApplicationReportPBImpl.java | 83 + .../impl/pb/ApplicationTimeoutPBImpl.java | 130 + .../hadoop/yarn/event/AsyncDispatcher.java | 6 +- .../yarn/util/ResourceCalculatorPlugin.java | 3 + .../resource/DefaultResourceCalculator.java | 19 +- .../resource/DominantResourceCalculator.java | 25 +- .../yarn/util/resource/ResourceCalculator.java | 17 +- .../hadoop/yarn/util/resource/Resources.java | 4 + .../src/main/resources/yarn-default.xml | 6 +- .../hadoop/yarn/api/TestPBImplRecords.java | 2 + .../hadoop/yarn/event/DrainDispatcher.java | 9 +- .../hadoop/yarn/event/TestAsyncDispatcher.java | 42 + .../util/resource/TestResourceCalculator.java | 79 + .../pom.xml | 1 - .../hadoop-yarn-server-common/pom.xml | 1 - .../hadoop-yarn-server-nodemanager/pom.xml | 1 - .../nodemanager/NodeResourceMonitorImpl.java | 5 + .../monitor/ContainersMonitorImpl.java | 39 +- .../util/NodeManagerHardwareUtils.java | 160 +- .../TestContainersMonitorResourceChange.java | 16 +- .../hadoop-yarn-server-resourcemanager/pom.xml | 1 - .../server/resourcemanager/ClientRMService.java | 2 +- ...pportunisticContainerAllocatorAMService.java | 13 +- .../server/resourcemanager/RMAppManager.java | 6 +- .../server/resourcemanager/RMAuditLogger.java | 1 + .../server/resourcemanager/RMServerUtils.java | 11 +- .../server/resourcemanager/rmapp/RMAppImpl.java | 17 + .../scheduler/AbstractYarnScheduler.java | 20 + .../scheduler/AppSchedulingInfo.java | 17 + .../scheduler/SchedulerApplicationAttempt.java | 16 + .../scheduler/SchedulerUtils.java | 59 +- .../scheduler/YarnScheduler.java | 8 + .../scheduler/capacity/CapacityScheduler.java | 4 +- .../scheduler/common/fica/FiCaSchedulerApp.java | 16 + .../distributed/NodeQueueLoadMonitor.java | 8 +- .../scheduler/fair/FSAppAttempt.java | 242 +- .../scheduler/fair/FSContext.java | 54 + .../scheduler/fair/FSLeafQueue.java | 319 +-- .../scheduler/fair/FSParentQueue.java | 52 +- .../scheduler/fair/FSPreemptionThread.java | 188 ++ .../resourcemanager/scheduler/fair/FSQueue.java | 37 +- .../scheduler/fair/FSQueueMetrics.java | 60 + .../scheduler/fair/FSSchedulerNode.java | 36 +- .../scheduler/fair/FSStarvedApps.java | 85 + .../scheduler/fair/FairScheduler.java | 309 +-- .../scheduler/fair/Schedulable.java | 29 +- .../scheduler/fifo/FifoScheduler.java | 4 +- .../webapp/JAXBContextResolver.java | 3 +- .../resourcemanager/webapp/RMWebServices.java | 186 ++ .../resourcemanager/webapp/dao/AppInfo.java | 24 + .../webapp/dao/AppTimeoutInfo.java | 71 + .../webapp/dao/AppTimeoutsInfo.java | 47 + ...pportunisticContainerAllocatorAMService.java | 97 +- .../yarn/server/resourcemanager/TestRMHA.java | 3 +- ...tyPreemptionPolicyForReservedContainers.java | 430 ---- .../server/resourcemanager/rmapp/MockRMApp.java | 3 +- .../rmapp/TestApplicationLifetimeMonitor.java | 30 +- .../scheduler/TestSchedulerUtils.java | 22 +- .../scheduler/fair/FairSchedulerTestBase.java | 39 +- .../fair/FairSchedulerWithMockPreemption.java | 58 + .../scheduler/fair/FakeSchedulable.java | 5 - .../scheduler/fair/TestFSAppStarvation.java | 256 ++ .../scheduler/fair/TestFSLeafQueue.java | 165 +- .../scheduler/fair/TestFairScheduler.java | 137 + .../fair/TestFairSchedulerPreemption.java | 1540 ++---------- .../fair/TestQueueManagerRealScheduler.java | 128 + .../scheduler/fair/TestSchedulingPolicy.java | 5 - .../webapp/TestRMWebServicesApps.java | 2 +- .../TestRMWebServicesAppsModification.java | 137 +- .../pom.xml | 1 - .../hadoop-yarn-server-tests/pom.xml | 1 - .../hadoop/yarn/server/MiniYARNCluster.java | 7 +- .../server/TestContainerManagerSecurity.java | 2 +- .../pom.xml | 1 - .../pom.xml | 1 - ...stTimelineReaderWebServicesHBaseStorage.java | 2 +- .../storage/DataGeneratorForTest.java | 4 +- .../storage/TestHBaseTimelineStorageApps.java | 8 +- .../TestHBaseTimelineStorageEntities.java | 6 +- .../flow/TestHBaseStorageFlowActivity.java | 6 +- .../storage/flow/TestHBaseStorageFlowRun.java | 14 +- .../flow/TestHBaseStorageFlowRunCompaction.java | 2 +- .../hadoop-yarn-server-timelineservice/pom.xml | 1 - .../storage/HBaseTimelineWriterImpl.java | 5 - .../hadoop-yarn-server-web-proxy/pom.xml | 1 - .../hadoop-yarn/hadoop-yarn-server/pom.xml | 1 - .../hadoop-yarn/hadoop-yarn-site/pom.xml | 1 - .../hadoop-yarn/hadoop-yarn-ui/pom.xml | 1 - hadoop-yarn-project/hadoop-yarn/pom.xml | 1 - hadoop-yarn-project/pom.xml | 1 - pom.xml | 2 + 286 files changed, 9225 insertions(+), 4927 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e93093e/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java ---------------------------------------------------------------------- diff --cc hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java index 9b8692a,f1cde4e..cb08e57 --- a/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java +++ b/hadoop-tools/hadoop-aws/src/main/java/org/apache/hadoop/fs/s3a/S3AFileSystem.java @@@ -660,17 -659,21 +668,24 @@@ public class S3AFileSystem extends File /** * The inner rename operation. See {@link #rename(Path, Path)} for * the description of the operation. + * This operation throws an exception on any failure which needs to be + * reported and downgraded to a failure. That is: if a rename - * @param src path to be renamed - * @param dst new path after rename + * @param source path to be renamed + * @param dest new path after rename - * @return true if rename is successful + * @throws RenameFailedException if some criteria for a state changing + * rename was not met. This means work didn't happen; it's not something + * which is reported upstream to the FileSystem APIs, for which the semantics + * of "false" are pretty vague. + * @throws FileNotFoundException there's no source file. * @throws IOException on IO failure. * @throws AmazonClientException on failures inside the AWS SDK */ - private boolean innerRename(Path source, Path dest) throws IOException, - AmazonClientException { - private boolean innerRename(Path src, Path dst) ++ private boolean innerRename(Path source, Path dest) + throws RenameFailedException, FileNotFoundException, IOException, + AmazonClientException { + Path src = qualify(source); + Path dst = qualify(dest); + LOG.debug("Rename path {} to {}", src, dst); incrementStatistic(INVOCATION_RENAME); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
