Merge branch 'HDFS-12943' into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/47dd00a8 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/47dd00a8 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/47dd00a8 Branch: refs/heads/trunk Commit: 47dd00a82e85076ce795c67eda2c9952fd6b282e Parents: 686fcd4 bd2a59e Author: Konstantin V Shvachko <[email protected]> Authored: Mon Dec 24 10:00:02 2018 -0800 Committer: Konstantin V Shvachko <[email protected]> Committed: Mon Dec 24 10:00:02 2018 -0800 ---------------------------------------------------------------------- .../apache/hadoop/ha/FailoverController.java | 2 +- .../main/java/org/apache/hadoop/ha/HAAdmin.java | 43 ++ .../org/apache/hadoop/ha/HAServiceProtocol.java | 18 + .../hadoop/ha/HAServiceProtocolHelper.java | 9 + .../org/apache/hadoop/ha/HAServiceTarget.java | 7 + ...HAServiceProtocolClientSideTranslatorPB.java | 16 + ...HAServiceProtocolServerSideTranslatorPB.java | 20 + .../hadoop/io/retry/RetryInvocationHandler.java | 5 + .../org/apache/hadoop/ipc/AlignmentContext.java | 96 +++++ .../main/java/org/apache/hadoop/ipc/Client.java | 38 +- .../org/apache/hadoop/ipc/ExternalCall.java | 3 +- .../ipc/ObserverRetryOnActiveException.java | 37 ++ .../apache/hadoop/ipc/ProtobufRpcEngine.java | 24 +- .../main/java/org/apache/hadoop/ipc/RPC.java | 49 ++- .../java/org/apache/hadoop/ipc/RpcEngine.java | 8 +- .../main/java/org/apache/hadoop/ipc/Server.java | 137 +++++- .../apache/hadoop/ipc/WritableRpcEngine.java | 47 ++- .../apache/hadoop/log/LogThrottlingHelper.java | 18 + .../java/org/apache/hadoop/util/ProtoUtil.java | 13 + .../src/main/proto/HAServiceProtocol.proto | 20 + .../src/main/proto/RpcHeader.proto | 2 + .../hadoop-common/src/site/markdown/Metrics.md | 5 + .../org/apache/hadoop/ha/DummyHAService.java | 18 +- .../org/apache/hadoop/ha/MiniZKFCCluster.java | 4 + .../java/org/apache/hadoop/ipc/TestRPC.java | 8 +- .../apache/hadoop/hdfs/ClientGSIContext.java | 90 ++++ .../java/org/apache/hadoop/hdfs/DFSClient.java | 27 ++ .../hadoop/hdfs/NameNodeProxiesClient.java | 23 +- .../hadoop/hdfs/protocol/ClientProtocol.java | 67 +++ .../ClientNamenodeProtocolTranslatorPB.java | 39 ++ .../ha/AbstractNNFailoverProxyProvider.java | 19 +- .../namenode/ha/ClientHAProxyFactory.java | 12 + .../namenode/ha/IPFailoverProxyProvider.java | 2 +- .../namenode/ha/ObserverReadProxyProvider.java | 357 ++++++++++++++++ ...ObserverReadProxyProviderWithIPFailover.java | 142 +++++++ .../hdfs/server/namenode/ha/ReadOnly.java | 57 +++ .../src/main/proto/ClientNamenodeProtocol.proto | 18 + .../hadoop/hdfs/protocol/TestReadOnly.java | 103 +++++ .../native/libhdfspp/lib/proto/CMakeLists.txt | 1 + .../FederationNamenodeServiceState.java | 3 + .../federation/router/RouterClientProtocol.java | 13 + .../federation/router/RouterRpcServer.java | 12 + .../org/apache/hadoop/hdfs/DFSConfigKeys.java | 3 + ...tNamenodeProtocolServerSideTranslatorPB.java | 49 +++ .../apache/hadoop/hdfs/protocolPB/PBHelper.java | 4 + .../hdfs/qjournal/client/AsyncLogger.java | 7 + .../hdfs/qjournal/client/AsyncLoggerSet.java | 14 + .../hdfs/qjournal/client/IPCLoggerChannel.java | 14 + .../qjournal/client/QuorumJournalManager.java | 137 +++++- .../qjournal/client/QuorumOutputStream.java | 13 +- .../qjournal/protocol/QJournalProtocol.java | 24 +- .../QJournalProtocolServerSideTranslatorPB.java | 14 + .../QJournalProtocolTranslatorPB.java | 20 + .../hadoop/hdfs/qjournal/server/Journal.java | 65 +++ .../hdfs/qjournal/server/JournalMetrics.java | 20 +- .../qjournal/server/JournalNodeRpcServer.java | 8 + .../qjournal/server/JournaledEditsCache.java | 417 +++++++++++++++++++ .../hdfs/server/common/HdfsServerConstants.java | 4 +- .../hdfs/server/datanode/BPServiceActor.java | 2 +- .../server/namenode/EditLogFileInputStream.java | 44 ++ .../hdfs/server/namenode/FSEditLogLoader.java | 47 ++- .../hadoop/hdfs/server/namenode/FSImage.java | 19 +- .../hdfs/server/namenode/FSNamesystem.java | 38 +- .../server/namenode/GlobalStateIdContext.java | 158 +++++++ .../hadoop/hdfs/server/namenode/NameNode.java | 41 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 22 +- .../namenode/RedundantEditLogInputStream.java | 15 +- .../hdfs/server/namenode/ha/EditLogTailer.java | 8 +- .../hdfs/server/namenode/ha/StandbyState.java | 19 +- .../hadoop/hdfs/tools/NNHAServiceTarget.java | 5 + .../hadoop-hdfs/src/main/proto/HdfsServer.proto | 1 + .../src/main/proto/QJournalProtocol.proto | 18 + .../src/main/resources/hdfs-default.xml | 16 +- .../src/site/markdown/HDFSCommands.md | 2 + .../markdown/HDFSHighAvailabilityWithQJM.md | 35 +- .../src/site/markdown/ObserverNameNode.md | 187 +++++++++ .../org/apache/hadoop/hdfs/DFSTestUtil.java | 12 +- .../org/apache/hadoop/hdfs/MiniDFSCluster.java | 16 +- .../hdfs/TestStateAlignmentContextWithHA.java | 347 +++++++++++++++ .../hadoop/hdfs/qjournal/MiniQJMHACluster.java | 4 +- .../client/TestQuorumJournalManager.java | 134 ++++++ .../client/TestQuorumJournalManagerUnit.java | 103 ++++- .../hdfs/qjournal/server/JournalTestUtil.java | 48 +++ .../hdfs/qjournal/server/TestJournal.java | 47 +++ .../server/TestJournaledEditsCache.java | 257 ++++++++++++ .../server/namenode/NNThroughputBenchmark.java | 6 +- .../hdfs/server/namenode/NameNodeAdapter.java | 13 + .../namenode/TestEditLogFileInputStream.java | 18 + .../server/namenode/TestFSEditLogLoader.java | 48 +++ .../hadoop/hdfs/server/namenode/TestFsck.java | 2 +- .../hdfs/server/namenode/ha/HATestUtil.java | 126 +++++- .../ha/TestConsistentReadsObserver.java | 235 +++++++++++ .../namenode/ha/TestMultiObserverNode.java | 159 +++++++ .../server/namenode/ha/TestObserverNode.java | 364 ++++++++++++++++ .../ha/TestObserverReadProxyProvider.java | 403 ++++++++++++++++++ .../namenode/ha/TestStandbyInProgressTail.java | 164 ++++++-- .../hadoop/hdfs/tools/TestDFSHAAdmin.java | 19 +- .../hdfs/tools/TestDFSHAAdminMiniCluster.java | 44 ++ .../server/resourcemanager/AdminService.java | 7 + 99 files changed, 5540 insertions(+), 158 deletions(-) ---------------------------------------------------------------------- --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
