Merge commit '96c4575d7373079becfa3e3db29ba98e6fb86388' into HDFS-12943 # Conflicts: # hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8aeb2d89 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8aeb2d89 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8aeb2d89 Branch: refs/heads/HDFS-12943 Commit: 8aeb2d8942116ec924f319569949516055f3f5c2 Parents: da18227 96c4575 Author: Konstantin V Shvachko <[email protected]> Authored: Sat Aug 25 17:28:33 2018 -0700 Committer: Konstantin V Shvachko <[email protected]> Committed: Sat Aug 25 17:28:33 2018 -0700 ---------------------------------------------------------------------- dev-support/byteman/README.md | 31 + dev-support/byteman/hadooprpc.btm | 44 ++ .../fs/CommonConfigurationKeysPublic.java | 9 + .../apache/hadoop/util/ShutdownHookManager.java | 169 ++++- .../src/main/resources/core-default.xml | 16 + .../site/markdown/InterfaceClassification.md | 7 +- .../hadoop/util/TestShutdownHookManager.java | 328 ++++++-- .../src/main/compose/ozone/docker-config | 2 + .../container/common/helpers/ContainerInfo.java | 1 + .../org/apache/hadoop/utils/db/DBStore.java | 93 +++ .../org/apache/hadoop/utils/db/RDBStore.java | 252 +++++++ .../hadoop/utils/db/RDBStoreIterator.java | 88 +++ .../org/apache/hadoop/utils/db/RDBTable.java | 173 +++++ .../java/org/apache/hadoop/utils/db/Table.java | 150 ++++ .../apache/hadoop/utils/db/TableIterator.java | 50 ++ .../apache/hadoop/utils/db/package-info.java | 22 + .../apache/hadoop/utils/db/TestRDBStore.java | 246 ++++++ .../hadoop/utils/db/TestRDBTableStore.java | 189 +++++ .../apache/hadoop/utils/db/package-info.java | 22 + .../apache/hadoop/hdds/scm/HddsServerUtil.java | 6 +- .../common/helpers/ContainerReport.java | 14 - .../common/helpers/KeyValueContainerReport.java | 117 +++ .../container/common/impl/ContainerData.java | 53 -- .../common/impl/ContainerDataYaml.java | 94 ++- .../container/common/impl/ContainerSet.java | 50 +- .../RandomContainerDeletionChoosingPolicy.java | 3 +- ...NOrderedContainerDeletionChoosingPolicy.java | 24 +- .../container/common/interfaces/Container.java | 39 +- .../ContainerDeletionChoosingPolicy.java | 13 + .../common/interfaces/ContainerPacker.java | 58 ++ .../report/CommandStatusReportPublisher.java | 2 +- .../common/report/ContainerReportPublisher.java | 2 +- .../common/report/NodeReportPublisher.java | 2 +- .../statemachine/DatanodeStateMachine.java | 3 +- .../statemachine/EndpointStateMachine.java | 9 +- .../container/keyvalue/KeyValueContainer.java | 174 ++++- .../keyvalue/KeyValueContainerData.java | 54 ++ .../container/keyvalue/KeyValueHandler.java | 19 +- .../container/keyvalue/TarContainerPacker.java | 249 +++++++ .../keyvalue/helpers/KeyValueContainerUtil.java | 22 +- .../common/TestKeyValueContainerData.java | 4 +- .../keyvalue/TestKeyValueContainer.java | 95 ++- .../keyvalue/TestTarContainerPacker.java | 231 ++++++ .../hdds/scm/container/ContainerMapping.java | 23 +- .../scm/container/closer/ContainerCloser.java | 9 +- .../scm/container/TestContainerMapping.java | 8 + .../federation/resolver/MountTableResolver.java | 37 +- .../server/federation/router/RBFConfigKeys.java | 4 + .../federation/router/RouterRpcClient.java | 18 +- .../federation/router/RouterRpcServer.java | 3 +- .../src/main/resources/hdfs-rbf-default.xml | 9 + .../hdfs/server/federation/MockResolver.java | 3 + .../resolver/TestMountTableResolver.java | 31 + .../server/federation/router/TestRouter.java | 18 + .../hdfs/qjournal/client/AsyncLogger.java | 3 +- .../hdfs/qjournal/client/AsyncLoggerSet.java | 4 +- .../hdfs/qjournal/client/IPCLoggerChannel.java | 5 +- .../qjournal/client/QuorumJournalManager.java | 4 +- .../qjournal/protocol/QJournalProtocol.java | 2 +- .../QJournalProtocolServerSideTranslatorPB.java | 2 +- .../QJournalProtocolTranslatorPB.java | 6 +- .../hadoop/hdfs/qjournal/server/JNStorage.java | 4 +- .../hadoop/hdfs/qjournal/server/Journal.java | 6 +- .../qjournal/server/JournalNodeRpcServer.java | 5 +- .../server/namenode/BackupJournalManager.java | 2 +- .../namenode/ErasureCodingPolicyManager.java | 15 +- .../server/namenode/FSDirErasureCodingOp.java | 22 +- .../hadoop/hdfs/server/namenode/FSEditLog.java | 5 +- .../hadoop/hdfs/server/namenode/FSImage.java | 5 +- .../hdfs/server/namenode/FSNamesystem.java | 30 +- .../server/namenode/FileJournalManager.java | 2 +- .../hdfs/server/namenode/JournalManager.java | 2 +- .../hadoop/hdfs/server/namenode/JournalSet.java | 2 +- .../hadoop/hdfs/server/namenode/NameNode.java | 4 +- .../hdfs/server/namenode/NameNodeRpcServer.java | 8 +- .../src/main/proto/QJournalProtocol.proto | 1 + .../qjournal/client/TestEpochsAreUnique.java | 2 +- .../hdfs/qjournal/client/TestQJMWithFaults.java | 6 +- .../client/TestQuorumJournalManager.java | 4 +- .../client/TestQuorumJournalManagerUnit.java | 4 +- .../hdfs/qjournal/server/TestJournal.java | 17 +- .../hdfs/qjournal/server/TestJournalNode.java | 4 +- .../qjournal/server/TestJournalNodeMXBean.java | 2 +- .../qjournal/server/TestJournalNodeSync.java | 2 +- .../server/namenode/TestGenericJournalConf.java | 2 +- .../server/namenode/TestNamenodeRetryCache.java | 2 +- .../namenode/ha/TestRetryCacheWithHA.java | 2 +- .../ozone/TestStorageContainerManager.java | 19 +- .../common/TestBlockDeletingService.java | 8 +- .../commandhandler/TestBlockDeletion.java | 19 +- .../hadoop/ozone/web/client/TestKeys.java | 10 +- .../contract/AbstractContractDistCpTest.java | 2 +- .../hadoop/yarn/conf/YarnConfiguration.java | 8 +- .../yarn/conf/TestYarnConfigurationFields.java | 2 + .../yarn/service/client/ApiServiceClient.java | 20 + .../hadoop/yarn/service/webapp/ApiServer.java | 12 +- .../hadoop/yarn/service/ClientAMService.java | 2 +- .../hadoop/yarn/service/ServiceEvent.java | 25 + .../hadoop/yarn/service/ServiceManager.java | 127 +++- .../hadoop/yarn/service/ServiceScheduler.java | 15 +- .../yarn/service/api/records/ServiceState.java | 2 +- .../yarn/service/client/ServiceClient.java | 100 ++- .../yarn/service/component/Component.java | 16 +- .../yarn/service/component/ComponentEvent.java | 10 + .../component/instance/ComponentInstance.java | 5 + .../yarn/service/utils/ServiceApiUtil.java | 44 ++ .../src/main/proto/ClientAMProtocol.proto | 1 + .../hadoop/yarn/service/TestServiceApiUtil.java | 653 ---------------- .../hadoop/yarn/service/TestServiceManager.java | 299 +++++--- .../yarn/service/TestYarnNativeServices.java | 35 + .../yarn/service/utils/TestServiceApiUtil.java | 743 +++++++++++++++++++ .../hadoop/yarn/client/cli/ApplicationCLI.java | 20 +- .../hadoop/yarn/client/cli/TestYarnCLI.java | 4 + .../hadoop/yarn/client/AMRMClientUtils.java | 47 ++ .../hadoop/yarn/client/api/AppAdminClient.java | 12 + .../hadoop/yarn/server/AMRMClientRelayer.java | 130 ++-- .../LocalityMulticastAMRMProxyPolicy.java | 64 +- .../utils/FederationStateStoreFacade.java | 9 + .../server/uam/UnmanagedApplicationManager.java | 2 +- .../yarn/server/TestAMRMClientRelayer.java | 53 +- .../TestLocalityMulticastAMRMProxyPolicy.java | 91 ++- .../utils/FederationPoliciesTestUtil.java | 7 +- .../server/nodemanager/DeletionService.java | 25 +- .../amrmproxy/FederationInterceptor.java | 6 +- .../containermanager/ContainerManagerImpl.java | 26 +- .../localizer/LocalResourcesTrackerImpl.java | 5 + .../localizer/ResourceLocalizationService.java | 68 +- .../recovery/NMLeveldbStateStoreService.java | 412 ++++++---- .../recovery/NMNullStateStoreService.java | 2 +- .../recovery/NMStateStoreService.java | 55 +- .../nodemanager/recovery/RecoveryIterator.java | 41 + .../security/NMContainerTokenSecretManager.java | 27 +- .../security/NMTokenSecretManagerInNM.java | 15 +- .../TestResourceLocalizationService.java | 12 +- .../recovery/NMMemoryStateStoreService.java | 82 +- .../TestNMLeveldbStateStoreService.java | 216 ++++-- .../ApplicationMasterService.java | 9 +- .../resourcemanager/RMActiveServiceContext.java | 16 + .../yarn/server/resourcemanager/RMContext.java | 8 +- .../server/resourcemanager/RMContextImpl.java | 14 +- .../server/resourcemanager/ResourceManager.java | 12 + .../scheduler/AbstractYarnScheduler.java | 10 + .../scheduler/AppSchedulingInfo.java | 28 +- .../scheduler/ClusterNodeTracker.java | 61 ++ .../scheduler/YarnScheduler.java | 10 + .../scheduler/activities/ActivitiesLogger.java | 32 +- .../scheduler/activities/ActivitiesManager.java | 8 +- .../scheduler/capacity/AbstractCSQueue.java | 16 +- .../scheduler/capacity/CSQueue.java | 6 + .../scheduler/capacity/CapacityScheduler.java | 77 +- .../CapacitySchedulerConfiguration.java | 116 +++ .../scheduler/capacity/LeafQueue.java | 49 +- .../scheduler/capacity/ParentQueue.java | 4 +- .../allocator/RegularContainerAllocator.java | 35 +- .../common/ApplicationSchedulingConfig.java | 4 + .../scheduler/common/fica/FiCaSchedulerApp.java | 23 + .../placement/AppPlacementAllocator.java | 6 + .../LocalityAppPlacementAllocator.java | 39 +- .../placement/MultiNodeLookupPolicy.java | 67 ++ .../placement/MultiNodePolicySpec.java | 56 ++ .../scheduler/placement/MultiNodeSorter.java | 167 +++++ .../placement/MultiNodeSortingManager.java | 139 ++++ .../ResourceUsageMultiNodeLookupPolicy.java | 79 ++ .../SingleConstraintAppPlacementAllocator.java | 179 ++--- .../webapp/RMAppAttemptBlock.java | 24 +- .../resourcemanager/webapp/dao/AppInfo.java | 10 + .../webapp/dao/ResourceRequestInfo.java | 52 +- .../yarn/server/resourcemanager/MockRM.java | 35 +- .../reservation/ReservationSystemTestUtil.java | 3 + .../scheduler/TestAppSchedulingInfo.java | 3 +- .../capacity/CapacitySchedulerTestBase.java | 13 + .../capacity/TestCapacityScheduler.java | 15 - .../TestCapacitySchedulerMultiNodes.java | 166 +++++ .../TestCapacitySchedulerNodeLabelUpdate.java | 70 ++ ...estSchedulingRequestContainerAllocation.java | 438 ++++++++++- ...stSingleConstraintAppPlacementAllocator.java | 78 -- .../reader/TimelineReaderWebServicesUtils.java | 6 +- 177 files changed, 7363 insertions(+), 2006 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs-rbf/src/main/java/org/apache/hadoop/hdfs/server/federation/router/RouterRpcServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLogger.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/AsyncLoggerSet.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/IPCLoggerChannel.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/client/QuorumJournalManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocol/QJournalProtocol.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolServerSideTranslatorPB.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/protocolPB/QJournalProtocolTranslatorPB.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/Journal.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/qjournal/server/JournalNodeRpcServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNode.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/main/proto/QJournalProtocol.proto ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java index b38a527,9e1e3bb..dd55af2 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/client/TestQuorumJournalManagerUnit.java @@@ -17,13 -17,11 +17,14 @@@ */ package org.apache.hadoop.hdfs.qjournal.client; +import static org.junit.Assert.assertEquals; import static org.junit.Assert.fail; import static org.mockito.Matchers.anyLong; + import static org.mockito.Matchers.anyBoolean; import static org.mockito.Matchers.eq; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; import java.io.IOException; import java.net.URI; import java.util.List; http://git-wip-us.apache.org/repos/asf/hadoop/blob/8aeb2d89/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java ---------------------------------------------------------------------- diff --cc hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java index d419f76,b8d2652..2f51275 --- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java +++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/qjournal/server/TestJournal.java @@@ -78,11 -71,9 +78,11 @@@ public class TestJournal public void setup() throws Exception { FileUtil.fullyDelete(TEST_LOG_DIR); conf = new Configuration(); + // Enable fetching edits via RPC + conf.setBoolean(DFSConfigKeys.DFS_HA_TAILEDITS_INPROGRESS_KEY, true); journal = new Journal(conf, TEST_LOG_DIR, JID, StartupOption.REGULAR, mockErrorReporter); - journal.format(FAKE_NSINFO); + journal.format(FAKE_NSINFO, false); } @After @@@ -444,41 -435,14 +444,52 @@@ } @Test + public void testReadFromCache() throws Exception { + journal.newEpoch(FAKE_NSINFO, 1); + journal.startLogSegment(makeRI(1), 1, + NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); + journal.journal(makeRI(2), 1, 1, 5, QJMTestUtil.createTxnData(1, 5)); + journal.journal(makeRI(3), 1, 6, 5, QJMTestUtil.createTxnData(6, 5)); + journal.journal(makeRI(4), 1, 11, 5, QJMTestUtil.createTxnData(11, 5)); + assertJournaledEditsTxnCountAndContents(1, 7, 7, + NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); + assertJournaledEditsTxnCountAndContents(1, 30, 15, + NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION); + + journal.finalizeLogSegment(makeRI(5), 1, 15); + int newLayoutVersion = NameNodeLayoutVersion.CURRENT_LAYOUT_VERSION - 1; + journal.startLogSegment(makeRI(6), 16, newLayoutVersion); + journal.journal(makeRI(7), 16, 16, 5, QJMTestUtil.createTxnData(16, 5)); + + assertJournaledEditsTxnCountAndContents(16, 10, 20, newLayoutVersion); + } + + private void assertJournaledEditsTxnCountAndContents(int startTxn, + int requestedMaxTxns, int expectedEndTxn, int layoutVersion) + throws Exception { + GetJournaledEditsResponseProto result = + journal.getJournaledEdits(startTxn, requestedMaxTxns); + int expectedTxnCount = expectedEndTxn - startTxn + 1; + ByteArrayOutputStream headerBytes = new ByteArrayOutputStream(); + EditLogFileOutputStream.writeHeader(layoutVersion, + new DataOutputStream(headerBytes)); + assertEquals(expectedTxnCount, result.getTxnCount()); + assertArrayEquals( + Bytes.concat( + headerBytes.toByteArray(), + QJMTestUtil.createTxnData(startTxn, expectedTxnCount)), + result.getEditLog().toByteArray()); + } + ++ @Test + public void testFormatNonEmptyStorageDirectoriesWhenforceOptionIsTrue() + throws Exception { + try { + // Format again here and to format the non-empty directories in + // journal node. + journal.format(FAKE_NSINFO, true); + } catch (IOException ioe) { + fail("Format should be success with force option."); + } + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
