This is an automated email from the ASF dual-hosted git repository. aengineer pushed a commit to branch HDDS-1880-Decom in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 88147d09b6a34f203596a26b0aef89c334d33894 Merge: 278699e ab7987c Author: Anu Engineer <[email protected]> AuthorDate: Fri Nov 8 09:22:35 2019 -0800 Merge branch 'master' into HDDS-1880-Decom BUILDING.txt | 511 --------------------- CONTRIBUTION.md | 206 +++++++++ README.md | 98 ++++ README.txt | 7 - .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 88 ++-- .../hadoop/hdds/scm/XceiverClientManager.java | 38 +- .../hdds/scm/client/ContainerOperationClient.java | 111 +++-- .../hadoop/hdds/scm/storage/BlockInputStream.java | 11 +- .../hadoop/hdds/scm/storage/ChunkInputStream.java | 6 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 111 ++++- .../apache/hadoop/hdds/scm/XceiverClientSpi.java | 2 +- .../scm/container/common/helpers/ExcludeList.java | 14 +- .../hdds/security/token/BlockTokenVerifier.java | 114 ++--- .../security/token/OzoneBlockTokenIdentifier.java | 3 +- .../hadoop/hdds/security/token/TokenVerifier.java | 13 +- .../authority/PKIProfiles/DefaultProfile.java | 4 + .../certificates/utils/CertificateSignRequest.java | 41 +- .../hadoop/hdds/utils/MetadataStoreBuilder.java | 4 +- .../org/apache/hadoop/hdds/utils/RocksDBStore.java | 8 +- .../org/apache/hadoop/hdds/utils/db/Codec.java | 6 + .../apache/hadoop/hdds/utils/db/CodecRegistry.java | 16 + .../apache/hadoop/hdds/utils/db/IntegerCodec.java | 5 + .../org/apache/hadoop/hdds/utils/db/LongCodec.java | 5 + .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 9 +- .../apache/hadoop/hdds/utils/db/StringCodec.java | 5 + .../apache/hadoop/hdds/utils/db/TypedTable.java | 3 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 7 - .../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 + .../common/src/main/resources/ozone-default.xml | 27 +- .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 21 + .../certificate/authority/TestDefaultCAServer.java | 1 + .../certificate/authority/TestDefaultProfile.java | 3 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 109 ++--- .../container/common/helpers/ContainerUtils.java | 2 +- .../ozone/container/common/impl/ContainerData.java | 41 +- .../common/impl/ContainerDataScanOrder.java | 57 +++ .../container/common/impl/ContainerDataYaml.java | 15 +- .../ozone/container/common/impl/ContainerSet.java | 2 + .../container/common/impl/HddsDispatcher.java | 41 +- .../container/common/interfaces/Container.java | 4 + .../transport/server/GrpcXceiverService.java | 16 - .../server/ServerCredentialInterceptor.java | 74 --- .../common/transport/server/XceiverServer.java | 89 ---- .../common/transport/server/XceiverServerGrpc.java | 18 +- .../server/ratis/ContainerStateMachine.java | 34 +- .../transport/server/ratis/XceiverServerRatis.java | 13 +- .../container/keyvalue/KeyValueContainer.java | 16 +- .../container/keyvalue/TarContainerPacker.java | 31 +- .../container/keyvalue/helpers/ChunkUtils.java | 23 +- .../keyvalue/impl/ChunkManagerDummyImpl.java | 17 +- .../container/ozoneimpl/ContainerController.java | 7 + .../container/ozoneimpl/ContainerDataScanner.java | 31 +- .../ozone/container/ozoneimpl/OzoneContainer.java | 7 +- .../replication/GrpcReplicationClient.java | 39 +- .../hadoop/ozone/TestHddsDatanodeService.java | 2 - .../hadoop/ozone/TestHddsSecureDatanodeInit.java | 1 - .../ozone/container/common/SCMTestUtils.java | 22 + .../common/impl/TestContainerDataYaml.java | 33 +- .../container/common/impl/TestContainerSet.java | 59 ++- .../container/common/impl/TestHddsDispatcher.java | 6 +- .../container/common/interfaces/TestHandler.java | 2 +- .../container/keyvalue/TestBlockManagerImpl.java | 11 +- .../container/keyvalue/TestChunkManagerImpl.java | 113 +++-- .../container/keyvalue/TestKeyValueContainer.java | 12 +- .../container/keyvalue/TestTarContainerPacker.java | 204 ++++++-- .../ozoneimpl/TestContainerScrubberMetrics.java | 138 +++--- hadoop-hdds/docs/content/beyond/Containers.md | 1 - hadoop-hdds/docs/content/beyond/RunningWithHDFS.md | 4 +- hadoop-hdds/docs/content/start/OnPrem.md | 16 - .../server/OzoneProtocolMessageDispatcher.java | 4 +- .../hadoop/hdds/server/events/EventQueue.java | 4 +- .../src/main/resources/webapps/datanode/dn.js | 2 - .../hdds/scm/container/ReplicationManager.java | 19 +- .../metrics/SCMContainerManagerMetrics.java | 3 +- .../algorithms/SCMContainerPlacementMetrics.java | 3 +- .../hadoop/hdds/scm/metadata/BigIntegerCodec.java | 5 + .../metadata/DeletedBlocksTransactionCodec.java | 5 + .../apache/hadoop/hdds/scm/metadata/LongCodec.java | 5 + .../hdds/scm/metadata/X509CertificateCodec.java | 5 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 42 +- .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 3 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 2 +- .../hdds/scm/pipeline/SCMPipelineMetrics.java | 3 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 7 +- .../hdds/scm/server/SCMContainerMetrics.java | 3 +- .../hdds/scm/server/StorageContainerManager.java | 10 +- .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 3 - .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 3 - .../hadoop/hdds/scm/block/TestBlockManager.java | 27 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 2 - .../hadoop/hdds/scm/container/MockNodeManager.java | 5 +- .../container/TestCloseContainerEventHandler.java | 5 +- .../scm/container/TestSCMContainerManager.java | 30 +- .../TestSCMContainerPlacementRackAware.java | 61 ++- .../hdds/scm/node/TestContainerPlacement.java | 10 +- .../scm/pipeline/TestRatisPipelineProvider.java | 181 ++++++++ .../org/apache/hadoop/hdds/scm/cli/SCMCLI.java | 89 +--- .../apache/hadoop/ozone/client/ObjectStore.java | 2 +- .../hadoop/ozone/client/io/KeyInputStream.java | 13 +- .../ozone/client/protocol/ClientProtocol.java | 2 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +- .../hadoop/ozone/om/codec/OmBucketInfoCodec.java | 4 + .../hadoop/ozone/om/codec/OmKeyInfoCodec.java | 5 + .../ozone/om/codec/OmMultipartKeyInfoCodec.java | 5 + .../hadoop/ozone/om/codec/OmPrefixInfoCodec.java | 5 + .../hadoop/ozone/om/codec/OmVolumeArgsCodec.java | 4 + .../ozone/om/codec/RepeatedOmKeyInfoCodec.java | 5 + .../hadoop/ozone/om/codec/S3SecretValueCodec.java | 5 + .../ozone/om/codec/TokenIdentifierCodec.java | 4 + .../hadoop/ozone/om/codec/UserVolumeInfoCodec.java | 4 + .../ozone/om/exceptions/NotLeaderException.java | 45 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 67 ++- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 28 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 53 ++- .../ozone/om/helpers/OmMultipartKeyInfo.java | 56 ++- .../hadoop/ozone/om/helpers/OmOzoneAclMap.java | 31 ++ .../hadoop/ozone/om/helpers/OmPrefixInfo.java | 18 + .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 29 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 2 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 78 ++-- .../apache/hadoop/ozone/util/OzoneVersionInfo.java | 3 +- .../apache/hadoop/ozone/web/utils/OzoneUtils.java | 8 +- .../src/main/proto/OzoneManagerProtocol.proto | 6 +- .../om/codec/TestOmMultipartKeyInfoCodec.java | 8 +- .../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 ++- .../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 83 ++++ .../ozone/om/helpers/TestOmMultipartKeyInfo.java | 64 +++ ...TestOmBucketInfo.java => TestOmPrefixInfo.java} | 42 +- .../hadoop/ozone/om/helpers/TestOmVolumeArgs.java | 87 ++++ .../checks/hadolint.sh} | 31 +- hadoop-ozone/dev-support/intellij/ozone-site.xml | 8 - .../dist/dev-support/bin/dist-layout-stitching | 35 +- .../main/compose/ozone-hdfs/docker-compose.yaml | 2 +- .../dist/src/main/compose/ozone-hdfs/docker-config | 2 - .../dist/src/main/compose/ozone-mr/common-config | 3 +- .../{ozones3/test.sh => ozone-om-ha-s3/.env} | 18 +- .../docker-compose.yaml | 10 + .../{ozone-om-ha => ozone-om-ha-s3}/docker-config | 9 +- .../compose/{ozones3 => ozone-om-ha-s3}/test.sh | 4 +- .../test.sh => ozone-om-ha/.ssh/authorized_keys} | 18 +- .../{ozones3/test.sh => ozone-om-ha/.ssh/config} | 20 +- .../test.sh => ozone-om-ha/.ssh/environment} | 18 +- .../dist/src/main/compose/ozone-om-ha/.ssh/id_rsa | 42 ++ .../test.sh => ozone-om-ha/.ssh/id_rsa.pub} | 18 +- .../dist/src/main/compose/ozone-om-ha/Dockerfile | 47 ++ .../main/compose/ozone-om-ha/docker-compose.yaml | 20 +- .../src/main/compose/ozone-om-ha/docker-config | 59 ++- .../{ozones3/test.sh => ozone-om-ha/startOM.sh} | 22 +- .../{ozones3/test.sh => ozone-om-ha/stopOM.sh} | 18 +- .../main/compose/{ozones3 => ozone-om-ha}/test.sh | 12 +- .../src/main/compose/ozone-recon/docker-config | 3 +- .../src/main/compose/ozone-topology/docker-config | 3 +- .../dist/src/main/compose/ozone/docker-config | 3 +- .../src/main/compose/ozoneblockade/docker-config | 1 - .../dist/src/main/compose/ozoneperf/README.md | 20 +- .../src/main/compose/ozoneperf/docker-compose.yaml | 9 - .../dist/src/main/compose/ozoneperf/docker-config | 3 +- .../{ozones3/test.sh => ozoneperf/freon-ockg.yaml} | 25 +- .../{ozones3/test.sh => ozoneperf/freon-rk.yaml} | 25 +- .../src/main/compose/ozones3-haproxy/docker-config | 3 +- .../dist/src/main/compose/ozones3/docker-config | 3 +- hadoop-ozone/dist/src/main/compose/ozones3/test.sh | 2 + .../src/main/compose/ozonescripts/docker-config | 1 - .../src/main/compose/ozonesecure-mr/docker-config | 3 +- .../src/main/compose/ozonesecure/docker-config | 3 +- .../src/main/k8s/definitions/ozone/config.yaml | 1 - .../examples/getting-started/config-configmap.yaml | 1 - .../k8s/examples/minikube/config-configmap.yaml | 1 - .../k8s/examples/ozone-dev/config-configmap.yaml | 1 - .../main/k8s/examples/ozone/config-configmap.yaml | 1 - .../dist/src/main/smoketest/basic/basic.robot | 2 +- .../src/main/smoketest/basic/ozone-shell.robot | 2 + .../dist/src/main/smoketest/omha/testOMHA.robot | 176 +++++++ .../dist/src/main/smoketest/s3/commonawslib.robot | 13 +- .../main/smoketest/security/ozone-secure-s3.robot | 2 +- .../conf/dn-audit-log4j2.properties | 0 .../dist/src/{main => shell}/conf/log4j.properties | 21 - .../conf/om-audit-log4j2.properties | 0 .../conf/ozone-shell-log4j.properties | 0 .../dist/src/{main => shell}/conf/ozone-site.xml | 0 .../conf/scm-audit-log4j2.properties | 0 .../dist/src/shell/hdds}/hadoop-config.cmd | 0 .../dist/src/shell/hdds}/hadoop-config.sh | 0 .../dist/src/shell/hdds}/hadoop-daemons.sh | 0 .../dist/src/shell/hdds}/hadoop-functions.sh | 0 .../dist/src/shell/hdds}/workers.sh | 0 .../src/main/bin => dist/src/shell/ozone}/ozone | 0 .../bin => dist/src/shell/ozone}/ozone-config.sh | 0 .../bin => dist/src/shell/ozone}/start-ozone.sh | 9 - .../bin => dist/src/shell/ozone}/stop-ozone.sh | 9 - .../shell}/shellprofile.d/hadoop-ozone-manager.sh | 8 +- .../src/shell}/shellprofile.d/hadoop-ozone.sh | 0 .../network-tests/src/test/compose/docker-config | 1 - .../hadoop/ozone/insight/BaseInsightPoint.java | 56 +-- .../apache/hadoop/ozone/insight/InsightPoint.java | 6 +- .../apache/hadoop/ozone/insight/LogSubcommand.java | 26 +- .../ozone/insight/datanode/RatisInsight.java | 5 + .../hadoop/ozone/insight/TestBaseInsightPoint.java | 73 +++ .../TestContainerStateManagerIntegration.java | 134 +++--- .../metrics/TestSCMContainerManagerMetrics.java | 6 +- .../scm/pipeline/TestRatisPipelineProvider.java | 207 --------- .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 6 +- .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 13 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 1 - .../hadoop/ozone/TestContainerOperations.java | 20 +- .../TestContainerStateMachineIdempotency.java | 5 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 3 - .../hadoop/ozone/TestStorageContainerManager.java | 11 +- .../ozone/client/rpc/Test2WayCommitInRatis.java | 4 +- .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 9 +- .../rpc/TestContainerStateMachineFailures.java | 2 +- .../ozone/client/rpc/TestKeyInputStream.java | 104 ++++- .../client/rpc/TestOzoneRpcClientAbstract.java | 2 +- .../ozone/client/rpc/TestWatchForCommit.java | 10 +- .../ozone/container/ContainerTestHelper.java | 34 +- .../transport/server/ratis/TestCSMMetrics.java | 2 +- .../container/metrics/TestContainerMetrics.java | 6 +- .../container/ozoneimpl/TestOzoneContainer.java | 2 +- .../container/server/TestContainerServer.java | 2 +- .../server/TestSecureContainerServer.java | 189 ++++---- .../ozone/om/TestOzoneManagerConfiguration.java | 53 ++- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 46 +- .../ozone/ozShell/TestOzoneDatanodeShell.java | 104 +---- .../apache/hadoop/ozone/ozShell/TestS3Shell.java | 4 +- .../hadoop/ozone/scm/TestAllocateContainer.java | 13 +- .../hadoop/ozone/scm/TestContainerSmallFile.java | 17 +- .../scm/TestGetCommittedBlockLengthAndPutKey.java | 17 +- .../hadoop/ozone/scm/TestXceiverClientManager.java | 46 +- .../hadoop/ozone/scm/TestXceiverClientMetrics.java | 9 +- .../scm/node/TestDecommissionAndMaintenance.java | 4 +- .../hadoop/ozone/scm/node/TestQueryNode.java | 5 +- .../security/acl/TestOzoneNativeAuthorizer.java | 91 +++- .../apache/hadoop/ozone/om/BucketManagerImpl.java | 2 +- .../java/org/apache/hadoop/ozone/om/IOzoneAcl.java | 2 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 31 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 9 +- .../apache/hadoop/ozone/om/PrefixManagerImpl.java | 2 +- .../apache/hadoop/ozone/om/VolumeManagerImpl.java | 5 +- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 36 +- .../hadoop/ozone/om/request/OMClientRequest.java | 32 +- .../om/request/bucket/OMBucketCreateRequest.java | 3 +- .../request/bucket/OMBucketSetPropertyRequest.java | 3 +- .../om/request/bucket/acl/OMBucketAclRequest.java | 3 +- .../ozone/om/request/file/OMFileRequest.java | 4 +- .../S3InitiateMultipartUploadRequest.java | 3 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 23 +- .../web/ozShell/bucket/AddAclBucketHandler.java | 6 +- .../web/ozShell/bucket/RemoveAclBucketHandler.java | 6 +- .../web/ozShell/bucket/SetAclBucketHandler.java | 6 +- .../ozone/web/ozShell/keys/AddAclKeyHandler.java | 6 +- .../ozone/web/ozShell/keys/GetKeyHandler.java | 2 +- .../web/ozShell/keys/RemoveAclKeyHandler.java | 6 +- .../ozone/web/ozShell/keys/SetAclKeyHandler.java | 6 +- .../web/ozShell/volume/AddAclVolumeHandler.java | 6 +- .../web/ozShell/volume/RemoveAclVolumeHandler.java | 6 +- .../web/ozShell/volume/SetAclVolumeHandler.java | 8 +- .../hadoop/ozone/om/TestS3BucketManager.java | 2 +- ...tOzoneManagerDoubleBufferWithDummyResponse.java | 69 ++- ...TestOzoneManagerDoubleBufferWithOMResponse.java | 3 +- .../request/TestOMClientRequestWithUserInfo.java | 13 +- .../om/request/key/TestOMAllocateBlockRequest.java | 4 +- .../om/request/key/TestOMKeyCommitRequest.java | 4 +- .../om/request/key/TestOMKeyCreateRequest.java | 4 +- .../s3/bucket/TestS3BucketDeleteRequest.java | 3 +- .../s3/bucket/TestS3BucketDeleteResponse.java | 3 +- .../s3/multipart/TestS3MultipartResponse.java | 4 +- .../security/TestOzoneBlockTokenSecretManager.java | 48 ++ .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 2 - .../org/apache/hadoop/ozone/recon/ReconUtils.java | 11 +- .../spi/impl/ContainerDBServiceProviderImpl.java | 2 +- .../recon/spi/impl/ContainerKeyPrefixCodec.java | 5 + .../s3/endpoint/TestAbortMultipartUpload.java | 7 +- .../hadoop/ozone/s3/endpoint/TestBucketDelete.java | 11 +- .../hadoop/ozone/s3/endpoint/TestBucketHead.java | 5 +- .../s3/endpoint/TestInitiateMultipartUpload.java | 9 +- .../hadoop/ozone/s3/endpoint/TestListParts.java | 29 +- .../s3/endpoint/TestMultipartUploadComplete.java | 33 +- .../s3/endpoint/TestMultipartUploadWithCopy.java | 32 +- .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 3 +- .../hadoop/ozone/s3/endpoint/TestPartUpload.java | 24 +- .../hadoop/ozone/s3/endpoint/TestRootList.java | 3 +- .../org/apache/hadoop/ozone/admin/OzoneAdmin.java | 2 +- .../ozone/admin/om/GetServiceRolesSubcommand.java | 2 +- .../apache/hadoop/ozone/admin/om/package-info.java | 2 +- .../apache/hadoop/ozone/admin/package-info.java | 2 +- .../hadoop/ozone/freon/BaseFreonGenerator.java | 40 ++ .../hadoop/ozone/freon/DatanodeChunkGenerator.java | 190 ++++++++ .../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +- .../apache/hadoop/ozone/freon/OmKeyGenerator.java | 8 +- .../ozone/freon/OzoneClientKeyGenerator.java | 8 +- .../GenerateOzoneRequiredConfigurations.java | 4 +- .../ozone/genesis/BenchMarkContainerStateMap.java | 11 +- .../ozone/genesis/BenchMarkDatanodeDispatcher.java | 2 +- .../hadoop/ozone/genesis/BenchMarkOMClient.java | 30 +- .../ozone/genesis/BenchMarkOzoneManager.java | 33 +- .../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 18 +- pom.xml | 2 +- 297 files changed, 4435 insertions(+), 2763 deletions(-) diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java index 03dd829,63fe152..003035c --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/container/placement/algorithms/TestSCMContainerPlacementRackAware.java @@@ -29,19 -34,8 +33,15 @@@ import org.apache.hadoop.hdds.scm.net.N import org.apache.hadoop.hdds.scm.net.NodeSchema; import org.apache.hadoop.hdds.scm.net.NodeSchemaManager; import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.hdds.scm.node.NodeStatus; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; +import org.mockito.Mockito; - import java.util.ArrayList; - import java.util.Arrays; - import java.util.Collection; - import java.util.List; - + import org.apache.commons.lang.StringUtils; import static org.apache.hadoop.hdds.scm.net.NetConstants.LEAF_SCHEMA; import static org.apache.hadoop.hdds.scm.net.NetConstants.RACK_SCHEMA; import static org.apache.hadoop.hdds.scm.net.NetConstants.ROOT_SCHEMA; diff --cc hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java index 0000000,6f0425d..fc16e98 mode 000000,100644..100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestRatisPipelineProvider.java @@@ -1,0 -1,180 +1,181 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + package org.apache.hadoop.hdds.scm.pipeline; + + import org.apache.hadoop.hdds.conf.OzoneConfiguration; + import org.apache.hadoop.hdds.protocol.DatanodeDetails; + import org.apache.hadoop.hdds.protocol.proto.HddsProtos; + import org.apache.hadoop.hdds.scm.TestUtils; + import org.apache.hadoop.hdds.scm.container.MockNodeManager; + import org.apache.hadoop.hdds.scm.node.NodeManager; ++import org.apache.hadoop.hdds.scm.node.NodeStatus; + import org.junit.Assume; + import org.junit.Before; + import org.junit.Test; + + import java.io.IOException; + import java.util.ArrayList; + import java.util.HashSet; + import java.util.List; + import java.util.Set; + + import static org.apache.commons.collections.CollectionUtils.intersection; + import static org.junit.Assert.assertEquals; + import static org.junit.Assert.assertTrue; + + /** + * Test for RatisPipelineProvider. + */ + public class TestRatisPipelineProvider { + + private static final HddsProtos.ReplicationType REPLICATION_TYPE = + HddsProtos.ReplicationType.RATIS; + + private NodeManager nodeManager; + private PipelineProvider provider; + private PipelineStateManager stateManager; + + @Before + public void init() throws Exception { + nodeManager = new MockNodeManager(true, 10); + stateManager = new PipelineStateManager(new OzoneConfiguration()); + provider = new MockRatisPipelineProvider(nodeManager, + stateManager, new OzoneConfiguration()); + } + + private void createPipelineAndAssertions( + HddsProtos.ReplicationFactor factor) throws IOException { + Pipeline pipeline = provider.create(factor); + assertPipelineProperties(pipeline, factor, REPLICATION_TYPE); + stateManager.addPipeline(pipeline); + + Pipeline pipeline1 = provider.create(factor); + assertPipelineProperties(pipeline1, factor, REPLICATION_TYPE); + // New pipeline should not overlap with the previous created pipeline + assertTrue( + intersection(pipeline.getNodes(), pipeline1.getNodes()) + .isEmpty()); + stateManager.addPipeline(pipeline1); + } + + @Test + public void testCreatePipelineWithFactor() throws IOException { + HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE; + Pipeline pipeline = provider.create(factor); + assertPipelineProperties(pipeline, factor, REPLICATION_TYPE); + stateManager.addPipeline(pipeline); + + factor = HddsProtos.ReplicationFactor.ONE; + Pipeline pipeline1 = provider.create(factor); + assertPipelineProperties(pipeline1, factor, REPLICATION_TYPE); + stateManager.addPipeline(pipeline1); + // New pipeline should overlap with the previous created pipeline, + // and one datanode should overlap between the two types. + assertEquals(1, + intersection(pipeline.getNodes(), pipeline1.getNodes()).size()); + } + + @Test + public void testCreatePipelineWithFactorThree() throws IOException { + createPipelineAndAssertions(HddsProtos.ReplicationFactor.THREE); + } + + @Test + public void testCreatePipelineWithFactorOne() throws IOException { + createPipelineAndAssertions(HddsProtos.ReplicationFactor.ONE); + } + + private List<DatanodeDetails> createListOfNodes(int nodeCount) { + List<DatanodeDetails> nodes = new ArrayList<>(); + for (int i = 0; i < nodeCount; i++) { + nodes.add(TestUtils.randomDatanodeDetails()); + } + return nodes; + } + + @Test + public void testCreatePipelineWithNodes() { + HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE; + Pipeline pipeline = + provider.create(factor, createListOfNodes(factor.getNumber())); + assertPipelineProperties(pipeline, factor, REPLICATION_TYPE); + + factor = HddsProtos.ReplicationFactor.ONE; + pipeline = provider.create(factor, createListOfNodes(factor.getNumber())); + assertPipelineProperties(pipeline, factor, REPLICATION_TYPE); + } + + @Test + public void testCreatePipelinesDnExclude() throws IOException { + List<DatanodeDetails> healthyNodes = - nodeManager.getNodes(HddsProtos.NodeState.HEALTHY); ++ nodeManager.getNodes(NodeStatus.inServiceHealthy()); + + Assume.assumeTrue(healthyNodes.size() == 8); + + HddsProtos.ReplicationFactor factor = HddsProtos.ReplicationFactor.THREE; + + // Use up first 3 DNs for an open pipeline. + List<DatanodeDetails> dns = healthyNodes.subList(0, 3); + addPipeline(dns, factor, Pipeline.PipelineState.OPEN, REPLICATION_TYPE); + Set<DatanodeDetails> membersOfOpenPipelines = new HashSet<>(dns); + + // Use up next 3 DNs for a closed pipeline. + dns = healthyNodes.subList(3, 6); + addPipeline(dns, factor, Pipeline.PipelineState.CLOSED, REPLICATION_TYPE); + Set<DatanodeDetails> membersOfClosedPipelines = new HashSet<>(dns); + + // only 2 healthy DNs left that are not part of any pipeline + Pipeline pipeline = provider.create(factor); + assertPipelineProperties(pipeline, factor, REPLICATION_TYPE); + + List<DatanodeDetails> nodes = pipeline.getNodes(); + + assertTrue( + "nodes of new pipeline cannot be from open pipelines", + nodes.stream().noneMatch(membersOfOpenPipelines::contains)); + + assertTrue( + "at least 1 node should have been from members of closed pipelines", + nodes.stream().anyMatch(membersOfClosedPipelines::contains)); + } + + private static void assertPipelineProperties( + Pipeline pipeline, HddsProtos.ReplicationFactor expectedFactor, + HddsProtos.ReplicationType expectedReplicationType) { + assertEquals(Pipeline.PipelineState.OPEN, pipeline.getPipelineState()); + assertEquals(expectedReplicationType, pipeline.getType()); + assertEquals(expectedFactor, pipeline.getFactor()); + assertEquals(expectedFactor.getNumber(), pipeline.getNodes().size()); + } + + private void addPipeline( + List<DatanodeDetails> dns, HddsProtos.ReplicationFactor factor, + Pipeline.PipelineState open, HddsProtos.ReplicationType replicationType) + throws IOException { + Pipeline openPipeline = Pipeline.newBuilder() + .setType(replicationType) + .setFactor(factor) + .setNodes(dns) + .setState(open) + .setId(PipelineID.randomId()) + .build(); + + stateManager.addPipeline(openPipeline); + } + } diff --cc hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java index 8649992,8c0fb03..75e2aa6 --- a/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java +++ b/hadoop-hdds/tools/src/main/java/org/apache/hadoop/hdds/scm/cli/SCMCLI.java @@@ -25,12 -27,9 +27,10 @@@ import org.apache.hadoop.hdds.HddsUtils import org.apache.hadoop.hdds.cli.GenericCli; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.conf.OzoneConfiguration; - import org.apache.hadoop.hdds.protocol.SCMSecurityProtocol; import org.apache.hadoop.hdds.scm.ScmConfigKeys; - import org.apache.hadoop.hdds.scm.XceiverClientManager; import org.apache.hadoop.hdds.scm.cli.container.ContainerCommands; import org.apache.hadoop.hdds.scm.cli.pipeline.PipelineCommands; +import org.apache.hadoop.hdds.scm.cli.node.DatanodeAdminCommands; import org.apache.hadoop.hdds.scm.client.ContainerOperationClient; import org.apache.hadoop.hdds.scm.client.ScmClient; import org.apache.hadoop.hdds.scm.container.ContainerInfo; diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java index bada595,0000000..5cf0864 mode 100644,000000..100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/scm/node/TestDecommissionAndMaintenance.java @@@ -1,137 -1,0 +1,135 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.ozone.scm.node; + +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.XceiverClientManager; +import org.apache.hadoop.hdds.scm.client.ContainerOperationClient; +import org.apache.hadoop.hdds.scm.node.NodeManager; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Arrays; +import java.util.List; +import java.util.concurrent.TimeUnit; + +import static java.util.concurrent.TimeUnit.SECONDS; +import static junit.framework.TestCase.assertEquals; +import static org.apache.hadoop.hdds.HddsConfigKeys.*; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.*; + +/** + * Test from the scmclient for decommission and maintenance. + */ + +public class TestDecommissionAndMaintenance { + private static final Logger LOG = + LoggerFactory.getLogger(TestDecommissionAndMaintenance.class); + + private static int numOfDatanodes = 5; + private MiniOzoneCluster cluster; + + private ContainerOperationClient scmClient; + + @Before + public void setUp() throws Exception { + OzoneConfiguration conf = new OzoneConfiguration(); + final int interval = 100; + + conf.setTimeDuration(OZONE_SCM_HEARTBEAT_PROCESS_INTERVAL, + interval, TimeUnit.MILLISECONDS); + conf.setTimeDuration(HDDS_HEARTBEAT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_PIPELINE_REPORT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_COMMAND_STATUS_REPORT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_CONTAINER_REPORT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(HDDS_NODE_REPORT_INTERVAL, 1, SECONDS); + conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 3, SECONDS); + conf.setTimeDuration(OZONE_SCM_DEADNODE_INTERVAL, 6, SECONDS); + + cluster = MiniOzoneCluster.newBuilder(conf) + .setNumDatanodes(numOfDatanodes) + .build(); + cluster.waitForClusterToBeReady(); - scmClient = new ContainerOperationClient(cluster - .getStorageContainerLocationClient(), - new XceiverClientManager(conf)); ++ scmClient = new ContainerOperationClient(conf); + } + + @After + public void tearDown() throws Exception { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testNodeCanBeDecommMaintAndRecommissioned() + throws IOException { + NodeManager nm = cluster.getStorageContainerManager().getScmNodeManager(); + + List<DatanodeDetails> dns = nm.getAllNodes(); + scmClient.decommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0)))); + + // Ensure one node is decommissioning + List<DatanodeDetails> decomNodes = nm.getNodes( + HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY); + assertEquals(1, decomNodes.size()); + + scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0)))); + + // Ensure zero nodes are now decommissioning + decomNodes = nm.getNodes( + HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY); + assertEquals(0, decomNodes.size()); + + scmClient.startMaintenanceNodes(Arrays.asList( + getDNHostAndPort(dns.get(0))), 10); + + // None are decommissioning + decomNodes = nm.getNodes( + HddsProtos.NodeOperationalState.DECOMMISSIONING, + HddsProtos.NodeState.HEALTHY); + assertEquals(0, decomNodes.size()); + + // One is in Maintenance + decomNodes = nm.getNodes( + HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + HddsProtos.NodeState.HEALTHY); + assertEquals(1, decomNodes.size()); + + scmClient.recommissionNodes(Arrays.asList(getDNHostAndPort(dns.get(0)))); + + // None are in maintenance + decomNodes = nm.getNodes( + HddsProtos.NodeOperationalState.ENTERING_MAINTENANCE, + HddsProtos.NodeState.HEALTHY); + assertEquals(0, decomNodes.size()); + } + + private String getDNHostAndPort(DatanodeDetails dn) { + return dn.getHostName()+":"+dn.getPorts().get(0).getValue(); + } + +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
