This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch HDDS-3698-nonrolling-upgrade in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 253cc5540f65e15f674f0f83e9c1bc395005cda6 Merge: cdb1471 48b84c5 Author: Ethan Rose <[email protected]> AuthorDate: Mon May 3 18:20:08 2021 -0400 Merge remote-tracking branch 'upstream/master' into HDDS-3698-nonrolling-upgrade-master-merge2 * upstream/master: (56 commits) HDDS-2212. Genconf tool should generate config files for secure clust… (#1788) HDDS-5166. Remove duplicate assignment of OZONE_OPTS for freon and sh (#2195) Revert "HDDS-5144. Create github check to alert when dependency tree is changed (#2177)" HDDS-4983. Display key offset for each block in command key info (#2051) HDDS-5144. Create github check to alert when dependency tree is changed (#2177) HDDS-4585. Support bucket acl operation in S3g (#1701) HDDS-5153. Decommissioning a dead node should complete immediately (#2190) HDDS-5147. Intermittent test failure in TestContainerDeletionChoosingPolicy#testRandomChoosingPolicy (#2188) HDDS-5152. Fix Suggested leader in Client. (#2189) HDDS-5148. Bump ratis version to 2.1.0-ff8aa66-SNAPSHOT (#2184) HDDS-4515. Datanodes should be able to persist and load CRL (#2181) HDDS-5060. [SCM HA Security] Make InterSCM grpc channel secure. (#2187) HDDS-5051. Ensure failover to suggested leader if any for NotLeaderException. (#2141) HDDS-5127. Fix getServiceList when SCM HA is enabled (#2173) HDDS-4889. Add simple CI check for docs (#2156) HDDS-5131. Use timeout in github actions (#2176) HDDS-5103. Fix Install Snapshot Mechanism in SCMStateMachine. (#2155) HDDS-5124. Use OzoneConsts.OZONE_TIME_ZONE instead of "GMT" (#2166) HDDS-5047. Refactor Pipeline to use ReplicationConfig instead of factor/type (#2096) HDDS-5083. Bump version of common-compress (#2139) ... Conflicts: hadoop-hdds/common/pom.xml hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/node/SCMNodeManager.java hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManager.java hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/node/TestSCMNodeManager.java hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java .github/workflows/cancel-ci.yaml | 41 ++ .github/workflows/post-commit.yml | 181 ++--- .gitignore | 2 + HISTORY.md | 4 +- hadoop-hdds/client/pom.xml | 4 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 7 + .../hadoop/hdds/scm/storage/BlockInputStream.java | 7 +- .../storage/DummyBlockInputStreamWithRetry.java | 7 +- hadoop-hdds/common/pom.xml | 12 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 3 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 104 ++- .../hdds/ratis/ServerNotLeaderException.java | 113 +++ .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 8 - .../hadoop/hdds/scm/ha/NonRetriableException.java | 23 +- .../scm/ha/RetriableWithNoFailoverException.java | 25 +- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 129 +++- .../org/apache/hadoop/hdds/scm/ha/SCMNodeInfo.java | 5 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 67 +- .../hadoop/hdds/security/x509/SecurityConfig.java | 13 +- .../hadoop/hdds/security/x509/crl}/CRLCodec.java | 2 +- .../hadoop/hdds/security/x509/crl/CRLInfo.java | 25 +- .../hdds/security/x509/crl}/CRLInfoCodec.java | 3 +- .../hdds/security/x509/crl/package-info.java | 23 +- .../org/apache/hadoop/hdds/utils/db/Codec.java | 0 .../apache/hadoop/hdds/utils/db/package-info.java | 23 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 8 + .../java/org/apache/hadoop/ozone/OzoneConsts.java | 12 + .../org/apache/hadoop/ozone/common/Storage.java | 12 + .../ozone/container/common/helpers/BlockData.java | 2 +- .../common/src/main/resources/ozone-default.xml | 154 +++-- .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 22 +- .../hdds/ratis/TestServerNotLeaderException.java | 82 +++ .../hadoop/hdds/scm/pipeline/MockPipeline.java | 12 +- hadoop-hdds/config/pom.xml | 4 +- hadoop-hdds/container-service/pom.xml | 12 +- .../hdds/datanode/metadata/CRLDBDefinition.java | 99 +++ .../hdds/datanode/metadata/DatanodeCRLStore.java | 84 +++ .../datanode/metadata/DatanodeCRLStoreImpl.java | 127 ++++ .../hdds/datanode/metadata/package-info.java | 23 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 23 +- .../common/statemachine/DatanodeStateMachine.java | 2 +- .../common/states/datanode/InitDatanodeState.java | 4 +- .../ozone/container/keyvalue/KeyValueHandler.java | 58 +- .../container/keyvalue/helpers/BlockUtils.java | 27 +- .../container/keyvalue/impl/BlockManagerImpl.java | 16 +- .../container/metadata/AbstractDatanodeStore.java | 26 +- .../metadata/TestDatanodeCRLStoreImpl.java | 127 ++++ .../hdds/datanode/metadata/package-info.java | 27 +- .../hadoop/ozone/TestHddsDatanodeService.java | 6 +- .../container/common/helpers/TestBlockData.java | 2 +- .../common/impl/TestContainerDataYaml.java | 13 +- .../impl/TestContainerDeletionChoosingPolicy.java | 31 +- .../container/keyvalue/TestKeyValueContainer.java | 43 +- .../TestKeyValueHandlerWithUnhealthyContainer.java | 18 +- .../container/ozoneimpl/TestContainerReader.java | 1 + hadoop-hdds/docs/content/feature/OM-HA.zh.md | 13 +- hadoop-hdds/docs/content/feature/SCM-HA.md | 2 +- hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 162 +++++ hadoop-hdds/docs/pom.xml | 4 +- hadoop-hdds/framework/pom.xml | 4 +- .../java/org/apache/hadoop/hdds}/ExitManager.java | 2 +- .../java/org/apache/hadoop/hdds}/NodeDetails.java | 2 +- .../java/org/apache/hadoop/hdds/package-info.java | 17 +- .../hadoop/hdds/protocol/SCMSecurityProtocol.java | 15 + .../SCMSecurityProtocolClientSideTranslatorPB.java | 37 + .../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 11 + .../scm/protocol/ScmBlockLocationProtocol.java | 25 +- ...lockLocationProtocolClientSideTranslatorPB.java | 53 +- ...inerLocationProtocolClientSideTranslatorPB.java | 1 + .../SCMBlockLocationFailoverProxyProvider.java | 59 +- .../hadoop/hdds/scm/proxy/SCMClientConfig.java | 27 +- .../SCMContainerLocationFailoverProxyProvider.java | 69 +- .../SCMSecurityProtocolFailoverProxyProvider.java | 79 ++- .../CRLInfo.java => certificate/CertInfo.java} | 97 +-- .../certificate/authority/CertificateServer.java | 15 + .../certificate/authority/CertificateStore.java | 28 + .../certificate/authority/DefaultApprover.java | 2 +- .../certificate/authority/DefaultCAServer.java | 28 +- .../certificate/authority/DefaultCRLApprover.java | 2 +- .../x509/certificate/client/CertificateClient.java | 17 + .../client/DefaultCertificateClient.java | 38 +- .../security/x509/certificate/package-info.java | 18 +- .../java/org/apache/hadoop/hdds/utils/HAUtils.java | 25 +- .../apache/hadoop/hdds/utils/HddsServerUtil.java | 98 ++- .../org/apache/hadoop/hdds/utils/db/DBProfile.java | 4 +- .../x509/certificate/authority/MockCAStore.java | 18 + .../certificate/authority/TestDefaultCAServer.java | 10 + .../x509/certificate/utils/TestCRLCodec.java | 1 + hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +- hadoop-hdds/interface-admin/pom.xml | 4 +- hadoop-hdds/interface-client/pom.xml | 4 +- .../interface-client/src/main/proto/hdds.proto | 9 + hadoop-hdds/interface-server/pom.xml | 4 +- .../src/main/proto/ScmServerSecurityProtocol.proto | 29 + hadoop-hdds/pom.xml | 10 +- hadoop-hdds/server-scm/pom.xml | 8 +- .../apache/hadoop/hdds/scm/block/BlockManager.java | 9 +- .../hadoop/hdds/scm/block/BlockManagerImpl.java | 45 +- .../hdds/scm/container/ContainerManagerImpl.java | 18 +- .../hdds/scm/container/ContainerManagerV2.java | 9 +- .../hdds/scm/container/ContainerStateManager.java | 31 +- .../hdds/scm/container/SCMContainerManager.java | 6 +- .../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 39 +- .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 28 +- .../hdds/scm/ha/InterSCMGrpcProtocolService.java | 23 + .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 18 + .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 19 +- .../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 3 +- .../apache/hadoop/hdds/scm/ha/SCMNodeDetails.java | 6 +- .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 48 +- .../hadoop/hdds/scm/ha/SCMSnapshotDownloader.java | 2 +- .../hadoop/hdds/scm/ha/SCMSnapshotProvider.java | 36 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 29 +- .../{CRLInfoCodec.java => CertInfoCodec.java} | 22 +- .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 19 +- .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 14 +- .../hdds/scm/node/NodeDecommissionManager.java | 57 +- .../hadoop/hdds/scm/node/SCMNodeManager.java | 62 +- .../scm/pipeline/BackgroundPipelineCreator.java | 41 +- .../scm/pipeline/BackgroundPipelineCreatorV2.java | 38 +- .../hadoop/hdds/scm/pipeline/PipelineFactory.java | 24 +- .../hadoop/hdds/scm/pipeline/PipelineManager.java | 39 +- .../hdds/scm/pipeline/PipelineManagerV2Impl.java | 74 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 18 +- .../hadoop/hdds/scm/pipeline/PipelineProvider.java | 29 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 12 +- .../hdds/scm/pipeline/PipelineStateManager.java | 34 +- .../scm/pipeline/PipelineStateManagerV2Impl.java | 35 +- .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 163 +---- .../hdds/scm/pipeline/RatisPipelineProvider.java | 54 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 14 +- .../hdds/scm/pipeline/SCMPipelineManager.java | 74 +- .../hdds/scm/pipeline/SCMPipelineMetrics.java | 4 +- .../hdds/scm/pipeline/SimplePipelineProvider.java | 27 +- .../hadoop/hdds/scm/pipeline/StateManager.java | 42 +- .../SCMSecurityProtocolServerSideTranslatorPB.java | 83 ++- ...lockLocationProtocolServerSideTranslatorPB.java | 20 +- ...inerLocationProtocolServerSideTranslatorPB.java | 9 +- .../scm/safemode/HealthyPipelineSafeModeRule.java | 6 +- .../safemode/OneReplicaPipelineSafeModeRule.java | 20 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 15 +- .../hadoop/hdds/scm/server/SCMCertStore.java | 106 ++- .../hdds/scm/server/SCMClientProtocolServer.java | 12 +- .../hdds/scm/server/SCMSecurityProtocolServer.java | 13 +- .../hadoop/hdds/scm/server/SCMStorageConfig.java | 19 +- .../hdds/scm/server/StorageContainerManager.java | 117 ++-- .../apache/hadoop/hdds/scm/TestHddsServerUtil.java | 31 +- .../hadoop/hdds/scm/TestHddsServerUtils.java | 21 +- .../scm/TestStorageContainerManagerHttpServer.java | 2 + .../java/org/apache/hadoop/hdds/scm/TestUtils.java | 21 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 95 +-- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 12 +- .../container/TestCloseContainerEventHandler.java | 10 +- .../scm/container/TestContainerManagerImpl.java | 18 +- .../scm/container/TestContainerStateManager.java | 15 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 7 +- .../hdds/scm/node/TestNodeDecommissionManager.java | 67 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 28 +- .../hdds/scm/pipeline/MockPipelineManager.java | 53 +- .../scm/pipeline/MockRatisPipelineProvider.java | 21 +- .../TestPipelineDatanodesIntersection.java | 13 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 83 +-- .../scm/pipeline/TestPipelinePlacementPolicy.java | 10 +- .../scm/pipeline/TestPipelineStateManager.java | 101 +-- .../scm/pipeline/TestRatisPipelineProvider.java | 100 +-- .../hdds/scm/pipeline/TestSCMPipelineManager.java | 71 +- ...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 6 + .../scm/pipeline/TestSimplePipelineProvider.java | 28 +- .../safemode/TestHealthyPipelineSafeModeRule.java | 26 +- .../TestOneReplicaPipelineSafeModeRule.java | 13 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 18 +- .../hadoop/hdds/scm/server/TestSCMCertStore.java | 36 +- hadoop-hdds/test-utils/pom.xml | 4 +- hadoop-hdds/tools/pom.xml | 4 +- .../hdds/scm/cli/datanode/ListInfoSubcommand.java | 6 +- .../scm/cli/pipeline/CreatePipelineSubcommand.java | 3 +- .../scm/cli/pipeline/ListPipelinesSubcommand.java | 4 +- hadoop-ozone/client/pom.xml | 4 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 15 +- .../hadoop/ozone/client/OzoneKeyLocation.java | 15 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 57 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 76 ++- .../hadoop/ozone/client/TestHddsClientUtils.java | 150 ++-- hadoop-ozone/common/pom.xml | 4 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 59 +- .../hadoop/ozone/om/helpers/OzoneAclUtil.java | 20 + .../ozone/security/acl/IAccessAuthorizer.java | 1 - .../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 7 +- hadoop-ozone/csi/pom.xml | 4 +- hadoop-ozone/datanode/pom.xml | 16 +- hadoop-ozone/dev-support/checks/_lib.sh | 142 ++++ hadoop-ozone/dev-support/checks/acceptance.sh | 5 + hadoop-ozone/dev-support/checks/bats.sh | 5 + .../dev-support/checks/{bats.sh => docs.sh} | 21 +- hadoop-ozone/dev-support/checks/findbugs.sh | 12 +- hadoop-ozone/dev-support/checks/kubernetes.sh | 16 + .../dist/dev-support/bin/dist-layout-stitching | 3 + hadoop-ozone/dist/pom.xml | 4 +- .../dist/src/main/assemblies/ozone-src.xml | 7 + .../src/main/compose/compatibility/docker-config | 1 + .../dist/src/main/compose/ozone-csi/docker-config | 1 + .../dist/src/main/compose/ozone-ha/docker-config | 1 + .../src/main/compose/ozone-om-ha/docker-config | 1 + .../src/main/compose/ozone-topology/docker-config | 1 + .../dist/src/main/compose/ozone/docker-config | 1 + .../src/main/compose/ozoneblockade/docker-config | 1 + .../src/main/compose/ozones3-haproxy/docker-config | 1 + .../src/main/compose/ozonescripts/docker-config | 1 + .../compose/ozonesecure-ha/docker-compose.yaml | 26 + .../src/main/compose/ozonesecure-ha/docker-config | 2 +- .../dist/src/main/compose/ozonesecure-ha/test.sh | 1 + .../src/main/compose/ozonesecure-mr/docker-config | 1 + .../main/compose/ozonesecure/docker-compose.yaml | 1 - .../src/main/compose/ozonesecure/docker-config | 1 + .../dist/src/main/compose/ozonesecure/test.sh | 51 +- .../dist/src/main/compose/restart/docker-config | 1 + hadoop-ozone/dist/src/main/compose/testlib.sh | 1 - .../compose/upgrade/compose/non-ha/docker-config | 1 + .../dist/src/main/compose/xcompat/docker-config | 1 + hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 15 +- .../src/main/license/bin/licenses/LICENSE-CDDL.txt | 133 ++++ ...-jakarta.validation-jakarta.validation-api.txt} | 241 ------- .../LICENSE-jakarta.ws.rs-jakarta.ws.rs-api.md | 637 +++++++++++++++++ .../LICENSE-javax.ws.rs-javax.ws.rs-api.txt | 759 --------------------- .../NOTICE-jakarta.ws.rs-jakarta.ws.rs-api.md | 61 ++ .../src/main/smoketest/admincli/pipeline.robot | 7 +- .../dist/src/main/smoketest/s3/commonawslib.robot | 2 +- hadoop-ozone/dist/src/shell/ozone/ozone | 2 - .../fault-injection-test/mini-chaos-tests/pom.xml | 4 +- .../fault-injection-test/network-tests/pom.xml | 2 +- .../network-tests/src/test/compose/docker-config | 1 + hadoop-ozone/fault-injection-test/pom.xml | 4 +- hadoop-ozone/insight/pom.xml | 12 +- hadoop-ozone/integration-test/pom.xml | 4 +- .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 1 - .../hadoop/hdds/scm/TestRatisPipelineLeader.java | 11 +- .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +- .../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +- .../metrics/TestSCMContainerManagerMetrics.java | 9 +- .../hdds/scm/pipeline/TestLeaderChoosePolicy.java | 25 +- .../hdds/scm/pipeline/TestNode2PipelineMap.java | 9 +- .../hadoop/hdds/scm/pipeline/TestNodeFailure.java | 7 +- .../hdds/scm/pipeline/TestPipelineClose.java | 10 +- .../TestRatisPipelineCreateAndDestroy.java | 30 +- .../hadoop/hdds/scm/pipeline/TestSCMRestart.java | 16 +- .../safemode/TestSCMSafeModeWithPipelineRules.java | 15 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 15 +- .../apache/hadoop/ozone/TestDelegationToken.java | 1 - .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 6 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 8 +- .../hadoop/ozone/TestSecureOzoneCluster.java | 6 +- .../hadoop/ozone/TestStorageContainerManager.java | 6 + .../ozone/client/CertificateClientTestImpl.java | 13 + .../TestContainerStateMachineFailureOnRead.java | 7 +- .../client/rpc/TestDeleteWithSlowFollower.java | 5 +- .../client/rpc/TestHybridPipelineOnDatanode.java | 2 +- .../client/rpc/TestOzoneAtRestEncryption.java | 25 + .../client/rpc/TestOzoneRpcClientAbstract.java | 120 +++- .../ozone/client/rpc/TestSecureOzoneRpcClient.java | 5 + .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 13 +- .../hadoop/ozone/om/TestOMRatisSnapshots.java | 2 +- .../ozone/om/TestOzoneManagerConfiguration.java | 1 - .../ozone/om/TestOzoneManagerRestInterface.java | 3 +- .../hadoop/ozone/om/TestSecureOzoneManager.java | 4 +- .../hadoop/ozone/recon/TestReconAsPassiveScm.java | 13 +- .../apache/hadoop/ozone/recon/TestReconTasks.java | 5 +- .../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 147 ++++ .../TestSCMContainerPlacementPolicyMetrics.java | 7 +- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 87 +-- .../org/apache/hadoop/ozone/scm/TestSCMMXBean.java | 9 +- .../hadoop/ozone/scm/TestXceiverClientGrpc.java | 7 +- .../ozone/scm/pipeline/TestSCMPipelineMetrics.java | 11 +- hadoop-ozone/interface-client/pom.xml | 4 +- hadoop-ozone/interface-storage/pom.xml | 4 +- hadoop-ozone/ozone-manager/pom.xml | 4 +- .../java/org/apache/hadoop/ozone/om/OMStorage.java | 17 - .../org/apache/hadoop/ozone/om/OzoneManager.java | 81 ++- .../hadoop/ozone/om/OzoneManagerStarter.java | 4 - .../apache/hadoop/ozone/om/ha/OMNodeDetails.java | 2 +- .../S3MultipartUploadCompleteRequest.java | 2 +- .../ozone/om/ScmBlockLocationTestingClient.java | 12 +- .../apache/hadoop/ozone/om/TestKeyManagerUnit.java | 9 +- .../ozone/om/TestOzoneManagerHttpServer.java | 2 + .../ozone/om/request/TestOMRequestUtils.java | 54 +- .../om/request/key/TestOMKeyCommitRequest.java | 78 ++- .../ozone/om/request/key/TestOMKeyRequest.java | 6 +- .../om/response/key/TestOMKeyDeleteResponse.java | 4 +- hadoop-ozone/ozonefs-common/pom.xml | 4 +- hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +- hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +- hadoop-ozone/ozonefs-shaded/pom.xml | 4 +- hadoop-ozone/ozonefs/pom.xml | 4 +- hadoop-ozone/pom.xml | 4 +- hadoop-ozone/recon-codegen/pom.xml | 2 +- hadoop-ozone/recon/pom.xml | 6 +- .../org/apache/hadoop/ozone/recon/ReconServer.java | 2 - .../hadoop/ozone/recon/api/NodeEndpoint.java | 6 +- .../hadoop/ozone/recon/api/PipelineEndpoint.java | 11 +- .../ozone/recon/scm/ReconPipelineFactory.java | 9 +- .../recon/scm/ReconPipelineReportHandler.java | 2 +- .../scm/ReconStorageContainerManagerFacade.java | 15 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 9 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 4 +- .../ozone/recon/scm/TestReconPipelineManager.java | 12 +- hadoop-ozone/s3gateway/pom.xml | 20 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 283 +++++++- .../s3/endpoint/ListMultipartUploadsResult.java | 53 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 3 +- .../endpoint/PutBucketAclRequestUnmarshaller.java | 85 +++ .../org/apache/hadoop/ozone/s3/endpoint/S3Acl.java | 342 ++++++++++ .../hadoop/ozone/s3/endpoint/S3BucketAcl.java | 235 +++++++ .../apache/hadoop/ozone/s3/endpoint/S3Owner.java | 72 ++ .../hadoop/ozone/s3/exception/S3ErrorTable.java | 5 + .../apache/hadoop/ozone/s3/util/RFC1123Util.java | 3 +- .../hadoop/ozone/client/OzoneBucketStub.java | 23 + .../hadoop/ozone/client/OzoneVolumeStub.java | 24 + .../hadoop/ozone/s3/endpoint/TestBucketAcl.java | 266 ++++++++ .../{TestBucketGet.java => TestBucketList.java} | 76 +-- .../hadoop/ozone/s3/endpoint/TestBucketPut.java | 6 +- .../ozone/s3/endpoint/TestPermissionCheck.java | 56 +- .../src/test/resources/groupAccessControlList.xml | 57 ++ .../src/test/resources/userAccessControlList.xml | 42 ++ hadoop-ozone/tools/pom.xml | 12 +- .../StorageContainerManagersCommandHandler.java | 2 +- .../apache/hadoop/ozone/debug/ChunkKeyHandler.java | 5 +- .../hadoop/ozone/freon/BaseFreonGenerator.java | 3 +- .../hadoop/ozone/freon/DatanodeChunkGenerator.java | 3 +- .../hadoop/ozone/freon/DatanodeChunkValidator.java | 5 +- .../ozone/freon/LeaderAppendLogEntryGenerator.java | 6 +- .../GenerateOzoneRequiredConfigurations.java | 44 +- .../ozone/genesis/BenchMarkContainerStateMap.java | 92 +-- .../ozone/genesis/BenchMarkOzoneManager.java | 5 +- .../apache/hadoop/ozone/genesis/BenchMarkSCM.java | 7 +- .../apache/hadoop/ozone/genesis/GenesisUtil.java | 5 +- .../TestGenerateOzoneRequiredConfigurations.java | 52 +- pom.xml | 42 +- 338 files changed, 7520 insertions(+), 3682 deletions(-) diff --cc hadoop-hdds/common/pom.xml index 4d6a0a2,cebee64..6b36972 --- a/hadoop-hdds/common/pom.xml +++ b/hadoop-hdds/common/pom.xml @@@ -207,18 -201,10 +206,17 @@@ https://maven.apache.org/xsd/maven-4.0. <scope>test</scope> </dependency> <dependency> + <groupId>com.codahale.metrics</groupId> + <artifactId>metrics-core</artifactId> + <version>3.0.2</version> + <scope>test</scope> + <!-- Needed for mocking RaftServerImpl --> + </dependency> + <dependency> <groupId>com.github.spotbugs</groupId> <artifactId>spotbugs-annotations</artifactId> - <scope>compile</scope> + <scope>test</scope> </dependency> - </dependencies> <build> diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index 997b65d,460803e..cf44a92 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@@ -440,7 -444,10 +444,11 @@@ public final class OzoneConfigKeys public static final boolean OZONE_OM_KEYNAME_CHARACTER_CHECK_ENABLED_DEFAULT = false; + public static final int OZONE_INIT_DEFAULT_LAYOUT_VERSION_DEFAULT = -1; + public static final String OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY = + "ozone.client.key.provider.cache.expiry"; + public static final long OZONE_CLIENT_KEY_PROVIDER_CACHE_EXPIRY_DEFAULT = + TimeUnit.DAYS.toMillis(10); // 10 days /** * There is no need to instantiate this class. diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index dd58cb8,efaeb23..5a9e71a --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@@ -435,6 -434,11 +439,14 @@@ public final class OzoneConsts public static final String SCM_SUB_CA_PREFIX = "scm-sub@"; public static final String SCM_ROOT_CA_PREFIX = "scm@"; + // Layout Version written into Meta Table ONLY during finalization. + public static final String LAYOUT_VERSION_KEY = "#LAYOUTVERSION"; ++ + // Kerberos constants + public static final String KERBEROS_CONFIG_VALUE = "kerberos"; + public static final String HTTP_AUTH_TYPE_SUFFIX = "http.auth.type"; + public static final String OZONE_SECURITY_ENABLED_SECURE = "true"; + public static final String OZONE_HTTP_SECURITY_ENABLED_SECURE = "true"; + public static final String OZONE_HTTP_FILTER_INITIALIZERS_SECURE = + "org.apache.hadoop.security.AuthenticationFilterInitializer"; } diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java index c9a407e,baa5889..d45ddcc --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/HddsDatanodeService.java @@@ -70,9 -71,7 +72,8 @@@ import com.google.common.base.Precondit import com.sun.jmx.mbeanserver.Introspector; import static org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec.getX509Certificate; import static org.apache.hadoop.hdds.security.x509.certificates.utils.CertificateSignRequest.getEncodedString; - import static org.apache.hadoop.hdds.utils.HAUtils.checkSecurityAndSCMHAEnabled; import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_DATANODE_PLUGINS_KEY; +import static org.apache.hadoop.ozone.common.Storage.StorageState.INITIALIZED; import static org.apache.hadoop.util.ExitUtil.terminate; import org.bouncycastle.pkcs.PKCS10CertificationRequest; import org.slf4j.Logger; @@@ -246,12 -245,10 +247,16 @@@ public class HddsDatanodeService extend } LOG.info("Hdds Datanode login successful."); } + + DatanodeLayoutStorage layoutStorage = new DatanodeLayoutStorage(conf, + datanodeDetails.getUuidString()); + if (layoutStorage.getState() != INITIALIZED) { + layoutStorage.initialize(); + } + + // initialize datanode CRL store + dnCRLStore = new DatanodeCRLStoreImpl(conf); + if (OzoneSecurityUtil.isSecurityEnabled(conf)) { initializeCertificateClient(conf); } diff --cc hadoop-hdds/interface-client/src/main/proto/hdds.proto index 7d746ad,e7e4502..14a2665 --- a/hadoop-hdds/interface-client/src/main/proto/hdds.proto +++ b/hadoop-hdds/interface-client/src/main/proto/hdds.proto @@@ -336,5 -323,13 +336,14 @@@ message UpgradeFinalizationStatus message CRLInfoProto { required string x509CRL = 1; required uint64 creationTimestamp = 2; + required int64 crlSequenceID = 3; + } + + /** + * Information for X509 Certificate. + */ + message CertInfoProto { + required string x509Certificate = 1; + required uint64 timestamp = 2; } + diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java index 05a130e,b7d8864..a0f7e0f --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/PipelineManagerV2Impl.java @@@ -152,17 -151,9 +156,17 @@@ public class PipelineManagerV2Impl impl throw new IOException("Pipeline creation is not allowed as safe mode " + "prechecks have not yet passed"); } + + if (freezePipelineCreation.get()) { + String message = "Cannot create new pipelines while pipeline creation " + + "is frozen."; + LOG.info(message); + throw new IOException(message); + } + lock.lock(); try { - Pipeline pipeline = pipelineFactory.create(type, factor); + Pipeline pipeline = pipelineFactory.create(replicationConfig); stateManager.addPipeline(pipeline.getProtobufMessage( ClientVersions.CURRENT_VERSION)); recordMetricsForPipeline(pipeline); diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java index cdc6e48,c96ce2d..ddf0d50 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/SCMPipelineManager.java @@@ -275,15 -270,9 +277,15 @@@ public class SCMPipelineManager impleme throw new IOException("Pipeline creation is not allowed as safe mode " + "prechecks have not yet passed"); } + if (freezePipelineCreation.get()) { + LOG.debug("Pipeline creation is frozen while an upgrade is in " + + "progress"); + throw new IOException("Pipeline creation is frozen while an upgrade " + + "is in progress"); + } lock.writeLock().lock(); try { - Pipeline pipeline = pipelineFactory.create(type, factor); + Pipeline pipeline = pipelineFactory.create(replicationConfig); if (pipelineStore != null) { pipelineStore.put(pipeline.getId(), pipeline); } diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java index 55f7f3e,4459902..6b8f192 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/SCMStorageConfig.java @@@ -28,8 -27,8 +28,10 @@@ import java.io.IOException import java.util.Properties; import java.util.UUID; +import static org.apache.hadoop.hdds.scm.ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION; +import static org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager.maxLayoutVersion; + import static org.apache.hadoop.ozone.OzoneConsts.SCM_HA; + import static org.apache.hadoop.ozone.OzoneConsts.STORAGE_DIR; import static org.apache.hadoop.ozone.OzoneConsts.PRIMARY_SCM_NODE_ID; import static org.apache.hadoop.ozone.OzoneConsts.SCM_CERT_SERIAL_ID; import static org.apache.hadoop.ozone.OzoneConsts.SCM_ID; diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 03272a5,453d421..143077a --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@@ -93,20 -99,12 +100,14 @@@ import org.apache.hadoop.hdds.scm.pipel import org.apache.hadoop.hdds.scm.pipeline.PipelineReportHandler; import org.apache.hadoop.hdds.scm.pipeline.choose.algorithms.PipelineChoosePolicyFactory; import org.apache.hadoop.hdds.scm.safemode.SCMSafeModeManager; +import org.apache.hadoop.hdds.scm.server.upgrade.SCMUpgradeFinalizer; import org.apache.hadoop.hdds.security.x509.SecurityConfig; import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateServer; - import org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore; import org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCAServer; - import org.apache.hadoop.hdds.security.x509.certificate.authority.PKIProfiles.DefaultProfile; - import org.apache.hadoop.hdds.security.x509.certificate.client.SCMCertificateClient; - import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec; import org.apache.hadoop.hdds.server.ServiceRuntimeInfoImpl; import org.apache.hadoop.hdds.server.events.EventPublisher; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; - import org.apache.hadoop.hdds.utils.HAUtils; - import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.utils.HddsVersionInfo; import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource; import org.apache.hadoop.io.IOUtils; diff --cc hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config index 744e458,e55477d..a6d4482 --- a/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config +++ b/hadoop-ozone/dist/src/main/compose/upgrade/compose/non-ha/docker-config @@@ -26,13 -24,13 +26,14 @@@ OZONE-SITE.XML_ozone.scm.pipeline.owner OZONE-SITE.XML_ozone.scm.names=scm OZONE-SITE.XML_ozone.scm.datanode.id.dir=/data OZONE-SITE.XML_ozone.scm.block.client.address=scm -OZONE-SITE.XML_ozone.metadata.dirs=/data/metadata -OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon +OZONE-SITE.XML_ozone.scm.container.size=1GB OZONE-SITE.XML_ozone.scm.client.address=scm + OZONE-SITE.XML_hdds.datanode.dir=/data/hdds -OZONE-SITE.XML_ozone.recon.address=recon:9891 + +OZONE-SITE.XML_ozone.recon.db.dir=/data/metadata/recon OZONE-SITE.XML_ozone.recon.om.snapshot.task.interval.delay=1m + OZONE-SITE.XML_hdds.scmclient.max.retry.timeout=30s OZONE_CONF_DIR=/etc/hadoop OZONE_LOG_DIR=/var/log/hadoop diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 9270e1c,2da2211..dbcc173 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@@ -241,12 -240,12 +243,12 @@@ public class MiniOzoneClusterImpl imple */ @Override public void waitForPipelineTobeReady(HddsProtos.ReplicationFactor factor, - int timeoutInMs) throws - TimeoutException, InterruptedException { + int timeoutInMs) throws + TimeoutException, InterruptedException { GenericTestUtils.waitFor(() -> { int openPipelineCount = scm.getPipelineManager(). - getPipelines(HddsProtos.ReplicationType.RATIS, - factor, Pipeline.PipelineState.OPEN).size(); + getPipelines(new RatisReplicationConfig(factor), + Pipeline.PipelineState.OPEN).size(); return openPipelineCount >= 1; }, 1000, timeoutInMs); } diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java index 676598d,3f9d2de..950d3c5 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OMStorage.java @@@ -27,10 -27,6 +27,9 @@@ import org.apache.hadoop.hdds.conf.Ozon import org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeType; import org.apache.hadoop.hdds.server.ServerUtils; import org.apache.hadoop.ozone.common.Storage; +import org.apache.hadoop.ozone.om.upgrade.OMLayoutVersionManager; + - import static org.apache.hadoop.ozone.OzoneConsts.SCM_ID; +import static org.apache.hadoop.ozone.om.OmGenericConfig.ConfigStrings.OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION; /** * OMStorage is responsible for management of the StorageDirectories used by @@@ -47,19 -43,9 +46,11 @@@ public class OMStorage extends Storage * @throws IOException if any directories are inaccessible. */ public OMStorage(OzoneConfiguration conf) throws IOException { - super(NodeType.OM, getOmDbDir(conf), STORAGE_DIR); + super(NodeType.OM, getOmDbDir(conf), STORAGE_DIR, + getInitLayoutVersion(conf, OZONE_OM_INIT_DEFAULT_LAYOUT_VERSION, + OMLayoutVersionManager::maxLayoutVersion)); } - public void setScmId(String scmId) throws IOException { - if (getState() == StorageState.INITIALIZED) { - throw new IOException("OM is already initialized."); - } else { - getStorageInfo().setProperty(SCM_ID, scmId); - } - } - public void setOmCertSerialId(String certSerialId) throws IOException { getStorageInfo().setProperty(OM_CERT_SERIAL_ID, certSerialId); } diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 9fd34fe,6c18e0e..f9ee84f --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@@ -161,9 -159,7 +162,9 @@@ import org.apache.hadoop.ozone.security import org.apache.hadoop.ozone.security.acl.OzoneObj.StoreType; import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; import org.apache.hadoop.ozone.security.acl.RequestContext; +import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer; +import org.apache.hadoop.ozone.upgrade.UpgradeFinalizer.StatusAndMessages; - import org.apache.hadoop.ozone.util.ExitManager; + import org.apache.hadoop.hdds.ExitManager; import org.apache.hadoop.ozone.util.OzoneVersionInfo; import org.apache.hadoop.security.SecurityUtil; import org.apache.hadoop.security.UserGroupInformation; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java index c25bbf1,99dc3e6..d6a9550 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManagerStarter.java @@@ -155,17 -130,8 +152,16 @@@ public class OzoneManagerStarter extend @Override public boolean init(OzoneConfiguration conf) throws IOException, AuthenticationException { - checkSecurityAndSCMHAEnabled(conf); return OzoneManager.omInit(conf); } - } + @Override + public void startAndCancelPrepare(OzoneConfiguration conf) + throws IOException, AuthenticationException { + OzoneManager om = OzoneManager.createOm(conf); + om.getPrepareState().cancelPrepare(); + om.start(); + om.join(); + } + } } diff --cc hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java index 2720a30,2272448..b82364f --- a/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java +++ b/hadoop-ozone/recon/src/main/java/org/apache/hadoop/ozone/recon/scm/ReconStorageContainerManagerFacade.java @@@ -57,7 -56,7 +56,8 @@@ import org.apache.hadoop.hdds.scm.safem import org.apache.hadoop.hdds.scm.server.OzoneStorageContainerManager; import org.apache.hadoop.hdds.scm.server.SCMStorageConfig; import org.apache.hadoop.hdds.server.events.EventQueue; +import org.apache.hadoop.hdds.upgrade.HDDSLayoutVersionManager; + import org.apache.hadoop.hdds.utils.HddsServerUtil; import org.apache.hadoop.hdds.utils.db.DBStore; import org.apache.hadoop.hdds.utils.db.DBStoreBuilder; import org.apache.hadoop.io.IOUtils; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
