This is an automated email from the ASF dual-hosted git repository. nanda pushed a commit to branch HDDS-2823 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 82c30a4d26be4a0be4c3e3e071f4add41c9c324e Merge: 3e7c427 d522884 Author: Nandakumar <[email protected]> AuthorDate: Sat Jun 27 20:14:12 2020 +0530 Merge branch 'master' into HDDS-2823 .github/workflows/post-commit.yml | 183 ++--- .github/workflows/pr.yml | 270 ------ .../apache/hadoop/hdds/conf/RatisClientConfig.java | 86 -- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 21 +- .../hadoop/hdds/scm/client/HddsClientUtils.java | 4 +- .../java/org/apache/hadoop/hdds/StringUtils.java | 5 + .../hadoop/hdds/conf/OzoneConfiguration.java | 6 + .../hdds/ratis/ContainerCommandRequestMessage.java | 6 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 214 +++-- .../hadoop/hdds/ratis/conf/RatisClientConfig.java | 183 +++++ .../hadoop/hdds/ratis}/conf/package-info.java | 2 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +- .../apache/hadoop/hdds/scm/client/ScmClient.java | 11 + .../org/apache/hadoop/hdds/scm/net/NetUtils.java | 12 +- .../protocol/StorageContainerLocationProtocol.java | 4 + .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 14 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 2 + .../ozone/common/IncrementalChunkBuffer.java | 12 +- .../common/src/main/resources/ozone-default.xml | 21 +- .../apache/hadoop/hdds/scm/net/TestNetUtils.java | 1 - .../server/ratis/ContainerStateMachine.java | 14 +- .../transport/server/ratis/XceiverServerRatis.java | 8 +- .../container/keyvalue/KeyValueBlockIterator.java | 3 +- .../ozone/container/keyvalue/KeyValueHandler.java | 2 +- .../keyvalue/impl/ChunkManagerDispatcher.java | 5 +- .../keyvalue/impl/ChunkManagerFactory.java | 13 +- .../keyvalue/impl/FilePerChunkStrategy.java | 45 +- .../common/impl/TestContainerPersistence.java | 7 +- .../container/keyvalue/ChunkLayoutTestInfo.java | 12 +- .../keyvalue/TestKeyValueBlockIterator.java | 3 +- .../keyvalue/TestKeyValueContainerCheck.java | 2 +- .../keyvalue/impl/AbstractTestChunkManager.java | 8 +- .../keyvalue/impl/CommonChunkManagerTestCases.java | 7 + hadoop-hdds/docs/content/beyond/RunningWithHDFS.md | 70 -- .../docs/content/beyond/RunningWithHDFS.zh.md | 64 -- hadoop-hdds/docs/content/interface/CSI.md | 84 ++ hadoop-hdds/docs/content/interface/OzoneFS.md | 53 +- hadoop-hdds/docs/content/interface/OzoneFS.zh.md | 37 +- .../docs/content/interface/pvtest-webui.png | Bin 0 -> 13830 bytes hadoop-hdds/docs/content/recipe/SparkOzoneFSK8S.md | 30 +- .../docs/content/recipe/SparkOzoneFSK8S.zh.md | 36 +- hadoop-hdds/docs/content/security/SecureOzone.md | 20 +- .../docs/content/security/SecureOzone.zh.md | 158 ++++ .../docs/content/security/SecuringOzoneHTTP.md | 141 ++++ hadoop-hdds/docs/content/tools/Admin.md | 2 +- hadoop-hdds/docs/content/tools/_index.zh.md | 62 ++ ...inerLocationProtocolClientSideTranslatorPB.java | 24 + .../hadoop/hdds/server/http/BaseHttpServer.java | 4 +- .../hadoop/hdds/server/http/HttpServer2.java | 22 +- .../org/apache/hadoop/hdds/utils/db/RDBTable.java | 11 +- .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +- .../src/main/proto/ScmAdminProtocol.proto | 17 +- .../interface-admin/src/main/proto/proto.lock | 48 ++ hadoop-hdds/interface-client/pom.xml | 6 +- .../ContainerPlacementPolicyFactory.java | 2 +- ...StoreRDBImpl.java => SCMMetadataStoreImpl.java} | 6 +- ...inerLocationProtocolServerSideTranslatorPB.java | 28 +- .../hdds/scm/safemode/ContainerSafeModeRule.java | 8 +- .../hdds/scm/safemode/DataNodeSafeModeRule.java | 8 +- .../scm/safemode/HealthyPipelineSafeModeRule.java | 9 +- .../safemode/OneReplicaPipelineSafeModeRule.java | 9 +- .../hdds/scm/safemode/SCMSafeModeManager.java | 15 + .../hadoop/hdds/scm/safemode/SafeModeExitRule.java | 5 + .../hdds/scm/server/SCMClientProtocolServer.java | 7 + .../hdds/scm/server/StorageContainerManager.java | 14 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 4 +- .../container/TestCloseContainerEventHandler.java | 28 +- .../scm/container/TestSCMContainerManager.java | 13 +- .../hdds/scm/node/TestContainerPlacement.java | 20 +- .../hdds/scm/pipeline/TestSCMPipelineManager.java | 36 +- .../safemode/TestHealthyPipelineSafeModeRule.java | 121 ++- .../TestOneReplicaPipelineSafeModeRule.java | 11 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 25 +- hadoop-hdds/test-utils/pom.xml | 6 + .../java/org/apache/hadoop/test/JacocoServer.java | 114 +++ .../hdds/scm/cli/ContainerOperationClient.java | 7 + .../hdds/scm/cli/SafeModeCheckSubcommand.java | 15 + .../apache/hadoop/ozone/client/OzoneBucket.java | 70 +- .../apache/hadoop/ozone/client/OzoneVolume.java | 69 +- .../ozone/client/protocol/ClientProtocol.java | 11 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 23 + .../common/dev-support/findbugsExcludeFile.xml | 6 - hadoop-ozone/common/pom.xml | 29 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 1 + .../hadoop/ozone/om/helpers/OmBucketInfo.java | 33 +- .../hadoop/ozone/om/helpers/OmVolumeArgs.java | 33 +- .../ozone/om/protocol/OzoneManagerProtocol.java | 10 + .../ozone/om/protocolPB/Hadoop3OmTransport.java | 58 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 29 + .../org/apache/hadoop/ozone/csi/CsiServer.java | 16 +- .../org/apache/hadoop/ozone/csi/NodeService.java | 17 +- hadoop-ozone/dev-support/checks/acceptance.sh | 5 +- hadoop-ozone/dev-support/checks/build.sh | 2 +- .../dev-support/checks/coverage.sh | 16 +- hadoop-ozone/dist/pom.xml | 53 ++ hadoop-ozone/dist/src/main/compose/ozone-csi/.env | 1 + .../src/main/compose/ozone-csi/docker-compose.yaml | 6 + hadoop-ozone/dist/src/main/compose/ozone-hdfs/.env | 18 - .../main/compose/ozone-hdfs/docker-compose.yaml | 70 -- .../dist/src/main/compose/ozone-hdfs/docker-config | 35 - .../dist/src/main/compose/ozone-mr/hadoop27/.env | 2 +- .../main/compose/ozone-mr/hadoop27/docker-config | 2 +- .../src/main/compose/ozone-mr/hadoop27/test.sh | 7 +- .../dist/src/main/compose/ozone-mr/hadoop31/.env | 1 + .../compose/ozone-mr/hadoop31/docker-compose.yaml | 6 + .../dist/src/main/compose/ozone-mr/hadoop32/.env | 1 + .../compose/ozone-mr/hadoop32/docker-compose.yaml | 4 + .../dist/src/main/compose/ozone-om-ha-s3/.env | 1 + .../compose/ozone-om-ha-s3/docker-compose.yaml | 8 + .../dist/src/main/compose/ozone-topology/.env | 1 + .../compose/ozone-topology/docker-compose.yaml | 14 + .../src/main/compose/ozone-topology/docker-config | 2 +- hadoop-ozone/dist/src/main/compose/ozone/.env | 1 + .../src/main/compose/ozone/docker-compose.yaml | 5 + .../dist/src/main/compose/ozone/docker-config | 2 + hadoop-ozone/dist/src/main/compose/ozone/test.sh | 5 +- .../src/main/compose/ozonescripts/docker-config | 9 +- .../dist/src/main/compose/ozonesecure-mr/.env | 1 + .../compose/ozonesecure-mr/docker-compose.yaml | 4 + .../dist/src/main/compose/ozonesecure-om-ha/.env | 1 + .../compose/ozonesecure-om-ha/docker-compose.yaml | 13 +- .../dist/src/main/compose/ozonesecure/.env | 1 + .../main/compose/ozonesecure/docker-compose.yaml | 5 + .../src/main/compose/ozonesecure/docker-config | 1 + .../dist/src/main/compose/ozonesecure/test.sh | 9 +- hadoop-ozone/dist/src/main/compose/test-all.sh | 18 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 15 +- .../k8s/definitions/ozone-csi/csi-controller.yaml | 2 +- .../main/k8s/definitions/ozone-csi/csi-node.yaml | 4 +- .../k8s/definitions/prometheus/deployment.yaml | 2 +- .../examples/ozone-dev/csi/csi-node-daemonset.yaml | 4 +- .../ozone-dev/csi/csi-provisioner-deployment.yaml | 2 +- .../examples/ozone-dev/prometheus-deployment.yaml | 2 +- .../k8s/examples/ozone/csi/csi-node-daemonset.yaml | 4 +- .../ozone/csi/csi-provisioner-deployment.yaml | 2 +- .../dist/src/main/smoketest/commonlib.robot | 6 +- hadoop-ozone/dist/src/main/smoketest/lib/fs.robot | 50 ++ .../dist/src/main/smoketest/lib/fs_tests.robot | 63 ++ .../src/main/smoketest/lib/string.robot} | 28 +- .../dist/src/main/smoketest/lib/string_tests.robot | 46 ++ .../dist/src/main/smoketest/ozonefs/ozonefs.robot | 217 ++--- .../dist/src/main/smoketest/ozonefs/setup.robot | 60 ++ hadoop-ozone/dist/src/shell/ozone/ozone | 4 +- .../ozone/insight/ConfigurationSubCommand.java | 31 +- .../ozone/insight/TestConfigurationSubCommand.java | 100 +++ .../hadoop/fs/ozone/TestOzoneFileSystem.java | 42 + .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 874 ++++++++++++++++++++ .../rooted/ITestRootedOzoneContractCreate.java | 49 ++ .../rooted/ITestRootedOzoneContractDelete.java | 49 ++ .../rooted/ITestRootedOzoneContractDistCp.java | 51 ++ .../ITestRootedOzoneContractGetFileStatus.java | 55 ++ .../rooted/ITestRootedOzoneContractMkdir.java | 49 ++ .../rooted/ITestRootedOzoneContractOpen.java | 48 ++ .../rooted/ITestRootedOzoneContractRename.java | 50 ++ .../rooted/ITestRootedOzoneContractRootDir.java | 61 ++ .../rooted/ITestRootedOzoneContractSeek.java | 48 ++ .../ozone/contract/rooted/RootedOzoneContract.java | 124 +++ .../ozone/client/rpc/Test2WayCommitInRatis.java | 6 - .../rpc/TestBlockOutputStreamWithFailures.java | 5 +- ...estBlockOutputStreamWithFailuresFlushDelay.java | 5 +- .../hadoop/ozone/client/rpc/TestCommitWatcher.java | 8 +- .../TestContainerStateMachineFailureOnRead.java | 6 +- .../rpc/TestContainerStateMachineFailures.java | 422 +++++----- .../client/rpc/TestDeleteWithSlowFollower.java | 8 +- .../client/rpc/TestFailureHandlingByClient.java | 8 +- .../rpc/TestFailureHandlingByClientFlushDelay.java | 8 +- .../ozone/client/rpc/TestKeyInputStream.java | 82 +- .../rpc/TestMultiBlockWritesWithDnFailures.java | 8 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 17 +- .../client/rpc/TestOzoneRpcClientWithRatis.java | 4 +- .../client/rpc/TestValidateBCSIDOnRestart.java | 258 ++++++ .../ozone/client/rpc/TestWatchForCommit.java | 170 ++-- .../apache/hadoop/ozone/container/TestHelper.java | 41 +- .../transport/server/ratis/TestCSMMetrics.java | 2 - .../hadoop/ozone/om/TestOMRatisSnapshots.java | 7 +- .../ozone/om/TestOzoneManagerHAWithData.java | 41 + .../ozone/om/TestOzoneManagerListVolumes.java | 62 +- .../ozone/om/TestOzoneManagerRestInterface.java | 25 - .../hadoop/ozone/scm/TestXceiverClientGrpc.java | 198 +++++ .../hadoop/ozone/shell/TestOzoneShellHA.java | 89 ++ .../src/test/resources/log4j.properties | 3 + .../dev-support/findbugsExcludeFile.xml | 1 - hadoop-ozone/interface-client/pom.xml | 77 ++ .../src/main/proto/OmClientProtocol.proto} | 15 + .../src/main/proto/Security.proto | 0 .../src/main/proto/proto.lock | 59 +- hadoop-ozone/ozone-manager/pom.xml | 7 + .../apache/hadoop/ozone/om/OMMetadataManager.java | 20 + .../hadoop/ozone/om/OmMetadataManagerImpl.java | 23 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 99 ++- .../hadoop/ozone/om/codec/OMDBDefinition.java | 161 ++++ .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 96 ++- .../om/ratis/utils/OzoneManagerRatisUtils.java | 3 + .../hadoop/ozone/om/request/OMClientRequest.java | 34 + .../om/request/bucket/OMBucketCreateRequest.java | 6 +- .../ozone/om/request/key/OMKeysDeleteRequest.java | 214 +++++ .../om/request/volume/OMVolumeCreateRequest.java | 8 +- .../om/request/volume/OMVolumeSetOwnerRequest.java | 22 +- .../om/request/volume/OMVolumeSetQuotaRequest.java | 17 + .../ozone/om/response/CleanupTableInfo.java} | 40 +- .../om/response/bucket/OMBucketCreateResponse.java | 4 + .../om/response/bucket/OMBucketDeleteResponse.java | 4 + .../bucket/OMBucketSetPropertyResponse.java | 4 + .../response/bucket/acl/OMBucketAclResponse.java | 5 +- .../response/file/OMDirectoryCreateResponse.java | 5 + .../om/response/key/OMAllocateBlockResponse.java | 4 + .../ozone/om/response/key/OMKeyCommitResponse.java | 6 + .../ozone/om/response/key/OMKeyCreateResponse.java | 4 + .../ozone/om/response/key/OMKeyDeleteResponse.java | 6 + .../ozone/om/response/key/OMKeyPurgeResponse.java | 6 +- .../ozone/om/response/key/OMKeyRenameResponse.java | 4 + .../om/response/key/OMKeysDeleteResponse.java | 133 +++ .../om/response/key/OMTrashRecoverResponse.java | 5 + .../om/response/key/acl/OMKeyAclResponse.java | 4 + .../key/acl/prefix/OMPrefixAclResponse.java | 4 + .../S3InitiateMultipartUploadResponse.java | 6 +- .../multipart/S3MultipartUploadAbortResponse.java | 7 + .../S3MultipartUploadCommitPartResponse.java | 7 + .../S3MultipartUploadCompleteResponse.java | 9 + .../response/s3/security/S3GetSecretResponse.java | 5 +- .../security/OMCancelDelegationTokenResponse.java | 5 +- .../security/OMGetDelegationTokenResponse.java | 6 +- .../security/OMRenewDelegationTokenResponse.java | 6 +- .../om/response/volume/OMVolumeAclOpResponse.java | 4 + .../om/response/volume/OMVolumeCreateResponse.java | 4 + .../om/response/volume/OMVolumeDeleteResponse.java | 4 + .../response/volume/OMVolumeSetOwnerResponse.java | 5 +- .../response/volume/OMVolumeSetQuotaResponse.java | 5 + ...tOzoneManagerDoubleBufferWithDummyResponse.java | 4 +- .../ozone/om/request/TestOMRequestUtils.java | 4 +- .../request/bucket/TestOMBucketCreateRequest.java | 2 + .../request/volume/TestOMVolumeCreateRequest.java | 7 + .../volume/TestOMVolumeSetOwnerRequest.java | 7 + .../volume/TestOMVolumeSetQuotaRequest.java | 7 + .../ozone/om/response/TestCleanupTableInfo.java | 69 ++ .../fs/ozone/BasicOzoneClientAdapterImpl.java | 24 + .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 63 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 912 +++++++++++++++++++++ ...System.java => BasicRootedOzoneFileSystem.java} | 320 +++++--- .../java/org/apache/hadoop/fs/ozone/OFSPath.java | 311 +++++++ .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 5 + .../fs/ozone/RootedOzoneClientAdapterImpl.java | 61 ++ .../org/apache/hadoop/fs/ozone/TestOFSPath.java | 154 ++++ hadoop-ozone/ozonefs-hadoop2/pom.xml | 12 + .../main/java/org/apache/hadoop/fs/ozone/OzFs.java | 5 + .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 15 +- hadoop-ozone/ozonefs-hadoop3/pom.xml | 17 +- .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 95 +++ .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 95 +++ hadoop-ozone/pom.xml | 6 + .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 6 + .../hadoop/ozone/s3/exception/S3ErrorTable.java | 6 + .../hadoop/ozone/client/ObjectStoreStub.java | 6 +- .../hadoop/ozone/client/OzoneVolumeStub.java | 10 +- .../hadoop/ozone/debug/DBDefinitionFactory.java | 49 ++ .../org/apache/hadoop/ozone/debug/DBScanner.java | 11 +- .../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 3 +- .../apache/hadoop/ozone/freon/S3KeyGenerator.java | 72 +- .../ozone/genesis/BenchmarkChunkManager.java | 2 +- .../apache/hadoop/ozone/genesis/GenesisUtil.java | 13 +- .../ozone/shell/volume/UpdateVolumeHandler.java | 4 +- pom.xml | 8 +- 262 files changed, 8358 insertions(+), 2158 deletions(-) diff --cc hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java index 69a8013,73113d1..20d0480 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/server/StorageContainerManager.java @@@ -1152,53 -1112,12 +1153,62 @@@ public final class StorageContainerMana return this.clusterMap; } + private static SCMRatisServer initializeRatisServer( + OzoneConfiguration conf, StorageContainerManager scm) throws IOException { + SCMNodeDetails scmNodeDetails = SCMNodeDetails + .initStandAlone(conf); + //TODO enable Ratis group + SCMRatisServer scmRatisServer = SCMRatisServer.newSCMRatisServer( + conf.getObject(SCMRatisServer.SCMRatisServerConfiguration.class), + scm, scmNodeDetails, Collections.EMPTY_LIST, + SCMRatisServer.getSCMRatisDirectory(conf)); + if (scmRatisServer != null) { + LOG.info("SCM Ratis server initialized at port {}", + scmRatisServer.getServerPort()); + } // TODO error handling for scmRatisServer creation failure + return scmRatisServer; + } + + @VisibleForTesting + public SCMRatisServer getScmRatisServer() { + return scmRatisServer; + } + + public void setScmRatisServer(SCMRatisServer scmRatisServer) { + this.scmRatisServer = scmRatisServer; + } + + @VisibleForTesting + public SCMRatisSnapshotInfo getSnapshotInfo() { + return scmRatisSnapshotInfo; + } + + @VisibleForTesting + public long getRatisSnapshotIndex() { + return scmRatisSnapshotInfo.getIndex(); + } + + /** + * Save ratis snapshot to SCM meta store and local disk. + */ + public TermIndex saveRatisSnapshot() throws IOException { + TermIndex snapshotIndex = scmRatisServer.getLastAppliedTermIndex(); + if (scmMetadataStore != null) { + // Flush the SCM state to disk + scmMetadataStore.getStore().flush(); + } + + scmRatisSnapshotInfo.saveRatisSnapshotToDisk(snapshotIndex); + + return snapshotIndex; + } ++ + /** + * Get the safe mode status of all rules. + * + * @return map of rule statuses. + */ + public Map<String, Pair<Boolean, String>> getRuleStatus() { + return scmSafeModeManager.getRuleStatus(); + } } diff --cc hadoop-ozone/interface-client/src/main/proto/proto.lock index 01eedd0,7e41dfa..91ec7f0 --- a/hadoop-ozone/interface-client/src/main/proto/proto.lock +++ b/hadoop-ozone/interface-client/src/main/proto/proto.lock @@@ -1215,6 -1229,6 +1229,11 @@@ "id": 9, "name": "updateID", "type": "uint64" ++ }, ++ { ++ "id": 10, ++ "name": "modificationTime", ++ "type": "uint64" } ] }, @@@ -1319,6 -1333,6 +1338,11 @@@ "id": 3, "name": "quotaInBytes", "type": "uint64" ++ }, ++ { ++ "id": 4, ++ "name": "modificationTime", ++ "type": "uint64" } ] }, @@@ -1490,6 -1504,6 +1514,11 @@@ "id": 10, "name": "updateID", "type": "uint64" ++ }, ++ { ++ "id": 11, ++ "name": "modificationTime", ++ "type": "uint64" } ] }, --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
