This is an automated email from the ASF dual-hosted git repository. pifta pushed a commit to branch HDDS-5447-httpfs in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 94d16737b2af25acd0cdeac3e8d1cd529d155d4f Merge: 121b54b aa8cfcd Author: Istvan Fajth <[email protected]> AuthorDate: Fri Sep 3 22:39:07 2021 +0200 Merge branch 'master' into HDDS-5447-httpfs .github/workflows/post-commit.yml | 130 ++-- SECURITY.md | 1 + dev-support/ci/lib/_all_libs.sh | 34 + dev-support/ci/lib/_initialization.sh | 140 ++++ dev-support/ci/lib/_script_init.sh | 42 ++ dev-support/ci/lib/_start_end.sh | 112 +++ dev-support/ci/lib/_traps.sh | 38 + dev-support/ci/lib/_verbosity.sh | 57 ++ dev-support/ci/selective_ci_checks.bats | 210 ++++++ dev-support/ci/selective_ci_checks.sh | 522 +++++++++++++ .../org/apache/hadoop/hdds/HddsConfigKeys.java | 5 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 10 + .../hadoop/hdds/client/ReplicationConfig.java | 10 +- .../hdds/client/ReplicationConfigValidator.java | 63 ++ .../apache/hadoop/hdds/recon/ReconConfigKeys.java | 10 + .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 85 +++ .../apache/hadoop/hdds/scm/client/ScmClient.java | 4 +- .../hadoop/hdds/scm/ha/SCMHAConfiguration.java | 320 -------- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 8 +- .../org/apache/hadoop/hdds/scm/net/InnerNode.java | 7 + .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 27 + .../hadoop/hdds/scm/net/NetworkTopology.java | 7 + .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 41 +- .../protocol/StorageContainerLocationProtocol.java | 11 +- .../ozone/upgrade/BasicUpgradeFinalizer.java | 18 +- .../common/src/main/resources/ozone-default.xml | 165 +++++ .../hadoop/hdds/client/TestReplicationConfig.java | 24 + .../client/TestReplicationConfigValidator.java | 73 ++ .../hdds/scm/net/TestNetworkTopologyImpl.java | 59 +- .../apache/hadoop/hdds/conf/TimeDurationUtil.java | 3 + .../container/common/helpers/ContainerUtils.java | 34 +- .../common/statemachine/DatanodeConfiguration.java | 69 ++ .../common/statemachine/DatanodeStateMachine.java | 11 +- .../commandhandler/DeleteBlocksCommandHandler.java | 276 +++++-- .../server/ratis/ContainerStateMachine.java | 19 +- .../transport/server/ratis/XceiverServerRatis.java | 18 +- .../container/keyvalue/KeyValueContainerCheck.java | 2 +- .../ozone/container/keyvalue/KeyValueHandler.java | 28 +- .../container/keyvalue/helpers/ChunkUtils.java | 18 +- .../keyvalue/helpers/KeyValueContainerUtil.java | 2 +- .../keyvalue/impl/FilePerBlockStrategy.java | 9 +- .../container/ozoneimpl/ContainerController.java | 24 +- .../replication/GrpcReplicationClient.java | 4 +- .../OnDemandContainerReplicationSource.java | 10 +- .../container/replication/ReplicationServer.java | 7 +- .../container/stream/DirectoryServerSource.java | 12 +- .../upgrade/DataNodeUpgradeFinalizer.java | 3 +- .../common/impl/TestContainerDataYaml.java | 35 +- .../TestCloseContainerCommandHandler.java | 33 +- hadoop-hdds/docs/content/concept/OzoneManager.md | 2 +- .../docs/content/concept/OzoneManager.zh.md | 2 +- .../docs/content/feature/Nonrolling-Upgrade.md | 107 +++ hadoop-hdds/docs/content/feature/OM-HA.zh.md | 4 +- hadoop-hdds/docs/content/feature/Quota.zh.md | 4 +- hadoop-hdds/docs/content/feature/Recon.zh.md | 2 +- hadoop-hdds/docs/content/feature/SCM-HA.md | 4 +- hadoop-hdds/docs/content/feature/SCM-HA.zh.md | 4 +- hadoop-hdds/docs/content/feature/_index.zh.md | 14 +- .../feature/how-to-do-a-nonrolling-upgrade.md | 54 -- hadoop-hdds/docs/content/interface/Ofs.md | 28 +- hadoop-hdds/docs/content/interface/ReconApi.md | 217 +++++- hadoop-hdds/docs/content/security/SecuringTDE.md | 54 ++ .../docs/themes/ozonedoc/layouts/index.html | 2 + .../container/common/helpers/MoveDataNodePair.java | 71 ++ .../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 6 + ...inerLocationProtocolClientSideTranslatorPB.java | 30 +- .../org/apache/hadoop/hdds/server/JsonUtils.java | 2 + .../hadoop/hdds/server/http/HttpServer2.java | 17 +- .../hadoop/hdds/server/http/TestHttpServer2.java | 54 ++ .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 2 +- .../src/main/proto/ScmAdminProtocol.proto | 4 +- .../src/main/proto/DatanodeClientProtocol.proto | 1 + .../interface-client/src/main/proto/hdds.proto | 5 + .../src/main/proto/SCMRatisProtocol.proto | 1 + .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 47 +- .../java/org/apache/hadoop/hdds/scm/ScmUtils.java | 10 + .../hadoop/hdds/scm/block/BlockManagerImpl.java | 27 +- .../hadoop/hdds/scm/block/DeletedBlockLogImpl.java | 280 ++++--- .../hdds/scm/block/DeletedBlockLogImplV2.java | 432 ----------- .../hdds/scm/block/SCMBlockDeletingService.java | 16 +- .../scm/block/ScmBlockDeletingServiceMetrics.java | 186 +++++ .../scm/command/CommandStatusReportHandler.java | 6 +- .../hdds/scm/container/ContainerManagerImpl.java | 67 +- .../hdds/scm/container/ContainerReplicaCount.java | 1 + .../hdds/scm/container/ContainerStateManager.java | 10 +- .../hdds/scm/container/ReplicationManager.java | 578 +++++++++++---- .../scm/container/balancer/ContainerBalancer.java | 635 ++++++++++++---- .../balancer/ContainerBalancerConfiguration.java | 206 +++++- .../balancer/ContainerBalancerMetrics.java | 13 +- .../ContainerBalancerSelectionCriteria.java | 169 +++++ .../container/balancer/ContainerMoveSelection.java | 55 ++ .../scm/container/balancer/FindTargetGreedy.java | 135 ++++ .../scm/container/balancer/FindTargetStrategy.java | 71 ++ .../algorithms/SCMContainerPlacementRackAware.java | 26 +- .../replication/ReplicationManagerMetrics.java | 189 +++++ .../scm/container/states/ContainerQueryKey.java | 33 +- .../scm/container/states/ContainerStateMap.java | 41 +- .../hadoop/hdds/scm/ha/InterSCMGrpcClient.java | 7 +- .../org/apache/hadoop/hdds/scm/ha/RatisUtil.java | 130 ++-- .../hdds/scm/ha/SCMHADBTransactionBufferImpl.java | 6 +- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 + .../hadoop/hdds/scm/ha/SCMRatisServerImpl.java | 26 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 6 +- .../hdds/scm/metadata/MoveDataNodePairCodec.java | 53 ++ .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +- .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 + .../hdds/scm/node/DatanodeAdminMonitorImpl.java | 18 + .../hadoop/hdds/scm/node/DeadNodeHandler.java | 13 + .../hdds/scm/node/HealthyReadOnlyNodeHandler.java | 12 + .../apache/hadoop/hdds/scm/node/NodeManager.java | 9 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 27 + .../hadoop/hdds/scm/pipeline/PipelineManager.java | 20 + ...ManagerV2Impl.java => PipelineManagerImpl.java} | 73 +- .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 8 +- .../hdds/scm/pipeline/PipelineReportHandler.java | 2 + .../hdds/scm/pipeline/SCMPipelineManager.java | 801 -------------------- .../pipeline/WritableRatisContainerProvider.java | 99 ++- ...inerLocationProtocolServerSideTranslatorPB.java | 40 +- .../hdds/scm/server/SCMClientProtocolServer.java | 56 +- .../hdds/scm/server/StorageContainerManager.java | 34 +- .../scm/server/upgrade/SCMUpgradeFinalizer.java | 5 +- .../ScmHAUnfinalizedStateValidationAction.java | 13 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 6 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 19 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 67 +- .../container/TestCloseContainerEventHandler.java | 6 +- .../hdds/scm/container/TestReplicationManager.java | 474 ++++++++++-- .../scm/container/TestSCMContainerManager.java | 6 +- .../container/balancer/TestContainerBalancer.java | 466 ++++++++++-- .../TestSCMContainerPlacementRackAware.java | 27 + .../TestSCMContainerPlacementRandom.java | 68 ++ .../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 25 - .../hdds/scm/node/TestContainerPlacement.java | 4 +- .../hdds/scm/node/TestDatanodeAdminMonitor.java | 1 + .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 36 +- .../hdds/scm/pipeline/MockPipelineManager.java | 20 + .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 40 +- .../hdds/scm/pipeline/TestSCMPipelineManager.java | 821 --------------------- ...TestSCMStoreImplWithOldPipelineIDKeyFormat.java | 6 + .../safemode/TestHealthyPipelineSafeModeRule.java | 14 +- .../TestOneReplicaPipelineSafeModeRule.java | 6 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 20 +- .../server/TestSCMUpdateServiceGrpcServer.java | 3 + .../ozone/container/common/TestEndPoint.java | 3 +- .../org/apache/hadoop/hdds/cli/OzoneAdmin.java | 7 + .../hdds/scm/cli/ContainerBalancerCommands.java | 19 +- .../scm/cli/ContainerBalancerStartSubcommand.java | 16 +- .../hdds/scm/cli/ContainerOperationClient.java | 26 +- .../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 45 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 41 +- .../hadoop/ozone/om/helpers/BucketLayout.java | 62 ++ .../hadoop/ozone/om/helpers/OmBucketInfo.java | 56 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 37 +- .../ozone/om/helpers/OmKeyLocationInfoGroup.java | 5 +- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 25 +- .../om/helpers/TestOmKeyLocationInfoGroup.java | 19 + hadoop-ozone/dev-support/docker/Dockerfile | 38 +- hadoop-ozone/dist/src/main/compose/ozone/README.md | 6 + .../dist/src/main/compose/ozone/docker-config | 2 + hadoop-ozone/dist/src/main/compose/ozone/run.sh | 4 + .../src/main/compose/ozonesecure-ha/docker-config | 3 + .../src/main/compose/ozonesecure/docker-config | 5 +- hadoop-ozone/dist/src/main/compose/test-all.sh | 3 +- hadoop-ozone/dist/src/main/compose/testlib.sh | 7 +- hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 4 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 46 +- .../dist/src/main/license/update-jar-report.sh | 2 +- .../dist/src/main/smoketest/basic/links.robot | 10 +- .../dist/src/main/smoketest/ozonefs/setup.robot | 2 +- .../dist/src/main/smoketest/s3/boto3.robot | 5 +- .../dist/src/main/smoketest/s3/boto_client.py | 10 +- .../dist/src/main/smoketest/s3/bucketcreate.robot | 3 +- .../dist/src/main/smoketest/s3/bucketdelete.robot | 3 +- .../dist/src/main/smoketest/s3/buckethead.robot | 3 +- .../dist/src/main/smoketest/s3/commonawslib.robot | 10 +- .../src/main/smoketest/s3/s3_compatbility_check.sh | 39 + hadoop-ozone/integration-test/pom.xml | 5 + .../hadoop/fs/ozone/TestOzoneFileSystem.java | 12 +- .../fs/ozone/TestOzoneFileSystemWithFSO.java | 3 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 39 +- .../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 21 + .../ozone/contract/ITestOzoneContractDistCp.java | 8 + .../rooted/ITestRootedOzoneContractDistCp.java | 8 + .../hadoop/hdds/scm/TestSCMInstallSnapshot.java | 10 +- .../apache/hadoop/hdds/scm/TestSCMSnapshot.java | 6 +- .../TestContainerStateManagerIntegration.java | 40 +- .../hdds/scm/pipeline/TestMultiRaftSetup.java | 173 +++++ .../hdds/scm/pipeline/TestPipelineClose.java | 2 + .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 24 +- .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 7 +- .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 +- .../hadoop/ozone/MiniOzoneClusterProvider.java | 283 +++++++ .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 21 +- .../ozone/TestContainerBalancerOperations.java | 11 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 40 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 11 +- .../hadoop/ozone/TestStorageContainerManager.java | 10 +- .../client/rpc/TestOzoneAtRestEncryption.java | 130 +++- .../ozone/container/TestContainerReplication.java | 48 -- .../commandhandler/TestBlockDeletion.java | 125 +++- .../server/TestSecureContainerServer.java | 6 + .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 3 - .../hadoop/ozone/om/TestOMRatisSnapshots.java | 13 +- .../hadoop/ozone/om/TestOMStartupWithLayout.java | 197 ----- .../hadoop/ozone/om/TestObjectStoreWithFSO.java | 76 +- .../hadoop/ozone/om/TestOmBlockVersioning.java | 21 +- .../hadoop/ozone/om/TestOzoneManagerHAWithACL.java | 24 +- .../ozone/om/TestOzoneManagerHAWithData.java | 32 +- .../ozone/recon/TestReconWithOzoneManagerFSO.java | 5 +- .../hadoop/ozone/scm/TestCloseContainer.java | 8 + .../hadoop/ozone/scm/TestFailoverWithSCMHA.java | 99 ++- .../ozone/scm/TestSCMInstallSnapshotWithHA.java | 19 +- .../scm/node/TestDecommissionAndMaintenance.java | 42 +- .../hadoop/ozone/shell/TestNSSummaryAdmin.java | 72 ++ .../hadoop/ozone/shell/TestOzoneShellHA.java | 9 +- .../src/main/proto/OmClientProtocol.proto | 7 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 31 + .../hadoop/ozone/om/TrashOzoneFileSystem.java | 4 +- .../apache/hadoop/ozone/om/TrashPolicyOzone.java | 12 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +- .../om/request/bucket/OMBucketCreateRequest.java | 102 +-- .../ozone/om/request/file/OMFileCreateRequest.java | 3 +- .../request/file/OMFileCreateRequestWithFSO.java | 3 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 3 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 3 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 24 +- .../request/s3/security/S3RevokeSecretRequest.java | 2 +- .../ozone/om/upgrade/OMUpgradeFinalizer.java | 3 +- .../om/request/key/TestOMKeyCreateRequest.java | 70 +- .../request/key/TestOMKeyCreateRequestWithFSO.java | 13 + .../fs/ozone/BasicOzoneClientAdapterImpl.java | 4 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 2 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 4 +- .../org/apache/hadoop/fs/ozone/OzoneFsShell.java | 4 +- .../apache/hadoop/ozone/recon/ReconHttpServer.java | 6 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 4 - .../hadoop/ozone/recon/api/NSSummaryEndpoint.java | 379 ++++++++-- .../hadoop/ozone/recon/api/types/DUResponse.java | 70 ++ .../recon/api/types/NamespaceSummaryResponse.java | 19 +- .../ozone/recon/api/types/ResponseStatus.java | 1 + .../ozone/recon/scm/ReconPipelineManager.java | 16 +- .../recon/scm/ReconPipelineReportHandler.java | 22 +- .../webapps/recon/ozone-recon-web/api/db.json | 324 +++++++- .../webapps/recon/ozone-recon-web/api/routes.json | 22 +- .../src/components/navBar/navBar.tsx | 5 + .../src/components/rightDrawer/rightDrawer.tsx | 72 ++ .../src/constants/breadcrumbs.constants.tsx | 3 +- .../webapps/recon/ozone-recon-web/src/routes.tsx | 5 + .../diskUsage/diskUsage.less} | 41 +- .../src/views/diskUsage/diskUsage.tsx | 444 +++++++++++ .../src/views/overview/overview.tsx | 3 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 23 + .../ozone/recon/api/TestNSSummaryEndpoint.java | 168 ++++- .../java/org/apache/hadoop/ozone/s3/Gateway.java | 29 + .../hadoop/ozone/s3/S3GatewayConfigKeys.java | 7 + .../ozone/admin/nssummary/DiskUsageSubCommand.java | 214 ++++++ .../admin/nssummary/FileSizeDistSubCommand.java | 129 ++++ .../ozone/admin/nssummary/NSSummaryAdmin.java | 137 ++++ .../ozone/admin/nssummary/NSSummaryCLIUtils.java | 174 +++++ .../admin/nssummary/QuotaUsageSubCommand.java | 117 +++ .../ozone/admin/nssummary/SummarySubCommand.java | 115 +++ .../ozone/admin/nssummary/package-info.java} | 14 +- .../ozone/genesis/BenchMarkContainerStateMap.java | 6 +- .../ozone/shell/bucket/CreateBucketHandler.java | 20 +- .../ozone/shell/bucket/InfoBucketHandler.java | 52 +- .../hadoop/ozone/shell/keys/CopyKeyHandler.java | 4 +- .../hadoop/ozone/shell/keys/PutKeyHandler.java | 4 +- .../ozone/shell/volume/CreateVolumeHandler.java | 4 +- pom.xml | 8 +- 270 files changed, 11100 insertions(+), 4512 deletions(-) diff --cc hadoop-ozone/dist/src/main/license/jar-report.txt index 4bf842a,ec949fd..a722de2 --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@@ -125,23 -121,21 +125,23 @@@ share/ozone/lib/jersey-media-json-jacks share/ozone/lib/jersey-server.jar share/ozone/lib/jersey-server.jar share/ozone/lib/jersey-servlet.jar - share/ozone/lib/jetty-http.v20201120.jar - share/ozone/lib/jetty-io.v20201120.jar - share/ozone/lib/jetty-security.v20201120.jar - share/ozone/lib/jetty-server.v20201120.jar - share/ozone/lib/jetty-servlet.v20201120.jar - share/ozone/lib/jetty-util-ajax.v20201120.jar - share/ozone/lib/jetty-util.v20201120.jar - share/ozone/lib/jetty-webapp.v20201120.jar - share/ozone/lib/jetty-xml.v20201120.jar + share/ozone/lib/jetty-http.jar + share/ozone/lib/jetty-io.jar + share/ozone/lib/jetty-security.jar + share/ozone/lib/jetty-server.jar + share/ozone/lib/jetty-servlet.jar + share/ozone/lib/jetty-util-ajax.jar + share/ozone/lib/jetty-util.jar + share/ozone/lib/jetty-webapp.jar + share/ozone/lib/jetty-xml.jar +share/ozone/lib/jline.jar share/ozone/lib/jmespath-java.jar share/ozone/lib/joda-time.jar share/ozone/lib/jooq-codegen.jar - share/ozone/lib/jooq-meta.jar share/ozone/lib/jooq.jar + share/ozone/lib/jooq-meta.jar share/ozone/lib/jsch.jar +share/ozone/lib/json-simple.jar share/ozone/lib/json-smart.jar share/ozone/lib/jsp-api.jar share/ozone/lib/jsr305.jar @@@ -171,18 -164,17 +171,18 @@@ share/ozone/lib/log4j.ja share/ozone/lib/metainf-services.jar share/ozone/lib/metrics-core.jar share/ozone/lib/netty-buffer.Final.jar - share/ozone/lib/netty-codec-http.Final.jar + share/ozone/lib/netty-codec.Final.jar share/ozone/lib/netty-codec-http2.Final.jar + share/ozone/lib/netty-codec-http.Final.jar share/ozone/lib/netty-codec-socks.Final.jar - share/ozone/lib/netty-codec.Final.jar share/ozone/lib/netty-common.Final.jar - share/ozone/lib/netty-handler-proxy.Final.jar share/ozone/lib/netty-handler.Final.jar + share/ozone/lib/netty-handler-proxy.Final.jar share/ozone/lib/netty-resolver.Final.jar + share/ozone/lib/netty-transport.Final.jar share/ozone/lib/netty-transport-native-epoll.Final.jar share/ozone/lib/netty-transport-native-unix-common.Final.jar - share/ozone/lib/netty-transport.Final.jar +share/ozone/lib/netty.Final.jar share/ozone/lib/nimbus-jose-jwt.jar share/ozone/lib/okhttp.jar share/ozone/lib/okio.jar --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
