This is an automated email from the ASF dual-hosted git repository. pifta pushed a commit to branch HDDS-5447-httpfs in repository https://gitbox.apache.org/repos/asf/ozone.git
commit c7f230f0506cddbfab15988f1b08cad41d30aa53 Merge: 08b028967d 21717321c6 Author: Istvan Fajth <[email protected]> AuthorDate: Wed Jan 4 17:02:47 2023 +0100 Merge branch 'master' into httpfs-merge .github/workflows/ci.yml | 2 +- .github/workflows/post-commit.yml | 1 + .github/workflows/scheduled_ci.yml | 1 + LICENSE.txt | 10 +- NOTICE.txt | 2 +- hadoop-hdds/annotations/pom.xml | 4 +- hadoop-hdds/client/pom.xml | 4 +- .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 74 + .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 2 +- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 7 +- .../hdds/scm/storage/AbstractDataStreamOutput.java | 131 + .../hdds/scm/storage/BlockDataStreamOutput.java | 733 ++++++ .../hdds/scm/storage/BlockExtendedInputStream.java | 11 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 5 - .../hdds/scm/storage/ByteBufferStreamOutput.java | 57 + .../hdds/scm/storage/MultipartInputStream.java | 161 +- ...tendedInputStream.java => PartInputStream.java} | 22 +- .../hadoop/hdds/scm/storage/StreamBuffer.java | 59 + .../hdds/scm/storage/StreamCommitWatcher.java | 208 ++ .../hadoop/ozone/client/io/ECBlockInputStream.java | 5 - .../ozone/client/io/ECBlockInputStreamProxy.java | 2 +- .../client/io/ECBlockReconstructedInputStream.java | 2 +- .../hadoop/ozone/client/io/ECStreamTestUtil.java | 5 - .../ozone/client/io/TestECBlockInputStream.java | 5 - hadoop-hdds/common/pom.xml | 6 +- .../org/apache/hadoop/hdds/HddsConfigKeys.java | 20 + .../hadoop/hdds/client/ReplicationConfig.java | 11 + .../hadoop/hdds/protocol/DatanodeDetails.java | 9 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 92 +- .../apache/hadoop/hdds/scm/PlacementPolicy.java | 18 +- .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 2 + .../scm/container/common/helpers/ExcludeList.java | 4 +- .../protocol/StorageContainerLocationProtocol.java | 7 + .../hdds/scm/storage/ContainerProtocolCalls.java | 14 +- .../hadoop/hdds/security/x509/SecurityConfig.java | 35 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 26 + .../org/apache/hadoop/ozone/audit/DNAction.java | 3 +- .../org/apache/hadoop/ozone/common/Checksum.java | 5 + .../helpers/ContainerCommandRequestPBHelper.java | 1 + .../common/src/main/resources/ozone-default.xml | 92 +- .../hadoop/ozone/audit/AuditLogTestUtils.java | 79 + .../ozone/container/ContainerTestHelper.java | 30 +- hadoop-hdds/config/pom.xml | 4 +- .../org/apache/hadoop/hdds/conf/ConfigTag.java | 3 +- hadoop-hdds/container-service/pom.xml | 8 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 17 +- .../ozone/container/common/impl/ContainerSet.java | 3 +- .../container/common/impl/HddsDispatcher.java | 21 +- .../common/interfaces/ContainerDispatcher.java | 10 + .../ozone/container/common/interfaces/Handler.java | 5 + .../common/statemachine/DatanodeStateMachine.java | 11 +- .../common/statemachine/StateContext.java | 36 +- .../DeleteContainerCommandHandler.java | 67 +- .../ReconstructECContainersCommandHandler.java | 6 +- .../ReplicateContainerCommandHandler.java | 3 +- .../states/endpoint/HeartbeatEndpointTask.java | 7 + .../common/transport/server/XceiverServerGrpc.java | 4 +- .../server/ratis/ContainerStateMachine.java | 79 +- .../common/transport/server/ratis/LocalStream.java | 58 + .../transport/server/ratis/XceiverServerRatis.java | 43 +- .../ECReconstructionCommandInfo.java | 28 +- .../ECReconstructionCoordinator.java | 12 + .../ECReconstructionCoordinatorTask.java | 29 +- .../reconstruction/ECReconstructionSupervisor.java | 11 +- .../ozone/container/keyvalue/KeyValueHandler.java | 52 + .../container/keyvalue/TarContainerPacker.java | 34 +- .../keyvalue/impl/ChunkManagerDispatcher.java | 16 + .../keyvalue/impl/FilePerBlockStrategy.java | 20 + .../keyvalue/impl/KeyValueStreamDataChannel.java | 277 ++ .../keyvalue/impl/StreamDataChannelBase.java | 96 + .../keyvalue/interfaces/ChunkManager.java | 13 + .../replication/ContainerReplicationSource.java | 5 +- .../replication/CopyContainerCompression.java | 78 + .../replication/GrpcReplicationClient.java | 20 +- .../replication/GrpcReplicationService.java | 8 +- .../OnDemandContainerReplicationSource.java | 19 +- .../container/replication/ReplicationServer.java | 26 +- .../replication/ReplicationSupervisor.java | 62 +- .../replication/ReplicationSupervisorMetrics.java | 5 +- .../container/replication/ReplicationTask.java | 34 +- .../replication/SimpleContainerDownloader.java | 12 +- .../hadoop/ozone/protocol/commands/SCMCommand.java | 35 + .../container/common/TestDatanodeStateMachine.java | 8 +- .../common/statemachine/TestStateContext.java | 85 +- .../TestDeleteContainerCommandHandler.java | 135 + .../states/endpoint/TestHeartbeatEndpointTask.java | 29 +- .../TestVolumeIOStatsWithPrometheusSink.java | 2 +- .../TestECReconstructionSupervisor.java | 96 +- .../container/keyvalue/TestKeyValueContainer.java | 141 +- .../container/keyvalue/TestTarContainerPacker.java | 37 +- .../impl/TestKeyValueStreamDataChannel.java | 313 +++ .../ReplicationSupervisorScheduling.java | 10 +- .../replication/TestReplicationSupervisor.java | 111 +- .../upgrade/TestDatanodeUpgradeToSchemaV3.java | 4 + .../upgrade/TestDatanodeUpgradeToScmHA.java | 2 +- .../content/feature/Streaming-Write-Pipeline.md | 136 + hadoop-hdds/docs/content/interface/Ofs.md | 32 +- hadoop-hdds/docs/content/interface/S3.zh.md | 2 +- hadoop-hdds/docs/content/security/SecuringTDE.md | 2 +- hadoop-hdds/docs/content/security/SecurityAcls.md | 9 +- .../docs/content/security/SecurityAcls.zh.md | 7 +- .../docs/content/security/SecurityWithRanger.md | 2 +- .../docs/content/security/SecurityWithRanger.zh.md | 2 +- hadoop-hdds/docs/pom.xml | 4 +- .../themes/ozonedoc/layouts/partials/header.html | 22 + hadoop-hdds/erasurecode/pom.xml | 4 +- .../framework/dev-support/findbugsExcludeFile.xml | 4 - hadoop-hdds/framework/pom.xml | 4 +- .../hdds/conf/DatanodeRatisServerConfig.java | 35 + ...inerLocationProtocolClientSideTranslatorPB.java | 20 + .../hadoop/hdds/security/ssl/KeyStoresFactory.java | 14 +- .../hdds/security/ssl/MonitoringTimerTask.java | 6 +- .../security/ssl/PemFileBasedKeyStoresFactory.java | 15 +- .../hdds/security/ssl/ReloadingX509KeyManager.java | 7 +- .../security/ssl/ReloadingX509TrustManager.java | 9 +- .../certificate/authority/DefaultCAServer.java | 108 +- .../x509/certificate/client/CertificateClient.java | 13 +- .../client/DefaultCertificateClient.java | 35 + .../hadoop/hdds/security/x509/keys/KeyCodec.java | 4 +- .../hdds/security/x509/keys/SecurityUtil.java | 34 + .../FixedThreadPoolWithAffinityExecutor.java | 7 +- .../hdds/server/events/SingleThreadExecutor.java | 6 +- .../hadoop/hdds/server/http/BaseHttpServer.java | 2 +- .../hdds/server/http/PrometheusMetricsSink.java | 109 +- .../hdds/server/http/RatisDropwizardExports.java | 59 +- .../hadoop/hdds/utils/DecayRpcSchedulerUtil.java | 120 + .../org/apache/hadoop/hdds/utils/MetricsUtil.java | 100 + .../hdds/utils/PrometheusMetricsSinkUtil.java | 116 + .../apache/hadoop/hdds/utils/UgiMetricsUtil.java | 68 + .../apache/hadoop/hdds/utils/db/RocksDatabase.java | 2 +- .../ssl/TestPemFileBasedKeyStoresFactory.java | 9 +- .../hdds/security/x509/CertificateClientTest.java | 17 + .../certificate/authority/TestDefaultCAServer.java | 99 +- ....java => TestPrometheusMetricsIntegration.java} | 148 +- .../hdds/utils/TestDecayRpcSchedulerUtil.java | 108 + .../hdds/utils/TestPrometheusMetricsSinkUtil.java | 229 ++ .../hadoop/hdds/utils/TestUgiMetricsUtil.java | 63 + hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +- hadoop-hdds/interface-admin/pom.xml | 4 +- .../src/main/proto/ScmAdminProtocol.proto | 1 + .../interface-admin/src/main/resources/proto.lock | 679 +++-- hadoop-hdds/interface-client/pom.xml | 4 +- .../src/main/proto/DatanodeClientProtocol.proto | 14 +- .../interface-client/src/main/resources/proto.lock | 991 ++++++-- hadoop-hdds/interface-server/pom.xml | 4 +- .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 4 + .../interface-server/src/main/resources/proto.lock | 840 ++++-- hadoop-hdds/pom.xml | 4 +- hadoop-hdds/server-scm/pom.xml | 4 +- .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 115 +- .../hdds/scm/container/ContainerManager.java | 18 + .../hdds/scm/container/ContainerManagerImpl.java | 21 + .../balancer/AbstractFindTargetGreedy.java | 31 +- .../scm/container/balancer/FindSourceGreedy.java | 20 +- .../ContainerPlacementStatusDefault.java | 36 +- .../algorithms/SCMContainerPlacementRackAware.java | 8 + .../SCMContainerPlacementRackScatter.java | 138 +- .../algorithms/SCMContainerPlacementRandom.java | 3 +- .../AbstractOverReplicationHandler.java | 4 +- .../replication/ContainerHealthResult.java | 35 +- .../replication/ContainerReplicaPendingOps.java | 45 + .../ContainerReplicaPendingOpsSubscriber.java} | 27 +- .../replication/ECMisReplicationHandler.java | 71 + .../replication/ECUnderReplicationHandler.java | 11 + .../replication/MisReplicationHandler.java | 188 ++ .../replication/OverReplicatedProcessor.java | 94 +- .../replication/RatisMisReplicationHandler.java | 78 + .../replication/RatisOverReplicationHandler.java | 247 ++ .../replication/RatisUnderReplicationHandler.java | 242 ++ .../container/replication/ReplicationManager.java | 73 +- .../container/replication/ReplicationQueue.java | 17 - .../replication/UnderReplicatedProcessor.java | 95 +- ...sor.java => UnhealthyReplicationProcessor.java} | 79 +- .../ClosedWithMismatchedReplicasHandler.java | 64 +- .../health/ClosedWithUnhealthyReplicasHandler.java | 4 +- .../health/ClosingContainerHandler.java | 7 + .../health/DeletingContainerHandler.java | 5 + .../health/ECReplicationCheckHandler.java | 18 +- .../replication/health/EmptyContainerHandler.java | 6 +- .../replication/health/OpenContainerHandler.java | 11 +- .../health/QuasiClosedContainerHandler.java | 4 + .../health/RatisReplicationCheckHandler.java | 14 + .../hdds/scm/pipeline/PipelinePlacementPolicy.java | 28 +- .../pipeline/PipelinePlacementPolicyFactory.java | 52 + .../hdds/scm/pipeline/RatisPipelineProvider.java | 34 +- ...inerLocationProtocolServerSideTranslatorPB.java | 17 + .../hdds/scm/server/SCMClientProtocolServer.java | 14 + .../hdds/scm/server/SCMDatanodeProtocolServer.java | 41 +- .../hdds/scm/server/StorageContainerManager.java | 34 +- .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 32 +- .../hdds/scm/TestSCMCommonPlacementPolicy.java | 275 +- .../hadoop/hdds/scm/block/TestBlockManager.java | 6 +- .../algorithms/TestContainerPlacementFactory.java | 13 +- .../TestContainerPlacementStatusDefault.java | 22 +- .../TestSCMContainerPlacementRackScatter.java | 96 +- .../TestContainerReplicaPendingOps.java | 77 + .../replication/TestECMisReplicationHandler.java | 174 ++ .../replication/TestECUnderReplicationHandler.java | 87 + .../replication/TestMisReplicationHandler.java | 182 ++ .../TestRatisMisReplicationHandler.java | 166 ++ .../TestRatisOverReplicationHandler.java | 281 ++ .../TestRatisUnderReplicationHandler.java | 228 ++ .../replication/TestReplicationManager.java | 114 + .../TestClosedWithMismatchedReplicasHandler.java | 16 +- .../health/TestECReplicationCheckHandler.java | 6 +- .../health/TestRatisReplicationCheckHandler.java | 5 +- .../hdds/scm/node/TestContainerPlacement.java | 4 +- .../scm/pipeline/TestPipelinePlacementFactory.java | 215 ++ .../scm/pipeline/TestPipelinePlacementPolicy.java | 15 +- .../scm/pipeline/TestRatisPipelineProvider.java | 22 +- .../safemode/TestHealthyPipelineSafeModeRule.java | 8 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 14 +- .../ozone/container/common/TestEndPoint.java | 8 + hadoop-hdds/test-utils/pom.xml | 5 +- .../org/apache/ozone/test/GenericTestUtils.java | 10 + hadoop-hdds/tools/pom.xml | 8 +- .../hadoop/hdds/scm/cli/cert/CertCommands.java | 1 + .../hadoop/hdds/scm/cli/cert/CleanExpired.java | 117 + .../hadoop/hdds/scm/cli/cert/TestCleanExpired.java | 100 + hadoop-ozone/client/pom.xml | 4 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 58 + .../client/io/BlockDataStreamOutputEntry.java | 287 +++ .../client/io/BlockDataStreamOutputEntryPool.java | 290 +++ .../client/io/ECBlockOutputStreamEntryPool.java | 4 +- .../hadoop/ozone/client/io/ECKeyOutputStream.java | 228 +- .../ozone/client/io/KeyDataStreamOutput.java | 510 ++++ .../hadoop/ozone/client/io/KeyInputStream.java | 436 +--- .../ozone/client/io/OzoneCryptoInputStream.java | 9 +- .../ozone/client/io/OzoneDataStreamOutput.java | 70 + .../ozone/client/protocol/ClientProtocol.java | 38 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 151 +- .../hadoop/ozone/client/TestOzoneECClient.java | 51 +- hadoop-ozone/common/pom.xml | 4 +- .../hadoop/ozone/om/helpers/OmBucketArgs.java | 7 +- .../hadoop/ozone/om/helpers/TestOmBucketArgs.java | 30 + hadoop-ozone/csi/pom.xml | 4 +- hadoop-ozone/datanode/pom.xml | 4 +- hadoop-ozone/dev-support/checks/coverage.sh | 6 +- hadoop-ozone/dev-support/intellij/ozone-site.xml | 20 + .../intellij/runConfigurations/Datanode2.xml | 2 +- .../intellij/runConfigurations/Datanode3.xml | 2 +- hadoop-ozone/dist/pom.xml | 13 +- .../main/compose/ozone-om-ha/docker-compose.yaml | 5 + .../dist/src/main/compose/ozone/docker-config | 2 + .../src/main/compose/ozonesecure/docker-config | 2 + hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 160 +- hadoop-ozone/dist/src/main/license/bin/NOTICE.txt | 2 +- .../main/license/bin/licenses/LICENSE-angular.txt | 2 +- .../src/main/license/bin/licenses/LICENSE-d3.txt | 2 +- ...{LICENSE-angular.txt => LICENSE-glyphicons.txt} | 4 +- .../main/license/bin/licenses/LICENSE-guava.txt | 13 + .../LICENSE-javax.activation-activation.txt | 3 - .../LICENSE-net.sf.jopt-simple-jopt-simple.txt | 24 - .../license/bin/licenses/LICENSE-org.aspectj.html | 91 - .../license/bin/licenses/LICENSE-org.aspectj.txt | 279 ++ .../bin/licenses/LICENSE-org.ow2.asm-asm.txt | 1 - .../license/bin/licenses/LICENSE-org.slf4j.txt | 2 +- ...y-misc.txt => NOTICE-ratis-thirdparty-misc.txt} | 24 +- hadoop-ozone/dist/src/main/license/jar-report.txt | 1 + .../main/license/src/licenses/LICENSE-angular.txt | 2 +- .../licenses/LICENSE-bootstrap.txt} | 4 +- .../src/main/license/src/licenses/LICENSE-d3.txt | 4 +- .../licenses/LICENSE-glyphicons.txt} | 4 +- .../main/license/src/licenses/LICENSE-guava.txt | 13 + .../main/license/src/licenses/LICENSE-jquery.txt | 2 +- .../src/main/license/src/licenses/LICENSE-nvd3.txt | 17 +- .../dist/src/main/license/update-jar-report.sh | 2 +- .../dist/src/main/smoketest/ozonefs/ozonefs.robot | 8 + .../fault-injection-test/mini-chaos-tests/pom.xml | 4 +- .../fault-injection-test/network-tests/pom.xml | 2 +- hadoop-ozone/fault-injection-test/pom.xml | 4 +- hadoop-ozone/httpfsgateway/pom.xml | 4 +- hadoop-ozone/insight/pom.xml | 4 +- hadoop-ozone/integration-test/pom.xml | 4 +- .../ozone/TestDirectoryDeletingServiceWithFSO.java | 4 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 81 +- .../fs/ozone/TestOzoneFileSystemWithStreaming.java | 158 ++ .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 42 +- .../hdds/scm/TestSCMDatanodeProtocolServer.java | 54 + .../hdds/scm/storage/TestContainerCommandsEC.java | 4 +- .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 20 + .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 21 + .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 4 + .../hadoop/ozone/TestOzoneConfigurationFields.java | 3 +- .../hadoop/ozone/TestStorageContainerManager.java | 2 +- .../ozone/client/CertificateClientTestImpl.java | 136 +- .../client/rpc/TestBlockDataStreamOutput.java | 273 ++ .../rpc/TestContainerStateMachineStream.java | 219 ++ .../ozone/client/rpc/TestECKeyOutputStream.java | 22 +- .../client/rpc/TestOzoneAtRestEncryption.java | 6 +- .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 441 ++-- .../client/rpc/TestOzoneRpcClientAbstract.java | 101 +- .../client/rpc/TestOzoneRpcClientWithRatis.java | 129 + .../client/rpc/read/TestChunkInputStream.java | 6 +- .../ozone/client/rpc/read/TestKeyInputStream.java | 2 +- .../ozone/container/TestECContainerRecovery.java | 2 + .../apache/hadoop/ozone/container/TestHelper.java | 42 +- .../ozoneimpl/TestOzoneContainerWithTLS.java | 187 +- .../server/TestSecureContainerServer.java | 4 + .../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 80 +- .../org/apache/hadoop/ozone/om/TestLDBCli.java | 15 + .../ozone/om/TestObjectStoreWithLegacyFS.java | 52 +- .../org/apache/hadoop/ozone/om/TestOmAcls.java | 171 +- .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 13 + .../ozone/parser/TestOzoneHARatisLogParser.java | 61 +- .../apache/hadoop/ozone/recon/TestReconTasks.java | 36 + .../hadoop/ozone/shell/TestOzoneShellHA.java | 22 + hadoop-ozone/interface-client/pom.xml | 4 +- .../interface-client/src/main/resources/proto.lock | 2672 ++++++++++++++++---- hadoop-ozone/interface-storage/pom.xml | 4 +- .../apache/hadoop/ozone/om/OMMetadataManager.java | 10 + hadoop-ozone/ozone-manager/pom.xml | 5 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 72 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 5 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 109 +- .../request/bucket/OMBucketSetPropertyRequest.java | 4 + .../om/request/file/OMDirectoryCreateRequest.java | 10 +- .../file/OMDirectoryCreateRequestWithFSO.java | 25 +- .../ozone/om/request/file/OMFileCreateRequest.java | 8 +- .../request/file/OMFileCreateRequestWithFSO.java | 3 +- .../om/request/key/OMAllocateBlockRequest.java | 3 +- .../key/OMDirectoriesPurgeRequestWithFSO.java | 9 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 19 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 3 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 7 +- .../S3InitiateMultipartUploadRequestWithFSO.java | 13 +- .../S3MultipartUploadCommitPartRequest.java | 8 +- .../S3MultipartUploadCompleteRequest.java | 3 +- .../response/file/OMDirectoryCreateResponse.java | 9 +- .../file/OMDirectoryCreateResponseWithFSO.java | 10 +- .../response/file/OMFileCreateResponseWithFSO.java | 6 + .../key/OMDirectoriesPurgeResponseWithFSO.java | 4 +- .../ozone/om/response/key/OMKeyCreateResponse.java | 7 + .../response/key/OMKeyDeleteResponseWithFSO.java | 4 +- .../response/key/OMKeysDeleteResponseWithFSO.java | 4 +- .../S3InitiateMultipartUploadResponseWithFSO.java | 12 +- .../ozone/om/service/DirectoryDeletingService.java | 149 +- .../webapps/ozoneManager/om-overview.html | 4 + .../apache/hadoop/ozone/om/TestChunkStreams.java | 150 +- .../ozone/om/request/OMRequestTestUtils.java | 12 + .../bucket/TestOMBucketSetPropertyRequest.java | 88 + .../request/file/TestOMDirectoryCreateRequest.java | 37 + .../file/TestOMDirectoryCreateRequestWithFSO.java | 39 + .../om/request/file/TestOMFileCreateRequest.java | 34 +- .../file/TestOMFileCreateRequestWithFSO.java | 31 + .../TestOMDirectoriesPurgeRequestAndResponse.java | 103 +- .../om/request/key/TestOMKeyCreateRequest.java | 32 + .../request/key/TestOMKeyCreateRequestWithFSO.java | 6 + .../TestS3MultipartUploadCompleteRequest.java | 6 +- ...estS3MultipartUploadCompleteRequestWithFSO.java | 6 + .../file/TestOMDirectoryCreateResponse.java | 19 +- .../file/TestOMDirectoryCreateResponseWithFSO.java | 19 +- .../key/TestOMKeysDeleteResponseWithFSO.java | 7 +- .../s3/multipart/TestS3MultipartResponse.java | 12 +- .../TestS3MultipartUploadAbortResponse.java | 3 +- .../TestS3MultipartUploadAbortResponseWithFSO.java | 12 +- hadoop-ozone/ozonefs-common/pom.xml | 8 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 27 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 8 + .../ozone/BasicRootedOzoneClientAdapterImpl.java | 36 +- .../fs/ozone/BasicRootedOzoneFileSystem.java | 47 +- .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 3 + .../hadoop/fs/ozone/OzoneFSDataStreamOutput.java | 103 + hadoop-ozone/ozonefs-hadoop2/pom.xml | 10 +- .../hadoop/fs/ozone/RootedOzoneFileSystem.java | 20 +- hadoop-ozone/ozonefs-hadoop3-client/pom.xml | 4 +- hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +- hadoop-ozone/ozonefs-shaded/pom.xml | 4 +- hadoop-ozone/ozonefs/pom.xml | 4 +- hadoop-ozone/pom.xml | 9 +- hadoop-ozone/recon-codegen/pom.xml | 3 +- hadoop-ozone/recon/pom.xml | 4 +- .../hadoop/ozone/recon/ReconServerConfigKeys.java | 12 + .../ozone/recon/api/ClusterStateEndpoint.java | 10 +- .../ozone/recon/api/handlers/BucketHandler.java | 26 - .../ozone/recon/api/handlers/FSOBucketHandler.java | 4 +- .../ozone/recon/api/handlers/KeyEntityHandler.java | 3 +- .../recon/api/handlers/LegacyBucketHandler.java | 4 +- .../recon/api/handlers/RootEntityHandler.java | 24 +- .../recon/api/types/ClusterStateResponse.java | 18 + .../ozone/recon/fsck/ContainerHealthTask.java | 67 +- .../hadoop/ozone/recon/scm/PipelineSyncTask.java | 33 +- .../recon/scm/ReconDatanodeProtocolServer.java | 2 +- .../ozone/recon/scm/ReconDeadNodeHandler.java | 12 +- .../ozone/recon/scm/ReconPipelineManager.java | 4 +- .../ozone/recon/scm/ReconStaleNodeHandler.java | 58 + .../scm/ReconStorageContainerManagerFacade.java | 180 +- .../recon/spi/StorageContainerServiceProvider.java | 21 + .../impl/StorageContainerServiceProviderImpl.java | 15 + .../resources/webapps/recon/ozone-recon-web/NOTICE | 5 - .../webapps/recon/ozone-recon-web/api/db.json | 93 +- .../recon/ozone-recon-web/src/utils/themeIcons.tsx | 6 +- .../src/views/datanodes/datanodes.tsx | 10 +- .../src/views/overview/overview.tsx | 13 +- .../src/views/pipelines/pipelines.tsx | 2 +- .../ozone/recon/OMMetadataManagerTestUtils.java | 4 +- .../recon/api/TestNSSummaryEndpointWithFSO.java | 99 +- .../recon/api/TestNSSummaryEndpointWithLegacy.java | 98 +- .../recon/api/TestTotalOpenContainerCount.java | 396 +++ .../ozone/recon/fsck/TestContainerHealthTask.java | 11 +- .../TestContainerHealthTaskRecordGenerator.java | 42 +- .../scm/AbstractReconContainerManagerTest.java | 4 +- hadoop-ozone/s3gateway/pom.xml | 5 +- .../hadoop/ozone/client/ClientProtocolStub.java | 24 + hadoop-ozone/tools/pom.xml | 4 +- .../org/apache/hadoop/ozone/debug/DBScanner.java | 31 +- .../ozone/debug/container/ExportSubcommand.java | 4 +- .../ozone/freon/ClosedContainerReplicator.java | 14 +- .../hadoop/ozone/freon/ContentGenerator.java | 18 + .../ozone/freon/OzoneClientKeyGenerator.java | 29 +- .../java/org/apache/hadoop/ozone/shell/Shell.java | 21 +- .../ozone/shell/bucket/ClearQuotaHandler.java | 12 +- .../hadoop/ozone/shell/bucket/SetQuotaHandler.java | 12 +- .../hadoop/ozone/shell/keys/GetKeyHandler.java | 4 +- .../hadoop/ozone/shell/keys/PutKeyHandler.java | 67 +- .../ozone/shell/volume/ClearQuotaHandler.java | 12 +- .../hadoop/ozone/shell/volume/SetQuotaHandler.java | 11 +- pom.xml | 154 +- 420 files changed, 20102 insertions(+), 3973 deletions(-) diff --cc hadoop-ozone/dist/pom.xml index 7afd0e7448,659c39c3d1..459cb6f5bd --- a/hadoop-ozone/dist/pom.xml +++ b/hadoop-ozone/dist/pom.xml @@@ -222,11 -222,6 +222,10 @@@ <dependency> <groupId>org.apache.ozone</groupId> <artifactId>ozone-insight</artifactId> + </dependency> + <dependency> + <groupId>org.apache.ozone</groupId> + <artifactId>ozone-httpfsgateway</artifactId> - <version>1.3.0-SNAPSHOT</version> </dependency> </dependencies> <profiles> diff --cc hadoop-ozone/dist/src/main/license/jar-report.txt index c53a764a34,c2561d8899..291dbd8e8d --- a/hadoop-ozone/dist/src/main/license/jar-report.txt +++ b/hadoop-ozone/dist/src/main/license/jar-report.txt @@@ -276,4 -265,4 +276,5 @@@ share/ozone/lib/token-provider.ja share/ozone/lib/txw2.jar share/ozone/lib/weld-servlet.Final.jar share/ozone/lib/woodstox-core.jar +share/ozone/lib/zookeeper.jar + share/ozone/lib/zstd-jni.jar diff --cc hadoop-ozone/httpfsgateway/pom.xml index 650192a9ab,0000000000..a7861cf86c mode 100644,000000..100644 --- a/hadoop-ozone/httpfsgateway/pom.xml +++ b/hadoop-ozone/httpfsgateway/pom.xml @@@ -1,316 -1,0 +1,316 @@@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +--> +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 + https://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.ozone</groupId> + <artifactId>ozone</artifactId> - <version>1.3.0-SNAPSHOT</version> ++ <version>1.4.0-SNAPSHOT</version> + </parent> + <artifactId>ozone-httpfsgateway</artifactId> - <version>1.3.0-SNAPSHOT</version> ++ <version>1.4.0-SNAPSHOT</version> + <packaging>jar</packaging> + + <name>Apache Ozone HttpFS</name> + <description>Apache Ozone HttpFS</description> + + <properties> + <httpfs.source.repository>REPO NOT AVAIL</httpfs.source.repository> + <httpfs.source.revision>REVISION NOT AVAIL</httpfs.source.revision> + <maven.build.timestamp.format>yyyy-MM-dd'T'HH:mm:ssZ</maven.build.timestamp.format> + <httpfs.build.timestamp>${maven.build.timestamp}</httpfs.build.timestamp> + </properties> + + <dependencies> + <dependency> + <groupId>org.apache.ozone</groupId> + <artifactId>ozone-filesystem-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.ozone</groupId> + <artifactId>ozone-filesystem</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-hdfs-client</artifactId> + </dependency> + <dependency> + <groupId>com.googlecode.json-simple</groupId> + <artifactId>json-simple</artifactId> + <version>1.1.1</version> + <scope>compile</scope> + <exclusions> + <exclusion> + <groupId>junit</groupId> + <artifactId>junit</artifactId> + </exclusion> + </exclusions> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-core</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-server</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>com.sun.jersey</groupId> + <artifactId>jersey-servlet</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>javax.servlet</groupId> + <artifactId>javax.servlet-api</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-server</artifactId> + </dependency> + <dependency> + <groupId>org.eclipse.jetty</groupId> + <artifactId>jetty-webapp</artifactId> + </dependency> + <dependency> + <groupId>commons-codec</groupId> + <artifactId>commons-codec</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>ch.qos.reload4j</groupId> + <artifactId>reload4j</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-api</artifactId> + <scope>compile</scope> + </dependency> + <dependency> + <groupId>org.slf4j</groupId> + <artifactId>slf4j-reload4j</artifactId> + <scope>runtime</scope> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>apache-curator</artifactId> + <version>2.4.0</version> + <type>pom</type> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-test</artifactId> + <version>2.4.0</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-client</artifactId> + <version>2.4.0</version> + </dependency> + <dependency> + <groupId>org.apache.curator</groupId> + <artifactId>curator-framework</artifactId> + <version>2.4.0</version> + </dependency> + <dependency> + <groupId>javax.xml.bind</groupId> + <artifactId>jaxb-api</artifactId> + <version>2.3.0</version> + </dependency> + </dependencies> + + <build> + <resources> + <resource> + <directory>src/main/resources</directory> + <filtering>true</filtering> + <includes> + <include>httpfs.properties</include> + </includes> + </resource> + <resource> + <directory>src/main/resources</directory> + <filtering>false</filtering> + <excludes> + <exclude>httpfs.properties</exclude> + </excludes> + </resource> + </resources> + <testResources> + <testResource> + <directory>${basedir}/src/test/resources</directory> + <filtering>false</filtering> + </testResource> + <testResource> + <directory>${basedir}/src/test/resources</directory> + <filtering>true</filtering> + </testResource> + </testResources> + + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-checkstyle-plugin</artifactId> + <version>${maven-checkstyle-plugin.version}</version> + <configuration> + <includeTestSourceDirectory>false</includeTestSourceDirectory> + </configuration> + </plugin> + <plugin> + <!-- workaround for filtered/unfiltered resources in same directory --> + <!-- remove when maven-eclipse-plugin 2.9 is available --> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-eclipse-plugin</artifactId> + <version>2.6</version> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-surefire-plugin</artifactId> + <configuration> + <testFailureIgnore>${ignoreTestFailure}</testFailureIgnore> + <threadCount>1</threadCount> + <forkedProcessTimeoutInSeconds>600</forkedProcessTimeoutInSeconds> + <properties> + <property> + <name>listener</name> + <value>org.apache.hadoop.test.TimedOutTestsListener</value> + </property> + </properties> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-javadoc-plugin</artifactId> + <executions> + <execution> + <goals> + <goal>javadoc-no-fork</goal> + </goals> + <phase>site</phase> + <configuration> + <quiet>true</quiet> + <verbose>false</verbose> + <source>${maven.compile.source}</source> + <charset>${maven.compile.encoding}</charset> + <groups> + <group> + <title>HttpFs API</title> + <packages>*</packages> + </group> + </groups> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-antrun-plugin</artifactId> + <version>1.8</version> + <executions> + <execution> + <id>create-web-xmls</id> + <phase>generate-test-resources</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <mkdir dir="${project.build.directory}/test-classes/webapp"/> + + <copy todir="${project.build.directory}/test-classes/webapp"> + <fileset dir="${basedir}/src/main/webapp"/> + </copy> + </target> + </configuration> + </execution> + <execution> + <id>site</id> + <phase>site</phase> + <goals> + <goal>run</goal> + </goals> + <configuration> + <target> + <xslt in="${basedir}/src/main/resources/httpfs-default.xml" + out="${project.build.directory}/site/httpfs-default.html" + style="${basedir}/src/site/configuration.xsl"/> + </target> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.github.spotbugs</groupId> + <artifactId>spotbugs-maven-plugin</artifactId> + <configuration> + <excludeFilterFile>${basedir}/dev-support/findbugsExcludeFile.xml</excludeFilterFile> + </configuration> + </plugin> + </plugins> + </build> + + <profiles> + <profile> + <id>dist</id> + <activation> + <activeByDefault>false</activeByDefault> + </activation> + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-assembly-plugin</artifactId> + <dependencies> + <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-assemblies</artifactId> + <version>${hadoop.version}</version> + </dependency> + </dependencies> + <executions> + <execution> + <id>dist</id> + <phase>package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <finalName>${project.artifactId}-${project.version}</finalName> + <appendAssemblyId>false</appendAssemblyId> + <attach>false</attach> + <descriptorRefs> + <descriptorRef>hadoop-httpfs-dist</descriptorRef> + </descriptorRefs> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> +</project> diff --cc hadoop-ozone/pom.xml index 6d643bc855,c3aefb4afc..73196476f7 --- a/hadoop-ozone/pom.xml +++ b/hadoop-ozone/pom.xml @@@ -236,6 -235,6 +236,11 @@@ <artifactId>ozone-insight</artifactId> <version>${hdds.version}</version> </dependency> ++ <dependency> ++ <groupId>org.apache.ozone</groupId> ++ <artifactId>ozone-httpfsgateway</artifactId> ++ <version>${ozone.version}</version> ++ </dependency> <dependency> <groupId>org.apache.ozone</groupId> <artifactId>ozone-recon</artifactId> --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
