This is an automated email from the ASF dual-hosted git repository. siyao pushed a commit to branch HDDS-4944 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 79a812af1f1156aed729e82a85b9492c00ef573f Merge: 35043683a6 c01fc7907a Author: Siyao Meng <[email protected]> AuthorDate: Mon May 23 12:30:16 2022 -0700 Merge remote-tracking branch 'asf/master' into HDDS-4944 Conflicts: hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/OzoneBucketStub.java hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestPermissionCheck.java hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/metrics/TestS3GatewayMetrics.java Change-Id: I05134acea37376d41d37bced6e40c700fbd9a0bf .github/workflows/post-commit.yml | 56 +- dev-support/annotations/pom.xml | 4 +- .../org/apache/ozone/annotations/package-info.java | 20 +- dev-support/ci/selective_ci_checks.bats | 24 +- dev-support/ci/selective_ci_checks.sh | 24 +- hadoop-hdds/client/pom.xml | 4 + .../hadoop/hdds/scm/ECXceiverClientGrpc.java | 60 + .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 42 + .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 17 +- .../hadoop/hdds/scm/XceiverClientManager.java | 13 +- .../scm/storage/BlockExtendedInputStream.java} | 16 +- .../hadoop/hdds/scm/storage/BlockInputStream.java | 56 +- .../hadoop/hdds/scm/storage/BlockOutputStream.java | 74 +- .../hdds/scm/storage/ECBlockOutputStream.java | 201 ++ .../hdds/scm/storage/ExtendedInputStream.java | 91 + .../ozone/client/io/BadDataLocationException.java | 56 + .../ozone/client/io/BlockInputStreamFactory.java | 55 + .../client/io/BlockInputStreamFactoryImpl.java | 92 + .../hadoop/ozone/client/io/ECBlockInputStream.java | 416 ++++ .../ozone/client/io/ECBlockInputStreamFactory.java | 58 + .../client/io/ECBlockInputStreamFactoryImpl.java | 100 + .../ozone/client/io/ECBlockInputStreamProxy.java | 239 ++ .../client/io/ECBlockReconstructedInputStream.java | 243 ++ .../io/ECBlockReconstructedStripeInputStream.java | 632 +++++ .../client/io/InsufficientLocationsException.java} | 26 +- .../hadoop/ozone/client/io}/package-info.java | 6 +- .../hadoop/ozone/client/io/ECStreamTestUtil.java | 407 +++ .../client/io/TestBlockInputStreamFactoryImpl.java | 108 + .../ozone/client/io/TestECBlockInputStream.java | 519 ++++ .../client/io/TestECBlockInputStreamProxy.java | 396 +++ .../io/TestECBlockReconstructedInputStream.java | 377 +++ .../TestECBlockReconstructedStripeInputStream.java | 721 ++++++ hadoop-hdds/common/pom.xml | 4 + .../java/org/apache/hadoop/hdds/HddsUtils.java | 20 + .../hdds/client/DefaultReplicationConfig.java | 134 + .../hadoop/hdds/client/ECReplicationConfig.java | 207 ++ .../hadoop/hdds/client/RatisReplicationConfig.java | 7 + .../hadoop/hdds/client/ReplicationConfig.java | 53 +- .../hadoop/hdds/client/ReplicationFactor.java | 20 + .../apache/hadoop/hdds/client/ReplicationType.java | 25 +- .../hdds/client/StandaloneReplicationConfig.java | 7 + .../hadoop/hdds/freon/FakeClusterTopology.java | 1 + .../org/apache/hadoop/hdds/scm/ScmConfigKeys.java | 10 +- .../apache/hadoop/hdds/scm/client/ScmClient.java | 7 +- .../hadoop/hdds/scm/container/ContainerInfo.java | 42 +- .../hdds/scm/container/ContainerReplicaInfo.java | 14 +- .../scm/container/ReplicationManagerReport.java | 44 +- .../scm/container/common/helpers/ExcludeList.java | 48 +- .../org/apache/hadoop/hdds/scm/ha/SCMHAUtils.java | 8 +- .../apache/hadoop/hdds/scm/net/InnerNodeImpl.java | 3 +- .../org/apache/hadoop/hdds/scm/net/NetUtils.java | 22 +- .../hadoop/hdds/scm/net/NetworkTopologyImpl.java | 23 +- .../java/org/apache/hadoop/hdds/scm/net/Node.java | 28 + .../org/apache/hadoop/hdds/scm/net/NodeImpl.java | 33 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 79 +- .../protocol/StorageContainerLocationProtocol.java | 23 + .../hadoop/hdds/scm/storage/BlockLocationInfo.java | 113 +- .../hdds/scm/storage/ContainerProtocolCalls.java | 66 +- .../org/apache/hadoop/hdds/server/JsonUtils.java | 0 .../hadoop/hdds/upgrade/HDDSLayoutFeature.java | 4 +- .../org/apache/hadoop/ozone/ClientVersion.java | 3 + .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 24 +- .../java/org/apache/hadoop/ozone/OzoneConsts.java | 4 + .../apache/hadoop/ozone/OzoneManagerVersion.java | 2 + .../java/org/apache/hadoop/ozone/ha/ConfUtils.java | 11 - .../apache/hadoop/ozone/util/OzoneNetUtils.java | 117 + .../common/src/main/resources/ozone-default.xml | 111 +- .../java/org/apache/hadoop/hdds/TestHddsUtils.java | 39 +- .../hdds/client/TestECReplicationConfig.java | 91 + .../hadoop/hdds/client/TestReplicationConfig.java | 214 +- .../org/apache/hadoop/hdds/scm/TestSCMHAUtils.java | 54 + .../hdds/scm/container/TestContainerInfo.java | 86 + .../scm/container/TestContainerReplicaInfo.java | 31 + .../container/TestReplicationManagerReport.java | 83 + .../container/common/helpers/TestExcludeList.java | 67 + .../hadoop/hdds/scm/container/package-info.java | 1 + .../hdds/scm/net/TestNetworkTopologyImpl.java | 48 +- .../hadoop/hdds/scm/pipeline/MockPipeline.java | 30 + .../hadoop/hdds/scm/pipeline/TestPipeline.java | 63 + .../hadoop/ozone/audit/TestOzoneAuditLogger.java | 43 +- .../apache/hadoop/ozone/ha/TestOzoneNetUtils.java} | 37 +- .../common/src/test/resources/auditlog.properties | 2 +- hadoop-hdds/container-service/pom.xml | 2 +- .../apache/hadoop/ozone/HddsDatanodeService.java | 3 + .../container/common/helpers/ContainerUtils.java | 5 +- .../container/common/impl/ContainerDataYaml.java | 55 +- .../common/impl/ContainerLayoutVersion.java | 10 +- .../container/common/impl/HddsDispatcher.java | 5 + .../common/report/ContainerReportPublisher.java | 1 + .../common/report/PipelineReportPublisher.java | 1 + .../common/statemachine/DatanodeStateMachine.java | 13 +- .../ReconstructECContainersCommandHandler.java | 78 + .../states/endpoint/HeartbeatEndpointTask.java | 2 +- .../server/ratis/ContainerStateMachine.java | 2 +- .../ECReconstructionCommandInfo.java | 82 + .../ECReconstructionCoordinatorTask.java | 52 + .../reconstruction/ECReconstructionSupervisor.java | 72 + .../container/ec/reconstruction/package-info.java | 34 +- .../container/keyvalue/KeyValueContainer.java | 1 + .../container/keyvalue/KeyValueContainerData.java | 10 + .../ozone/container/keyvalue/KeyValueHandler.java | 15 + .../container/keyvalue/helpers/ChunkUtils.java | 20 +- .../keyvalue/impl/FilePerBlockStrategy.java | 2 +- .../ozoneimpl/ContainerDataScrubberMetrics.java | 1 + .../commands/ReconstructECContainersCommand.java | 193 ++ .../TestSchemaOneBackwardsCompatibility.java | 3 +- .../common/impl/TestContainerDataYaml.java | 46 +- .../states/endpoint/TestHeartbeatEndpointTask.java | 4 +- .../TestECReconstructionSupervisor.java | 53 + .../upgrade/TestDatanodeUpgradeToScmHA.java | 17 +- .../TestReconstructionECContainersCommands.java | 97 + .../docs/content/feature/EC-Chunk-Layout.png | Bin 0 -> 179275 bytes .../content/feature/EC-Reads-With-No-Failures.png | Bin 0 -> 194800 bytes .../content/feature/EC-Reconstructional-Read.png | Bin 0 -> 221681 bytes .../EC-Write-Block-Allocation-in-Containers.png | Bin 0 -> 194715 bytes hadoop-hdds/docs/content/feature/ErasureCoding.md | 212 ++ hadoop-hdds/docs/content/feature/PrefixFSO.md | 4 +- .../docs/content/security/SecurityWithRanger.md | 7 + .../docs/content/security/SecurityWithRanger.zh.md | 9 +- hadoop-hdds/docs/dev-support/bin/generate-site.sh | 12 +- .../docs/dev-support/bin/make_images_responsive.py | 57 - hadoop-hdds/docs/pom.xml | 12 +- .../_markup/render-image.html} | 3 +- .../client => hadoop-hdds/erasurecode}/pom.xml | 45 +- .../rawcoder/HadoopNativeECAccessorUtil.java | 62 + .../io/erasurecode/rawcoder}/package-info.java | 12 +- .../apache/ozone/erasurecode/BufferAllocator.java | 91 + .../apache/ozone/erasurecode/CodecRegistry.java | 170 ++ .../java/org/apache/ozone/erasurecode/ECChunk.java | 113 + .../apache/ozone/erasurecode}/package-info.java | 14 +- .../rawcoder/AbstractNativeRawDecoder.java | 102 + .../rawcoder/AbstractNativeRawEncoder.java | 99 + .../rawcoder/ByteArrayDecodingState.java | 132 + .../rawcoder/ByteArrayEncodingState.java | 105 + .../rawcoder/ByteBufferDecodingState.java | 143 ++ .../rawcoder/ByteBufferEncodingState.java | 110 + .../ozone/erasurecode/rawcoder/CoderUtil.java | 174 ++ .../ozone/erasurecode/rawcoder/DecodingState.java | 52 + .../erasurecode/rawcoder/DummyRawDecoder.java | 45 + .../erasurecode/rawcoder/DummyRawEncoder.java | 45 + .../rawcoder/DummyRawErasureCoderFactory.java | 52 + .../ozone/erasurecode/rawcoder/EncodingState.java | 47 + .../erasurecode/rawcoder/ErasureCodeNative.java | 96 + .../erasurecode/rawcoder/NativeRSRawDecoder.java | 63 + .../erasurecode/rawcoder/NativeRSRawEncoder.java | 63 + .../rawcoder/NativeRSRawErasureCoderFactory.java | 53 + .../erasurecode/rawcoder/NativeXORRawDecoder.java | 58 + .../erasurecode/rawcoder/NativeXORRawEncoder.java | 58 + .../rawcoder/NativeXORRawErasureCoderFactory.java | 53 + .../ozone/erasurecode/rawcoder/RSRawDecoder.java | 177 ++ .../ozone/erasurecode/rawcoder/RSRawEncoder.java | 77 + .../rawcoder/RSRawErasureCoderFactory.java | 52 + .../rawcoder/RawErasureCoderFactory.java | 56 + .../erasurecode/rawcoder/RawErasureDecoder.java | 217 ++ .../erasurecode/rawcoder/RawErasureEncoder.java | 193 ++ .../ozone/erasurecode/rawcoder/XORRawDecoder.java | 88 + .../ozone/erasurecode/rawcoder/XORRawEncoder.java | 86 + .../rawcoder/XORRawErasureCoderFactory.java | 53 + .../ozone/erasurecode/rawcoder/package-info.java} | 44 +- .../ozone/erasurecode/rawcoder/util/CodecUtil.java | 111 + .../ozone/erasurecode/rawcoder/util/DumpUtil.java | 99 + .../ozone/erasurecode/rawcoder/util/GF256.java | 333 +++ .../erasurecode/rawcoder/util/GaloisField.java | 565 +++++ .../ozone/erasurecode/rawcoder/util/RSUtil.java | 188 ++ .../erasurecode/rawcoder/util}/package-info.java | 10 +- ...one.erasurecode.rawcoder.RawErasureCoderFactory | 17 + .../org/apache/ozone/erasurecode/DumpUtil.java | 96 + .../ozone/erasurecode/TestCodecRegistry.java | 159 ++ .../apache/ozone/erasurecode/TestCoderBase.java | 524 ++++ .../apache/ozone/erasurecode/package-info.java} | 9 +- .../rawcoder/RawErasureCoderBenchmark.java | 412 ++++ .../rawcoder/TestCodecRawCoderMapping.java | 68 + .../erasurecode/rawcoder/TestDummyRawCoder.java | 96 + .../erasurecode/rawcoder/TestNativeRSRawCoder.java | 129 + .../rawcoder/TestNativeXORRawCoder.java} | 28 +- .../erasurecode/rawcoder/TestRSRawCoder.java} | 18 +- .../erasurecode/rawcoder/TestRSRawCoderBase.java | 123 + .../erasurecode/rawcoder/TestRawCoderBase.java | 353 +++ .../rawcoder/TestRawErasureCoderBenchmark.java} | 31 +- .../erasurecode/rawcoder/TestXORRawCoder.java} | 13 +- .../erasurecode/rawcoder/TestXORRawCoderBase.java | 65 + .../ozone/erasurecode/rawcoder}/package-info.java | 12 +- hadoop-hdds/framework/pom.xml | 4 + .../hadoop/hdds/scm/metadata/SCMMetadataStore.java | 3 + ...lockLocationProtocolClientSideTranslatorPB.java | 8 + ...inerLocationProtocolClientSideTranslatorPB.java | 36 +- .../SCMBlockLocationFailoverProxyProvider.java | 3 + .../client/DefaultCertificateClient.java | 9 +- .../exceptions/TestSCMExceptionResultCodes.java | 19 +- .../token/TestOzoneBlockTokenIdentifier.java | 16 +- .../hdds/security/token/TokenVerifierTests.java | 4 +- .../certificate/authority/TestDefaultCAServer.java | 40 +- .../certificate/authority/TestDefaultProfile.java | 22 +- .../client/TestCertificateClientInit.java | 73 +- .../client/TestDefaultCertificateClient.java | 60 +- .../x509/certificate/utils/TestCRLCodec.java | 26 +- .../certificate/utils/TestCertificateCodec.java | 41 +- .../certificates/TestCertificateSignRequest.java | 60 +- .../x509/certificates/TestRootCertificate.java | 68 +- .../security/x509/keys/TestHDDSKeyGenerator.java | 14 +- .../hdds/security/x509/keys/TestKeyCodec.java | 49 +- .../apache/hadoop/hdds/server/TestJsonUtils.java | 8 +- .../apache/hadoop/hdds/server/TestServerUtils.java | 28 +- .../hadoop/hdds/server/events/TestEventQueue.java | 28 +- .../hdds/server/events/TestEventQueueChain.java | 2 +- .../hdds/server/events/TestEventWatcher.java | 46 +- .../hdds/server/http/TestBaseHttpServer.java | 8 +- .../hadoop/hdds/server/http/TestHtmlQuoting.java | 31 +- .../hdds/server/http/TestHttpRequestLog.java | 16 +- .../server/http/TestHttpRequestLogAppender.java | 12 +- .../hadoop/hdds/server/http/TestHttpServer2.java | 4 +- .../hdds/server/http/TestProfileServlet.java | 39 +- .../server/http/TestPrometheusMetricsSink.java | 47 +- .../server/http/TestRatisDropwizardExports.java | 13 +- .../hdds/server/http/TestRatisNameRewrite.java | 65 +- .../hadoop/hdds/utils/db/TestDBConfigFromFile.java | 33 +- .../hadoop/hdds/utils/db/TestDBStoreBuilder.java | 94 +- .../apache/hadoop/hdds/utils/db/TestRDBStore.java | 224 +- .../hadoop/hdds/utils/db/TestRDBStoreIterator.java | 20 +- .../hadoop/hdds/utils/db/TestRDBTableStore.java | 129 +- .../hdds/utils/db/TestTypedRDBTableStore.java | 78 +- .../hadoop/hdds/utils/db/cache/TestTableCache.java | 237 +- .../src/main/proto/ScmAdminProtocol.proto | 2 + hadoop-hdds/interface-client/pom.xml | 2 +- .../src/main/proto/DatanodeClientProtocol.proto | 4 + .../interface-client/src/main/proto/hdds.proto | 22 +- hadoop-hdds/interface-server/pom.xml | 2 +- .../src/main/proto/SCMRatisProtocol.proto | 1 + .../proto/ScmServerDatanodeHeartbeatProtocol.proto | 24 +- .../src/main/proto/ScmServerProtocol.proto | 6 +- hadoop-hdds/pom.xml | 7 + hadoop-hdds/server-scm/pom.xml | 2 +- .../hadoop/hdds/scm/SCMCommonPlacementPolicy.java | 6 +- .../container/AbstractContainerReportHandler.java | 88 +- .../hdds/scm/container/ContainerManagerImpl.java | 38 +- .../hdds/scm/container/ContainerReplica.java | 26 +- .../hdds/scm/container/ContainerReplicaCount.java | 3 +- .../scm/container/ECContainerReplicaCount.java | 313 +++ .../scm/container/balancer/ContainerBalancer.java | 135 +- .../balancer/ContainerBalancerMetrics.java | 70 +- .../ContainerBalancerSelectionCriteria.java | 2 +- .../ContainerPlacementPolicyFactory.java | 28 +- .../algorithms/SCMContainerPlacementMetrics.java | 10 + .../SCMContainerPlacementRackScatter.java | 329 +++ .../container/replication/ContainerReplicaOp.java | 63 + .../replication/ContainerReplicaPendingOps.java | 227 ++ ...tivityStatusMXBean.java => InflightAction.java} | 27 +- .../LegacyReplicationManager.java} | 379 +-- .../replication/ReplicationActivityStatus.java | 104 - .../container/replication/ReplicationManager.java | 453 ++++ .../replication/ReplicationManagerMetrics.java | 1 - .../apache/hadoop/hdds/scm/events/SCMEvents.java | 10 + .../apache/hadoop/hdds/scm/ha/HASecurityUtils.java | 2 +- .../hadoop/hdds/scm/ha/SCMHAManagerImpl.java | 2 + .../hadoop/hdds/scm/ha/SCMHAManagerStub.java | 17 +- .../hadoop/hdds/scm/ha/SCMHANodeDetails.java | 12 +- .../hadoop/hdds/scm/ha/SequenceIdGenerator.java | 15 +- .../apache/hadoop/hdds/scm/ha/StatefulService.java | 80 + .../hdds/scm/ha/StatefulServiceStateManager.java | 61 + .../scm/ha/StatefulServiceStateManagerImpl.java | 124 + .../hadoop/hdds/scm/metadata/ByteStringCodec.java | 69 + .../hadoop/hdds/scm/metadata/SCMDBDefinition.java | 13 +- .../hdds/scm/metadata/SCMMetadataStoreImpl.java | 13 + .../hdds/scm/node/CommandQueueReportHandler.java | 49 + .../hdds/scm/node/DatanodeAdminMonitorImpl.java | 2 +- .../apache/hadoop/hdds/scm/node/DatanodeInfo.java | 61 + .../hdds/scm/node/NodeDecommissionManager.java | 2 +- .../apache/hadoop/hdds/scm/node/NodeManager.java | 20 + .../hadoop/hdds/scm/node/SCMNodeManager.java | 44 + .../hadoop/hdds/scm/node/SCMNodeMetrics.java | 16 + .../scm/pipeline/BackgroundPipelineCreator.java | 31 +- .../scm/pipeline/BackgroundPipelineScrubber.java | 183 ++ .../hdds/scm/pipeline/ECPipelineProvider.java | 126 + .../hadoop/hdds/scm/pipeline/PipelineFactory.java | 30 +- .../hadoop/hdds/scm/pipeline/PipelineManager.java | 24 +- .../hdds/scm/pipeline/PipelineManagerImpl.java | 84 +- .../hadoop/hdds/scm/pipeline/PipelineProvider.java | 23 +- .../hdds/scm/pipeline/PipelineStateManager.java | 5 + .../scm/pipeline/PipelineStateManagerImpl.java | 20 + .../hadoop/hdds/scm/pipeline/PipelineStateMap.java | 44 +- .../hdds/scm/pipeline/RatisPipelineProvider.java | 26 +- .../hdds/scm/pipeline/SimplePipelineProvider.java | 20 + .../scm/pipeline/WritableContainerFactory.java | 8 + .../scm/pipeline/WritableECContainerProvider.java | 239 ++ .../algorithms/RandomPipelineChoosePolicy.java | 1 + ...lockLocationProtocolServerSideTranslatorPB.java | 17 +- ...inerLocationProtocolServerSideTranslatorPB.java | 255 +- .../scm/server/OzoneStorageContainerManager.java | 2 +- .../hdds/scm/server/SCMBlockProtocolServer.java | 4 +- .../hdds/scm/server/SCMClientProtocolServer.java | 106 +- .../hadoop/hdds/scm/server/SCMConfigurator.java | 2 +- .../scm/server/SCMDatanodeHeartbeatDispatcher.java | 20 + .../hdds/scm/server/StorageContainerManager.java | 40 +- .../scm/server/StorageContainerManagerStarter.java | 3 + .../org/apache/hadoop/hdds/scm/HddsTestUtils.java | 28 +- .../hdds/scm/TestSCMCommonPlacementPolicy.java | 75 + .../hadoop/hdds/scm/block/TestBlockManager.java | 12 +- .../hadoop/hdds/scm/block/TestDeletedBlockLog.java | 48 +- .../command/TestCommandStatusReportHandler.java | 10 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 27 + .../hdds/scm/container/SimpleMockNodeManager.java | 20 + .../container/TestCloseContainerEventHandler.java | 39 +- .../scm/container/TestContainerActionsHandler.java | 2 +- .../scm/container/TestContainerManagerImpl.java | 109 +- .../scm/container/TestContainerReportHandler.java | 207 +- .../scm/container/TestContainerStateManager.java | 18 +- .../TestIncrementalContainerReportHandler.java | 34 +- .../hdds/scm/container/TestReplicationManager.java | 573 +++-- .../scm/container/TestUnknownContainerReport.java | 10 +- .../container/balancer/TestContainerBalancer.java | 55 +- .../container/balancer/TestFindTargetStrategy.java | 26 +- .../algorithms/TestContainerPlacementFactory.java | 43 +- .../TestContainerPlacementStatusDefault.java | 8 +- .../TestSCMContainerPlacementCapacity.java | 28 +- .../TestSCMContainerPlacementRackAware.java | 6 + ...a => TestSCMContainerPlacementRackScatter.java} | 284 +-- .../TestSCMContainerPlacementRandom.java | 36 +- .../TestContainerReplicaPendingOps.java | 226 ++ .../replication/TestReplicationManagerMetrics.java | 27 +- .../states/TestContainerReplicaCount.java | 12 +- .../states/TestECContainerReplicaCount.java | 432 ++++ .../hdds/scm/ha/TestReplicationAnnotation.java | 20 +- .../apache/hadoop/hdds/scm/ha/TestSCMContext.java | 10 +- .../hadoop/hdds/scm/ha/TestSCMHAConfiguration.java | 74 +- .../hadoop/hdds/scm/ha/TestSCMRatisRequest.java | 30 +- .../hadoop/hdds/scm/ha/TestSCMRatisResponse.java | 25 +- .../hadoop/hdds/scm/ha/TestSCMServiceManager.java | 6 +- .../hdds/scm/ha/TestSequenceIDGenerator.java | 76 +- .../ha/TestStatefulServiceStateManagerImpl.java | 89 + .../hadoop/hdds/scm/ha/io/TestBigIntegerCodec.java | 6 +- .../hdds/scm/ha/io/TestX509CertificateCodec.java | 13 +- .../hdds/scm/metadata/TestPipelineIDCodec.java | 6 +- ...ler.java => TestCommandQueueReportHandler.java} | 106 +- .../hdds/scm/node/TestContainerPlacement.java | 7 +- .../hdds/scm/node/TestDatanodeAdminMonitor.java | 17 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 38 +- .../hdds/scm/node/TestNodeDecommissionManager.java | 10 +- .../hdds/scm/node/TestNodeReportHandler.java | 22 +- .../hadoop/hdds/scm/node/TestNodeStateManager.java | 25 +- .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 37 +- .../hadoop/hdds/scm/node/TestStatisticsUpdate.java | 28 +- .../hdds/scm/node/states/TestNodeStateMap.java | 12 +- .../hdds/scm/pipeline/MockPipelineManager.java | 57 +- .../pipeline/TestBackgroundPipelineScrubber.java | 109 + .../hdds/scm/pipeline/TestECPipelineProvider.java | 154 ++ .../scm/pipeline/TestPipelineActionHandler.java | 2 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 290 ++- .../scm/pipeline/TestPipelineStateManagerImpl.java | 78 +- .../hdds/scm/pipeline/TestPipelineStateMap.java | 99 + .../scm/pipeline/TestRatisPipelineProvider.java | 93 +- .../scm/pipeline/TestSimplePipelineProvider.java | 44 +- .../pipeline/TestWritableECContainerProvider.java | 446 ++++ .../TestPipelineChoosePolicyFactory.java | 14 +- .../choose/algorithms/TestLeaderChoosePolicy.java | 26 +- .../safemode/TestHealthyPipelineSafeModeRule.java | 23 +- .../TestOneReplicaPipelineSafeModeRule.java | 6 +- .../hdds/scm/safemode/TestSCMSafeModeManager.java | 21 +- .../scm/server/TestSCMBlockProtocolServer.java | 22 +- .../hdds/scm/server/TestSCMContainerMetrics.java | 2 +- .../server/TestSCMDatanodeHeartbeatDispatcher.java | 61 +- .../server/TestStorageContainerManagerStarter.java | 18 +- .../ozone/container/common/TestEndPoint.java | 84 +- .../placement/TestContainerPlacement.java | 10 +- .../container/placement/TestDatanodeMetrics.java | 6 +- .../testutils/ReplicationNodeManagerMock.java | 21 + .../hadoop/ozone/scm/node/TestSCMNodeMetrics.java | 31 +- .../org/apache/ozone/test/GenericTestUtils.java | 1 + hadoop-hdds/tools/pom.xml | 7 + .../hdds/scm/cli/ContainerOperationClient.java | 6 +- .../hdds/scm/cli/container/InfoSubcommand.java | 10 +- .../hdds/scm/cli/container/ListSubcommand.java | 28 +- .../hdds/scm/cli/container/ReportSubcommand.java | 12 + .../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 162 +- .../scm/cli/pipeline/CreatePipelineSubcommand.java | 9 +- .../scm/cli/pipeline/ListPipelinesSubcommand.java | 78 +- .../hdds/scm/cli/container/TestInfoSubCommand.java | 109 +- .../scm/cli/container/TestReportSubCommand.java | 36 +- .../datanode/TestContainerBalancerSubCommand.java | 14 +- .../cli/datanode/TestDecommissionSubCommand.java | 16 +- .../scm/cli/datanode/TestListInfoSubcommand.java | 14 +- .../cli/datanode/TestMaintenanceSubCommand.java | 16 +- .../cli/datanode/TestRecommissionSubCommand.java | 14 +- .../scm/cli/datanode/TestUsageInfoSubcommand.java | 112 + .../cli/pipeline/TestListPipelinesSubCommand.java | 238 ++ hadoop-ozone/client/pom.xml | 14 + .../org/apache/hadoop/ozone/client/BucketArgs.java | 25 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 47 +- .../org/apache/hadoop/ozone/client/OzoneKey.java | 3 + .../ozone/client/io/BlockOutputStreamEntry.java | 7 +- .../client/io/BlockOutputStreamEntryPool.java | 11 +- .../ozone/client/io/ECBlockOutputStreamEntry.java | 455 ++++ .../client/io/ECBlockOutputStreamEntryPool.java | 89 + .../hadoop/ozone/client/io/ECKeyOutputStream.java | 625 +++++ .../hadoop/ozone/client/io/KeyInputStream.java | 67 +- .../hadoop/ozone/client/io/KeyOutputStream.java | 46 +- .../ozone/client/protocol/ClientProtocol.java | 11 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 228 +- .../hadoop/ozone/client/MockBlockAllocator.java | 5 +- .../hadoop/ozone/client/MockDatanodeStorage.java | 29 +- .../hadoop/ozone/client/MockOmTransport.java | 154 +- .../ozone/client/MockXceiverClientFactory.java | 47 +- .../hadoop/ozone/client/MockXceiverClientSpi.java | 11 +- .../client/MultiNodePipelineBlockAllocator.java | 143 ++ .../ozone/client/SinglePipelineBlockAllocator.java | 30 +- .../hadoop/ozone/client/TestOzoneClient.java | 49 +- .../hadoop/ozone/client/TestOzoneECClient.java | 1113 +++++++++ .../ozone/client/io/BlockStreamAccessor.java | 51 + .../client/io/TestECBlockOutputStreamEntry.java | 128 + .../ozone/client/io/TestKeyInputStreamEC.java | 115 + hadoop-ozone/common/pom.xml | 50 + .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 43 +- .../hadoop/ozone/om/exceptions/OMException.java | 1 + .../ozone/om/ha/GrpcOMFailoverProxyProvider.java | 143 ++ .../ozone/om/ha/OMFailoverProxyProvider.java | 22 +- .../hadoop/ozone/om/helpers/BucketLayout.java | 4 + .../hadoop/ozone/om/helpers/OmBucketArgs.java | 64 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 72 +- .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 23 +- .../hadoop/ozone/om/helpers/OmKeyLocationInfo.java | 212 +- .../ozone/om/helpers/OmMultipartKeyInfo.java | 19 +- .../apache/hadoop/ozone/om/helpers/QuotaUtil.java | 62 + .../hadoop/ozone/om/lock/OzoneManagerLock.java | 13 +- .../hadoop/ozone/om/lock/OzoneManagerLockUtil.java | 13 +- .../ozone/om/protocolPB/GrpcOmTransport.java | 329 +++ .../om/protocolPB/GrpcOmTransportFactory.java} | 24 +- .../ozone/om/protocolPB/OmTransportFactory.java | 34 +- ...OzoneManagerProtocolClientSideTranslatorPB.java | 60 +- .../apache/hadoop/ozone/protocolPB/OMPBHelper.java | 54 + .../hadoop/ozone/om/helpers/TestOmBucketInfo.java | 56 + .../hadoop/ozone/om/helpers/TestOmKeyInfo.java | 68 +- .../ozone/om/helpers/TestOmMultipartKeyInfo.java | 112 +- .../hadoop/ozone/om/helpers/TestQuotaUtil.java | 99 + .../hadoop/ozone/om/lock/TestOzoneManagerLock.java | 325 +++ .../ozone/om/protocolPB/TestS3GrpcOmTransport.java | 222 ++ .../common/src/test/resources/log4j.properties | 21 + hadoop-ozone/csi/pom.xml | 17 +- hadoop-ozone/dev-support/checks/build.sh | 2 +- hadoop-ozone/dev-support/checks/integration.sh | 2 +- hadoop-ozone/dev-support/checks/rat.sh | 15 +- .../dashboards/Ozone - Overall Metrics.json | 2591 ++++++++++++++++++++ .../dist/src/main/compose/compatibility/test.sh | 2 + .../dist/src/main/compose/ozone-ha/docker-config | 1 + .../src/main/compose/ozone-om-ha/docker-config | 1 + hadoop-ozone/dist/src/main/compose/ozone/README.md | 3 +- .../dist/src/main/compose/ozone/docker-config | 1 + .../dist/src/main/compose/ozone/prometheus.yml | 11 + hadoop-ozone/dist/src/main/compose/ozone/test.sh | 13 +- .../dist/src/main/compose/ozonescripts/test.sh | 1 + .../src/main/compose/ozonesecure-ha/docker-config | 1 + .../src/main/compose/ozonesecure/docker-config | 2 +- .../dist/src/main/compose/ozonesecure/test.sh | 12 +- .../compose/upgrade/compose/ha/docker-compose.yaml | 22 +- .../main/compose/upgrade/compose/ha/docker-config | 3 - .../src/main/compose/upgrade/compose/ha/load.sh | 2 +- .../upgrade/compose/non-ha/docker-compose.yaml | 22 +- .../compose/upgrade/compose/non-ha/docker-config | 2 +- .../main/compose/upgrade/compose/non-ha/load.sh | 2 +- hadoop-ozone/dist/src/main/compose/upgrade/test.sh | 10 +- .../dist/src/main/compose/upgrade/testlib.sh | 8 + .../non-rolling-upgrade/1.2.1-1.3.0/callback.sh | 84 + .../upgrade/upgrades/non-rolling-upgrade/driver.sh | 1 - .../dist/src/main/compose/xcompat/docker-config | 2 + hadoop-ozone/dist/src/main/compose/xcompat/test.sh | 43 + hadoop-ozone/dist/src/main/license/bin/LICENSE.txt | 4 + hadoop-ozone/dist/src/main/license/jar-report.txt | 7 +- ....io.erasurecode.rawcoder.RawErasureCoderFactory | 18 + .../src/main/smoketest/admincli/container.robot | 4 +- .../buckethead.robot => basic/single_node.robot} | 22 +- .../src/main/smoketest/ec/backward-compat.robot | 103 + .../dist/src/main/smoketest/ec/basic.robot | 64 + .../dist/src/main/smoketest/ec/lib.resource | 42 + .../src/main/smoketest/ec/upgrade-ec-check.robot | 45 + .../dist/src/main/smoketest/ozone-lib/shell.robot | 38 + .../dist/src/main/smoketest/ozonefs/ozonefs.robot | 4 +- .../dist/src/main/smoketest/s3/bucketcreate.robot | 1 + .../dist/src/main/smoketest/s3/bucketdelete.robot | 1 + .../dist/src/main/smoketest/s3/buckethead.robot | 5 +- .../dist/src/main/smoketest/s3/commonawslib.robot | 6 + .../dist/src/main/smoketest/s3/webui.robot | 1 + .../dist/src/shell/conf/dn-audit-log4j2.properties | 2 +- .../dist/src/shell/conf/om-audit-log4j2.properties | 2 +- .../src/shell/conf/s3g-audit-log4j2.properties | 2 +- .../src/shell/conf/scm-audit-log4j2.properties | 2 +- .../apache/hadoop/ozone/MiniOzoneChaosCluster.java | 2 +- hadoop-ozone/insight/pom.xml | 2 +- .../ozone/insight/scm/ReplicaManagerInsight.java | 2 +- hadoop-ozone/integration-test/pom.xml | 5 + .../ozone/TestDirectoryDeletingServiceWithFSO.java | 40 +- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 75 +- .../hadoop/fs/ozone/TestRootedDDSWithFSO.java | 10 +- .../hadoop/fs/ozone/TestRootedOzoneFileSystem.java | 237 +- .../fs/ozone/TestRootedOzoneFileSystemWithFSO.java | 15 +- .../hadoop/hdds/scm/TestRatisPipelineLeader.java | 13 +- .../metrics/TestSCMContainerManagerMetrics.java | 2 - .../hdds/scm/pipeline/TestMultiRaftSetup.java | 1 - .../safemode/TestSCMSafeModeWithPipelineRules.java | 2 +- .../hdds/scm/storage/TestContainerCommandsEC.java | 255 ++ .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 - .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 25 +- .../java/org/apache/hadoop/ozone/TestDataUtil.java | 18 +- .../hadoop/ozone/TestOzoneConfigurationFields.java | 9 + .../hadoop/ozone/TestSecureOzoneCluster.java | 2 - .../hadoop/ozone/TestStorageContainerManager.java | 91 +- .../apache/hadoop/ozone/client/rpc/TestBCSID.java | 4 +- .../rpc/TestContainerReplicationEndToEnd.java | 2 +- .../client/rpc/TestContainerStateMachine.java | 2 - .../rpc/TestContainerStateMachineFailures.java | 2 - .../rpc/TestContainerStateMachineFlushDelay.java | 2 - .../ozone/client/rpc/TestECKeyOutputStream.java | 421 ++++ .../rpc/TestOzoneClientMultipartUploadWithFSO.java | 18 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 1 - .../client/rpc/TestOzoneRpcClientAbstract.java | 492 ++-- .../client/rpc/read/TestChunkInputStream.java | 7 +- .../ozone/client/rpc/read/TestInputStreamBase.java | 25 +- .../ozone/client/rpc/read/TestKeyInputStream.java | 59 +- .../ozone/container/TestContainerReplication.java | 2 +- .../apache/hadoop/ozone/container/TestHelper.java | 9 + .../commandhandler/TestBlockDeletion.java | 7 +- .../commandhandler/TestDeleteContainerHandler.java | 42 +- .../ozone/dn/ratis/TestDnRatisLogParser.java | 2 - .../hadoop/ozone/freon/TestDataValidate.java | 45 +- .../freon/TestFreonWithDatanodeFastRestart.java | 25 +- .../ozone/freon/TestFreonWithPipelineDestroy.java | 23 +- .../ozone/freon/TestOmBucketReadWriteFileOps.java | 326 +++ .../ozone/freon/TestOmBucketReadWriteKeyOps.java | 306 +++ .../hadoop/ozone/freon/TestRandomKeyGenerator.java | 169 +- .../apache/hadoop/ozone/om/OMUpgradeTestUtils.java | 88 + .../hadoop/ozone/om/TestOMBucketLayoutUpgrade.java | 272 ++ .../hadoop/ozone/om/TestOMRatisSnapshots.java | 295 ++- .../hadoop/ozone/om/TestOMUpgradeFinalization.java | 52 +- .../apache/hadoop/ozone/om/TestObjectStore.java | 19 +- .../org/apache/hadoop/ozone/om/TestOmMetrics.java | 48 +- .../apache/hadoop/ozone/om/TestOzoneManagerHA.java | 16 +- .../ozone/om/TestOzoneManagerHAMetadataOnly.java | 5 +- .../ozone/om/TestOzoneManagerHAWithData.java | 1 - .../hadoop/ozone/om/TestOzoneManagerPrepare.java | 8 + .../ozone/recon/TestReconWithOzoneManagerFSO.java | 3 +- .../ozone/recon/TestReconWithOzoneManagerHA.java | 2 - .../hadoop/ozone/scm/TestCloseContainer.java | 2 +- .../scm/node/TestDecommissionAndMaintenance.java | 4 +- .../hadoop/ozone/shell/TestOzoneShellHA.java | 160 +- hadoop-ozone/interface-client/pom.xml | 30 +- .../src/main/proto/OmClientProtocol.proto | 44 +- .../apache/hadoop/ozone/om/OMMetadataManager.java | 8 +- hadoop-ozone/ozone-manager/pom.xml | 13 +- .../hadoop/ozone/om/GrpcOzoneManagerServer.java | 140 ++ .../org/apache/hadoop/ozone/om/KeyManager.java | 10 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 15 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 67 +- .../hadoop/ozone/om/OpenKeyCleanupService.java | 3 +- .../apache/hadoop/ozone/om/OzoneConfigUtil.java | 66 + .../org/apache/hadoop/ozone/om/OzoneManager.java | 316 ++- .../hadoop/ozone/om/OzoneManagerServiceGrpc.java | 98 + .../hadoop/ozone/om/OzoneManagerStarter.java | 3 + .../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 12 +- .../ozone/om/lock/OBSKeyPathLockStrategy.java | 101 + .../hadoop/ozone/om/lock/OzoneLockProvider.java | 52 + .../hadoop/ozone/om/lock/OzoneLockStrategy.java | 49 + .../ozone/om/lock/RegularBucketLockStrategy.java | 82 + .../apache/hadoop/ozone/om/lock}/package-info.java | 6 +- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 7 +- .../ozone/om/ratis/OzoneManagerStateMachine.java | 38 +- .../hadoop/ozone/om/request/OMClientRequest.java | 6 +- .../om/request/bucket/OMBucketCreateRequest.java | 57 +- .../request/bucket/OMBucketSetPropertyRequest.java | 38 + .../om/request/file/OMDirectoryCreateRequest.java | 96 +- .../ozone/om/request/file/OMFileCreateRequest.java | 67 +- .../request/file/OMFileCreateRequestWithFSO.java | 19 +- .../om/request/key/OMAllocateBlockRequest.java | 52 +- .../request/key/OMAllocateBlockRequestWithFSO.java | 19 +- .../ozone/om/request/key/OMKeyCommitRequest.java | 72 +- .../om/request/key/OMKeyCommitRequestWithFSO.java | 24 +- .../ozone/om/request/key/OMKeyCreateRequest.java | 62 +- .../om/request/key/OMKeyCreateRequestWithFSO.java | 19 +- .../ozone/om/request/key/OMKeyRenameRequest.java | 15 +- .../hadoop/ozone/om/request/key/OMKeyRequest.java | 45 +- .../om/request/key/OMOpenKeysDeleteRequest.java | 26 +- .../S3InitiateMultipartUploadRequest.java | 45 +- .../S3InitiateMultipartUploadRequestWithFSO.java | 13 +- .../multipart/S3MultipartUploadAbortRequest.java | 34 +- .../S3MultipartUploadCommitPartRequest.java | 40 +- .../S3MultipartUploadCompleteRequest.java | 60 +- .../om/request/validation/RequestValidations.java | 19 +- .../om/request/validation/ValidationContext.java | 24 +- .../response/file/OMFileCreateResponseWithFSO.java | 5 - .../om/response/key/OMAllocateBlockResponse.java | 14 +- .../key/OMAllocateBlockResponseWithFSO.java | 10 +- .../ozone/om/response/key/OMKeyCreateResponse.java | 5 - .../om/response/key/OMOpenKeysDeleteResponse.java | 13 +- .../hadoop/ozone/om/upgrade/OMLayoutFeature.java | 10 +- .../ozone/om/upgrade/OMLayoutFeatureAspect.java | 4 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 14 +- .../protocolPB/OzoneManagerRequestHandler.java | 231 +- .../ozone/om/TestGrpcOzoneManagerServer.java | 62 + .../hadoop/ozone/om/TestOmMetadataManager.java | 107 +- .../hadoop/ozone/om/TestOzoneConfigUtil.java | 105 + .../hadoop/ozone/om/failover/TestOMFailovers.java | 2 +- .../ozone/om/lock/TestOzoneLockProvider.java | 108 + ...TestOzoneManagerDoubleBufferWithOMResponse.java | 2 +- .../ozone/om/request/OMRequestTestUtils.java | 16 - .../om/request/key/TestOMKeyCreateRequest.java | 81 +- .../request/key/TestOMKeyRenameRequestWithFSO.java | 99 + .../ozone/om/request/key/TestOMKeyRequest.java | 31 +- .../request/key/TestOMOpenKeysDeleteRequest.java | 55 +- .../s3/multipart/TestS3MultipartRequest.java | 3 + .../TestS3MultipartUploadCompleteRequest.java | 8 + .../request/validation/TestRequestValidations.java | 95 +- .../ozone/om/response/TestCleanupTableInfo.java | 3 + .../response/key/TestOMAllocateBlockResponse.java | 2 +- .../key/TestOMAllocateBlockResponseWithFSO.java | 2 +- .../response/key/TestOMOpenKeysDeleteResponse.java | 59 +- .../hadoop/ozone/security/acl/TestParentAcl.java | 2 + hadoop-ozone/ozonefs-common/pom.xml | 2 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 72 +- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 45 +- .../apache/hadoop/fs/ozone/OzoneClientUtils.java | 124 + .../hadoop/fs/ozone/TestOzoneClientUtils.java | 206 +- hadoop-ozone/ozonefs-shaded/pom.xml | 4 + hadoop-ozone/pom.xml | 12 + hadoop-ozone/recon/pom.xml | 2 +- .../org/apache/hadoop/ozone/recon/ReconServer.java | 3 + .../hadoop/ozone/recon/api/NodeEndpoint.java | 11 +- .../hadoop/ozone/recon/api/PipelineEndpoint.java | 5 +- .../ozone/recon/api/types/DatanodePipeline.java | 14 +- .../ozone/recon/api/types/PipelineMetadata.java | 19 +- .../ozone/recon/scm/ReconContainerManager.java | 7 +- .../ozone/recon/scm/ReconPipelineFactory.java | 19 + .../ozone/recon/scm/ReconPipelineManager.java | 5 +- .../scm/ReconStorageContainerManagerFacade.java | 9 +- .../src/views/datanodes/datanodes.tsx | 2 +- .../src/views/pipelines/pipelines.tsx | 2 +- .../hadoop/ozone/recon/api/TestEndpoints.java | 4 +- .../scm/AbstractReconContainerManagerTest.java | 8 +- hadoop-ozone/s3gateway/pom.xml | 26 +- .../java/org/apache/hadoop/ozone/s3/Gateway.java | 5 +- .../apache/hadoop/ozone/s3/OzoneClientCache.java | 172 ++ .../hadoop/ozone/s3/OzoneClientProducer.java | 41 +- .../apache/hadoop/ozone/s3/RequestIdentifier.java | 8 +- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 73 +- .../hadoop/ozone/s3/endpoint/EndpointBase.java | 58 +- .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 176 +- .../hadoop/ozone/s3/metrics/S3GatewayMetrics.java | 80 + .../apache/hadoop/ozone/s3/util/S3StorageType.java | 13 +- .../org/apache/hadoop/ozone/s3/util/S3Utils.java | 87 +- ....hadoop.ozone.om.protocolPB.OmTransportFactory} | 8 +- .../hadoop/ozone/client/OzoneBucketStub.java | 78 +- .../ozone/protocolPB/TestGrpcOmTransport.java | 88 + .../hadoop/ozone/s3/TestS3GatewayAuditLog.java | 25 +- .../s3/endpoint/TestInitiateMultipartUpload.java | 35 +- .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 31 +- .../ozone/s3/endpoint/TestPermissionCheck.java | 4 +- .../ozone/s3/metrics/TestS3GatewayMetrics.java | 369 ++- .../apache/hadoop/ozone/s3/util/TestS3Utils.java | 143 ++ hadoop-ozone/tools/pom.xml | 2 +- .../ozone/freon/AbstractOmBucketReadWriteOps.java | 228 ++ .../hadoop/ozone/freon/BaseFreonGenerator.java | 13 +- .../hadoop/ozone/freon/ChunkManagerDiskWrite.java | 1 + .../hadoop/ozone/freon/ContentGenerator.java | 1 + .../hadoop/ozone/freon/DatanodeBlockPutter.java | 1 + .../hadoop/ozone/freon/DatanodeChunkGenerator.java | 1 + .../hadoop/ozone/freon/DatanodeChunkValidator.java | 32 +- .../freon/FollowerAppendLogEntryGenerator.java | 3 +- .../java/org/apache/hadoop/ozone/freon/Freon.java | 4 +- .../ozone/freon/FreonReplicationOptions.java | 73 + .../hadoop/ozone/freon/HadoopDirTreeGenerator.java | 1 + .../ozone/freon/HadoopNestedDirGenerator.java | 1 + .../ozone/freon/LeaderAppendLogEntryGenerator.java | 3 +- .../ozone/freon/OmBucketReadWriteFileOps.java | 108 + .../ozone/freon/OmBucketReadWriteKeyOps.java | 145 ++ .../apache/hadoop/ozone/freon/OmKeyGenerator.java | 22 +- .../ozone/freon/OzoneClientKeyGenerator.java | 23 +- .../ozone/freon/OzoneClientKeyValidator.java | 2 + .../hadoop/ozone/freon/RandomKeyGenerator.java | 110 +- .../apache/hadoop/ozone/freon/S3KeyGenerator.java | 1 + .../hadoop/ozone/freon/SCMThroughputBenchmark.java | 1 + .../containergenerator/GeneratorDatanode.java | 1 + .../hadoop/ozone/shell/ReplicationOptions.java | 87 + .../ozone/shell/ShellReplicationOptions.java} | 23 +- .../hadoop/ozone/shell/bucket/BucketCommands.java | 1 + .../ozone/shell/bucket/CreateBucketHandler.java | 31 +- .../shell/bucket/SetReplicationConfigHandler.java | 54 + .../hadoop/ozone/shell/keys/CopyKeyHandler.java | 20 +- .../hadoop/ozone/shell/keys/PutKeyHandler.java | 20 +- pom.xml | 87 +- 685 files changed, 40052 insertions(+), 5758 deletions(-) diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java index f0a26df46f,f1c3d6578b..067777bb79 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java @@@ -473,55 -455,6 +474,58 @@@ public final class OzoneConsts public static final String DELEGATION_TOKEN_SERVICE = "service"; public static final String DELEGATION_TOKEN_RENEWER = "renewer"; + // EC Constants + public static final String BLOCK_GROUP_LEN_KEY_IN_PUT_BLOCK = "blockGroupLen"; ++ + public static final String OZONE_OM_RANGER_ADMIN_CREATE_USER_HTTP_ENDPOINT = + "/service/xusers/secure/users"; + + // Ideally we should use /addUsersAndGroups endpoint for add user to role, + // but it always return 405 somehow. + // https://ranger.apache.org/apidocs/resource_RoleREST.html + // #resource_RoleREST_addUsersAndGroups_PUT + public static final String OZONE_OM_RANGER_ADMIN_ROLE_ADD_USER_HTTP_ENDPOINT = + "/service/roles/roles/"; + + public static final String OZONE_OM_RANGER_ADMIN_GET_USER_HTTP_ENDPOINT = + "/service/xusers/users/?name="; + + public static final String OZONE_OM_RANGER_ADMIN_DELETE_USER_HTTP_ENDPOINT = + "/service/xusers/secure/users/id/"; + + public static final String OZONE_OM_RANGER_ADMIN_CREATE_ROLE_HTTP_ENDPOINT = + "/service/roles/roles"; + + public static final String OZONE_OM_RANGER_ADMIN_GET_ROLE_HTTP_ENDPOINT = + "/service/roles/roles/name/"; + + public static final String OZONE_OM_RANGER_ADMIN_DELETE_GROUP_HTTP_ENDPOINT = + "/service/xusers/secure/groups/id/"; + + public static final String OZONE_OM_RANGER_ADMIN_DELETE_ROLE_HTTP_ENDPOINT = + "/service/roles/roles/"; + + public static final String OZONE_OM_RANGER_ADMIN_CREATE_POLICY_HTTP_ENDPOINT = + "/service/public/v2/api/policy"; + + public static final String OZONE_OM_RANGER_ADMIN_GET_POLICY_HTTP_ENDPOINT = + "/service/public/v2/api/policy/?policyName="; + + public static final String OZONE_OM_RANGER_ADMIN_GET_POLICY_ID_HTTP_ENDPOINT = + "/service/public/v2/api/policy/?policyId="; + + public static final String OZONE_OM_RANGER_ADMIN_DELETE_POLICY_HTTP_ENDPOINT = + "/service/plugins/policies/"; + + public static final String OZONE_OM_RANGER_OZONE_SERVICE_ENDPOINT = + "/service/plugins/services/"; + + public static final String OZONE_OM_RANGER_DOWNLOAD_ENDPOINT = + "/service/plugins/secure/policies/download/cm_ozone" + + "?supportsPolicyDeltas=true&lastKnownVersion="; + + public static final String OZONE_OM_RANGER_ALL_POLICIES_ENDPOINT = + "/service/plugins/policies/service/"; + + public static final String OZONE_TENANT_RANGER_POLICY_LABEL = "OzoneTenant"; } diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 3077bc73c1,c7764e9d5b..db0882c80b --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@@ -77,8 -86,10 +86,11 @@@ import org.apache.hadoop.ozone.client.O import org.apache.hadoop.ozone.client.OzoneMultipartUploadList; import org.apache.hadoop.ozone.client.OzoneMultipartUploadPartListParts; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.TenantArgs; import org.apache.hadoop.ozone.client.VolumeArgs; + import org.apache.hadoop.ozone.client.io.BlockInputStreamFactory; + import org.apache.hadoop.ozone.client.io.BlockInputStreamFactoryImpl; + import org.apache.hadoop.ozone.client.io.ECKeyOutputStream; import org.apache.hadoop.ozone.client.io.KeyInputStream; import org.apache.hadoop.ozone.client.io.KeyOutputStream; import org.apache.hadoop.ozone.client.io.LengthInputStream; diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 291eb6f28d,bc1ed8d6f5..aa34f049f4 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@@ -312,44 -341,11 +341,52 @@@ public final class OMConfigKeys public static final int OZONE_OM_UNFLUSHED_TRANSACTION_MAX_COUNT_DEFAULT = 10000; + /** + * This configuration shall be enabled to utilize the functionality of the + * fine-grained KEY_PATH_LOCK. + */ + public static final String OZONE_OM_KEY_PATH_LOCK_ENABLED = + "ozone.om.key.path.lock.enabled"; + public static final boolean OZONE_OM_KEY_PATH_LOCK_ENABLED_DEFAULT = false; ++ + public static final String OZONE_OM_MULTITENANCY_ENABLED = + "ozone.om.multitenancy.enabled"; + public static final boolean OZONE_OM_MULTITENANCY_ENABLED_DEFAULT = false; + + /** + * Temporary configuration properties for Ranger REST use in multitenancy. + */ + public static final String OZONE_RANGER_OM_IGNORE_SERVER_CERT = + "ozone.om.ranger.ignore.cert"; + public static final boolean OZONE_RANGER_OM_IGNORE_SERVER_CERT_DEFAULT = + true; + public static final String OZONE_RANGER_OM_CONNECTION_TIMEOUT = + "ozone.om.ranger.connection.timeout"; + public static final String OZONE_RANGER_OM_CONNECTION_TIMEOUT_DEFAULT = "5s"; + public static final String OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT = + "ozone.om.ranger.connection.request.timeout"; + public static final String + OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT_DEFAULT = "5s"; + public static final String OZONE_OM_RANGER_HTTPS_ADMIN_API_USER = + "ozone.om.ranger.https.admin.api.user"; + // TODO: Note this should be removed once Ranger Java Client is in place. + // And Ranger SPNEGO auth (ranger.spnego.kerberos.principal ?) should be used + // instead. Or keep this solely for dev testing. See HDDS-5836. + public static final String OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD = + "ozone.om.ranger.https.admin.api.passwd"; + public static final String OZONE_RANGER_HTTPS_ADDRESS_KEY = + "ozone.om.ranger.https-address"; + public static final String OZONE_RANGER_SERVICE = + "ozone.om.ranger.service"; + + public static final String OZONE_OM_MULTITENANCY_RANGER_SYNC_INTERVAL + = "ozone.om.multitenancy.ranger.sync.interval"; + public static final TimeDuration + OZONE_OM_MULTITENANCY_RANGER_SYNC_INTERVAL_DEFAULT + = TimeDuration.valueOf(600, TimeUnit.SECONDS); + public static final String OZONE_OM_MULTITENANCY_RANGER_SYNC_TIMEOUT + = "ozone.om.multitenancy.ranger.sync.timeout"; + public static final TimeDuration + OZONE_OM_MULTITENANCY_RANGER_SYNC_TIMEOUT_DEFAULT + = TimeDuration.valueOf(10, TimeUnit.SECONDS); } diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java index 717df88b48,168b2c8bb4..a669f8a7df --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/exceptions/OMException.java @@@ -243,19 -243,6 +243,20 @@@ public class OMException extends IOExce LAYOUT_FEATURE_FINALIZATION_FAILED, PREPARE_FAILED, NOT_SUPPORTED_OPERATION_WHEN_PREPARED, - NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION ++ NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION, + + TENANT_NOT_FOUND, + TENANT_ALREADY_EXISTS, INVALID_TENANT_ID, + + ACCESS_ID_NOT_FOUND, + TENANT_USER_ACCESS_ID_ALREADY_EXISTS, + INVALID_TENANT_USERNAME, + INVALID_ACCESS_ID, + TENANT_AUTHORIZER_ERROR, + + VOLUME_IS_REFERENCED, + TENANT_NOT_EMPTY, + + FEATURE_NOT_ENABLED } } diff --cc hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh index 0000000000,d54c9bc89b..f46f067914 mode 000000,100755..100755 --- a/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh +++ b/hadoop-ozone/dist/src/main/compose/upgrade/upgrades/non-rolling-upgrade/1.2.1-1.3.0/callback.sh @@@ -1,0 -1,84 +1,84 @@@ + #!/usr/bin/env bash + # Licensed to the Apache Software Foundation (ASF) under one + # or more contributor license agreements. See the NOTICE file + # distributed with this work for additional information + # regarding copyright ownership. The ASF licenses this file + # to you under the Apache License, Version 2.0 (the + # "License"); you may not use this file except in compliance + # with the License. You may obtain a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, + # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + # See the License for the specific language governing permissions and + # limitations under the License. + + source "$TEST_DIR"/testlib.sh + + # Helper function, not a callback. + _check_hdds_mlvs() { + mlv="$1" + check_scm_mlv scm "$mlv" + check_dn_mlv dn1 "$mlv" + check_dn_mlv dn2 "$mlv" + check_dn_mlv dn3 "$mlv" + } + + # Helper function, not a callback. + _check_om_mlvs() { + mlv="$1" + check_om_mlv om1 "$mlv" + check_om_mlv om2 "$mlv" + check_om_mlv om3 "$mlv" + } + + setup() { + export OZONE_OM_PREPARE='true' + } + + with_old_version() { + generate old1 + validate old1 + } + + with_new_version_pre_finalized() { + _check_hdds_mlvs 2 + _check_om_mlvs 0 + + validate old1 + # HDDS-6261: overwrite the same keys intentionally + generate old1 --exclude create-volume-and-bucket + + generate new1 + validate new1 + + check_ec_is_disabled + } + + with_old_version_downgraded() { + validate old1 + validate new1 + + generate old2 + validate old2 + + # HDDS-6261: overwrite the same keys again to trigger the precondition check + # that exists <= 1.1.0 OM + generate old1 --exclude create-volume-and-bucket + } + + with_new_version_finalized() { + _check_hdds_mlvs 3 - _check_om_mlvs 2 ++ _check_om_mlvs 3 + + validate old1 + validate new1 + validate old2 + + generate new2 + validate new2 + + check_ec_is_enabled + } diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java index 65aae1e7ed,21edfe5d15..ad025d19f1 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/upgrade/TestHDDSUpgrade.java @@@ -26,8 -26,8 +26,7 @@@ import static org.apache.hadoop.hdds.pr import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY; import static org.apache.hadoop.hdds.protocol.proto.HddsProtos.NodeState.HEALTHY_READONLY; import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_DATANODE_PIPELINE_LIMIT; - import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_HA_ENABLE_KEY; import static org.apache.hadoop.hdds.scm.pipeline.Pipeline.PipelineState.OPEN; -import static org.apache.hadoop.hdds.upgrade.HDDSLayoutFeature.INITIAL_VERSION; import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_COMPLETE_FINALIZATION; import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_POST_FINALIZE_UPGRADE; import static org.apache.hadoop.ozone.upgrade.InjectedUpgradeFinalizationExecutor.UpgradeTestInjectionPoints.AFTER_PRE_FINALIZE_UPGRADE; diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index 15eda5a4b9,3b9d3df975..f9245268f1 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@@ -113,13 -116,9 +120,15 @@@ public class TestOzoneConfigurationFiel ReconServerConfigKeys.RECON_OM_SNAPSHOT_TASK_FLUSH_PARAM, OMConfigKeys.OZONE_OM_RATIS_SNAPSHOT_AUTO_TRIGGER_THRESHOLD_KEY, OMConfigKeys.OZONE_OM_HA_PREFIX, + OMConfigKeys.OZONE_OM_TRANSPORT_CLASS, - OMConfigKeys.OZONE_OM_GRPC_PORT_KEY ++ OMConfigKeys.OZONE_OM_GRPC_PORT_KEY, // TODO HDDS-2856 + OMConfigKeys.OZONE_RANGER_OM_IGNORE_SERVER_CERT, + OMConfigKeys.OZONE_RANGER_OM_CONNECTION_TIMEOUT, + OMConfigKeys.OZONE_RANGER_OM_CONNECTION_REQUEST_TIMEOUT, + OMConfigKeys.OZONE_RANGER_HTTPS_ADDRESS_KEY, + OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_USER, + OMConfigKeys.OZONE_OM_RANGER_HTTPS_ADMIN_API_PASSWD )); } } diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java index fdbf17d9a4,e38c413e74..69977648a1 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java @@@ -102,9 -108,8 +109,9 @@@ public class TestOzoneShellHA private static String testFilePathString; private static OzoneConfiguration conf = null; private static MiniOzoneCluster cluster = null; - private static OzoneShell ozoneShell = null; - private static OzoneAdmin ozoneAdminShell = null; - private static S3Shell s3Shell = null; + private OzoneShell ozoneShell = null; + private OzoneAdmin ozoneAdminShell = null; ++ private S3Shell s3Shell = null; private final ByteArrayOutputStream out = new ByteArrayOutputStream(); private final ByteArrayOutputStream err = new ByteArrayOutputStream(); @@@ -170,6 -173,8 +175,9 @@@ @Before public void setup() throws UnsupportedEncodingException { + ozoneShell = new OzoneShell(); + ozoneAdminShell = new OzoneAdmin(); ++ s3Shell = new S3Shell(); System.setOut(new PrintStream(out, false, DEFAULT_ENCODING)); System.setErr(new PrintStream(err, false, DEFAULT_ENCODING)); } diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index deb77db639,d91d8519a9..c6a6745d5f --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@@ -411,21 -356,7 +411,22 @@@ enum Status LAYOUT_FEATURE_FINALIZATION_FAILED = 72; PREPARE_FAILED = 73; NOT_SUPPORTED_OPERATION_WHEN_PREPARED = 74; + NOT_SUPPORTED_OPERATION_PRIOR_FINALIZATION = 75; + - TENANT_NOT_FOUND = 75; - TENANT_ALREADY_EXISTS = 76; - INVALID_TENANT_ID = 77; ++ TENANT_NOT_FOUND = 76; ++ TENANT_ALREADY_EXISTS = 77; ++ INVALID_TENANT_ID = 78; + - ACCESS_ID_NOT_FOUND = 78; - TENANT_USER_ACCESS_ID_ALREADY_EXISTS = 79; - INVALID_TENANT_USERNAME = 80; - INVALID_ACCESS_ID = 81; - TENANT_AUTHORIZER_ERROR = 82; ++ ACCESS_ID_NOT_FOUND = 79; ++ TENANT_USER_ACCESS_ID_ALREADY_EXISTS = 80; ++ INVALID_TENANT_USERNAME = 81; ++ INVALID_ACCESS_ID = 82; ++ TENANT_AUTHORIZER_ERROR = 83; + - VOLUME_IS_REFERENCED = 83; - TENANT_NOT_EMPTY = 84; ++ VOLUME_IS_REFERENCED = 84; ++ TENANT_NOT_EMPTY = 85; + - FEATURE_NOT_ENABLED = 85; ++ FEATURE_NOT_ENABLED = 86; } /** diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index d2848748a1,4e051875d7..ef2e8d70e9 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@@ -67,10 -69,8 +69,11 @@@ import org.apache.hadoop.hdds.protocol. import org.apache.hadoop.hdds.protocolPB.SCMSecurityProtocolClientSideTranslatorPB; import org.apache.hadoop.hdds.scm.ScmInfo; import org.apache.hadoop.hdds.scm.client.HddsClientUtils; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.Table.KeyValue; +import org.apache.hadoop.hdds.utils.db.TableIterator; import org.apache.hadoop.ozone.OzoneManagerVersion; + import org.apache.hadoop.ozone.util.OzoneNetUtils; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.hdds.scm.ha.SCMNodeInfo; import org.apache.hadoop.hdds.scm.protocol.ScmBlockLocationProtocol; @@@ -127,12 -124,9 +130,13 @@@ import org.apache.hadoop.ozone.om.helpe import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.S3VolumeContext; import org.apache.hadoop.ozone.om.helpers.ServiceInfo; import org.apache.hadoop.ozone.om.helpers.ServiceInfoEx; +import org.apache.hadoop.ozone.om.helpers.TenantStateList; +import org.apache.hadoop.ozone.om.helpers.TenantUserInfoValue; +import org.apache.hadoop.ozone.om.helpers.TenantUserList; + import org.apache.hadoop.ozone.om.lock.OzoneLockProvider; import org.apache.hadoop.ozone.om.protocol.OMInterServiceProtocol; import org.apache.hadoop.ozone.om.protocol.OMConfiguration; import org.apache.hadoop.ozone.om.protocolPB.OMInterServiceProtocolClientSideImpl; @@@ -300,6 -301,7 +314,7 @@@ public final class OzoneManager extend private final Text omRpcAddressTxt; private OzoneConfiguration configuration; private RPC.Server omRpcServer; - private GrpcOzoneManagerServer omS3gGrpcServer; ++ private GrpcOzoneManagerServer omS3gGrpcServer; private InetSocketAddress omRpcAddress; private String omId; @@@ -339,7 -340,9 +354,9 @@@ private JvmPauseMonitor jvmPauseMonitor; private final SecurityConfig secConfig; private S3SecretManager s3SecretManager; + private final boolean isOmGrpcServerEnabled; private volatile boolean isOmRpcServerRunning = false; - private volatile boolean isOmGrpcServerRunning = false; ++ private volatile boolean isOmGrpcServerRunning = false; private String omComponent; private OzoneManagerProtocolServerSideTranslatorPB omServerProtocol; @@@ -3910,21 -3741,20 +4057,21 @@@ throws IOException { Pair<String, String> resolved; - try { - if (isAclEnabled) { - InetAddress remoteIp = Server.getRemoteIp(); - resolved = resolveBucketLink(requested, new HashSet<>(), - getRemoteUser(), - remoteIp, - remoteIp != null ? remoteIp.getHostName() : - omRpcAddress.getHostName()); - } else { - resolved = resolveBucketLink(requested, new HashSet<>(), - null, null, null); + if (isAclEnabled) { - UserGroupInformation ugi = Server.getRemoteUser(); ++ UserGroupInformation ugi = getRemoteUser(); + if (getS3Auth() != null) { + ugi = UserGroupInformation.createRemoteUser( + OzoneAclUtils.accessIdToUserPrincipal(getS3Auth().getAccessId())); } - } catch (Throwable t) { - throw t; + InetAddress remoteIp = Server.getRemoteIp(); + resolved = resolveBucketLink(requested, new HashSet<>(), + ugi, + remoteIp, + remoteIp != null ? remoteIp.getHostName() : + omRpcAddress.getHostName()); + } else { + resolved = resolveBucketLink(requested, new HashSet<>(), + null, null, null); } return new ResolvedBucket(requested, resolved); } diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java index 60a888e446,7b1f4185b6..4302327cf3 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/upgrade/OMLayoutFeature.java @@@ -31,8 -31,13 +31,14 @@@ public enum OMLayoutFeature implements ////////////////////////////// ////////////////////////////// INITIAL_VERSION(0, "Initial Layout Version"), - // TODO: Make this 2 after bringing in EC - MULTITENANCY_SCHEMA(1, "Multi-Tenancy Schema"); + ERASURE_CODED_STORAGE_SUPPORT(1, "Ozone version with built in support for" + + " Erasure Coded block data storage."), + + BUCKET_LAYOUT_SUPPORT(2, "Ozone version supporting bucket " + + "layouts and introducing the FILE_SYSTEM_OPTIMIZED and OBJECT_STORE " + - "bucket layout types."); ++ "bucket layout types."), + ++ MULTITENANCY_SCHEMA(3, "Multi-Tenancy Schema"); /////////////////////////////// ///////////////////////////// // Example OM Layout Feature with Actions diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java index 4dffc39e22,ecec94592d..94b9b6d56c --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerRequestHandler.java @@@ -48,8 -48,11 +51,12 @@@ import org.apache.hadoop.ozone.om.ratis import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; + import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator; + import org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase; + import org.apache.hadoop.ozone.om.request.validation.ValidationCondition; + import org.apache.hadoop.ozone.om.request.validation.ValidationContext; import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.upgrade.DisallowedUntilLayoutVersion; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CheckVolumeAccessResponse; @@@ -78,15 -80,12 +86,17 @@@ import org.apache.hadoop.ozone.protocol import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.MultipartUploadListPartsResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; + import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OzoneFileStatusProto; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PrepareStatusResponse; + import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.RepeatedKeyInfo; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.ServiceListResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.GetS3VolumeContextResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantGetUserInfoRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantGetUserInfoResponse; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserRequest; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.TenantListUserResponse; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type; import org.apache.hadoop.ozone.security.acl.OzoneObjInfo; diff --cc hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java index 891c48ca96,e9ddc08815..ca8d32b48d --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneClientProducer.java @@@ -71,9 -66,7 +66,7 @@@ public class OzoneClientProducer @Produces public synchronized OzoneClient createClient() throws WebApplicationException, IOException { - if (client == null) { - client = createOzoneClient(); - } - client = getClient(ozoneConfiguration); ++ client = getClient(ozoneConfiguration); return client; } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
