This is an automated email from the ASF dual-hosted git repository. sodonnell pushed a commit to branch HDDS-3816-ec in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 73bddf36d009983d0e5a38c0e6043e8ae24e585e Merge: a1d7292 937a3d0 Author: S O'Donnell <[email protected]> AuthorDate: Wed Dec 8 21:07:54 2021 +0000 Updates to address compile issues after merge hadoop-hdds/client/pom.xml | 4 +- .../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 38 +- hadoop-hdds/common/pom.xml | 4 +- .../java/org/apache/hadoop/hdds/HddsUtils.java | 71 ++ .../hadoop/hdds/client/RatisReplicationConfig.java | 27 +- .../hdds/client/ReplicatedReplicationConfig.java} | 23 +- .../hadoop/hdds/client/ReplicationConfig.java | 116 ++- .../hdds/client/ReplicationConfigValidator.java | 5 +- .../hdds/client/StandaloneReplicationConfig.java | 23 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 1 + .../apache/hadoop/hdds/scm/XceiverClientSpi.java | 6 +- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 92 +- .../hadoop/hdds/utils/ResourceLimitCache.java | 7 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 6 + .../common/src/main/resources/ozone-default.xml | 18 + .../hadoop/hdds/client/TestReplicationConfig.java | 276 ++++-- .../hadoop/hdds/utils/TestResourceLimitCache.java | 46 +- hadoop-hdds/config/pom.xml | 4 +- hadoop-hdds/container-service/pom.xml | 4 +- .../container/common/helpers/ContainerUtils.java | 68 -- .../container/common/impl/HddsDispatcher.java | 4 +- .../common/statemachine/DatanodeConfiguration.java | 43 + .../common/transport/server/XceiverServerGrpc.java | 52 +- .../server/ratis/ContainerStateMachine.java | 73 +- .../container/keyvalue/KeyValueContainer.java | 4 + .../common/helpers/TestContainerUtils.java | 2 +- hadoop-hdds/docs/pom.xml | 4 +- hadoop-hdds/erasurecode/pom.xml | 4 +- hadoop-hdds/framework/pom.xml | 4 +- .../java/org/apache/hadoop/hdds/ExitManager.java | 12 +- .../scm/protocol/ScmBlockLocationProtocol.java | 2 +- hadoop-hdds/hadoop-dependency-client/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-server/pom.xml | 4 +- hadoop-hdds/hadoop-dependency-test/pom.xml | 4 +- hadoop-hdds/interface-admin/pom.xml | 4 +- .../interface-admin/src/main/resources/proto.lock | 504 ++++++++++ hadoop-hdds/interface-client/pom.xml | 4 +- .../interface-client/src/main/proto/hdds.proto | 11 + .../interface-client/src/main/resources/proto.lock | 408 +++++++- hadoop-hdds/interface-server/pom.xml | 4 +- .../interface-server/src/main/resources/proto.lock | 1024 ++++++++++++++++++++ hadoop-hdds/pom.xml | 4 +- hadoop-hdds/server-scm/pom.xml | 4 +- .../scm/container/CloseContainerEventHandler.java | 12 +- .../hdds/scm/container/ReplicationManager.java | 10 +- .../scm/container/balancer/ContainerBalancer.java | 209 ++-- .../balancer/ContainerBalancerConfiguration.java | 75 +- .../ContainerBalancerSelectionCriteria.java | 22 +- .../scm/container/balancer/FindSourceGreedy.java | 158 +++ .../scm/container/balancer/FindSourceStrategy.java | 67 ++ .../scm/container/balancer/FindTargetGreedy.java | 121 ++- .../scm/container/balancer/FindTargetStrategy.java | 32 +- .../algorithms/SCMContainerPlacementCapacity.java | 2 +- .../org/apache/hadoop/hdds/scm/ha/SCMContext.java | 23 +- .../apache/hadoop/hdds/scm/ha/SCMStateMachine.java | 7 + .../scm/pipeline/BackgroundPipelineCreator.java | 2 +- .../hdds/scm/pipeline/PipelineManagerImpl.java | 25 +- .../hdds/scm/server/SCMClientProtocolServer.java | 4 +- .../scm/server/upgrade/SCMUpgradeFinalizer.java | 5 +- .../hadoop/hdds/scm/container/MockNodeManager.java | 6 +- .../container/balancer/TestContainerBalancer.java | 74 ++ .../TestSCMContainerPlacementCapacity.java | 6 +- .../hdds/scm/node/TestContainerPlacement.java | 2 +- .../hadoop/hdds/scm/node/TestDeadNodeHandler.java | 6 + .../hadoop/hdds/scm/node/TestSCMNodeManager.java | 11 +- .../scm/pipeline/MockRatisPipelineProvider.java | 2 +- .../hdds/scm/pipeline/TestPipelineManagerImpl.java | 59 +- .../scm/pipeline/TestPipelineStateManagerImpl.java | 8 +- .../placement/TestContainerPlacement.java | 4 +- hadoop-hdds/test-utils/pom.xml | 4 +- hadoop-hdds/tools/pom.xml | 4 +- .../hdds/scm/cli/datanode/UsageInfoSubcommand.java | 41 +- hadoop-ozone/client/pom.xml | 4 +- .../org/apache/hadoop/ozone/client/BucketArgs.java | 22 +- .../apache/hadoop/ozone/client/OzoneBucket.java | 47 +- .../ozone/client/protocol/ClientProtocol.java | 1 + .../apache/hadoop/ozone/client/rpc/RpcClient.java | 62 +- .../hadoop/ozone/client/MockOmTransport.java | 8 +- .../hadoop/ozone/client/rpc/RpcClientTest.java | 217 +++++ hadoop-ozone/common/pom.xml | 4 +- .../main/java/org/apache/hadoop/ozone/OmUtils.java | 82 ++ .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 11 + .../hadoop/ozone/om/helpers/OMNodeDetails.java | 50 +- .../hadoop/ozone/om/helpers/OmBucketInfo.java | 31 +- .../ozone/om/helpers/OmMultipartKeyInfo.java | 2 +- .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 17 + .../hadoop/ozone/om/helpers/ServiceInfo.java | 41 +- .../hadoop/ozone/om/protocol/OMAdminProtocol.java | 23 +- .../hadoop/ozone/om/protocol/OMConfiguration.java | 92 ++ .../protocolPB/OMAdminProtocolClientSideImpl.java | 136 +++ .../ozone/om/protocolPB/OMAdminProtocolPB.java | 37 + ...OzoneManagerProtocolClientSideTranslatorPB.java | 4 +- hadoop-ozone/csi/pom.xml | 4 +- hadoop-ozone/datanode/pom.xml | 4 +- hadoop-ozone/dist/pom.xml | 4 +- .../dist/src/main/compose/ozone-mr/common-config | 1 - hadoop-ozone/dist/src/main/compose/ozone/README.md | 6 - .../src/main/compose/ozone/docker-compose.yaml | 10 - hadoop-ozone/dist/src/main/compose/ozone/run.sh | 4 - hadoop-ozone/dist/src/main/compose/ozone/test.sh | 29 +- .../src/main/compose/ozonesecure/docker-config | 1 - .../dist/src/main/compose/xcompat/docker-config | 1 + hadoop-ozone/dist/src/main/k8s/examples/testlib.sh | 5 +- .../main/smoketest/basic/ozone-shell-single.robot | 2 +- .../src/main/smoketest/basic/ozone-shell.robot | 15 +- .../dist/src/main/smoketest/createmrenv.robot | 2 +- .../dist/src/main/smoketest/freon/generate.robot | 9 + .../dist/src/main/smoketest/freon/validate.robot | 9 + .../dist/src/main/smoketest/ozonefs/setup.robot | 12 +- .../dist/src/main/smoketest/s3/objectdelete.robot | 6 +- .../main/smoketest/security/ozone-secure-fs.robot | 10 +- .../fault-injection-test/mini-chaos-tests/pom.xml | 4 +- .../fault-injection-test/network-tests/pom.xml | 2 +- hadoop-ozone/fault-injection-test/pom.xml | 4 +- hadoop-ozone/insight/pom.xml | 4 +- hadoop-ozone/integration-test/pom.xml | 4 +- .../hadoop/fs/ozone/TestOzoneFSInputStream.java | 4 + .../hadoop/fs/ozone/TestOzoneFileInterfaces.java | 45 + .../hadoop/fs/ozone/TestOzoneFileSystem.java | 7 +- .../fs/ozone/TestOzoneFileSystemMissingParent.java | 2 + .../fs/ozone/TestOzoneFileSystemWithLinks.java | 248 +++++ .../apache/hadoop/fs/ozone/TestOzoneFsHAURLs.java | 5 + .../hadoop/fs/ozone/contract/OzoneContract.java | 3 + .../hdds/scm/pipeline/TestMultiRaftSetup.java | 4 +- .../hadoop/hdds/upgrade/TestHDDSUpgrade.java | 2 +- .../hadoop/ozone/MiniOzoneHAClusterImpl.java | 150 ++- .../hadoop/ozone/TestOzoneConfigurationFields.java | 5 + .../rpc/TestContainerStateMachineFailures.java | 20 +- .../ozone/client/rpc/TestECKeyOutputStream.java | 4 +- .../client/rpc/TestOzoneRpcClientAbstract.java | 9 +- .../ozone/freon/TestHadoopDirTreeGenerator.java | 5 +- .../ozone/freon/TestHadoopNestedDirGenerator.java | 3 + .../apache/hadoop/ozone/om/TestBucketOwner.java | 244 +++++ .../ozone/om/TestOMStartupWithBucketLayout.java | 161 +++ .../apache/hadoop/ozone/om/TestObjectStore.java | 130 +++ .../org/apache/hadoop/ozone/om/TestOmAcls.java | 6 +- .../hadoop/ozone/om/TestOzoneManagerBootstrap.java | 184 +++- hadoop-ozone/interface-client/pom.xml | 4 +- .../src/main/proto/OMAdminProtocol.proto | 65 ++ .../src/main/proto/OmClientProtocol.proto | 4 +- .../interface-client/src/main/resources/proto.lock | 826 +++++++++++++++- hadoop-ozone/interface-storage/pom.xml | 4 +- hadoop-ozone/ozone-manager/pom.xml | 4 +- .../apache/hadoop/ozone/om/BucketManagerImpl.java | 58 ++ .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 2 +- .../apache/hadoop/ozone/om/OMPolicyProvider.java | 5 +- .../apache/hadoop/ozone/om/OMStarterInterface.java | 2 +- .../org/apache/hadoop/ozone/om/OzoneAclUtils.java | 125 +++ .../org/apache/hadoop/ozone/om/OzoneManager.java | 367 ++++--- .../hadoop/ozone/om/OzoneManagerStarter.java | 59 +- .../apache/hadoop/ozone/om/OzoneManagerUtils.java | 125 +++ .../apache/hadoop/ozone/om/ha/OMHANodeDetails.java | 6 +- .../ozone/om/ratis/OzoneManagerRatisServer.java | 24 +- .../om/ratis/utils/OzoneManagerRatisUtils.java | 219 +---- .../hadoop/ozone/om/request/OMClientRequest.java | 90 +- .../ozone/om/request/OMKeyRequestFactory.java | 139 +++ .../om/request/file/OMDirectoryCreateRequest.java | 29 +- .../ozone/om/request/file/OMFileCreateRequest.java | 13 + .../om/request/key/OMAllocateBlockRequest.java | 14 + .../ozone/om/request/key/OMKeyCommitRequest.java | 13 + .../ozone/om/request/key/OMKeyCreateRequest.java | 14 + .../ozone/om/request/key/OMKeyDeleteRequest.java | 14 + .../ozone/om/request/key/OMKeyRenameRequest.java | 14 + .../hadoop/ozone/om/request/key/OMKeyRequest.java | 5 +- .../om/request/key/OMPathsPurgeRequestWithFSO.java | 5 +- .../ozone/om/request/key/acl/OMKeyAclRequest.java | 25 +- .../om/request/key/acl/OMKeyAclRequestWithFSO.java | 7 +- .../om/request/key/acl/OMKeyAddAclRequest.java | 5 +- .../request/key/acl/OMKeyAddAclRequestWithFSO.java | 5 +- .../om/request/key/acl/OMKeyRemoveAclRequest.java | 4 +- .../key/acl/OMKeyRemoveAclRequestWithFSO.java | 6 +- .../om/request/key/acl/OMKeySetAclRequest.java | 4 +- .../request/key/acl/OMKeySetAclRequestWithFSO.java | 5 +- .../S3InitiateMultipartUploadRequest.java | 15 +- .../multipart/S3MultipartUploadAbortRequest.java | 15 + .../S3MultipartUploadCommitPartRequest.java | 15 + .../S3MultipartUploadCompleteRequest.java | 16 +- .../protocolPB/OMAdminProtocolServerSideImpl.java | 66 ++ .../protocolPB/OzoneManagerRequestHandler.java | 2 +- .../hadoop/ozone/protocolPB/RequestHandler.java | 4 +- .../hadoop/ozone/om/TestOzoneManagerStarter.java | 4 +- .../ozone/om/request/TestOMRequestUtils.java | 4 +- .../ozone/om/request/key/TestOMKeyAclRequest.java | 10 +- .../om/request/key/TestOMKeyAclRequestWithFSO.java | 16 +- .../s3/multipart/TestS3MultipartRequest.java | 7 +- .../ozone/om/response/TestCleanupTableInfo.java | 7 +- hadoop-ozone/ozonefs-common/pom.xml | 4 +- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 16 +- .../ozone/BasicRootedOzoneClientAdapterImpl.java | 14 +- .../apache/hadoop/fs/ozone/OzoneClientUtils.java | 67 ++ hadoop-ozone/ozonefs-hadoop2/pom.xml | 4 +- hadoop-ozone/ozonefs-hadoop3/pom.xml | 4 +- hadoop-ozone/ozonefs-shaded/pom.xml | 4 +- hadoop-ozone/ozonefs/pom.xml | 4 +- hadoop-ozone/pom.xml | 4 +- hadoop-ozone/recon-codegen/pom.xml | 2 +- .../recon/schema/ContainerSchemaDefinition.java | 3 +- hadoop-ozone/recon/pom.xml | 2 +- .../codec/ContainerReplicaHistoryListCodec.java | 45 +- .../ozone/recon/fsck/ContainerHealthStatus.java | 17 +- .../persistence/ContainerHealthSchemaManager.java | 12 +- .../ozone/recon/persistence/ContainerHistory.java | 9 +- .../ozone/recon/scm/ContainerReplicaHistory.java | 27 +- .../recon/scm/ContainerReplicaHistoryList.java | 32 +- .../ozone/recon/scm/ReconContainerManager.java | 21 +- .../scm/ReconStorageContainerManagerFacade.java | 5 +- .../impl/ReconContainerMetadataManagerImpl.java | 4 +- .../ozone/recon/spi/impl/ReconDBDefinition.java | 13 +- .../ozone/recon/api/TestContainerEndpoint.java | 26 +- .../ozone/recon/fsck/TestContainerHealthTask.java | 16 +- .../ozone/recon/scm/TestReconContainerManager.java | 13 +- .../ozone/recon/scm/TestReconPipelineManager.java | 5 + hadoop-ozone/s3gateway/pom.xml | 4 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 5 + .../hadoop/ozone/s3/endpoint/ObjectEndpoint.java | 9 +- .../apache/hadoop/ozone/s3/util/S3StorageType.java | 22 +- .../s3/endpoint/TestAbortMultipartUpload.java | 2 + .../s3/endpoint/TestInitiateMultipartUpload.java | 2 + .../hadoop/ozone/s3/endpoint/TestListParts.java | 2 + .../s3/endpoint/TestMultipartUploadComplete.java | 2 + .../s3/endpoint/TestMultipartUploadWithCopy.java | 2 + .../hadoop/ozone/s3/endpoint/TestObjectDelete.java | 2 + .../hadoop/ozone/s3/endpoint/TestObjectGet.java | 2 + .../hadoop/ozone/s3/endpoint/TestObjectHead.java | 2 + .../hadoop/ozone/s3/endpoint/TestObjectPut.java | 2 + .../hadoop/ozone/s3/endpoint/TestPartUpload.java | 2 + .../ozone/s3/endpoint/TestPermissionCheck.java | 4 + hadoop-ozone/tools/pom.xml | 4 +- .../apache/hadoop/ozone/debug/PrefixParser.java | 8 +- .../hadoop/ozone/freon/DatanodeChunkGenerator.java | 8 +- .../hadoop/ozone/freon/DatanodeChunkValidator.java | 155 ++- .../java/org/apache/hadoop/ozone/freon/Freon.java | 3 +- .../hadoop/ozone/freon/SCMThroughputBenchmark.java | 908 +++++++++++++++++ .../ozone/genesis/BenchMarkContainerStateMap.java | 2 +- .../ozone/shell/bucket/CreateBucketHandler.java | 19 +- .../hadoop/ozone/shell/keys/CopyKeyHandler.java | 18 +- .../hadoop/ozone/shell/keys/PutKeyHandler.java | 18 +- pom.xml | 8 +- 238 files changed, 8854 insertions(+), 1532 deletions(-) diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java index a521195,61aa7fa..8e368fc --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/client/ReplicationConfig.java @@@ -64,45 -71,15 +71,37 @@@ public interface ReplicationConfig } static ReplicationConfig getDefault(ConfigurationSource config) { - String replication = config.get(OzoneConfigKeys.OZONE_REPLICATION); - String replType = config.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE); - ReplicationConfig replicationConfig = null; - if (replication != null && replType != null) { - replicationConfig = ReplicationConfig - .fromTypeAndString(ReplicationType.valueOf(replType), replication); - } - return replicationConfig; + String replication = + config.get(OZONE_REPLICATION, OZONE_REPLICATION_DEFAULT); + return parse(null, replication, config); } + /** + * Helper method to serialize from proto. + * <p> + * This uses either the old type/factor or the new ecConfig depends on the + * type. + */ + static ReplicationConfig fromProto( + HddsProtos.ReplicationType type, + HddsProtos.ReplicationFactor factor, + HddsProtos.ECReplicationConfig ecConfig) { + switch (type) { + case EC: + return new ECReplicationConfig(ecConfig); + case RATIS: + case STAND_ALONE: - return fromTypeAndFactor(type, factor); ++ return fromProtoTypeAndFactor(type, factor); + default: + throw new UnsupportedOperationException( + "Not supported replication: " + type); + } + } + static HddsProtos.ReplicationFactor getLegacyFactor( ReplicationConfig replicationConfig) { - if (replicationConfig instanceof RatisReplicationConfig) { - return ((RatisReplicationConfig) replicationConfig) - .getReplicationFactor(); - } else if (replicationConfig instanceof StandaloneReplicationConfig) { - return ((StandaloneReplicationConfig) replicationConfig) + if (replicationConfig instanceof ReplicatedReplicationConfig) { + return ((ReplicatedReplicationConfig) replicationConfig) .getReplicationFactor(); } throw new UnsupportedOperationException( @@@ -113,39 -91,71 +113,73 @@@ /** * Create new replication config with adjusted replication factor. * <p> - * Used by hadoop file system. Some replication scheme (like EC) may not + * Used by hadoop file system. Some replication schemes (like EC) may not * support changing the replication. + * <p> + * Based on the provided configuration the adjusted ReplicationConfig is + * validated against the ozone.replication.allowed-configs property, and if + * the new config is not allowed the method throws an + * IllegalArgumentException. */ static ReplicationConfig adjustReplication( - ReplicationConfig replicationConfig, short replication) { - switch (replicationConfig.getReplicationType()) { - case RATIS: - return new RatisReplicationConfig( - org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor - .valueOf(replication)); - case STAND_ALONE: - return new StandaloneReplicationConfig( - org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor - .valueOf(replication)); - default: - return replicationConfig; - } + ReplicationConfig config, short replication, ConfigurationSource conf) { + return parse( + ReplicationType.valueOf(config.getReplicationType().toString()), + Short.toString(replication), conf); } - static ReplicationConfig fromTypeAndString(ReplicationType replicationType, - String replication) { - switch (replicationType) { + /** + * Parses the string representation of the replication configuration that is + * defined by the ReplicationType parameter. + * The configuration object is necessary to check if the parsed + * ReplicationConfig object is allowed based on the + * ozone.replication.allowed-configs property. + * @param type the ReplicationType to parse from the replication string + * @param replication the replication String that for example contains the + * replication factor for RATIS replication. + * @param config the current Ozone configuration to apply validation on the + * parsed object. + * @return a validated ReplicationConfig object that is allowed based on the + * system's configuration. + * @throws IllegalArgumentException if the parsed ReplicationConfig is not + * allowed by the ozone.replication.allowed-configs property, or + * if the give replication type or replication can not be parsed. + * @throws NullPointerException if the ReplicationConfig was not created + * for the type. + */ + static ReplicationConfig parse(ReplicationType type, String replication, + ConfigurationSource config) { + if (type == null) { + type = ReplicationType.valueOf( + config.get(OZONE_REPLICATION_TYPE, OZONE_REPLICATION_TYPE_DEFAULT)); + } + replication = Objects.toString(replication, + config.get(OZONE_REPLICATION, OZONE_REPLICATION_DEFAULT)); + + ReplicationConfig replicationConfig; + switch (type) { case RATIS: - return new RatisReplicationConfig(replication); case STAND_ALONE: - return new StandaloneReplicationConfig(replication); + ReplicationFactor factor; + try { + factor = ReplicationFactor.valueOf(Integer.parseInt(replication)); + } catch (NumberFormatException ex) { + factor = ReplicationFactor.valueOf(replication); + } + replicationConfig = fromTypeAndFactor(type, factor); + break; + case EC: + return new ECReplicationConfig(replication); default: - throw new UnsupportedOperationException( - "String based replication config initialization is not supported for " - + replicationType); + throw new RuntimeException("Replication type" + type + " can not" + + "be parsed."); } + + ReplicationConfigValidator validator = + config.getObject(ReplicationConfigValidator.class); + validator.validate(replicationConfig); + + return replicationConfig; } /** diff --cc hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java index 2163539,044f151..2d14d4e --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/hdds/scm/pipeline/Pipeline.java @@@ -286,61 -261,11 +286,15 @@@ public final class Pipeline return replicationConfig; } - public static Pipeline getFromProtobuf(HddsProtos.Pipeline pipeline) - throws UnknownPipelineStateException { - Preconditions.checkNotNull(pipeline, "Pipeline is null"); - - Map<DatanodeDetails, Integer> nodes = new LinkedHashMap<>(); - int index = 0; - int repIndexListLength = pipeline.getMemberReplicaIndexesCount(); - for (DatanodeDetailsProto member : pipeline.getMembersList()) { - int repIndex = 0; - if (index < repIndexListLength) { - repIndex = pipeline.getMemberReplicaIndexes(index); - } - nodes.put(DatanodeDetails.getFromProtoBuf(member), repIndex); - index++; - } - UUID leaderId = null; - if (pipeline.hasLeaderID128()) { - HddsProtos.UUID uuid = pipeline.getLeaderID128(); - leaderId = new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits()); - } else if (pipeline.hasLeaderID() && - StringUtils.isNotEmpty(pipeline.getLeaderID())) { - leaderId = UUID.fromString(pipeline.getLeaderID()); - } - - UUID suggestedLeaderId = null; - if (pipeline.hasSuggestedLeaderID()) { - HddsProtos.UUID uuid = pipeline.getSuggestedLeaderID(); - suggestedLeaderId = - new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits()); - } - - final ReplicationConfig config = ReplicationConfig - .fromProto(pipeline.getType(), pipeline.getFactor(), - pipeline.getEcReplicationConfig()); - return new Builder().setId(PipelineID.getFromProtobuf(pipeline.getId())) - .setReplicationConfig(config) - .setState(PipelineState.fromProtobuf(pipeline.getState())) - .setNodes(new ArrayList<>(nodes.keySet())) - .setReplicaIndexes(nodes) - .setLeaderId(leaderId) - .setSuggestedLeaderId(suggestedLeaderId) - .setNodesInOrder(pipeline.getMemberOrdersList()) - .setCreateTimestamp(pipeline.getCreationTimeStamp()) - .build(); - } - public HddsProtos.Pipeline getProtobufMessage(int clientVersion) throws UnknownPipelineStateException { + List<HddsProtos.DatanodeDetailsProto> members = new ArrayList<>(); + List<Integer> memberReplicaIndexes = new ArrayList<>(); + for (DatanodeDetails dn : nodeStatus.keySet()) { members.add(dn.toProto(clientVersion)); + memberReplicaIndexes.add(replicaIndexes.getOrDefault(dn, 0)); } HddsProtos.Pipeline.Builder builder = HddsProtos.Pipeline.newBuilder() @@@ -394,6 -313,43 +348,52 @@@ return builder.build(); } + public static Pipeline getFromProtobuf(HddsProtos.Pipeline pipeline) + throws UnknownPipelineStateException { + Preconditions.checkNotNull(pipeline, "Pipeline is null"); + - List<DatanodeDetails> nodes = new ArrayList<>(); ++ Map<DatanodeDetails, Integer> nodes = new LinkedHashMap<>(); ++ int index = 0; ++ int repIndexListLength = pipeline.getMemberReplicaIndexesCount(); + for (DatanodeDetailsProto member : pipeline.getMembersList()) { - nodes.add(DatanodeDetails.getFromProtoBuf(member)); ++ int repIndex = 0; ++ if (index < repIndexListLength) { ++ repIndex = pipeline.getMemberReplicaIndexes(index); ++ } ++ nodes.put(DatanodeDetails.getFromProtoBuf(member), repIndex); ++ index++; + } + UUID leaderId = null; + if (pipeline.hasLeaderID128()) { + HddsProtos.UUID uuid = pipeline.getLeaderID128(); + leaderId = new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits()); + } else if (pipeline.hasLeaderID() && + StringUtils.isNotEmpty(pipeline.getLeaderID())) { + leaderId = UUID.fromString(pipeline.getLeaderID()); + } + + UUID suggestedLeaderId = null; + if (pipeline.hasSuggestedLeaderID()) { + HddsProtos.UUID uuid = pipeline.getSuggestedLeaderID(); + suggestedLeaderId = + new UUID(uuid.getMostSigBits(), uuid.getLeastSigBits()); + } + + final ReplicationConfig config = ReplicationConfig - .fromProtoTypeAndFactor(pipeline.getType(), pipeline.getFactor()); ++ .fromProto(pipeline.getType(), pipeline.getFactor(), ++ pipeline.getEcReplicationConfig()); + return new Builder().setId(PipelineID.getFromProtobuf(pipeline.getId())) + .setReplicationConfig(config) + .setState(PipelineState.fromProtobuf(pipeline.getState())) - .setNodes(nodes) ++ .setNodes(new ArrayList<>(nodes.keySet())) ++ .setReplicaIndexes(nodes) + .setLeaderId(leaderId) + .setSuggestedLeaderId(suggestedLeaderId) + .setNodesInOrder(pipeline.getMemberOrdersList()) + .setCreateTimestamp(pipeline.getCreationTimeStamp()) + .build(); + } + @Override public boolean equals(Object o) { if (this == o) { diff --cc hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfig.java index a85415e,9adf8f7..6e6f7b5 --- a/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfig.java +++ b/hadoop-hdds/common/src/test/java/org/apache/hadoop/hdds/client/TestReplicationConfig.java @@@ -27,13 -33,41 +33,40 @@@ import static org.junit.Assert.assertTh /** * Test replicationConfig. */ + @RunWith(Parameterized.class) public class TestReplicationConfig { + @SuppressWarnings("checkstyle:VisibilityModifier") + @Parameterized.Parameter() + public String type; + + @SuppressWarnings("checkstyle:VisibilityModifier") + @Parameterized.Parameter(1) + public String factor; + + @SuppressWarnings("checkstyle:VisibilityModifier") + @Parameterized.Parameter(2) + public Class<?> replicationConfigClass; + + @Parameterized.Parameters(name = "{0}/{1}") + public static Object[][] parameters() { + return new Object[][] { + {"RATIS", "ONE", RatisReplicationConfig.class }, + {"RATIS", "THREE", RatisReplicationConfig.class}, + {"STAND_ALONE", "ONE", StandaloneReplicationConfig.class}, + {"STAND_ALONE", "THREE", StandaloneReplicationConfig.class} + }; + } + @Test - public void testGetDefaultShouldCreateReplicationConfigFromDefaultConf() { + public void testGetDefaultShouldReturnNullIfNotSetClientSide() { OzoneConfiguration conf = new OzoneConfiguration(); + ReplicationConfig replicationConfig = ReplicationConfig.getDefault(conf); - Assert.assertNull(replicationConfig); - + validate(replicationConfig, + org.apache.hadoop.hdds.client.ReplicationType.RATIS, + org.apache.hadoop.hdds.client.ReplicationFactor.THREE, + RatisReplicationConfig.class); } @Test diff --cc hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java index 521d563,032705d..b0ed3be --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/helpers/ContainerUtils.java @@@ -46,8 -46,6 +46,7 @@@ import org.apache.hadoop.ozone.OzoneCon import org.apache.hadoop.ozone.container.common.impl.ContainerData; import org.apache.hadoop.ozone.container.common.impl.ContainerDataYaml; import org.apache.hadoop.ozone.container.common.impl.ContainerSet; +import org.apache.hadoop.ozone.container.keyvalue.KeyValueContainerData; - import org.apache.ratis.thirdparty.com.google.protobuf.ByteString; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; diff --cc hadoop-hdds/erasurecode/pom.xml index 40149fe,dbf5df3..ca2e3cb --- a/hadoop-hdds/erasurecode/pom.xml +++ b/hadoop-hdds/erasurecode/pom.xml @@@ -19,14 -19,13 +19,14 @@@ https://maven.apache.org/xsd/maven-4.0. <modelVersion>4.0.0</modelVersion> <parent> <groupId>org.apache.ozone</groupId> - <artifactId>ozone</artifactId> + <artifactId>hdds</artifactId> - <version>1.2.0-SNAPSHOT</version> + <version>1.3.0-SNAPSHOT</version> </parent> - <artifactId>ozone-client</artifactId> + <artifactId>hdds-erasurecode</artifactId> - <version>1.2.0-SNAPSHOT</version> + <version>1.3.0-SNAPSHOT</version> - <description>Apache Ozone Client</description> - <name>Apache Ozone Client</name> + <description>Apache Ozone Distributed Data Store Earsurecode utils + </description> + <name>Apache Ozone HDDS Erasurecode</name> <packaging>jar</packaging> <dependencies> diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java index 62f61f6,43d28f0..6876059 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/BucketArgs.java @@@ -80,15 -80,15 +82,16 @@@ public final class BucketArgs * @param sourceBucket * @param quotaInBytes Bucket quota in bytes. * @param quotaInNamespace Bucket quota in counts. - * @param bucketLayout Bucket Layouts. + * @param bucketLayout bucket layout. + * @param owner owner of the bucket. + * @param defaultReplicationConfig default replication config. */ @SuppressWarnings("parameternumber") private BucketArgs(Boolean versioning, StorageType storageType, List<OzoneAcl> acls, Map<String, String> metadata, String bucketEncryptionKey, String sourceVolume, String sourceBucket, long quotaInBytes, long quotaInNamespace, BucketLayout bucketLayout, - DefaultReplicationConfig defaultReplicationConfig) { - String owner) { ++ String owner, DefaultReplicationConfig defaultReplicationConfig) { this.acls = acls; this.versioning = versioning; this.storageType = storageType; @@@ -99,7 -99,7 +102,8 @@@ this.quotaInBytes = quotaInBytes; this.quotaInNamespace = quotaInNamespace; this.bucketLayout = bucketLayout; + this.owner = owner; + this.defaultReplicationConfig = defaultReplicationConfig; } /** @@@ -205,7 -204,7 +216,8 @@@ private long quotaInBytes; private long quotaInNamespace; private BucketLayout bucketLayout; + private String owner; + private DefaultReplicationConfig defaultReplicationConfig; public Builder() { metadata = new HashMap<>(); @@@ -263,13 -262,11 +275,17 @@@ return this; } + public BucketArgs.Builder setOwner(String ownerName) { + owner = ownerName; + return this; + } + + public BucketArgs.Builder setDefaultReplicationConfig( + DefaultReplicationConfig defaultReplConfig) { + defaultReplicationConfig = defaultReplConfig; + return this; + } + - /** * Constructs the BucketArgs. * @return instance of BucketArgs. @@@ -277,7 -274,7 +293,7 @@@ public BucketArgs build() { return new BucketArgs(versioning, storageType, acls, metadata, bucketEncryptionKey, sourceVolume, sourceBucket, quotaInBytes, - quotaInNamespace, bucketLayout, defaultReplicationConfig); - quotaInNamespace, bucketLayout, owner); ++ quotaInNamespace, bucketLayout, owner, defaultReplicationConfig); } } } diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java index 37098a9,23cf922..38ee2a7 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/OzoneBucket.java @@@ -223,39 -225,22 +227,54 @@@ public class OzoneBucket extends WithMe this.bucketLayout = bucketLayout; } + @SuppressWarnings("parameternumber") + public OzoneBucket(ConfigurationSource conf, ClientProtocol proxy, + String volumeName, String bucketName, StorageType storageType, + Boolean versioning, long creationTime, long modificationTime, + Map<String, String> metadata, String encryptionKeyName, + String sourceVolume, String sourceBucket, long usedBytes, + long usedNamespace, long quotaInBytes, long quotaInNamespace, - BucketLayout bucketLayout, ++ BucketLayout bucketLayout, String owner, + DefaultReplicationConfig defaultReplicationConfig) { + this(conf, proxy, volumeName, bucketName, storageType, versioning, + creationTime, modificationTime, metadata, encryptionKeyName, + sourceVolume, sourceBucket, usedBytes, usedNamespace, quotaInBytes, - quotaInNamespace); ++ quotaInNamespace, bucketLayout, owner); + this.bucketLayout = bucketLayout; + if (defaultReplicationConfig != null) { + this.defaultReplication = + defaultReplicationConfig.getType() == ReplicationType.EC ? + defaultReplicationConfig.getEcReplicationConfig() : + ReplicationConfig + .fromTypeAndFactor(defaultReplicationConfig.getType(), + defaultReplicationConfig.getFactor()); + } else { + // This can happen when talk to old server. So, using old client side + // defaults. - this.defaultReplication = ReplicationConfig.fromTypeAndString( ++ this.defaultReplication = ReplicationConfig.parse( + ReplicationType.valueOf( + conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE, + OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT)), + conf.get(OzoneConfigKeys.OZONE_REPLICATION, - OzoneConfigKeys.OZONE_REPLICATION_DEFAULT)); ++ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT), conf); + } + } + @SuppressWarnings("checkstyle:ParameterNumber") + public OzoneBucket(ConfigurationSource conf, ClientProtocol proxy, + String volumeName, String bucketName, StorageType storageType, + Boolean versioning, long creationTime, long modificationTime, + Map<String, String> metadata, String encryptionKeyName, + String sourceVolume, String sourceBucket, long usedBytes, + long usedNamespace, long quotaInBytes, long quotaInNamespace, + BucketLayout bucketLayout, String owner) { + this(conf, proxy, volumeName, bucketName, storageType, versioning, + creationTime, modificationTime, metadata, encryptionKeyName, + sourceVolume, sourceBucket, usedBytes, usedNamespace, quotaInBytes, + quotaInNamespace, bucketLayout); + this.owner = owner; + } + /** * Constructs OzoneBucket instance. * @param conf Configuration object. diff --cc hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java index 7fdb8fc,9ca8693..8a5ab08 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java @@@ -510,15 -549,10 +553,16 @@@ public class RpcClient implements Clien builder.setBucketEncryptionKey(bek); } + DefaultReplicationConfig defaultReplicationConfig = + bucketArgs.getDefaultReplicationConfig(); + if (defaultReplicationConfig != null) { + builder.setDefaultReplicationConfig(defaultReplicationConfig); + } + - LOG.info("Creating Bucket: {}/{}, with Versioning {} and " + - "Storage Type set to {} and Encryption set to {} ", - volumeName, bucketName, isVersionEnabled, storageType, bek != null); + LOG.info("Creating Bucket: {}/{}, with {} as owner and Versioning {} and " + + "Storage Type set to {} and Encryption set to {} ", + volumeName, bucketName, owner, isVersionEnabled, + storageType, bek != null); ozoneManagerClient.createBucket(builder.build()); } @@@ -759,7 -778,7 +803,8 @@@ bucketInfo.getQuotaInBytes(), bucketInfo.getQuotaInNamespace(), bucketInfo.getBucketLayout(), - bucketInfo.getOwner() ++ bucketInfo.getOwner(), + bucketInfo.getDefaultReplicationConfig() ); } diff --cc hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockOmTransport.java index 0a2fc41,17a7f6b..be16e1f --- a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockOmTransport.java +++ b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockOmTransport.java @@@ -17,12 -17,6 +17,13 @@@ */ package org.apache.hadoop.ozone.client; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; ++import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.om.protocolPB.OmTransport; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @@@ -255,28 -189,9 +253,26 @@@ public class MockOmTransport implement } private InfoBucketResponse infoBucket(InfoBucketRequest infoBucketRequest) { + BucketInfo bucketInfo = buckets.get(infoBucketRequest.getVolumeName()) + .get(infoBucketRequest.getBucketName()); + if(!bucketInfo.hasDefaultReplicationConfig()) { + final ReplicationConfig replicationConfig = ReplicationConfig - .fromTypeAndString(ReplicationType - .valueOf(OZONE_SERVER_DEFAULT_REPLICATION_TYPE_DEFAULT), - OZONE_SERVER_DEFAULT_REPLICATION_DEFAULT); ++ .getDefault(new OzoneConfiguration()); + + bucketInfo = bucketInfo.toBuilder().setDefaultReplicationConfig( + new DefaultReplicationConfig( + ReplicationType.fromProto(replicationConfig.getReplicationType()), + replicationConfig + .getReplicationType() != HddsProtos.ReplicationType.EC ? + ReplicationFactor + .valueOf(replicationConfig.getRequiredNodes()) : + null, replicationConfig + .getReplicationType() == HddsProtos.ReplicationType.EC ? + (ECReplicationConfig) replicationConfig : + null).toProto()).build(); + } return InfoBucketResponse.newBuilder() - .setBucketInfo(buckets.get(infoBucketRequest.getVolumeName()) - .get(infoBucketRequest.getBucketName())) + .setBucketInfo(bucketInfo) .build(); } diff --cc hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java index a1da197,786bb74..a843490 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmBucketInfo.java @@@ -118,8 -109,8 +120,10 @@@ public final class OmBucketInfo extend * @param usedBytes - Bucket Quota Usage in bytes. * @param quotaInBytes Bucket quota in bytes. * @param quotaInNamespace Bucket quota in counts. - * @param bucketLayout Bucket Layout. + * @param bucketLayout bucket layout. + * @param owner owner of the bucket. + * @param defaultReplicationConfig default replication config. ++ * @param bucketLayout Bucket Layout. */ @SuppressWarnings("checkstyle:ParameterNumber") private OmBucketInfo(String volumeName, @@@ -140,7 -131,7 +144,8 @@@ long quotaInBytes, long quotaInNamespace, BucketLayout bucketLayout, - String owner) { ++ String owner, + DefaultReplicationConfig defaultReplicationConfig) { this.volumeName = volumeName; this.bucketName = bucketName; this.acls = acls; @@@ -159,7 -150,7 +164,8 @@@ this.quotaInBytes = quotaInBytes; this.quotaInNamespace = quotaInNamespace; this.bucketLayout = bucketLayout; + this.owner = owner; + this.defaultReplicationConfig = defaultReplicationConfig; } /** @@@ -391,20 -372,7 +401,21 @@@ .setQuotaInBytes(quotaInBytes) .setQuotaInNamespace(quotaInNamespace) .setBucketLayout(bucketLayout) - .setOwner(owner); ++ .setOwner(owner) + .setDefaultReplicationConfig(defaultReplicationConfig); + } + + public void setDefaultReplicationConfig(ReplicationConfig replicationConfig) { + this.defaultReplicationConfig = new DefaultReplicationConfig( + ReplicationType.fromProto(replicationConfig.getReplicationType()), + replicationConfig + .getReplicationType() == HddsProtos.ReplicationType.EC ? + null : + ReplicationFactor.valueOf(replicationConfig.getRequiredNodes()), + replicationConfig + .getReplicationType() == HddsProtos.ReplicationType.EC ? + ((ECReplicationConfig) replicationConfig) : + null); } /** @@@ -429,7 -397,7 +440,8 @@@ private long quotaInBytes; private long quotaInNamespace; private BucketLayout bucketLayout; + private String owner; + private DefaultReplicationConfig defaultReplicationConfig; public Builder() { //Default values @@@ -553,12 -521,11 +565,17 @@@ return this; } + public Builder setOwner(String ownerName) { + this.owner = ownerName; + return this; + } + + public Builder setDefaultReplicationConfig( + DefaultReplicationConfig defaultReplConfig) { + this.defaultReplicationConfig = defaultReplConfig; + return this; + } + /** * Constructs the OmBucketInfo. * @return instance of OmBucketInfo. @@@ -572,8 -539,8 +589,8 @@@ return new OmBucketInfo(volumeName, bucketName, acls, isVersionEnabled, storageType, creationTime, modificationTime, objectID, updateID, metadata, bekInfo, sourceVolume, sourceBucket, usedBytes, - usedNamespace, quotaInBytes, quotaInNamespace, bucketLayout, - usedNamespace, quotaInBytes, quotaInNamespace, bucketLayout, owner); ++ usedNamespace, quotaInBytes, quotaInNamespace, bucketLayout, owner, + defaultReplicationConfig); } } @@@ -721,8 -686,8 +744,9 @@@ Objects.equals(sourceVolume, that.sourceVolume) && Objects.equals(sourceBucket, that.sourceBucket) && Objects.equals(metadata, that.metadata) && - Objects.equals(bekInfo, that.bekInfo) && Objects - .equals(defaultReplicationConfig, this.defaultReplicationConfig); + Objects.equals(bekInfo, that.bekInfo) && - Objects.equals(owner, that.owner); ++ Objects.equals(owner, that.owner) && ++ Objects.equals(defaultReplicationConfig, this.defaultReplicationConfig); } @Override @@@ -750,7 -715,7 +774,8 @@@ ", quotaInBytes=" + quotaInBytes + ", quotaInNamespace=" + quotaInNamespace + ", bucketLayout=" + bucketLayout + + ", owner=" + owner + + ", defaultReplicationConfig=" + defaultReplicationConfig + '}'; } } diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index 55715bd,8462caa..4de0f00 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@@ -66,11 -65,10 +66,15 @@@ public class TestOzoneConfigurationFiel ".duration"); // Deprecated config configurationPropsToSkipCompare .add(ScmConfig.ConfigStrings.HDDS_SCM_INIT_DEFAULT_LAYOUT_VERSION); + // Currently replication and type configs moved to server side. + configurationPropsToSkipCompare + .add(OzoneConfigKeys.OZONE_REPLICATION); + configurationPropsToSkipCompare + .add(OzoneConfigKeys.OZONE_REPLICATION_TYPE); + configurationPropsToSkipCompare + .add(OzoneConfigKeys.OZONE_OM_CLIENT_PROTOCOL_VERSION_KEY); + configurationPropsToSkipCompare + .add(OzoneConfigKeys.OZONE_OM_CLIENT_PROTOCOL_VERSION); // This property is tested in TestHttpServer2 instead xmlPropsToSkipCompare.add(HttpServer2.HTTP_IDLE_TIMEOUT_MS_KEY); addPropertiesNotInXml(); diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java index 3549532,0000000..73e02a5 mode 100644,000000..100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestECKeyOutputStream.java @@@ -1,323 -1,0 +1,325 @@@ +/** + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ +package org.apache.hadoop.ozone.client.rpc; + +import org.apache.hadoop.conf.StorageUnit; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.client.ECReplicationConfig; +import org.apache.hadoop.hdds.client.RatisReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationType; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.DatanodeDetails; +import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos; ++import org.apache.hadoop.hdds.protocol.proto.HddsProtos; +import org.apache.hadoop.hdds.scm.OzoneClientConfig; +import org.apache.hadoop.ozone.MiniOzoneCluster; +import org.apache.hadoop.ozone.OzoneConfigKeys; +import org.apache.hadoop.ozone.client.BucketArgs; +import org.apache.hadoop.ozone.client.ObjectStore; +import org.apache.hadoop.ozone.client.OzoneBucket; +import org.apache.hadoop.ozone.client.OzoneClient; +import org.apache.hadoop.ozone.client.OzoneClientFactory; +import org.apache.hadoop.ozone.client.OzoneKey; +import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.ECKeyOutputStream; +import org.apache.hadoop.ozone.client.io.KeyOutputStream; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; +import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.container.TestHelper; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.BeforeClass; +import org.junit.Test; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_SCM_WATCHER_TIMEOUT; +import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL; + +/** + * Tests key output stream. + */ +public class TestECKeyOutputStream { + private static MiniOzoneCluster cluster; + private static OzoneConfiguration conf = new OzoneConfiguration(); + private static OzoneClient client; + private static ObjectStore objectStore; + private static int chunkSize; + private static int flushSize; + private static int maxFlushSize; + private static int blockSize; + private static String volumeName; + private static String bucketName; + private static String keyString; + private static int dataBlocks = 3; + private static int inputSize = dataBlocks * chunkSize; + private static byte[][] inputChunks = new byte[dataBlocks][chunkSize]; + + /** + * Create a MiniDFSCluster for testing. + */ + @BeforeClass + public static void init() throws Exception { + chunkSize = 1024; + flushSize = 2 * chunkSize; + maxFlushSize = 2 * flushSize; + blockSize = 2 * maxFlushSize; + + OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class); + clientConfig.setChecksumType(ContainerProtos.ChecksumType.NONE); + clientConfig.setStreamBufferFlushDelay(false); + conf.setFromObject(clientConfig); + + conf.setTimeDuration(HDDS_SCM_WATCHER_TIMEOUT, 1000, TimeUnit.MILLISECONDS); + // If SCM detects dead node too quickly, then container would be moved to + // closed state and all in progress writes will get exception. To avoid + // that, we are just keeping higher timeout and none of the tests depending + // on deadnode detection timeout currently. + conf.setTimeDuration(OZONE_SCM_STALENODE_INTERVAL, 300, TimeUnit.SECONDS); + conf.setTimeDuration("hdds.ratis.raft.server.rpc.slowness.timeout", 300, + TimeUnit.SECONDS); + conf.setTimeDuration( + "hdds.ratis.raft.server.notification.no-leader.timeout", 300, + TimeUnit.SECONDS); + conf.setQuietMode(false); + conf.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 4, + StorageUnit.MB); + + cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(10) + .setTotalPipelineNumLimit(10).setBlockSize(blockSize) + .setChunkSize(chunkSize).setStreamBufferFlushSize(flushSize) + .setStreamBufferMaxSize(maxFlushSize) + .setStreamBufferSizeUnit(StorageUnit.BYTES).build(); + cluster.waitForClusterToBeReady(); + client = OzoneClientFactory.getRpcClient(conf); + objectStore = client.getObjectStore(); + keyString = UUID.randomUUID().toString(); + volumeName = "testeckeyoutputstream"; + bucketName = volumeName; + objectStore.createVolume(volumeName); + objectStore.getVolume(volumeName).createBucket(bucketName); + initInputChunks(); + } + + /** + * Shutdown MiniDFSCluster. + */ + @AfterClass + public static void shutdown() { + if (cluster != null) { + cluster.shutdown(); + } + } + + @Test + public void testCreateKeyWithECReplicationConfig() throws Exception { + try (OzoneOutputStream key = TestHelper + .createKey(keyString, new ECReplicationConfig(3, 2, + ECReplicationConfig.EcCodec.RS, chunkSize), inputSize, + objectStore, volumeName, bucketName)) { + Assert.assertTrue(key.getOutputStream() instanceof ECKeyOutputStream); + } + } + + @Test + public void testCreateKeyWithOutBucketDefaults() throws Exception { + OzoneVolume volume = objectStore.getVolume(volumeName); + OzoneBucket bucket = volume.getBucket(bucketName); + try (OzoneOutputStream out = bucket.createKey("myKey", inputSize)) { + Assert.assertTrue(out.getOutputStream() instanceof KeyOutputStream); + for (int i = 0; i < inputChunks.length; i++) { + out.write(inputChunks[i]); + } + } + } + + @Test + public void testCreateKeyWithBucketDefaults() throws Exception { + String myBucket = UUID.randomUUID().toString(); + OzoneVolume volume = objectStore.getVolume(volumeName); + final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); + bucketArgs.setDefaultReplicationConfig( + new DefaultReplicationConfig(ReplicationType.EC, + new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, + chunkSize))); + + volume.createBucket(myBucket, bucketArgs.build()); + OzoneBucket bucket = volume.getBucket(myBucket); + + try (OzoneOutputStream out = bucket.createKey(keyString, inputSize)) { + Assert.assertTrue(out.getOutputStream() instanceof ECKeyOutputStream); + for (int i = 0; i < inputChunks.length; i++) { + out.write(inputChunks[i]); + } + } + byte[] buf = new byte[chunkSize]; + try (OzoneInputStream in = bucket.readKey(keyString)) { + for (int i=0; i< inputChunks.length; i++) { + int read = in.read(buf, 0, chunkSize); + Assert.assertEquals(chunkSize, read); + Assert.assertTrue(Arrays.equals(buf, inputChunks[i])); + } + } + } + + @Test + public void testCreateRatisKeyAndWithECBucketDefaults() throws Exception { + OzoneBucket bucket = getOzoneBucket(); + try (OzoneOutputStream out = bucket + .createKey("testCreateRatisKeyAndWithECBucketDefaults", 2000, - new RatisReplicationConfig("3"), new HashMap<>())) { ++ new RatisReplicationConfig(HddsProtos.ReplicationFactor.THREE), ++ new HashMap<>())) { + Assert.assertTrue(out.getOutputStream() instanceof KeyOutputStream); + for (int i = 0; i < inputChunks.length; i++) { + out.write(inputChunks[i]); + } + } + } + + @Test + public void test13ChunksInSingleWriteOp() throws IOException { + testMultipleChunksInSingleWriteOp(13); + } + + @Test + public void test15ChunksInSingleWriteOp() throws IOException { + testMultipleChunksInSingleWriteOp(15); + } + + @Test + public void test20ChunksInSingleWriteOp() throws IOException { + testMultipleChunksInSingleWriteOp(20); + } + + @Test + public void test21ChunksInSingleWriteOp() throws IOException { + testMultipleChunksInSingleWriteOp(21); + } + + public void testMultipleChunksInSingleWriteOp(int numChunks) + throws IOException { + byte[] inputData = getInputBytes(numChunks); + final OzoneBucket bucket = getOzoneBucket(); + String keyName = "testMultipleChunksInSingleWriteOp" + numChunks; + try (OzoneOutputStream out = bucket.createKey(keyName, 4096, + new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, + chunkSize), new HashMap<>())) { + out.write(inputData); + } + + validateContent(inputData, bucket, bucket.getKey(keyName)); + } + + private void validateContent(byte[] inputData, OzoneBucket bucket, + OzoneKey key) throws IOException { + try (OzoneInputStream is = bucket.readKey(key.getName())) { + byte[] fileContent = new byte[inputData.length]; + Assert.assertEquals(inputData.length, is.read(fileContent)); + Assert.assertEquals(new String(inputData, UTF_8), + new String(fileContent, UTF_8)); + } + } + + private OzoneBucket getOzoneBucket() throws IOException { + String myBucket = UUID.randomUUID().toString(); + OzoneVolume volume = objectStore.getVolume(volumeName); + final BucketArgs.Builder bucketArgs = BucketArgs.newBuilder(); + bucketArgs.setDefaultReplicationConfig( + new DefaultReplicationConfig(ReplicationType.EC, + new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, + chunkSize))); + + volume.createBucket(myBucket, bucketArgs.build()); + return volume.getBucket(myBucket); + } + + private static void initInputChunks() { + for (int i = 0; i < dataBlocks; i++) { + inputChunks[i] = getBytesWith(i + 1, chunkSize); + } + } + + private static byte[] getBytesWith(int singleDigitNumber, int total) { + StringBuilder builder = new StringBuilder(singleDigitNumber); + for (int i = 1; i <= total; i++) { + builder.append(singleDigitNumber); + } + return builder.toString().getBytes(UTF_8); + } + + @Test + public void testWriteShouldSucceedWhenDNKilled() throws Exception { + int numChunks = 3; + byte[] inputData = getInputBytes(numChunks); + final OzoneBucket bucket = getOzoneBucket(); + String keyName = "testWriteShouldSucceedWhenDNKilled" + numChunks; + try { + try (OzoneOutputStream out = bucket.createKey(keyName, 1024, + new ECReplicationConfig(3, 2, ECReplicationConfig.EcCodec.RS, + chunkSize), new HashMap<>())) { + out.write(inputData); + // Kill a node from first pipeline + DatanodeDetails nodeToKill = + ((ECKeyOutputStream) out.getOutputStream()).getStreamEntries() + .get(0).getPipeline().getFirstNode(); + cluster.shutdownHddsDatanode(nodeToKill); + + out.write(inputData); + // Check the second blockGroup pipeline to make sure that the failed not + // is not selected. + Assert.assertFalse( + ((ECKeyOutputStream) out.getOutputStream()).getStreamEntries() + .get(1).getPipeline().getNodes().contains(nodeToKill)); + } + + try (OzoneInputStream is = bucket.readKey(keyName)) { + // TODO: this skip can be removed once read handles online recovery. + long skip = is.skip(inputData.length); + Assert.assertTrue(skip == inputData.length); + // All nodes available in second block group. So, lets assert. + byte[] fileContent = new byte[inputData.length]; + Assert.assertEquals(inputData.length, is.read(fileContent)); + Assert.assertEquals(new String(inputData, UTF_8), + new String(fileContent, UTF_8)); + } + } finally { + // TODO: optimize to just start the killed DN back. + resetCluster(); + } + } + + private void resetCluster() throws Exception { + cluster.shutdown(); + init(); + } + + private byte[] getInputBytes(int numChunks) { + byte[] inputData = new byte[numChunks * chunkSize]; + for (int i = 0; i < numChunks; i++) { + int start = (i * chunkSize); + Arrays.fill(inputData, start, start + chunkSize - 1, + String.valueOf(i % 9).getBytes(UTF_8)[0]); + } + return inputData; + } + +} diff --cc hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java index e8c0953,5a96d68..4ddefad --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOmAcls.java @@@ -145,10 -144,11 +145,10 @@@ public class TestOmAcls logCapturer.clearOutput(); TestOmAcls.aclAllow = false; - OzoneTestUtils.expectOmException(ResultCodes.PERMISSION_DENIED, () -> TestDataUtil.createKey(bucket, "testKey", "testcontent")); - assertTrue(logCapturer.getOutput().contains("doesn't have CREATE " + - "permission to access key")); + assertTrue(logCapturer.getOutput().contains("doesn't have READ " + + "permission to access volume")); } /** diff --cc hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index e6afa99,1dd922c..7161859 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@@ -545,7 -545,7 +545,8 @@@ message BucketInfo optional int64 quotaInNamespace = 16 [default = -2]; optional uint64 usedNamespace = 17; optional BucketLayoutProto bucketLayout = 18; - optional hadoop.hdds.DefaultReplicationConfig defaultReplicationConfig = 19; + optional string owner = 19; ++ optional hadoop.hdds.DefaultReplicationConfig defaultReplicationConfig = 20; } enum StorageTypeProto { diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 9bde8f9,2107754..ffed22e --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@@ -60,8 -60,6 +60,7 @@@ import org.apache.hadoop.fs.FileSystem import org.apache.hadoop.hdds.HddsConfigKeys; import org.apache.hadoop.hdds.HddsUtils; import org.apache.hadoop.hdds.annotation.InterfaceAudience; +import org.apache.hadoop.hdds.client.ReplicationConfig; - import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.ConfigurationException; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@@ -3410,15 -3612,6 +3619,10 @@@ public final class OzoneManager extend .getTrimmed(OZONE_OM_METADATA_LAYOUT, OZONE_OM_METADATA_LAYOUT_DEFAULT); } + public ReplicationConfig getDefaultReplicationConfig() { - String replication = configuration.get(OZONE_SERVER_DEFAULT_REPLICATION_KEY, - OZONE_SERVER_DEFAULT_REPLICATION_DEFAULT); - String type = configuration.get(OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY, - OZONE_SERVER_DEFAULT_REPLICATION_TYPE_DEFAULT); - return ReplicationConfig - .fromTypeAndString(ReplicationType.valueOf(type), replication); ++ return ReplicationConfig.getDefault(configuration); + } + public String getOMDefaultBucketLayout() { return this.defaultBucketLayout; } diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java index 1105166,6c2a862..91d4ed3 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMDirectoryCreateRequest.java @@@ -28,10 -29,9 +29,11 @@@ import java.util.Map import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneAcl; + import org.apache.hadoop.ozone.om.OzoneManagerUtils; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfoGroup; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; @@@ -356,35 -352,22 +358,37 @@@ public class OMDirectoryCreateRequest e } private static OmKeyInfo.Builder dirKeyInfoBuilderNoACL(String keyName, - KeyArgs keyArgs, long objectId) { + KeyArgs keyArgs, long objectId, + ReplicationConfig serverDefaultReplConfig) { String dirName = OzoneFSUtils.addTrailingSlashIfNeeded(keyName); - return new OmKeyInfo.Builder() - .setVolumeName(keyArgs.getVolumeName()) - .setBucketName(keyArgs.getBucketName()) - .setKeyName(dirName) - .setOmKeyLocationInfos(Collections.singletonList( - new OmKeyLocationInfoGroup(0, new ArrayList<>()))) - .setCreationTime(keyArgs.getModificationTime()) - .setModificationTime(keyArgs.getModificationTime()) - .setDataSize(0) - .setReplicationConfig(ReplicationConfig - .fromProtoTypeAndFactor(keyArgs.getType(), keyArgs.getFactor())) - .setObjectID(objectId) - .setUpdateID(objectId); + OmKeyInfo.Builder keyInfoBuilder = - new OmKeyInfo.Builder().setVolumeName(keyArgs.getVolumeName()) - .setBucketName(keyArgs.getBucketName()).setKeyName(dirName) ++ new OmKeyInfo.Builder() ++ .setVolumeName(keyArgs.getVolumeName()) ++ .setBucketName(keyArgs.getBucketName()) ++ .setKeyName(dirName) + .setOmKeyLocationInfos(Collections.singletonList( + new OmKeyLocationInfoGroup(0, new ArrayList<>()))) + .setCreationTime(keyArgs.getModificationTime()) - .setModificationTime(keyArgs.getModificationTime()).setDataSize(0); ++ .setModificationTime(keyArgs.getModificationTime()) ++ .setDataSize(0); + if (keyArgs.getFactor() != null && keyArgs + .getFactor() != HddsProtos.ReplicationFactor.ZERO && keyArgs + .getType() != HddsProtos.ReplicationType.EC) { + // Factor available and not an EC replication config. + keyInfoBuilder.setReplicationConfig(ReplicationConfig - .fromTypeAndFactor(keyArgs.getType(), keyArgs.getFactor())); ++ .fromProtoTypeAndFactor(keyArgs.getType(), keyArgs.getFactor())); + } else if (keyArgs.getType() == HddsProtos.ReplicationType.EC) { + // Found EC type + keyInfoBuilder.setReplicationConfig( + new ECReplicationConfig(keyArgs.getEcReplicationConfig())); + } else { + // default type + keyInfoBuilder.setReplicationConfig(serverDefaultReplConfig); + } + - keyInfoBuilder.setObjectID(objectId) - .setUpdateID(objectId); ++ keyInfoBuilder.setObjectID(objectId); + return keyInfoBuilder; } static long getMaxNumOfRecursiveDirs() { diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java index 295028d,ad18b8f..bc70862 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMAllocateBlockRequest.java @@@ -25,7 -26,7 +26,8 @@@ import java.util.Map import com.google.common.base.Optional; import com.google.common.base.Preconditions; +import org.apache.hadoop.hdds.client.ReplicationConfig; + import org.apache.hadoop.ozone.om.OzoneManagerUtils; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java index e2ac0a8,151564c..b128259 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/s3/multipart/S3InitiateMultipartUploadRequest.java @@@ -23,8 -23,8 +23,9 @@@ import com.google.common.base.Precondit import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.ozone.audit.OMAction; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OzoneConfigUtil; import org.apache.hadoop.ozone.om.OzoneManager; + import org.apache.hadoop.ozone.om.OzoneManagerUtils; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.BucketLayout; @@@ -169,15 -170,10 +171,14 @@@ public class S3InitiateMultipartUploadR // care of in the final complete multipart upload. AWS S3 behavior is // also like this, even when key exists in a bucket, user can still // initiate MPU. -- - final ReplicationConfig replicationConfig = - ReplicationConfig.fromProtoTypeAndFactor( - keyArgs.getType(), keyArgs.getFactor()); + final OmBucketInfo bucketInfo = omMetadataManager.getBucketTable() + .get(omMetadataManager.getBucketKey(volumeName, bucketName)); + final ReplicationConfig replicationConfig = OzoneConfigUtil + .resolveReplicationConfigPreference(keyArgs.getType(), + keyArgs.getFactor(), keyArgs.getEcReplicationConfig(), + bucketInfo != null ? + bucketInfo.getDefaultReplicationConfig() : + null, ozoneManager.getDefaultReplicationConfig()); multipartKeyInfo = new OmMultipartKeyInfo.Builder() .setUploadID(keyArgs.getMultipartUploadID()) diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java index c72aae2,a89a5bd..004ff75 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3MultipartRequest.java @@@ -22,8 -22,6 +22,7 @@@ package org.apache.hadoop.ozone.om.requ import java.io.IOException; import java.util.List; +import org.apache.hadoop.hdds.client.ReplicationConfig; - import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.ozone.om.helpers.BucketLayout; import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.junit.After; @@@ -87,10 -83,6 +84,8 @@@ public class TestS3MultipartRequest when(ozoneManager.getMetadataManager()).thenReturn(omMetadataManager); auditLogger = Mockito.mock(AuditLogger.class); when(ozoneManager.getAuditLogger()).thenReturn(auditLogger); + when(ozoneManager.getDefaultReplicationConfig()).thenReturn( - ReplicationConfig.fromTypeAndString(ReplicationType - .valueOf(OZONE_SERVER_DEFAULT_REPLICATION_TYPE_DEFAULT), - OZONE_SERVER_DEFAULT_REPLICATION_DEFAULT)); ++ ReplicationConfig.getDefault(ozoneConfiguration)); Mockito.doNothing().when(auditLogger).logWrite(any(AuditMessage.class)); when(ozoneManager.resolveBucketLink(any(KeyArgs.class), any(OMClientRequest.class))) diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index 65f2d5a,8d98882..0597eea --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@@ -21,8 -21,6 +21,7 @@@ import com.google.common.base.Optional import com.google.common.collect.Iterators; import org.apache.commons.lang3.tuple.Pair; import org.apache.hadoop.hdds.client.BlockID; +import org.apache.hadoop.hdds.client.ReplicationConfig; - import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; @@@ -138,10 -134,6 +135,8 @@@ public class TestCleanupTableInfo ); when(om.getAclsEnabled()).thenReturn(false); when(om.getAuditLogger()).thenReturn(mock(AuditLogger.class)); + when(om.getDefaultReplicationConfig()).thenReturn(ReplicationConfig - .fromTypeAndString(ReplicationType - .valueOf(OZONE_SERVER_DEFAULT_REPLICATION_TYPE_DEFAULT), - OZONE_SERVER_DEFAULT_REPLICATION_DEFAULT)); ++ .getDefault(new OzoneConfiguration())); addVolumeToMetaTable(aVolumeArgs()); addBucketToMetaTable(aBucketInfo()); } diff --cc hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index 38b1a06,2c3a465..508e22d --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@@ -209,8 -211,8 +222,7 @@@ public class BasicOzoneClientAdapterImp ReplicationConfig customReplicationConfig = ReplicationConfig.adjustReplication( - replConfig, replication - replicationConfig, replication, config -- ); ++ replConfig, replication, config); ozoneOutputStream = bucket.createFile(key, 0, customReplicationConfig, overWrite, recursive); diff --cc hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 1a619f1,2577105..b314f2e --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@@ -344,7 -336,8 +355,8 @@@ public class BasicRootedOzoneClientAdap || replication == ReplicationFactor.THREE.getValue()) { ozoneOutputStream = bucket.createFile(key, 0, - ReplicationConfig.adjustReplication(replConfig, replication), + ReplicationConfig.adjustReplication( - replicationConfig, replication, config), ++ replConfig, replication, config), overWrite, recursive); } else { ozoneOutputStream = diff --cc hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java index cce8948,5c07662..072b907 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/shell/bucket/CreateBucketHandler.java @@@ -18,14 -18,8 +18,15 @@@ package org.apache.hadoop.ozone.shell.bucket; import com.google.common.base.Strings; +import org.apache.hadoop.hdds.client.DefaultReplicationConfig; +import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.client.OzoneQuota; +import org.apache.hadoop.hdds.client.ReplicationConfig; +import org.apache.hadoop.hdds.client.ReplicationFactor; +import org.apache.hadoop.hdds.client.ReplicationType; ++import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.StorageType; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.client.BucketArgs; import org.apache.hadoop.ozone.client.OzoneBucket; @@@ -110,26 -103,6 +121,26 @@@ public class CreateBucketHandler extend } } + if(replicationType!=null) { + if (replication != null) { + ReplicationConfig replicationConfig = ReplicationConfig - .fromTypeAndString(ReplicationType.valueOf(replicationType), - replication); ++ .parse(ReplicationType.valueOf(replicationType), ++ replication, new OzoneConfiguration()); + boolean isEC = replicationConfig + .getReplicationType() == HddsProtos.ReplicationType.EC; + bb.setDefaultReplicationConfig(new DefaultReplicationConfig( + ReplicationType.fromProto(replicationConfig.getReplicationType()), + isEC ? + null : + ReplicationFactor.valueOf(replicationConfig.getRequiredNodes()), + isEC ? (ECReplicationConfig) replicationConfig : null)); + } else { + throw new IOException( + "Replication can't be null. Replication type passed was : " + + replicationType); + } + } + if (!Strings.isNullOrEmpty(quotaOptions.getQuotaInBytes())) { bb.setQuotaInBytes(OzoneQuota.parseSpaceQuota( quotaOptions.getQuotaInBytes()).getQuotaInBytes()); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
