This is an automated email from the ASF dual-hosted git repository. sodonnell pushed a commit to branch HDDS-3816-ec in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 0c5c859b99e367f70210bbb9793eb8f26c616c7a Author: S O'Donnell <[email protected]> AuthorDate: Fri Jul 30 11:06:35 2021 +0100 Fix issues caused by merging master into branch --- .../apache/hadoop/hdds/scm/storage/BlockOutputStream.java | 8 -------- .../hadoop/hdds/scm/pipeline/ECPipelineProvider.java | 11 +++++++++-- .../hadoop/hdds/scm/pipeline/TestECPipelineProvider.java | 15 +++++++++++---- .../hadoop/ozone/client/io/BlockOutputStreamEntry.java | 10 ++-------- .../src/main/proto/OmClientProtocol.proto | 2 +- 5 files changed, 23 insertions(+), 23 deletions(-) diff --git a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java index c251c46..ebcd962 100644 --- a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java +++ b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java @@ -213,7 +213,6 @@ public class BlockOutputStream extends OutputStream { return ioException.get(); } -<<<<<<< HEAD XceiverClientSpi getXceiverClientSpi(){ return this.xceiverClient; } @@ -230,13 +229,6 @@ public class BlockOutputStream extends OutputStream { return this.responseExecutor; } - @VisibleForTesting - public Map<Long, List<ChunkBuffer>> getCommitIndex2flushedDataMap() { - return commitWatcher.getCommitIndex2flushedDataMap(); - } - -======= ->>>>>>> master @Override public void write(int b) throws IOException { checkOpen(); diff --git a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java index d8fd343..f674b6f 100644 --- a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/pipeline/ECPipelineProvider.java @@ -19,8 +19,10 @@ package org.apache.hadoop.hdds.scm.pipeline; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.ConfigurationSource; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.node.NodeManager; import java.io.IOException; @@ -40,6 +42,7 @@ public class ECPipelineProvider extends PipelineProvider<ECReplicationConfig> { private final ConfigurationSource conf; private final PlacementPolicy placementPolicy; + private final long containerSizeBytes; public ECPipelineProvider(NodeManager nodeManager, StateManager stateManager, @@ -48,6 +51,9 @@ public class ECPipelineProvider extends PipelineProvider<ECReplicationConfig> { super(nodeManager, stateManager); this.conf = conf; this.placementPolicy = placementPolicy; + this.containerSizeBytes = (long) this.conf + .getStorageSize(ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, StorageUnit.BYTES); } @Override @@ -61,8 +67,9 @@ public class ECPipelineProvider extends PipelineProvider<ECReplicationConfig> { protected Pipeline create(ECReplicationConfig replicationConfig, List<DatanodeDetails> excludedNodes, List<DatanodeDetails> favoredNodes) throws IOException { - List<DatanodeDetails> dns = placementPolicy.chooseDatanodes(excludedNodes, - favoredNodes, replicationConfig.getRequiredNodes(), 0); + List<DatanodeDetails> dns = placementPolicy + .chooseDatanodes(excludedNodes, favoredNodes, + replicationConfig.getRequiredNodes(), 0, this.containerSizeBytes); return create(replicationConfig, dns); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java index 326bc1c..eb88a60 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/pipeline/TestECPipelineProvider.java @@ -20,9 +20,11 @@ package org.apache.hadoop.hdds.scm.pipeline; import org.apache.hadoop.hdds.client.ECReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.conf.StorageUnit; import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.MockDatanodeDetails; import org.apache.hadoop.hdds.scm.PlacementPolicy; +import org.apache.hadoop.hdds.scm.ScmConfigKeys; import org.apache.hadoop.hdds.scm.node.NodeManager; import org.junit.Assert; import org.junit.Before; @@ -47,16 +49,20 @@ public class TestECPipelineProvider { private NodeManager nodeManager = Mockito.mock(NodeManager.class); private StateManager stateManager = Mockito.mock(StateManager.class); private PlacementPolicy placementPolicy = Mockito.mock(PlacementPolicy.class); - + private long containerSizeBytes; @Before public void setup() throws IOException { conf = new OzoneConfiguration(); provider = new ECPipelineProvider( nodeManager, stateManager, conf, placementPolicy); - + this.containerSizeBytes = (long) this.conf.getStorageSize( + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE, + ScmConfigKeys.OZONE_SCM_CONTAINER_SIZE_DEFAULT, + StorageUnit.BYTES); // Placement policy will always return EC number of random nodes. Mockito.when(placementPolicy.chooseDatanodes(Mockito.anyList(), - Mockito.anyList(), Mockito.anyInt(), Mockito.anyLong())) + Mockito.anyList(), Mockito.anyInt(), Mockito.anyLong(), + Mockito.anyLong())) .thenAnswer(invocation -> { List<DatanodeDetails> dns = new ArrayList<>(); for (int i=0; i<(int)invocation.getArguments()[2]; i++) { @@ -64,6 +70,7 @@ public class TestECPipelineProvider { } return dns; }); + } @@ -99,7 +106,7 @@ public class TestECPipelineProvider { pipeline.getNodes().size()); verify(placementPolicy).chooseDatanodes(excludedNodes, favoredNodes, - ecConf.getRequiredNodes(), 0); + ecConf.getRequiredNodes(), 0, containerSizeBytes); } } diff --git a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java index 3fa339a..2d151ed 100644 --- a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java +++ b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/BlockOutputStreamEntry.java @@ -96,19 +96,13 @@ public class BlockOutputStreamEntry extends OutputStream { */ private void checkStream() throws IOException { if (this.outputStream == null) { -<<<<<<< HEAD this.outputStream = createOutputStream(); -======= - this.outputStream = - new RatisBlockOutputStream(blockID, xceiverClientManager, - pipeline, bufferPool, config, token); ->>>>>>> master } } BlockOutputStream createOutputStream() throws IOException { - return new BlockOutputStream(blockID, xceiverClientManager, - pipeline, bufferPool, config, token); + return new RatisBlockOutputStream(blockID, xceiverClientManager, pipeline, + bufferPool, config, token); } @Override diff --git a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto index 3706a62..a330e78 100644 --- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto +++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto @@ -772,7 +772,7 @@ message KeyArgs { // When it is a head operation which is to check whether key exist optional bool headOp = 18; - optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 18; + optional hadoop.hdds.ECReplicationConfig ecReplicationConfig = 19; } message KeyLocation { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
