This is an automated email from the ASF dual-hosted git repository. captainzmc pushed a commit to branch HDDS-4454 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 51b5395786363085d4a5cf3322d1a2d4730c6999 Author: Sadanand Shenoy <[email protected]> AuthorDate: Fri Apr 8 17:38:09 2022 +0530 HDDS-5666. Add option to createKey via streaming api in Freon (#2574) --- .../hadoop/ozone/freon/ContentGenerator.java | 18 ++++++++++++++ .../ozone/freon/OzoneClientKeyGenerator.java | 29 +++++++++++++++++++++- 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java index 92f7ae4b2e..b01c12f6b3 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java @@ -18,10 +18,12 @@ package org.apache.hadoop.ozone.freon; import java.io.IOException; import java.io.OutputStream; +import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; /** * Utility class to write random keys from a limited buffer. @@ -81,6 +83,22 @@ public class ContentGenerator { } } + /** + * Write the required bytes to the streaming output stream. + */ + public void write(OzoneDataStreamOutput out) throws IOException { + for (long nrRemaining = keySize; + nrRemaining > 0; nrRemaining -= bufferSize) { + int curSize = (int) Math.min(bufferSize, nrRemaining); + for (int i = 0; i < curSize; i += copyBufferSize) { + ByteBuffer bb = + ByteBuffer.wrap(buffer, i, Math.min(copyBufferSize, curSize - i)); + out.write(bb); + } + } + out.close(); + } + @VisibleForTesting byte[] getBuffer() { return buffer; diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java index 74cd0d0b37..6ab5c03009 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OzoneClientKeyGenerator.java @@ -24,10 +24,12 @@ import java.util.concurrent.Callable; import org.apache.hadoop.hdds.cli.HddsVersionProvider; import org.apache.hadoop.hdds.client.ReplicationConfig; import org.apache.hadoop.hdds.conf.OzoneConfiguration; +import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneClient; import com.codahale.metrics.Timer; +import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; import picocli.CommandLine.Command; import picocli.CommandLine.Mixin; import picocli.CommandLine.Option; @@ -74,6 +76,12 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator @Mixin private FreonReplicationOptions replication; + @Option( + names = {"--enable-streaming", "--stream"}, + description = "Specify whether the write will be through ratis streaming" + ) + private boolean enableRatisStreaming = false; + private Timer timer; private OzoneBucket bucket; @@ -101,7 +109,11 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator timer = getMetrics().timer("key-create"); - runTests(this::createKey); + if (enableRatisStreaming) { + runTests(this::createStreamKey); + } else { + runTests(this::createKey); + } } return null; } @@ -118,4 +130,19 @@ public class OzoneClientKeyGenerator extends BaseFreonGenerator return null; }); } + + private void createStreamKey(long counter) throws Exception { + final ReplicationConfig replicationConfig = ReplicationConfig + .fromProtoTypeAndFactor(HddsProtos.ReplicationType.RATIS, + HddsProtos.ReplicationFactor.THREE); + final String key = generateObjectName(counter); + + timer.time(() -> { + try (OzoneDataStreamOutput stream = bucket + .createStreamKey(key, keySize, replicationConfig, metadata)) { + contentGenerator.write(stream); + } + return null; + }); + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
