This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch HDDS-3698-upgrade in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit cfdaaa1daa9aaf689006ebce3a2b02aa345df3dd Author: Elek, Márton <[email protected]> AuthorDate: Mon Aug 24 15:14:41 2020 +0200 HDDS-4094. Support byte-level write in Freon HadoopFsGenerator (#1310) --- .../hadoop/ozone/freon/ContentGenerator.java | 31 +++++++- .../hadoop/ozone/freon/HadoopFsGenerator.java | 12 +++- .../hadoop/ozone/freon/TestContentGenerator.java | 82 ++++++++++++++++++++++ 3 files changed, 119 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java index c6ec60e..542634c 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/ContentGenerator.java @@ -20,6 +20,7 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.charset.StandardCharsets; +import com.google.common.annotations.VisibleForTesting; import org.apache.commons.lang3.RandomStringUtils; /** @@ -38,15 +39,25 @@ public class ContentGenerator { */ private int bufferSize; + /** + * Number of bytes to write in one call. + * <p> + * Should be no larger than the bufferSize. + */ + private final int copyBufferSize; + private final byte[] buffer; ContentGenerator(long keySize, int bufferSize) { + this(keySize, bufferSize, bufferSize); + } + + ContentGenerator(long keySize, int bufferSize, int copyBufferSize) { this.keySize = keySize; this.bufferSize = bufferSize; - + this.copyBufferSize = copyBufferSize; buffer = RandomStringUtils.randomAscii(bufferSize) .getBytes(StandardCharsets.UTF_8); - } /** @@ -56,7 +67,21 @@ public class ContentGenerator { for (long nrRemaining = keySize; nrRemaining > 0; nrRemaining -= bufferSize) { int curSize = (int) Math.min(bufferSize, nrRemaining); - outputStream.write(buffer, 0, curSize); + if (copyBufferSize == 1) { + for (int i = 0; i < curSize; i++) { + outputStream.write(buffer[i]); + } + } else { + for (int i = 0; i < curSize; i += copyBufferSize) { + outputStream.write(buffer, i, + Math.min(copyBufferSize, curSize - i)); + } + } } } + + @VisibleForTesting + byte[] getBuffer() { + return buffer; + } } diff --git a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java index 548f829..925ba7d 100644 --- a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java +++ b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/HadoopFsGenerator.java @@ -57,10 +57,15 @@ public class HadoopFsGenerator extends BaseFreonGenerator private int fileSize; @Option(names = {"--buffer"}, - description = "Size of buffer used to generated the key content.", - defaultValue = "4096") + description = "Size of buffer used store the generated key content", + defaultValue = "10240") private int bufferSize; + @Option(names = {"--copy-buffer"}, + description = "Size of bytes written to the output in one operation", + defaultValue = "4096") + private int copyBufferSize; + private ContentGenerator contentGenerator; private Timer timer; @@ -76,7 +81,8 @@ public class HadoopFsGenerator extends BaseFreonGenerator fileSystem = FileSystem.get(URI.create(rootPath), configuration); - contentGenerator = new ContentGenerator(fileSize, bufferSize); + contentGenerator = + new ContentGenerator(fileSize, bufferSize, copyBufferSize); timer = getMetrics().timer("file-create"); diff --git a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestContentGenerator.java b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestContentGenerator.java new file mode 100644 index 0000000..d61be3a --- /dev/null +++ b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/freon/TestContentGenerator.java @@ -0,0 +1,82 @@ +package org.apache.hadoop.ozone.freon; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with this + * work for additional information regarding copyright ownership. The ASF + * licenses this file to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations under + * the License. + */ + +import org.junit.Assert; +import org.junit.Test; + +/** + * Tests for the ContentGenerator class of Freon. + */ +public class TestContentGenerator { + + @Test + public void writeWrite() throws IOException { + ContentGenerator generator = new ContentGenerator(1024, 1024); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + generator.write(output); + Assert.assertArrayEquals(generator.getBuffer(), output.toByteArray()); + } + + @Test + public void writeWithSmallerBuffers() throws IOException { + ContentGenerator generator = new ContentGenerator(10000, 1024, 3); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + generator.write(baos); + + Assert.assertEquals(10000, baos.toByteArray().length); + } + + @Test + public void writeWithByteLevelWrite() throws IOException { + ContentGenerator generator = new ContentGenerator(1024, 1024, 1); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + generator.write(output); + Assert.assertArrayEquals(generator.getBuffer(), output.toByteArray()); + } + + @Test + public void writeWithSmallBuffer() throws IOException { + ContentGenerator generator = new ContentGenerator(1024, 1024, 10); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + generator.write(output); + Assert.assertArrayEquals(generator.getBuffer(), output.toByteArray()); + } + + @Test + public void writeWithDistinctSizes() throws IOException { + ContentGenerator generator = new ContentGenerator(20, 8, 3); + ByteArrayOutputStream output = new ByteArrayOutputStream(); + + generator.write(output); + + byte[] expected = new byte[20]; + byte[] buffer = generator.getBuffer(); + System.arraycopy(buffer, 0, expected, 0, buffer.length); + System.arraycopy(buffer, 0, expected, 8, buffer.length); + System.arraycopy(buffer, 0, expected, 16, 4); + Assert.assertArrayEquals(expected, output.toByteArray()); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
