This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new aee577c HDDS-5825: EC: ECKeyOutputStream#close fails if we write the
partial chunk (#2720)
aee577c is described below
commit aee577c17123fd28c5b2b8bc9a2c45bc81809df0
Author: Uma Maheswara Rao G <[email protected]>
AuthorDate: Thu Oct 7 16:32:33 2021 -0700
HDDS-5825: EC: ECKeyOutputStream#close fails if we write the partial chunk
(#2720)
Co-authored-by: Uma Maheswara Rao G <[email protected]>
---
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 8 ++++----
.../hadoop/ozone/client/TestOzoneECClient.java | 23 ++++++++++++++++++++++
2 files changed, 27 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
index 61107b0..02c3ec3 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
@@ -234,7 +234,7 @@ public class ECKeyOutputStream extends KeyOutputStream {
// executePutBlock for all.
// TODO: we should alter the put block calls to share CRC to each stream.
blockOutputStreamEntryPool.executePutBlockForAll();
- ecChunkBufferCache.clear();
+ ecChunkBufferCache.clear(parityCellSize);
// check if block ends?
if (shouldEndBlockGroup()) {
@@ -691,9 +691,9 @@ public class ECKeyOutputStream extends KeyOutputStream {
return pos;
}
- private void clear() {
- clearBuffers(cellSize, dataBuffers);
- clearBuffers(cellSize, parityBuffers);
+ private void clear(int size) {
+ clearBuffers(size, dataBuffers);
+ clearBuffers(size, parityBuffers);
}
private void release() {
diff --git
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
index daf1789..6eee2ab 100644
---
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
+++
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
@@ -275,6 +275,29 @@ public class TestOzoneECClient {
}
@Test
+ public void testMultipleChunksWithPartialChunkInSigleWripeOp()
+ throws IOException {
+ final int partialChunkLen = 10;
+ final int numFullChunks = 9;
+ final int inputBuffLen = (numFullChunks * chunkSize) + partialChunkLen;
+ byte[] inputData = new byte[inputBuffLen];
+ for (int i = 0; i < numFullChunks; i++) {
+ int start = (i * chunkSize);
+ Arrays.fill(inputData, start, start + chunkSize - 1,
+ String.valueOf(i).getBytes(UTF_8)[0]);
+ }
+ //fill the last partial chunk as well.
+ Arrays.fill(inputData, (numFullChunks * chunkSize),
+ ((numFullChunks * chunkSize)) + partialChunkLen - 1, (byte) 1);
+ final OzoneBucket bucket = writeIntoECKey(inputData, keyName,
+ new DefaultReplicationConfig(ReplicationType.EC,
+ new ECReplicationConfig(dataBlocks, parityBlocks,
+ ECReplicationConfig.EcCodec.RS, chunkSize)));
+ OzoneKey key = bucket.getKey(keyName);
+ validateContent(inputData, bucket, key);
+ }
+
+ @Test
public void testCommitKeyInfo()
throws IOException {
final OzoneBucket bucket = writeIntoECKey(inputChunks, keyName,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
