This is an automated email from the ASF dual-hosted git repository.
sodonnell pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new 5d2fa94 HDDS-6256. EC: Parity blocks are incorrectly padded with
zeros to the chunk size (#3043)
5d2fa94 is described below
commit 5d2fa9494985edd2ef844e1a8335501d0e76f468
Author: Uma Maheswara Rao G <[email protected]>
AuthorDate: Mon Feb 7 03:33:50 2022 -0800
HDDS-6256. EC: Parity blocks are incorrectly padded with zeros to the chunk
size (#3043)
---
.../hadoop/ozone/client/io/ECKeyOutputStream.java | 8 +--
.../hadoop/ozone/client/MockDatanodeStorage.java | 10 +++
.../hadoop/ozone/client/TestOzoneECClient.java | 74 ++++++++++++++++++++++
3 files changed, 87 insertions(+), 5 deletions(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
index db51a65..d738436 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java
@@ -380,8 +380,7 @@ public class ECKeyOutputStream extends KeyOutputStream {
for (int i =
numDataBlks; i < (this.numDataBlks + this.numParityBlks); i++) {
// Move the stream entry cursor to parity block index
- handleParityWrite(i, parityBuffers[i - numDataBlks].array(), 0,
- parityCellSize, true);
+ handleParityWrite(i, parityCellSize, true);
}
}
@@ -396,8 +395,7 @@ public class ECKeyOutputStream extends KeyOutputStream {
return pos;
}
- private void handleParityWrite(int currIdx, byte[] b, int off, long len,
- boolean isFullCell) throws IOException {
+ private void handleParityWrite(int currIdx, long len, boolean isFullCell) {
handleOutputStreamWrite(currIdx, len, isFullCell, true);
blockOutputStreamEntryPool.getCurrentStreamEntry().useNextBlockStream();
}
@@ -418,7 +416,7 @@ public class ECKeyOutputStream extends KeyOutputStream {
try {
// Since it's a fullcell, let's write all content from buffer.
writeToOutputStream(current, len, bytesToWrite.array(),
- bytesToWrite.array().length, 0, isParity);
+ bytesToWrite.limit(), 0, isParity);
} catch (Exception e) {
markStreamAsFailed(e);
}
diff --git
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockDatanodeStorage.java
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockDatanodeStorage.java
index ce70c15..41de363 100644
---
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockDatanodeStorage.java
+++
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/MockDatanodeStorage.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.client;
import org.apache.commons.collections.map.HashedMap;
+import org.apache.hadoop.hdds.client.BlockID;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.BlockData;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.ChunkInfo;
import
org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos.DatanodeBlockID;
@@ -33,6 +34,7 @@ import java.util.Map;
public class MockDatanodeStorage {
private final Map<DatanodeBlockID, BlockData> blocks = new HashedMap();
+ private final Map<BlockID, String> fullBlockData = new HashMap<>();
private final Map<String, ChunkInfo> chunks = new HashMap<>();
@@ -61,6 +63,10 @@ public class MockDatanodeStorage {
data.put(createKey(blockID, chunkInfo),
ByteString.copyFrom(bytes.toByteArray()));
chunks.put(createKey(blockID, chunkInfo), chunkInfo);
+ fullBlockData
+ .put(new BlockID(blockID.getContainerID(), blockID.getLocalID()),
+ fullBlockData.getOrDefault(blockID, "")
+ .concat(bytes.toStringUtf8()));
}
public ChunkInfo readChunkInfo(
@@ -85,4 +91,8 @@ public class MockDatanodeStorage {
return this.data;
}
+ public String getFullBlockData(BlockID blockID) {
+ return this.fullBlockData.get(blockID);
+ }
+
}
diff --git
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
index 2804199..c2a9c71 100644
---
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
+++
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.ozone.client;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.hadoop.conf.StorageUnit;
+import org.apache.hadoop.hdds.client.BlockID;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
import org.apache.hadoop.hdds.client.ReplicationType;
@@ -49,6 +50,7 @@ import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.UUID;
@@ -283,6 +285,78 @@ public class TestOzoneECClient {
}
@Test
+ public void testSmallerThanChunkSize() throws IOException {
+ byte[] firstSmallChunk = new byte[chunkSize - 1];
+ Arrays.fill(firstSmallChunk, 0, firstSmallChunk.length - 1,
+ Byte.parseByte("1"));
+
+ writeIntoECKey(firstSmallChunk, keyName,
+ new DefaultReplicationConfig(ReplicationType.EC,
+ new ECReplicationConfig(dataBlocks, parityBlocks,
+ ECReplicationConfig.EcCodec.RS, chunkSize)));
+ OzoneManagerProtocolProtos.KeyLocationList blockList =
+ transportStub.getKeys().get(volumeName).get(bucketName).get(keyName)
+ .getKeyLocationListList().get(0);
+
+ Map<DatanodeDetails, MockDatanodeStorage> storages =
+ ((MockXceiverClientFactory) factoryStub).getStorages();
+ OzoneManagerProtocolProtos.KeyLocation keyLocations =
+ blockList.getKeyLocations(0);
+
+ List<MockDatanodeStorage> dns = new ArrayList<>();
+ for (int i = 0; i < dataBlocks + parityBlocks; i++) {
+ HddsProtos.DatanodeDetailsProto member =
+ blockList.getKeyLocations(0).getPipeline().getMembers(i);
+ MockDatanodeStorage mockDatanodeStorage =
+ storages.get(getMatchingStorage(storages, member.getUuid()));
+ dns.add(mockDatanodeStorage);
+ }
+ String firstBlockData = dns.get(0).getFullBlockData(new BlockID(
+ keyLocations.getBlockID().getContainerBlockID().getContainerID(),
+ keyLocations.getBlockID().getContainerBlockID().getLocalID()));
+
+ Assert.assertArrayEquals(firstSmallChunk, firstBlockData.getBytes(UTF_8));
+
+ final ByteBuffer[] dataBuffers = new ByteBuffer[dataBlocks];
+ dataBuffers[0] = ByteBuffer.wrap(firstSmallChunk);
+ //Let's pad the remaining length equal to firstSmall chunk len
+ for (int i = 1; i < dataBlocks; i++) {
+ dataBuffers[i] = ByteBuffer.allocate(firstSmallChunk.length);
+ Arrays.fill(dataBuffers[i].array(), 0, firstSmallChunk.length, (byte) 0);
+ }
+
+ final ByteBuffer[] parityBuffers = new ByteBuffer[parityBlocks];
+ for (int i = 0; i < parityBlocks; i++) {
+ parityBuffers[i] = ByteBuffer.allocate(firstSmallChunk.length);
+ }
+ encoder.encode(dataBuffers, parityBuffers);
+
+ //Lets assert the parity data.
+ for (int i = dataBlocks; i < dataBlocks + parityBlocks; i++) {
+ String parityBlockData = dns.get(i).getFullBlockData(new BlockID(
+ keyLocations.getBlockID().getContainerBlockID().getContainerID(),
+ keyLocations.getBlockID().getContainerBlockID().getLocalID()));
+ String expected =
+ new String(parityBuffers[i - dataBlocks].array(), UTF_8);
+ Assert.assertEquals(expected, parityBlockData);
+ Assert.assertEquals(expected.length(), parityBlockData.length());
+
+ }
+ }
+
+ private static DatanodeDetails getMatchingStorage(
+ Map<DatanodeDetails, MockDatanodeStorage> storages, String uuid) {
+ Iterator<DatanodeDetails> iterator = storages.keySet().iterator();
+ while (iterator.hasNext()) {
+ DatanodeDetails dn = iterator.next();
+ if (dn.getUuid().toString().equals(uuid)) {
+ return dn;
+ }
+ }
+ return null;
+ }
+
+ @Test
public void testMultipleChunksWithPartialChunkInSingleWriteOp()
throws IOException {
final int partialChunkLen = 10;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
