This is an automated email from the ASF dual-hosted git repository.
umamahesh pushed a commit to branch HDDS-3816-ec
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-3816-ec by this push:
new 6f64eb7 HDDS-6342. EC: Fix large write with multiple stripes upon
stripe failure. (#3108)
6f64eb7 is described below
commit 6f64eb77d579629fd79705b112d9a5e01937dd84
Author: Gui Hecheng <[email protected]>
AuthorDate: Fri Feb 18 06:12:51 2022 +0800
HDDS-6342. EC: Fix large write with multiple stripes upon stripe failure.
(#3108)
---
.../ozone/client/io/ECBlockOutputStreamEntry.java | 2 +-
.../hadoop/ozone/client/TestOzoneECClient.java | 80 ++++++++++++++++++++++
2 files changed, 81 insertions(+), 1 deletion(-)
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntry.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntry.java
index 3a86bd5..88f8d2e 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntry.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECBlockOutputStreamEntry.java
@@ -212,7 +212,7 @@ public class ECBlockOutputStreamEntry extends
BlockOutputStreamEntry {
if (isWritingParity()) {
return;
}
- this.successfulBlkGrpAckedLen += len;
+ this.successfulBlkGrpAckedLen = len;
}
/**
diff --git
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
index 758e3bd..8b2a8a1 100644
---
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
+++
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestOzoneECClient.java
@@ -763,6 +763,86 @@ public class TestOzoneECClient {
}
}
+ @Test
+ public void testLargeWriteOfMultipleStripesWithStripeFailure()
+ throws IOException {
+ close();
+ OzoneConfiguration con = new OzoneConfiguration();
+ // block size of 3KB could hold 3 full stripes
+ con.setStorageSize(OzoneConfigKeys.OZONE_SCM_BLOCK_SIZE, 3,
StorageUnit.KB);
+ con.setInt(OzoneConfigKeys.OZONE_CLIENT_MAX_EC_STRIPE_WRITE_RETRIES, 3);
+ MultiNodePipelineBlockAllocator blkAllocator =
+ new MultiNodePipelineBlockAllocator(con, dataBlocks + parityBlocks,
+ 15);
+ createNewClient(con, blkAllocator);
+
+ store.createVolume(volumeName);
+ OzoneVolume volume = store.getVolume(volumeName);
+ volume.createBucket(bucketName);
+ OzoneBucket bucket = volume.getBucket(bucketName);
+
+ // should write > 1 full stripe to trigger potential issue
+ int numFullStripesBeforeFailure = 2;
+ int numChunksToWriteAfterFailure = dataBlocks;
+ int numExpectedBlockGrps = 2;
+ // fail the DNs for parity blocks
+ int[] nodesIndexesToMarkFailure = {3, 4};
+
+ try (OzoneOutputStream out = bucket.createKey(keyName,
+ 1024 * dataBlocks * numFullStripesBeforeFailure
+ + numChunksToWriteAfterFailure,
+ new ECReplicationConfig(dataBlocks, parityBlocks,
+ ECReplicationConfig.EcCodec.RS,
+ chunkSize), new HashMap<>())) {
+ for (int j = 0; j < numFullStripesBeforeFailure; j++) {
+ for (int i = 0; i < dataBlocks; i++) {
+ out.write(inputChunks[i]);
+ }
+ }
+
+ List<DatanodeDetails> failedDNs = new ArrayList<>();
+ List<HddsProtos.DatanodeDetailsProto> dns = allocator.getClusterDns();
+ for (int j = 0; j < nodesIndexesToMarkFailure.length; j++) {
+ failedDNs.add(DatanodeDetails
+ .getFromProtoBuf(dns.get(nodesIndexesToMarkFailure[j])));
+ }
+
+ // First let's set storage as bad
+ ((MockXceiverClientFactory) factoryStub).setFailedStorages(failedDNs);
+
+ for (int i = 0; i < numChunksToWriteAfterFailure; i++) {
+ out.write(inputChunks[i % dataBlocks]);
+ }
+ }
+
+ final OzoneKeyDetails key = bucket.getKey(keyName);
+ // Data supposed to store in single block group. Since we introduced the
+ // failures after first stripe, the second stripe data should have been
+ // written into new block group. So, we should have numExpectedBlockGrps.
+ // That means two keyLocations.
+ Assert
+ .assertEquals(numExpectedBlockGrps, key.getOzoneKeyLocations().size());
+ try (OzoneInputStream is = bucket.readKey(keyName)) {
+ byte[] fileContent = new byte[chunkSize];
+ for (int i = 0; i < dataBlocks * numFullStripesBeforeFailure; i++) {
+ Assert.assertEquals(inputChunks[i % dataBlocks].length,
+ is.read(fileContent));
+ Assert.assertTrue(
+ "Expected: " + new String(inputChunks[i % dataBlocks], UTF_8)
+ + " \n " + "Actual: " + new String(fileContent, UTF_8),
+ Arrays.equals(inputChunks[i % dataBlocks], fileContent));
+ }
+ for (int i = 0; i < numChunksToWriteAfterFailure; i++) {
+ Assert.assertEquals(inputChunks[i % dataBlocks].length,
+ is.read(fileContent));
+ Assert.assertTrue(
+ "Expected: " + new String(inputChunks[i % dataBlocks],
+ UTF_8) + " \n " + "Actual: " + new String(fileContent, UTF_8),
+ Arrays.equals(inputChunks[i % dataBlocks], fileContent));
+ }
+ }
+ }
+
@Test(expected = NotImplementedException.class)
public void testFlushShouldThrowNotImplementedException() throws IOException
{
store.createVolume(volumeName);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
