This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a5fccbc1d6 HDDS-10682. EC Reconstruction creates empty chunks at the
end of blocks with partial stripes (#6515)
a5fccbc1d6 is described below
commit a5fccbc1d6539cdccf5366edf26382a139897789
Author: Stephen O'Donnell <[email protected]>
AuthorDate: Fri Apr 12 21:02:33 2024 +0100
HDDS-10682. EC Reconstruction creates empty chunks at the end of blocks
with partial stripes (#6515)
---
.../ec/reconstruction/ECReconstructionCoordinator.java | 10 +++++++---
.../hadoop/hdds/scm/storage/TestContainerCommandsEC.java | 14 ++++++++------
2 files changed, 15 insertions(+), 9 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java
index 774c7d8345..8fadd19b67 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECReconstructionCoordinator.java
@@ -323,9 +323,13 @@ public class ECReconstructionCoordinator implements
Closeable {
}
// TODO: can be submitted in parallel
for (int i = 0; i < bufs.length; i++) {
- CompletableFuture<ContainerProtos.ContainerCommandResponseProto>
- future = targetBlockStreams[i].write(bufs[i]);
- checkFailures(targetBlockStreams[i], future);
+ if (bufs[i].remaining() != 0) {
+ // If the buffer is empty, we don't need to write it as it
will cause
+ // an empty chunk to be added to the end of the block.
+
CompletableFuture<ContainerProtos.ContainerCommandResponseProto>
+ future = targetBlockStreams[i].write(bufs[i]);
+ checkFailures(targetBlockStreams[i], future);
+ }
bufs[i].clear();
}
length -= readLen;
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
index 5254f8748c..b4814d7b5e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/storage/TestContainerCommandsEC.java
@@ -136,7 +136,9 @@ public class TestContainerCommandsEC {
private static final int EC_CHUNK_SIZE = 1024 * 1024;
private static final int STRIPE_DATA_SIZE = EC_DATA * EC_CHUNK_SIZE;
private static final int NUM_DN = EC_DATA + EC_PARITY + 3;
- private static byte[][] inputChunks = new byte[EC_DATA][EC_CHUNK_SIZE];
+ // Data slots are EC_DATA + 1 so we can generate enough data to have a full
stripe
+ // plus one extra chunk.
+ private static byte[][] inputChunks = new byte[EC_DATA + 1][EC_CHUNK_SIZE];
// Each key size will be in range [min, max), min inclusive, max exclusive
private static final int[][] KEY_SIZE_RANGES =
@@ -621,13 +623,13 @@ public class TestContainerCommandsEC {
testECReconstructionCoordinator(missingIndexes, 1);
}
- @Test
- void testECReconstructParityWithPartialStripe()
- throws Exception {
- testECReconstructionCoordinator(ImmutableList.of(4, 5), 1);
+ @ParameterizedTest
+ @MethodSource("recoverableMissingIndexes")
+ void testECReconstructionCoordinatorWithFullAndPartialStripe(List<Integer>
missingIndexes)
+ throws Exception {
+ testECReconstructionCoordinator(missingIndexes, 4);
}
-
static Stream<List<Integer>> recoverableMissingIndexes() {
return Stream
.concat(IntStream.rangeClosed(1, 5).mapToObj(ImmutableList::of), Stream
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
