sodonnel commented on code in PR #7401:
URL: https://github.com/apache/ozone/pull/7401#discussion_r1878275690
##########
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECValidator.java:
##########
@@ -0,0 +1,152 @@
+package org.apache.hadoop.ozone.container.ec.reconstruction;
+
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.scm.storage.ECBlockOutputStream;
+import org.apache.hadoop.ozone.common.Checksum;
+import org.apache.hadoop.ozone.common.ChecksumData;
+import org.apache.hadoop.ozone.common.ChunkBuffer;
+import org.apache.hadoop.ozone.common.OzoneChecksumException;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class ECValidator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ECValidator.class);
+ private final boolean isValidationEnabled;
+ private Collection<Integer> reconstructionIndexes;
+ private final int parityCount;
+ private long blockLength;
+ private final ECReplicationConfig ecReplicationConfig;
+ private int ecChunkSize;
+
+ ECValidator(OzoneClientConfig config, ECReplicationConfig ecReplConfig) {
+ // We fetch the configuration value beforehand to avoid re-fetching on
every validation call
+ isValidationEnabled = config.getEcReconstructionValidation();
+ ecReplicationConfig = ecReplConfig;
+ parityCount = ecReplConfig.getParity();
+ ecChunkSize = ecReplConfig.getEcChunkSize();
+ }
+
+ public void setReconstructionIndexes(Collection<Integer>
reconstructionIndexes) {
+ this.reconstructionIndexes = reconstructionIndexes;
+ }
+
+ public void setBlockLength(long blockLength) {
+ this.blockLength = blockLength;
+ }
+
+ /**
+ * Validate the expected checksum data for a chunk with the corresponding
checksum in original stripe checksum
+ * Note: The stripe checksum is a combination of all the checksums of all
the chunks in the stripe
+ * @param recreatedChunkChecksum Stores the {@link
ContainerProtos.ChecksumData} of the recreated chunk to verify
+ * @param stripeChecksum Stores the {@link ByteBuffer} of stripe
checksum
+ * @param chunkIndex Stores the index of the recreated chunk we
are comparing
+ * @param checksumSize Stores the length of the stripe checksum
+ * @throws OzoneChecksumException If there is a mismatch in the recreated
chunk vs stripe checksum, or if there is any
+ * internal error while performing {@link
ByteBuffer} operations
+ */
+ private void validateChecksumInStripe(ContainerProtos.ChecksumData
recreatedChunkChecksum,
+ ByteBuffer stripeChecksum, int
chunkIndex, int checksumSize)
+ throws OzoneChecksumException {
+
+ int bytesPerChecksum = recreatedChunkChecksum.getBytesPerChecksum();
+ int parityLength = (int) (Math.ceil((double)ecChunkSize /
bytesPerChecksum) * 4L * parityCount);
+ // Ignore the parity bits
+ stripeChecksum.limit(checksumSize - parityLength);
+
+ // If we have a 100 bytes per checksum, and a chunk of size 1000 bytes, it
means there are total 10 checksums
+ // for each chunk that is present. So the 1st chunk will have 10 checksums
together to form a single chunk checksum.
+ // For each chunk we will have:
+ // Checksum of length = (chunkIdx * numOfChecksumPerChunk)
+ // Number of Checksums per Chunk = (chunkSize / bytesPerChecksum)
+ // So the checksum should start from (numOfBytesPerChecksum * (chunkIdx *
numOfChecksumPerChunk)
+
+ int checksumIdxStart = (ecChunkSize * chunkIndex);
Review Comment:
This does not align with the comment above, as we are not considering
numOfBytesPerChecksum or numOfChecksumPerChunk ?
Also I am not sure about the above calculation.
What if the bytes per checksum is 100, and the chunksize is 1000, but only
80 bytes were written? I that case, we would expect a stripe (for EC-3-2) that
looks like:
Index_1: 80 bytes of data, 4 bytes of checksum.
Index_2: 0 bytes
Index_3: 0 bytes
Index_4: 80 bytes of data, 4 bytes of checksum.
Index_5: 80 bytes of data, 4 bytes of checksum.
Similar, if you have 1080 bytes written, then index 1 and 2 will have data,
but index 2 has shorter data and a shorter checksum. The logic is different
(and simpler) for a full stripe than a partial stripe.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
