sodonnel commented on code in PR #7401:
URL: https://github.com/apache/ozone/pull/7401#discussion_r1878249082
##########
hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/ec/reconstruction/ECValidator.java:
##########
@@ -0,0 +1,152 @@
+package org.apache.hadoop.ozone.container.ec.reconstruction;
+
+import org.apache.hadoop.hdds.client.ECReplicationConfig;
+import org.apache.hadoop.hdds.protocol.datanode.proto.ContainerProtos;
+import org.apache.hadoop.hdds.scm.OzoneClientConfig;
+import org.apache.hadoop.hdds.scm.storage.ECBlockOutputStream;
+import org.apache.hadoop.ozone.common.Checksum;
+import org.apache.hadoop.ozone.common.ChecksumData;
+import org.apache.hadoop.ozone.common.ChunkBuffer;
+import org.apache.hadoop.ozone.common.OzoneChecksumException;
+import org.apache.hadoop.ozone.container.common.helpers.BlockData;
+import org.apache.hadoop.ozone.container.common.helpers.ChunkInfo;
+import org.apache.hadoop.ozone.container.common.interfaces.Container;
+import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.nio.BufferUnderflowException;
+import java.nio.ByteBuffer;
+import java.util.*;
+import java.util.stream.Collectors;
+
+public class ECValidator {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ECValidator.class);
+ private final boolean isValidationEnabled;
+ private Collection<Integer> reconstructionIndexes;
+ private final int parityCount;
+ private long blockLength;
+ private final ECReplicationConfig ecReplicationConfig;
+ private int ecChunkSize;
+
+ ECValidator(OzoneClientConfig config, ECReplicationConfig ecReplConfig) {
+ // We fetch the configuration value beforehand to avoid re-fetching on
every validation call
+ isValidationEnabled = config.getEcReconstructionValidation();
+ ecReplicationConfig = ecReplConfig;
+ parityCount = ecReplConfig.getParity();
+ ecChunkSize = ecReplConfig.getEcChunkSize();
+ }
+
+ public void setReconstructionIndexes(Collection<Integer>
reconstructionIndexes) {
+ this.reconstructionIndexes = reconstructionIndexes;
+ }
+
+ public void setBlockLength(long blockLength) {
+ this.blockLength = blockLength;
+ }
+
+ /**
+ * Validate the expected checksum data for a chunk with the corresponding
checksum in original stripe checksum
+ * Note: The stripe checksum is a combination of all the checksums of all
the chunks in the stripe
+ * @param recreatedChunkChecksum Stores the {@link
ContainerProtos.ChecksumData} of the recreated chunk to verify
+ * @param stripeChecksum Stores the {@link ByteBuffer} of stripe
checksum
+ * @param chunkIndex Stores the index of the recreated chunk we
are comparing
+ * @param checksumSize Stores the length of the stripe checksum
+ * @throws OzoneChecksumException If there is a mismatch in the recreated
chunk vs stripe checksum, or if there is any
+ * internal error while performing {@link
ByteBuffer} operations
+ */
+ private void validateChecksumInStripe(ContainerProtos.ChecksumData
recreatedChunkChecksum,
+ ByteBuffer stripeChecksum, int
chunkIndex, int checksumSize)
+ throws OzoneChecksumException {
+
+ int bytesPerChecksum = recreatedChunkChecksum.getBytesPerChecksum();
+ int parityLength = (int) (Math.ceil((double)ecChunkSize /
bytesPerChecksum) * 4L * parityCount);
+ // Ignore the parity bits
+ stripeChecksum.limit(checksumSize - parityLength);
Review Comment:
Why are we limiting due to parity? We could be reconstructing a parity
index, and it should have checksum too. Or, does the stripe checksum not
contain the parity checksums? I cannot remember how this was designed, but if
you are reducing the effective stripeChecksum length to remove parity, then
parity is likely included in the stripechecksum.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
