Fix Adler32 digest for compressed sstables patch by jeff jirsa; reviewed by benedict for CASSANDRA-8778
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/0e621317 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/0e621317 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/0e621317 Branch: refs/heads/trunk Commit: 0e621317edb2cffac2a02d0ac13bd4a370373c73 Parents: 59ed491 Author: Jeff Jirsa <j...@jeffjirsa.net> Authored: Thu Mar 5 20:05:05 2015 +0000 Committer: Benedict Elliott Smith <bened...@apache.org> Committed: Thu Mar 5 20:05:05 2015 +0000 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../io/compress/CompressedSequentialWriter.java | 2 +- .../io/util/ChecksummedSequentialWriter.java | 2 +- .../io/util/DataIntegrityMetadata.java | 22 ++++++++++++++++++-- 4 files changed, 23 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/0e621317/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 57dd97e..59f9207 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -30,6 +30,7 @@ * markCompacting only succeeds if the exact SSTableReader instances being marked are in the live set (CASSANDRA-8689) * cassandra-stress support for varint (CASSANDRA-8882) + * Fix Adler32 digest for compressed sstables (CASSANDRA-8778) Merged from 2.0: * Fix regression in mixed single and multi-column relation support for SELECT statements (CASSANDRA-8613) http://git-wip-us.apache.org/repos/asf/cassandra/blob/0e621317/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java index ad53e83..6152c5f 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java +++ b/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java @@ -132,7 +132,7 @@ public class CompressedSequentialWriter extends SequentialWriter // write data itself out.write(compressed.buffer, 0, compressedLength); // write corresponding checksum - crcMetadata.append(compressed.buffer, 0, compressedLength); + crcMetadata.append(compressed.buffer, 0, compressedLength, true); lastFlushOffset += compressedLength + 4; } catch (IOException e) http://git-wip-us.apache.org/repos/asf/cassandra/blob/0e621317/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java b/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java index f4281b2..526347b 100644 --- a/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java +++ b/src/java/org/apache/cassandra/io/util/ChecksummedSequentialWriter.java @@ -37,7 +37,7 @@ public class ChecksummedSequentialWriter extends SequentialWriter protected void flushData() { super.flushData(); - crcMetadata.append(buffer, 0, validBufferBytes); + crcMetadata.append(buffer, 0, validBufferBytes, false); } public void writeFullChecksum(Descriptor descriptor) http://git-wip-us.apache.org/repos/asf/cassandra/blob/0e621317/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java index 797b964..e6f5083 100644 --- a/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java +++ b/src/java/org/apache/cassandra/io/util/DataIntegrityMetadata.java @@ -23,6 +23,7 @@ import java.io.DataOutput; import java.io.File; import java.io.IOError; import java.io.IOException; +import java.nio.ByteBuffer; import java.nio.file.Files; import java.util.zip.Adler32; import java.util.zip.Checksum; @@ -107,15 +108,32 @@ public class DataIntegrityMetadata } } - public void append(byte[] buffer, int start, int end) + // checksumIncrementalResult indicates if the checksum we compute for this buffer should itself be + // included in the full checksum, translating to if the partial checksum is serialized along with the + // data it checksums (in which case the file checksum as calculated by external tools would mismatch if + // we did not include it), or independently. + + // CompressedSequentialWriters serialize the partial checksums inline with the compressed data chunks they + // corroborate, whereas ChecksummedSequentialWriters serialize them to a different file. + public void append(byte[] buffer, int start, int end, boolean checksumIncrementalResult) { try { + int incrementalChecksumValue; + incrementalChecksum.update(buffer, start, end); - incrementalOut.writeInt((int) incrementalChecksum.getValue()); + incrementalChecksumValue = (int) incrementalChecksum.getValue(); + incrementalOut.writeInt((int) incrementalChecksumValue); incrementalChecksum.reset(); fullChecksum.update(buffer, start, end); + + if (checksumIncrementalResult) + { + ByteBuffer byteBuffer = ByteBuffer.allocate(4); + byteBuffer.putInt((int) incrementalChecksumValue); + fullChecksum.update(byteBuffer.array(), 0, byteBuffer.array().length); + } } catch (IOException e) {