Updated Branches: refs/heads/trunk e1b199c05 -> 7f747c9d9
introduce 'crc_check_chance' in CompressionParameters to support a checksum percentage checking chance similarly to read-repair patch by Vijay; reviewed by Pavel Yaskevich for CASSANDRA-3611 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7f747c9d Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7f747c9d Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7f747c9d Branch: refs/heads/trunk Commit: 7f747c9d9addb87d3e6f9747483c413ea2918724 Parents: e1b199c Author: Pavel Yaskevich <[email protected]> Authored: Wed Dec 28 19:02:16 2011 +0200 Committer: Pavel Yaskevich <[email protected]> Committed: Wed Dec 28 19:04:55 2011 +0200 ---------------------------------------------------------------------- CHANGES.txt | 3 + .../io/compress/CompressedRandomAccessReader.java | 14 +++-- .../io/compress/CompressionParameters.java | 43 +++++++++------ 3 files changed, 37 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/7f747c9d/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 4637665..6baaa20 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -33,6 +33,9 @@ * fix minor issues reported by FindBugs (CASSANDRA-3658) * global key/row caches (CASSANDRA-3143) * optimize memtable iteration during range scan (CASSANDRA-3638) + * introduce 'crc_check_chance' in CompressionParameters to support + a checksum percentage checking chance similarly to read-repair (CASSANDRA-3611) + 1.0.7 * allow configuring bloom_filter_fp_chance (CASSANDRA-3497) http://git-wip-us.apache.org/repos/asf/cassandra/blob/7f747c9d/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java b/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java index 3bed2c3..b7128f3 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java +++ b/src/java/org/apache/cassandra/io/compress/CompressedRandomAccessReader.java @@ -88,14 +88,16 @@ public class CompressedRandomAccessReader extends RandomAccessReader validBufferBytes = metadata.compressor().uncompress(compressed, 0, chunk.length, buffer, 0); - checksum.update(buffer, 0, validBufferBytes); + if (metadata.parameters.crcChance > FBUtilities.threadLocalRandom().nextDouble()) + { + checksum.update(buffer, 0, validBufferBytes); - if (checksum(chunk) != (int) checksum.getValue()) - throw new CorruptedBlockException(getPath(), chunk); - - // reset checksum object back to the original (blank) state - checksum.reset(); + if (checksum(chunk) != (int) checksum.getValue()) + throw new CorruptedBlockException(getPath(), chunk); + // reset checksum object back to the original (blank) state + checksum.reset(); + } // buffer offset is always aligned bufferOffset = current & ~(buffer.length - 1); http://git-wip-us.apache.org/repos/asf/cassandra/blob/7f747c9d/src/java/org/apache/cassandra/io/compress/CompressionParameters.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/compress/CompressionParameters.java b/src/java/org/apache/cassandra/io/compress/CompressionParameters.java index 2240af2..2336ef6 100644 --- a/src/java/org/apache/cassandra/io/compress/CompressionParameters.java +++ b/src/java/org/apache/cassandra/io/compress/CompressionParameters.java @@ -33,12 +33,15 @@ import org.apache.cassandra.config.ConfigurationException; public class CompressionParameters { public final static int DEFAULT_CHUNK_LENGTH = 65536; + public final static double DEFAULT_CRC_CHECK_CHANCE = 1.0; public static final String SSTABLE_COMPRESSION = "sstable_compression"; public static final String CHUNK_LENGTH_KB = "chunk_length_kb"; + public static final String CRC_CHECK_CHANCE = "crc_check_chance"; public final ICompressor sstableCompressor; private final Integer chunkLength; + public final double crcChance; public final Map<String, String> otherOptions; // Unrecognized options, can be use by the compressor public static CompressionParameters create(Map<? extends CharSequence, ? extends CharSequence> opts) throws ConfigurationException @@ -49,7 +52,7 @@ public class CompressionParameters options.remove(SSTABLE_COMPRESSION); options.remove(CHUNK_LENGTH_KB); CompressionParameters cp = new CompressionParameters(sstableCompressionClass, parseChunkLength(chunkLength), options); - cp.validateChunkLength(); + cp.validate(); return cp; } @@ -68,6 +71,8 @@ public class CompressionParameters this.sstableCompressor = sstableCompressor; this.chunkLength = chunkLength; this.otherOptions = otherOptions; + String chance = otherOptions.get(CRC_CHECK_CHANCE); + this.crcChance = (chance == null) ? DEFAULT_CRC_CHECK_CHANCE : Double.parseDouble(chance); } public int chunkLength() @@ -164,27 +169,31 @@ public class CompressionParameters // chunkLength must be a power of 2 because we assume so when // computing the chunk number from an uncompressed file offset (see // CompressedRandomAccessReader.decompresseChunk()) - private void validateChunkLength() throws ConfigurationException + private void validate() throws ConfigurationException { - if (chunkLength == null) - return; // chunk length not set, this is fine, default will be used - - if (chunkLength <= 0) - throw new ConfigurationException("Invalid negative or null " + CHUNK_LENGTH_KB); - - int c = chunkLength; - boolean found = false; - while (c != 0) + // if chunk length was not set (chunkLength == null), this is fine, default will be used + if (chunkLength != null) { - if ((c & 0x01) != 0) + if (chunkLength <= 0) + throw new ConfigurationException("Invalid negative or null " + CHUNK_LENGTH_KB); + + int c = chunkLength; + boolean found = false; + while (c != 0) { - if (found) - throw new ConfigurationException(CHUNK_LENGTH_KB + " must be a power of 2"); - else - found = true; + if ((c & 0x01) != 0) + { + if (found) + throw new ConfigurationException(CHUNK_LENGTH_KB + " must be a power of 2"); + else + found = true; + } + c >>= 1; } - c >>= 1; } + + if (crcChance > 1.0d || crcChance < 0.0d) + throw new ConfigurationException("crc_check_chance should be between 0.0 to 1.0"); } public Map<CharSequence, CharSequence> asAvroOptions()
