Repository: cassandra Updated Branches: refs/heads/trunk 7f989edd2 -> 912fdb3ea
Switch to CRC32 for sstable metadata checksums patch by Aleksey Yeschenko; reviewed by Marcus Eriksson for CASSANDRA-13953 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/912fdb3e Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/912fdb3e Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/912fdb3e Branch: refs/heads/trunk Commit: 912fdb3ea4ff58ae9e1fe0c9b230ec76273ff601 Parents: 7f989ed Author: Aleksey Yeschenko <[email protected]> Authored: Fri Oct 13 14:26:02 2017 +0100 Committer: Aleksey Yeschenko <[email protected]> Committed: Tue Oct 17 16:48:32 2017 +0100 ---------------------------------------------------------------------- CHANGES.txt | 2 +- .../io/sstable/metadata/MetadataSerializer.java | 157 ++++++++++++------- .../legacy_na_clust/na-1-big-CompressionInfo.db | Bin 87 -> 87 bytes .../legacy_na_clust/na-1-big-Data.db | Bin 5256 -> 5215 bytes .../legacy_na_clust/na-1-big-Digest.crc32 | 2 +- .../legacy_na_clust/na-1-big-Index.db | Bin 157553 -> 157553 bytes .../legacy_na_clust/na-1-big-Statistics.db | Bin 7103 -> 7095 bytes .../legacy_na_clust/na-1-big-TOC.txt | 8 +- .../na-1-big-CompressionInfo.db | Bin 87 -> 87 bytes .../legacy_na_clust_compact/na-1-big-Data.db | Bin 5286 -> 5261 bytes .../na-1-big-Digest.crc32 | 2 +- .../legacy_na_clust_compact/na-1-big-Index.db | Bin 157553 -> 157553 bytes .../na-1-big-Statistics.db | Bin 7103 -> 7095 bytes .../legacy_na_clust_compact/na-1-big-TOC.txt | 8 +- .../na-1-big-CompressionInfo.db | Bin 79 -> 79 bytes .../legacy_na_clust_counter/na-1-big-Data.db | Bin 5797 -> 5781 bytes .../na-1-big-Digest.crc32 | 2 +- .../legacy_na_clust_counter/na-1-big-Index.db | Bin 157553 -> 157553 bytes .../na-1-big-Statistics.db | Bin 7112 -> 7104 bytes .../legacy_na_clust_counter/na-1-big-TOC.txt | 8 +- .../na-1-big-CompressionInfo.db | Bin 79 -> 79 bytes .../na-1-big-Data.db | Bin 5817 -> 5545 bytes .../na-1-big-Digest.crc32 | 2 +- .../na-1-big-Index.db | Bin 157553 -> 157553 bytes .../na-1-big-Statistics.db | Bin 7112 -> 7104 bytes .../na-1-big-TOC.txt | 8 +- .../legacy_na_simple/na-1-big-Data.db | Bin 88 -> 89 bytes .../legacy_na_simple/na-1-big-Digest.crc32 | 2 +- .../legacy_na_simple/na-1-big-Statistics.db | Bin 4656 -> 4648 bytes .../legacy_na_simple/na-1-big-TOC.txt | 8 +- .../legacy_na_simple_compact/na-1-big-Data.db | Bin 91 -> 90 bytes .../na-1-big-Digest.crc32 | 2 +- .../na-1-big-Statistics.db | Bin 4697 -> 4689 bytes .../legacy_na_simple_compact/na-1-big-TOC.txt | 8 +- .../legacy_na_simple_counter/na-1-big-Data.db | Bin 139 -> 146 bytes .../na-1-big-Digest.crc32 | 2 +- .../na-1-big-Statistics.db | Bin 4665 -> 4657 bytes .../legacy_na_simple_counter/na-1-big-TOC.txt | 8 +- .../na-1-big-Data.db | Bin 142 -> 148 bytes .../na-1-big-Digest.crc32 | 2 +- .../na-1-big-Statistics.db | Bin 4706 -> 4698 bytes .../na-1-big-TOC.txt | 8 +- .../metadata/MetadataSerializerTest.java | 33 ++-- 43 files changed, 162 insertions(+), 110 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 0efa8b9..bea407e 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,7 +1,7 @@ 4.0 + * Checksum sstable metadata (CASSANDRA-13321, CASSANDRA-13593) * Add result set metadata to prepared statement MD5 hash calculation (CASSANDRA-10786) * Refactor GcCompactionTest to avoid boxing (CASSANDRA-13941) - * Checksum sstable metadata (CASSANDRA-13321) * Expose recent histograms in JmxHistograms (CASSANDRA-13642) * Fix buffer length comparison when decompressing in netty-based streaming (CASSANDRA-13899) * Properly close StreamCompressionInputStream to release any ByteBuf (CASSANDRA-13906) http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java index f17157c..9499ff3 100644 --- a/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java +++ b/src/java/org/apache/cassandra/io/sstable/metadata/MetadataSerializer.java @@ -19,11 +19,9 @@ package org.apache.cassandra.io.sstable.metadata; import java.io.*; import java.util.*; +import java.util.zip.CRC32; import com.google.common.collect.Lists; -import com.google.common.hash.HashFunction; -import com.google.common.hash.Hasher; -import com.google.common.hash.Hashing; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,11 +39,13 @@ import org.apache.cassandra.io.util.BufferedDataOutputStreamPlus; import org.apache.cassandra.io.util.RandomAccessReader; import org.apache.cassandra.utils.FBUtilities; +import static org.apache.cassandra.utils.FBUtilities.updateChecksumInt; + /** - * Metadata serializer for SSTables {@code version >= 'k'}. + * Metadata serializer for SSTables {@code version >= 'na'}. * * <pre> - * File format := | number of components (4 bytes) | toc | component1 | c1 hash | component2 | c2 hash | ... | + * File format := | number of components (4 bytes) | crc | toc | crc | component1 | c1 crc | component2 | c2 crc | ... | * toc := | component type (4 bytes) | position of component | * </pre> * @@ -54,31 +54,40 @@ import org.apache.cassandra.utils.FBUtilities; public class MetadataSerializer implements IMetadataSerializer { private static final Logger logger = LoggerFactory.getLogger(MetadataSerializer.class); - private static final HashFunction hashFunction = Hashing.md5(); + + private static final int CHECKSUM_LENGTH = 4; // CRC32 public void serialize(Map<MetadataType, MetadataComponent> components, DataOutputPlus out, Version version) throws IOException { boolean checksum = version.hasMetadataChecksum(); + CRC32 crc = new CRC32(); // sort components by type List<MetadataComponent> sortedComponents = Lists.newArrayList(components.values()); Collections.sort(sortedComponents); // write number of component out.writeInt(components.size()); + updateChecksumInt(crc, components.size()); + maybeWriteChecksum(crc, out, version); + // build and write toc - int lastPosition = 4 + (8 * sortedComponents.size()); + int lastPosition = 4 + (8 * sortedComponents.size()) + (checksum ? 2 * CHECKSUM_LENGTH : 0); Map<MetadataType, Integer> sizes = new EnumMap<>(MetadataType.class); for (MetadataComponent component : sortedComponents) { MetadataType type = component.getType(); // serialize type out.writeInt(type.ordinal()); + updateChecksumInt(crc, type.ordinal()); // serialize position out.writeInt(lastPosition); + updateChecksumInt(crc, lastPosition); int size = type.serializer.serializedSize(version, component); - lastPosition += size + (checksum ? 8 : 0); // checksum is long + lastPosition += size + (checksum ? CHECKSUM_LENGTH : 0); sizes.put(type, size); } + maybeWriteChecksum(crc, out, version); + // serialize components for (MetadataComponent component : sortedComponents) { @@ -89,11 +98,18 @@ public class MetadataSerializer implements IMetadataSerializer bytes = dob.getData(); } out.write(bytes); - if (checksum) - out.writeLong(hashFunction.hashBytes(bytes).asLong()); + + crc.reset(); crc.update(bytes); + maybeWriteChecksum(crc, out, version); } } + private static void maybeWriteChecksum(CRC32 crc, DataOutputPlus out, Version version) throws IOException + { + if (version.hasMetadataChecksum()) + out.writeInt((int) crc.getValue()); + } + public Map<MetadataType, MetadataComponent> deserialize( Descriptor descriptor, EnumSet<MetadataType> types) throws IOException { Map<MetadataType, MetadataComponent> components; @@ -120,66 +136,87 @@ public class MetadataSerializer implements IMetadataSerializer return deserialize(descriptor, EnumSet.of(type)).get(type); } - public Map<MetadataType, MetadataComponent> deserialize(Descriptor descriptor, FileDataInput in, EnumSet<MetadataType> types) throws IOException + public Map<MetadataType, MetadataComponent> deserialize(Descriptor descriptor, + FileDataInput in, + EnumSet<MetadataType> selectedTypes) + throws IOException { - int totalSize = (int) in.bytesRemaining(); - Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class); - // read number of components - int numComponents = in.readInt(); - // read toc - Map<MetadataType, Integer> toc = new EnumMap<>(MetadataType.class); - MetadataType[] values = MetadataType.values(); - Map<MetadataType, Integer> lengths = new EnumMap<>(MetadataType.class); - int start = 0; - MetadataType lastType = null; - for (int i = 0; i < numComponents; i++) + boolean isChecksummed = descriptor.version.hasMetadataChecksum(); + CRC32 crc = new CRC32(); + + /* + * Read TOC + */ + + int length = (int) in.bytesRemaining(); + + int count = in.readInt(); + updateChecksumInt(crc, count); + maybeValidateChecksum(crc, in, descriptor); + + int[] ordinals = new int[count]; + int[] offsets = new int[count]; + int[] lengths = new int[count]; + + for (int i = 0; i < count; i++) { - int metadataTypeId = in.readInt(); - int position = in.readInt(); - - toc.put(values[metadataTypeId], position); - if (lastType != null) - lengths.put(lastType, position - start); - start = position; - lastType = values[metadataTypeId]; + ordinals[i] = in.readInt(); + updateChecksumInt(crc, ordinals[i]); + + offsets[i] = in.readInt(); + updateChecksumInt(crc, offsets[i]); } - lengths.put(lastType, totalSize - start); - for (MetadataType type : types) + maybeValidateChecksum(crc, in, descriptor); + + lengths[count - 1] = length - offsets[count - 1]; + for (int i = 0; i < count - 1; i++) + lengths[i] = offsets[i + 1] - offsets[i]; + + /* + * Read components + */ + + MetadataType[] allMetadataTypes = MetadataType.values(); + + Map<MetadataType, MetadataComponent> components = new EnumMap<>(MetadataType.class); + + for (int i = 0; i < count; i++) { - Integer offset = toc.get(type); - if (offset != null) - { - in.seek(offset); - - if (descriptor.version.hasMetadataChecksum()) - { - int size = lengths.get(type) - 8; // 8 bytes checksum - byte[] bytes = new byte[size]; - in.readFully(bytes); - MetadataComponent component; - try (DataInputBuffer dib = new DataInputBuffer(bytes)) - { - component = type.serializer.deserialize(descriptor.version, dib); - } - long writtenChecksum = in.readLong(); - if (writtenChecksum != hashFunction.hashBytes(bytes).asLong()) - { - String filename = descriptor.filenameFor(Component.STATS); - throw new CorruptSSTableException(new IOException("Checksums do not match for " + filename), filename); - } - components.put(type, component); - } - else - { - MetadataComponent component = type.serializer.deserialize(descriptor.version, in); - components.put(type, component); - } + MetadataType type = allMetadataTypes[ordinals[i]]; + if (!selectedTypes.contains(type)) + { + in.skipBytes(lengths[i]); + continue; } + + byte[] buffer = new byte[isChecksummed ? lengths[i] - CHECKSUM_LENGTH : lengths[i]]; + in.readFully(buffer); + + crc.reset(); crc.update(buffer); + maybeValidateChecksum(crc, in, descriptor); + + components.put(type, type.serializer.deserialize(descriptor.version, new DataInputBuffer(buffer))); } + return components; } + private static void maybeValidateChecksum(CRC32 crc, FileDataInput in, Descriptor descriptor) throws IOException + { + if (!descriptor.version.hasMetadataChecksum()) + return; + + int actualChecksum = (int) crc.getValue(); + int expectedChecksum = in.readInt(); + + if (actualChecksum != expectedChecksum) + { + String filename = descriptor.filenameFor(Component.STATS); + throw new CorruptSSTableException(new IOException("Checksums do not match for " + filename), filename); + } + } + public void mutateLevel(Descriptor descriptor, int newLevel) throws IOException { logger.trace("Mutating {} to level {}", descriptor.filenameFor(Component.STATS), newLevel); http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db index c7fc204..935d863 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-CompressionInfo.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db index 36f9800..9982578 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 index 9adb8f5..1e6cbfc 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Digest.crc32 @@ -1 +1 @@ -829376603 \ No newline at end of file +4285275084 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db index d3b366d..25b063c 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Index.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db index f750d82..59969c2 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt index 0251bd9..734a80d 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db +TOC.txt +Filter.db CompressionInfo.db -Statistics.db Summary.db -TOC.txt +Data.db +Statistics.db Digest.crc32 -Filter.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db index 1ef9177..6d3ac8e 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-CompressionInfo.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db index 1d130d4..fd6aebb 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 index a9900da..239b372 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Digest.crc32 @@ -1 +1 @@ -818525950 \ No newline at end of file +3704038982 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db index e04d3db..2e20951 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Index.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db index f2da58d..cd90137 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt index 0251bd9..734a80d 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_compact/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db +TOC.txt +Filter.db CompressionInfo.db -Statistics.db Summary.db -TOC.txt +Data.db +Statistics.db Digest.crc32 -Filter.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db index 449f507..d6a1ff8 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-CompressionInfo.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db index 55be94d..03babd4 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 index 0608cac..7bd9e39 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Digest.crc32 @@ -1 +1 @@ -1278737554 \ No newline at end of file +1093063834 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db index 646d4d4..5ad5400 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Index.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db index 5692214..9347a2c 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt index 0251bd9..734a80d 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db +TOC.txt +Filter.db CompressionInfo.db -Statistics.db Summary.db -TOC.txt +Data.db +Statistics.db Digest.crc32 -Filter.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db index 405b9fb..1b1f2da 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-CompressionInfo.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db index 1f4e26a..1976981 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 index 3ae0adf..22ed33b 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Digest.crc32 @@ -1 +1 @@ -2193718788 \ No newline at end of file +2742231118 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db index 041117e..022bc3d 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Index.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db index e899252..d393159 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt index 0251bd9..734a80d 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_clust_counter_compact/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db +TOC.txt +Filter.db CompressionInfo.db -Statistics.db Summary.db -TOC.txt +Data.db +Statistics.db Digest.crc32 -Filter.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db index cd52dab..414ff07 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 index 1275575..1dfec4f 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Digest.crc32 @@ -1 +1 @@ -3074491452 \ No newline at end of file +2354437953 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db index 4205870..33f0516 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt index 0251bd9..734a80d 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db +TOC.txt +Filter.db CompressionInfo.db -Statistics.db Summary.db -TOC.txt +Data.db +Statistics.db Digest.crc32 -Filter.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db index 8adbf54..b9b836f 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 index 138bc9c..669bbcf 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Digest.crc32 @@ -1 +1 @@ -1825853914 \ No newline at end of file +1732739494 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db index 7e25613..b112e58 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt index 0251bd9..734a80d 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_compact/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db +TOC.txt +Filter.db CompressionInfo.db -Statistics.db Summary.db -TOC.txt +Data.db +Statistics.db Digest.crc32 -Filter.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db index f3e6359..ed0dadb 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 index 2ad4fa0..1480f5d 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Digest.crc32 @@ -1 +1 @@ -3361465012 \ No newline at end of file +2023874595 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db index 04cacf9..124dacd 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt index 0251bd9..734a80d 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db +TOC.txt +Filter.db CompressionInfo.db -Statistics.db Summary.db -TOC.txt +Data.db +Statistics.db Digest.crc32 -Filter.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db index 72772f0..366315f 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 index b44a733..839639c 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Digest.crc32 @@ -1 +1 @@ -2410392183 \ No newline at end of file +138332031 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db index c431a59..613f2ad 100644 Binary files a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db and b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt index 0251bd9..734a80d 100644 --- a/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt +++ b/test/data/legacy-sstables/na/legacy_tables/legacy_na_simple_counter_compact/na-1-big-TOC.txt @@ -1,8 +1,8 @@ Index.db -Data.db +TOC.txt +Filter.db CompressionInfo.db -Statistics.db Summary.db -TOC.txt +Data.db +Statistics.db Digest.crc32 -Filter.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/912fdb3e/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java index 95d2a77..7d8150c 100644 --- a/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/metadata/MetadataSerializerTest.java @@ -98,8 +98,13 @@ public class MetadataSerializerTest String partitioner = RandomPartitioner.class.getCanonicalName(); double bfFpChance = 0.1; - Map<MetadataType, MetadataComponent> originalMetadata = collector.finalizeMetadata(partitioner, bfFpChance, 0, null, SerializationHeader.make(cfm, Collections.emptyList())); - return originalMetadata; + return collector.finalizeMetadata(partitioner, bfFpChance, 0, null, SerializationHeader.make(cfm, Collections.emptyList())); + } + + @Test + public void testMaReadMa() throws IOException + { + testOldReadsNew("ma", "ma"); } @Test @@ -115,15 +120,27 @@ public class MetadataSerializerTest } @Test + public void testMbReadMb() throws IOException + { + testOldReadsNew("mb", "mb"); + } + + @Test public void testMbReadMc() throws IOException { testOldReadsNew("mb", "mc"); } @Test - public void testNaReadMc() throws IOException + public void testMcReadMc() throws IOException + { + testOldReadsNew("mc", "mc"); + } + + @Test + public void testNaReadNa() throws IOException { - testOldReadsNew("mc", "na"); + testOldReadsNew("na", "na"); } public void testOldReadsNew(String oldV, String newV) throws IOException @@ -146,11 +163,9 @@ public class MetadataSerializerTest for (MetadataType type : MetadataType.values()) { assertEquals(deserializedLa.get(type), deserializedLb.get(type)); - if (!originalMetadata.get(type).equals(deserializedLb.get(type))) - { - // Currently only STATS can be different. Change if no longer the case - assertEquals(MetadataType.STATS, type); - } + + if (MetadataType.STATS != type) + assertEquals(originalMetadata.get(type), deserializedLb.get(type)); } } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
