Repository: cassandra Updated Branches: refs/heads/cassandra-3.0 387258024 -> 62092e45c refs/heads/cassandra-3.11 e4cef08d0 -> 6214b1ec8 refs/heads/trunk baffdeae8 -> ef404053b
Fix current position calculation in OldFormatDeserializer Patch by Sam Tunnicliffe; reviewed by Jeff Jirsa for CASSANDRA-13525 This patch also adds a unit test for CASSANDRA-13236. Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/62092e45 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/62092e45 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/62092e45 Branch: refs/heads/cassandra-3.0 Commit: 62092e45c8bbb75ac9f680188b3746913602507b Parents: 3872580 Author: Sam Tunnicliffe <[email protected]> Authored: Wed May 10 18:10:51 2017 +0100 Committer: Sam Tunnicliffe <[email protected]> Committed: Wed May 24 19:01:25 2017 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/db/UnfilteredDeserializer.java | 28 +++++++++++-- ...es-legacy_ka_indexed-ka-1-CompressionInfo.db | Bin 0 -> 91 bytes ...legacy_tables-legacy_ka_indexed-ka-1-Data.db | Bin 0 -> 196203 bytes ...cy_tables-legacy_ka_indexed-ka-1-Digest.sha1 | 1 + ...gacy_tables-legacy_ka_indexed-ka-1-Filter.db | Bin 0 -> 16 bytes ...egacy_tables-legacy_ka_indexed-ka-1-Index.db | Bin 0 -> 332 bytes ..._tables-legacy_ka_indexed-ka-1-Statistics.db | Bin 0 -> 4450 bytes ...acy_tables-legacy_ka_indexed-ka-1-Summary.db | Bin 0 -> 92 bytes ...legacy_tables-legacy_ka_indexed-ka-1-TOC.txt | 8 ++++ ...cy_ka_indexed_static-ka-1-CompressionInfo.db | Bin 0 -> 91 bytes ...tables-legacy_ka_indexed_static-ka-1-Data.db | Bin 0 -> 196269 bytes ...es-legacy_ka_indexed_static-ka-1-Digest.sha1 | 1 + ...bles-legacy_ka_indexed_static-ka-1-Filter.db | Bin 0 -> 16 bytes ...ables-legacy_ka_indexed_static-ka-1-Index.db | Bin 0 -> 332 bytes ...-legacy_ka_indexed_static-ka-1-Statistics.db | Bin 0 -> 4446 bytes ...les-legacy_ka_indexed_static-ka-1-Summary.db | Bin 0 -> 92 bytes ...tables-legacy_ka_indexed_static-ka-1-TOC.txt | 8 ++++ .../cassandra/io/sstable/LegacySSTableTest.java | 39 +++++++++++++++++++ 19 files changed, 83 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 98a1808..fe6ab09 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0.14 + * ReverseIndexedReader may drop rows during 2.1 to 3.0 upgrade (CASSANDRA-13525) * Fix repair process violating start/end token limits for small ranges (CASSANDRA-13052) * Add storage port options to sstableloader (CASSANDRA-13518) * Properly handle quoted index names in cqlsh DESCRIBE output (CASSANDRA-12847) http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java index 92690e1..ea65633 100644 --- a/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java +++ b/src/java/org/apache/cassandra/db/UnfilteredDeserializer.java @@ -256,6 +256,10 @@ public abstract class UnfilteredDeserializer // The position in the input after the last data consumption (readNext/skipNext). private long lastConsumedPosition; + // Tracks the size of the last LegacyAtom read from disk, because this needs to be accounted + // for when marking lastConsumedPosition after readNext/skipNext + private long bytesReadForNextAtom; + private OldFormatDeserializer(CFMetaData metadata, DataInputPlus in, SerializationHelper helper, @@ -272,7 +276,10 @@ public abstract class UnfilteredDeserializer { try { - return LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic); + long pos = currentPosition(); + LegacyLayout.LegacyAtom atom = LegacyLayout.readLegacyAtom(metadata, in, readAllAsDynamic); + bytesReadForNextAtom = currentPosition() - pos; + return atom; } catch (IOException e) { @@ -359,7 +366,7 @@ public abstract class UnfilteredDeserializer throw new IllegalStateException(); Unfiltered toReturn = next; next = null; - lastConsumedPosition = currentPosition(); + lastConsumedPosition = currentPosition() - bytesReadForNextAtom(); return toReturn; } @@ -368,7 +375,7 @@ public abstract class UnfilteredDeserializer if (!hasNext()) throw new UnsupportedOperationException(); next = null; - lastConsumedPosition = currentPosition(); + lastConsumedPosition = currentPosition() - bytesReadForNextAtom(); } public long bytesReadForUnconsumedData() @@ -379,6 +386,21 @@ public abstract class UnfilteredDeserializer return currentPosition() - lastConsumedPosition; } + // Reading/skipping an Unfiltered consumes LegacyAtoms from the underlying legacy atom iterator + // e.g. hasNext() -> iterator.hasNext() -> iterator.readRow() -> atoms.next() + // The stop condition of the loop which groups legacy atoms into rows causes that AtomIterator + // to read in the first atom which doesn't belong in the row. So by that point, our position + // is actually past the end of the next Unfiltered. To compensate, we record the size of + // the last LegacyAtom read and subtract it from the current position when we calculate lastConsumedPosition. + // If we don't, then when reading an indexed block, we can over correct and may think that we've + // exhausted the block before we actually have. + private long bytesReadForNextAtom() + { + // If we've read anything at all then we will have recorded this in bytesReadForNextAtom, + // but being extra careful here just incase this method is called before any reads happen. + return iterator.atoms.next == null ? 0 : bytesReadForNextAtom; + } + public void clearState() { next = null; http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db new file mode 100644 index 0000000..ecd3ddb Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-CompressionInfo.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db new file mode 100644 index 0000000..d1e4e2f Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1 new file mode 100644 index 0000000..bce117c --- /dev/null +++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Digest.sha1 @@ -0,0 +1 @@ +76435450 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db new file mode 100644 index 0000000..00a88b4 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Filter.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db new file mode 100644 index 0000000..9ba4894 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Index.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db new file mode 100644 index 0000000..a57d32b Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db new file mode 100644 index 0000000..d60d8f4 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-Summary.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt new file mode 100644 index 0000000..25fc863 --- /dev/null +++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed/legacy_tables-legacy_ka_indexed-ka-1-TOC.txt @@ -0,0 +1,8 @@ +Data.db +Summary.db +TOC.txt +Statistics.db +Digest.sha1 +Filter.db +Index.db +CompressionInfo.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db new file mode 100644 index 0000000..09c4cfa Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-CompressionInfo.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db new file mode 100644 index 0000000..40ee3c6 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Data.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1 ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1 b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1 new file mode 100644 index 0000000..55ac08c --- /dev/null +++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Digest.sha1 @@ -0,0 +1 @@ +3851004816 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db new file mode 100644 index 0000000..00a88b4 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Filter.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db new file mode 100644 index 0000000..fb6ceed Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Index.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db new file mode 100644 index 0000000..b08f500 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Statistics.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db new file mode 100644 index 0000000..d60d8f4 Binary files /dev/null and b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-Summary.db differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt ---------------------------------------------------------------------- diff --git a/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt new file mode 100644 index 0000000..6865eca --- /dev/null +++ b/test/data/legacy-sstables/ka/legacy_tables/legacy_ka_indexed_static/legacy_tables-legacy_ka_indexed_static-ka-1-TOC.txt @@ -0,0 +1,8 @@ +CompressionInfo.db +Summary.db +Data.db +Index.db +Statistics.db +TOC.txt +Digest.sha1 +Filter.db http://git-wip-us.apache.org/repos/asf/cassandra/blob/62092e45/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index cefee52..f4f6e85 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -146,6 +146,45 @@ public class LegacySSTableTest verifyReads(legacyVersion); } } + @Test + public void testReverseIterationOfLegacyIndexedSSTable() throws Exception + { + // During upgrades from 2.1 to 3.0, reverse queries can drop rows before upgradesstables is completed + QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_indexed (" + + " p int," + + " c int," + + " v1 int," + + " v2 int," + + " PRIMARY KEY(p, c)" + + ")"); + loadLegacyTable("legacy_%s_indexed%s", "ka", ""); + UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * " + + "FROM legacy_tables.legacy_ka_indexed " + + "WHERE p=1 " + + "ORDER BY c DESC"); + Assert.assertEquals(5000, rs.size()); + } + + @Test + public void testReadingLegacyIndexedSSTableWithStaticColumns() throws Exception + { + // During upgrades from 2.1 to 3.0, reading from tables with static columns errors before upgradesstables + // is completed + QueryProcessor.executeInternal("CREATE TABLE legacy_tables.legacy_ka_indexed_static (" + + " p int," + + " c int," + + " v1 int," + + " v2 int," + + " s1 int static," + + " s2 int static," + + " PRIMARY KEY(p, c)" + + ")"); + loadLegacyTable("legacy_%s_indexed_static%s", "ka", ""); + UntypedResultSet rs = QueryProcessor.executeInternal("SELECT * " + + "FROM legacy_tables.legacy_ka_indexed_static " + + "WHERE p=1 "); + Assert.assertEquals(5000, rs.size()); + } private void streamLegacyTables(String legacyVersion) throws Exception { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
