Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/260475d0 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/260475d0 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/260475d0 Branch: refs/heads/cassandra-3.11 Commit: 260475d0f2b1f9a8344a3cfb38f6766c9241b52e Parents: 5dcb354 ae88fd6 Author: Stefania Alborghetti <[email protected]> Authored: Fri Sep 8 12:43:30 2017 +0800 Committer: Stefania Alborghetti <[email protected]> Committed: Fri Sep 8 12:43:30 2017 +0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../io/sstable/IndexSummaryBuilder.java | 70 ++++++++--- .../cassandra/io/sstable/IndexSummaryTest.java | 123 +++++++++++++++++++ 3 files changed, 179 insertions(+), 15 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/260475d0/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 75d9bd6,f4360be..23e6c8b --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,14 -1,4 +1,15 @@@ -3.0.15 +3.11.1 + * Add a compaction option to TWCS to ignore sstables overlapping checks (CASSANDRA-13418) + * BTree.Builder memory leak (CASSANDRA-13754) + * Revert CASSANDRA-10368 of supporting non-pk column filtering due to correctness (CASSANDRA-13798) + * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938) + * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744) + * "ignore" option is ignored in sstableloader (CASSANDRA-13721) + * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652) + * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512) + * Properly evict pstmts from prepared statements cache (CASSANDRA-13641) +Merged from 3.0: ++ * Avoid assertion error when IndexSummary > 2G (CASSANDRA-12014) * Change repair midpoint logging for tiny ranges (CASSANDRA-13603) * Better handle corrupt final commitlog segment (CASSANDRA-11995) * StreamingHistogram is not thread safe (CASSANDRA-13756) http://git-wip-us.apache.org/repos/asf/cassandra/blob/260475d0/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java index f3757a0,ad08ba0..ab0486d --- a/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/IndexSummaryTest.java @@@ -45,14 -44,126 +45,137 @@@ import static org.junit.Assert.* public class IndexSummaryTest { + private final static Random random = new Random(); - private final static IPartitioner partitioner = Util.testPartitioner(); ++ + @BeforeClass + public static void initDD() + { + DatabaseDescriptor.daemonInitialization(); ++ ++ final long seed = System.nanoTime(); ++ System.out.println("Using seed: " + seed); ++ random.setSeed(seed); + } + + IPartitioner partitioner = Util.testPartitioner(); + @BeforeClass + public static void setup() + { + final long seed = System.nanoTime(); + System.out.println("Using seed: " + seed); + random.setSeed(seed); + } + + @Test + public void testIndexSummaryKeySizes() throws IOException + { + testIndexSummaryProperties(32, 100); + testIndexSummaryProperties(64, 100); + testIndexSummaryProperties(100, 100); + testIndexSummaryProperties(1000, 100); + testIndexSummaryProperties(10000, 100); + } + + private void testIndexSummaryProperties(int keySize, int numKeys) throws IOException + { + final int minIndexInterval = 1; + final List<DecoratedKey> keys = new ArrayList<>(numKeys); + + try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL)) + { + for (int i = 0; i < numKeys; i++) + { + byte[] randomBytes = new byte[keySize]; + random.nextBytes(randomBytes); + DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes)); + keys.add(key); + builder.maybeAddEntry(key, i); + } + + try(IndexSummary indexSummary = builder.build(partitioner)) + { + assertEquals(numKeys, keys.size()); + assertEquals(minIndexInterval, indexSummary.getMinIndexInterval()); + assertEquals(numKeys, indexSummary.getMaxNumberOfEntries()); + assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount()); + + for (int i = 0; i < numKeys; i++) + assertEquals(keys.get(i).getKey(), ByteBuffer.wrap(indexSummary.getKey(i))); + } + } + } + + /** + * Test an index summary whose total size is bigger than 2GB, + * the index summary builder should log an error but it should still + * create an index summary, albeit one that does not cover the entire sstable. + */ + @Test + public void tesLargeIndexSummary() throws IOException + { + final int numKeys = 1000000; + final int keySize = 3000; + final int minIndexInterval = 1; + + try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL)) + { + for (int i = 0; i < numKeys; i++) + { + byte[] randomBytes = new byte[keySize]; + random.nextBytes(randomBytes); + DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes)); + builder.maybeAddEntry(key, i); + } + + try (IndexSummary indexSummary = builder.build(partitioner)) + { + assertNotNull(indexSummary); + assertEquals(numKeys, indexSummary.getMaxNumberOfEntries()); + assertEquals(numKeys + 1, indexSummary.getEstimatedKeyCount()); + } + } + } + + /** + * Test an index summary whose total size is bigger than 2GB, + * having updated IndexSummaryBuilder.defaultExpectedKeySize to match the size, + * the index summary should be downsampled automatically. + */ + @Test + public void tesLargeIndexSummaryWithExpectedSizeMatching() throws IOException + { + final int numKeys = 1000000; + final int keySize = 3000; + final int minIndexInterval = 1; + + long oldExpectedKeySize = IndexSummaryBuilder.defaultExpectedKeySize; + IndexSummaryBuilder.defaultExpectedKeySize = 3000; + + try (IndexSummaryBuilder builder = new IndexSummaryBuilder(numKeys, minIndexInterval, BASE_SAMPLING_LEVEL)) + { + for (int i = 0; i < numKeys; i++) + { + byte[] randomBytes = new byte[keySize]; + random.nextBytes(randomBytes); + DecoratedKey key = partitioner.decorateKey(ByteBuffer.wrap(randomBytes)); + builder.maybeAddEntry(key, i); + } + + try (IndexSummary indexSummary = builder.build(partitioner)) + { + assertNotNull(indexSummary); + assertEquals(minIndexInterval * 2, indexSummary.getMinIndexInterval()); + assertEquals(numKeys / 2, indexSummary.getMaxNumberOfEntries()); + assertEquals(numKeys + 2, indexSummary.getEstimatedKeyCount()); + } + } + finally + { + IndexSummaryBuilder.defaultExpectedKeySize = oldExpectedKeySize; + } + } + @Test public void testGetKey() { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
