Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/bf7d4334 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/bf7d4334 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/bf7d4334 Branch: refs/heads/trunk Commit: bf7d4334edf7f845f1abdc6ed2285a1738788b13 Parents: 251449f 3128802 Author: Marcus Eriksson <marc...@apache.org> Authored: Tue Apr 26 08:28:33 2016 +0200 Committer: Marcus Eriksson <marc...@apache.org> Committed: Tue Apr 26 08:28:33 2016 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../compaction/LeveledCompactionStrategy.java | 15 ++++- .../db/compaction/LeveledManifest.java | 15 ++++- .../LongLeveledCompactionStrategyTest.java | 64 ++++++++++++++++++++ 4 files changed, 91 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/bf7d4334/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index ea21ee7,06b27b4..7d6760b --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -23,25 -8,16 +23,26 @@@ Merged from 2.1 * cqlsh COPY FROM fails with []{} chars in UDT/tuple fields/values (CASSANDRA-11633) * clqsh: COPY FROM throws TypeError with Cython extensions enabled (CASSANDRA-11574) * cqlsh: COPY FROM ignores NULL values in conversion (CASSANDRA-11549) - * (cqlsh) Fix potential COPY deadlock when parent process is terminating child - processes (CASSANDRA-11505) + * Validate levels when building LeveledScanner to avoid overlaps with orphaned sstables (CASSANDRA-9935) -2.2.6 - * Allow only DISTINCT queries with partition keys restrictions (CASSANDRA-11339) - * CqlConfigHelper no longer requires both a keystore and truststore to work (CASSANDRA-11532) - * Make deprecated repair methods backward-compatible with previous notification service (CASSANDRA-11430) - * IncomingStreamingConnection version check message wrong (CASSANDRA-11462) +3.0.5 + * Fix rare NPE on schema upgrade from 2.x to 3.x (CASSANDRA-10943) + * Improve backoff policy for cqlsh COPY FROM (CASSANDRA-11320) + * Improve IF NOT EXISTS check in CREATE INDEX (CASSANDRA-11131) + * Upgrade ohc to 0.4.3 + * Enable SO_REUSEADDR for JMX RMI server sockets (CASSANDRA-11093) + * Allocate merkletrees with the correct size (CASSANDRA-11390) + * Support streaming pre-3.0 sstables (CASSANDRA-10990) + * Add backpressure to compressed commit log (CASSANDRA-10971) + * SSTableExport supports secondary index tables (CASSANDRA-11330) + * Fix sstabledump to include missing info in debug output (CASSANDRA-11321) + * Establish and implement canonical bulk reading workload(s) (CASSANDRA-10331) + * Fix paging for IN queries on tables without clustering columns (CASSANDRA-11208) + * Remove recursive call from CompositesSearcher (CASSANDRA-11304) + * Fix filtering on non-primary key columns for queries without index (CASSANDRA-6377) + * Fix sstableloader fail when using materialized view (CASSANDRA-11275) +Merged from 2.2: * DatabaseDescriptor should log stacktrace in case of Eception during seed provider creation (CASSANDRA-11312) * Use canonical path for directory in SSTable descriptor (CASSANDRA-10587) * Add cassandra-stress keystore option (CASSANDRA-9325) http://git-wip-us.apache.org/repos/asf/cassandra/blob/bf7d4334/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java index 5a5cbef,bdc9141..cd74620 --- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java @@@ -216,8 -215,10 +216,10 @@@ public class LeveledCompactionStrategy return maxSSTableSizeInMB * 1024L * 1024L; } - public ScannerList getScanners(Collection<SSTableReader> sstables, Range<Token> range) + public ScannerList getScanners(Collection<SSTableReader> sstables, Collection<Range<Token>> ranges) { + Set<SSTableReader>[] sstablesPerLevel = manifest.getSStablesPerLevelSnapshot(); + Multimap<Integer, SSTableReader> byLevel = ArrayListMultimap.create(); for (SSTableReader sstable : sstables) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/bf7d4334/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index b0cd077,3b207da..2bfe88f --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@@ -25,10 -25,9 +25,11 @@@ import com.google.common.base.Predicate import com.google.common.base.Predicates; import com.google.common.collect.ImmutableSortedSet; import com.google.common.collect.Iterables; + import com.google.common.collect.Lists; import com.google.common.collect.Sets; import com.google.common.primitives.Ints; + +import org.apache.cassandra.db.PartitionPosition; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.slf4j.Logger; import org.slf4j.LoggerFactory; http://git-wip-us.apache.org/repos/asf/cassandra/blob/bf7d4334/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java ---------------------------------------------------------------------- diff --cc test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java index 96ee072,5439a72..79497aa --- a/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java +++ b/test/long/org/apache/cassandra/db/compaction/LongLeveledCompactionStrategyTest.java @@@ -21,19 -21,22 +21,26 @@@ import java.nio.ByteBuffer import java.util.*; import java.util.concurrent.*; -import org.apache.cassandra.db.columniterator.OnDiskAtomIterator; ++import com.google.common.collect.Lists; ++ ++import org.apache.cassandra.db.lifecycle.SSTableSet; ++import org.apache.cassandra.db.rows.UnfilteredRowIterator; + import org.apache.cassandra.io.sstable.ISSTableScanner; import org.apache.cassandra.io.sstable.format.SSTableReader; import org.junit.BeforeClass; import org.junit.Test; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; -import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.UpdateBuilder; import org.apache.cassandra.db.*; import org.apache.cassandra.exceptions.ConfigurationException; -import org.apache.cassandra.locator.SimpleStrategy; +import org.apache.cassandra.schema.CompactionParams; +import org.apache.cassandra.schema.KeyspaceParams; import org.apache.cassandra.utils.FBUtilities; + import static org.junit.Assert.assertTrue; + public class LongLeveledCompactionStrategyTest { public static final String KEYSPACE1 = "LongLeveledCompactionStrategyTest"; @@@ -134,4 -140,62 +141,61 @@@ } } } + + @Test + public void testLeveledScanner() throws Exception + { + testParallelLeveledCompaction(); + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore store = keyspace.getColumnFamilyStore(CF_STANDARDLVL); + store.disableAutoCompaction(); + - WrappingCompactionStrategy strategy = ((WrappingCompactionStrategy) store.getCompactionStrategy()); - LeveledCompactionStrategy lcs = (LeveledCompactionStrategy) strategy.getWrappedStrategies().get(1); ++ LeveledCompactionStrategy lcs = (LeveledCompactionStrategy)store.getCompactionStrategyManager().getStrategies().get(1); + + ByteBuffer value = ByteBuffer.wrap(new byte[10 * 1024]); // 10 KB value + + // Adds 10 partitions + for (int r = 0; r < 10; r++) + { + DecoratedKey key = Util.dk(String.valueOf(r)); - Mutation rm = new Mutation(KEYSPACE1, key.getKey()); ++ UpdateBuilder builder = UpdateBuilder.create(store.metadata, key); + for (int c = 0; c < 10; c++) - { - rm.add(CF_STANDARDLVL, Util.cellname("column" + c), value, 0); - } ++ builder.newRow("column" + c).add("val", value); ++ ++ Mutation rm = new Mutation(builder.build()); + rm.apply(); + } + + //Flush sstable + store.forceBlockingFlush(); + - Collection<SSTableReader> allSSTables = store.getSSTables(); ++ Iterable<SSTableReader> allSSTables = store.getSSTables(SSTableSet.LIVE); + for (SSTableReader sstable : allSSTables) + { + if (sstable.getSSTableLevel() == 0) + { + System.out.println("Mutating L0-SSTABLE level to L1 to simulate a bug: " + sstable.getFilename()); + sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 1); + sstable.reloadSSTableMetadata(); + } + } + - try (AbstractCompactionStrategy.ScannerList scannerList = lcs.getScanners(allSSTables)) ++ try (AbstractCompactionStrategy.ScannerList scannerList = lcs.getScanners(Lists.newArrayList(allSSTables))) + { + //Verify that leveled scanners will always iterate in ascending order (CASSANDRA-9935) + for (ISSTableScanner scanner : scannerList.scanners) + { + DecoratedKey lastKey = null; + while (scanner.hasNext()) + { - OnDiskAtomIterator row = scanner.next(); ++ UnfilteredRowIterator row = scanner.next(); + if (lastKey != null) + { - assertTrue("row " + row.getKey() + " received out of order wrt " + lastKey, row.getKey().compareTo(lastKey) >= 0); ++ assertTrue("row " + row.partitionKey() + " received out of order wrt " + lastKey, row.partitionKey().compareTo(lastKey) >= 0); + } - lastKey = row.getKey(); ++ lastKey = row.partitionKey(); + } + } + } + } }
