Merge branch 'cassandra-2.0' into cassandra-2.1
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/f18fb044 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/f18fb044 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/f18fb044 Branch: refs/heads/trunk Commit: f18fb044527115d9dfe8586bf1615825463b1156 Parents: c6e4379 b325316 Author: Aleksey Yeschenko <[email protected]> Authored: Tue Apr 21 14:41:08 2015 +0300 Committer: Aleksey Yeschenko <[email protected]> Committed: Tue Apr 21 14:41:08 2015 +0300 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/db/index/SecondaryIndex.java | 10 +++ .../db/index/SecondaryIndexManager.java | 19 +++++ .../cassandra/db/ColumnFamilyStoreTest.java | 77 +++++++++----------- .../db/index/PerRowSecondaryIndexTest.java | 11 ++- 5 files changed, 74 insertions(+), 44 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/f18fb044/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 2777d79,5e1e62c..a40530a --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,86 -1,5 +1,87 @@@ -2.0.15: +2.1.5 + * Re-add deprecated cold_reads_to_omit param for backwards compat (CASSANDRA-9203) + * Make anticompaction visible in compactionstats (CASSANDRA-9098) + * Improve nodetool getendpoints documentation about the partition + key parameter (CASSANDRA-6458) + * Don't check other keyspaces for schema changes when an user-defined + type is altered (CASSANDRA-9187) + * Allow takeColumnFamilySnapshot to take a list of tables (CASSANDRA-8348) + * Limit major sstable operations to their canonical representation (CASSANDRA-8669) + * cqlsh: Add tests for INSERT and UPDATE tab completion (CASSANDRA-9125) + * cqlsh: quote column names when needed in COPY FROM inserts (CASSANDRA-9080) + * Add generate-idea-files target to build.xml (CASSANDRA-9123) + * Do not load read meter for offline operations (CASSANDRA-9082) + * cqlsh: Make CompositeType data readable (CASSANDRA-8919) + * cqlsh: Fix display of triggers (CASSANDRA-9081) + * Fix NullPointerException when deleting or setting an element by index on + a null list collection (CASSANDRA-9077) + * Buffer bloom filter serialization (CASSANDRA-9066) + * Fix anti-compaction target bloom filter size (CASSANDRA-9060) + * Make FROZEN and TUPLE unreserved keywords in CQL (CASSANDRA-9047) + * Prevent AssertionError from SizeEstimatesRecorder (CASSANDRA-9034) + * Avoid overwriting index summaries for sstables with an older format that + does not support downsampling; rebuild summaries on startup when this + is detected (CASSANDRA-8993) + * Fix potential data loss in CompressedSequentialWriter (CASSANDRA-8949) + * Make PasswordAuthenticator number of hashing rounds configurable (CASSANDRA-8085) + * Fix AssertionError when binding nested collections in DELETE (CASSANDRA-8900) + * Check for overlap with non-early sstables in LCS (CASSANDRA-8739) + * Only calculate max purgable timestamp if we have to (CASSANDRA-8914) + * (cqlsh) Greatly improve performance of COPY FROM (CASSANDRA-8225) + * IndexSummary effectiveIndexInterval is now a guideline, not a rule (CASSANDRA-8993) + * Use correct bounds for page cache eviction of compressed files (CASSANDRA-8746) + * SSTableScanner enforces its bounds (CASSANDRA-8946) + * Cleanup cell equality (CASSANDRA-8947) + * Introduce intra-cluster message coalescing (CASSANDRA-8692) + * DatabaseDescriptor throws NPE when rpc_interface is used (CASSANDRA-8839) + * Don't check if an sstable is live for offline compactions (CASSANDRA-8841) + * Don't set clientMode in SSTableLoader (CASSANDRA-8238) + * Fix SSTableRewriter with disabled early open (CASSANDRA-8535) + * Allow invalidating permissions and cache time (CASSANDRA-8722) + * Log warning when queries that will require ALLOW FILTERING in Cassandra 3.0 + are executed (CASSANDRA-8418) + * Fix cassandra-stress so it respects the CL passed in user mode (CASSANDRA-8948) + * Fix rare NPE in ColumnDefinition#hasIndexOption() (CASSANDRA-8786) + * cassandra-stress reports per-operation statistics, plus misc (CASSANDRA-8769) + * Add SimpleDate (cql date) and Time (cql time) types (CASSANDRA-7523) + * Use long for key count in cfstats (CASSANDRA-8913) + * Make SSTableRewriter.abort() more robust to failure (CASSANDRA-8832) + * Remove cold_reads_to_omit from STCS (CASSANDRA-8860) + * Make EstimatedHistogram#percentile() use ceil instead of floor (CASSANDRA-8883) + * Fix top partitions reporting wrong cardinality (CASSANDRA-8834) + * Fix rare NPE in KeyCacheSerializer (CASSANDRA-8067) + * Pick sstables for validation as late as possible inc repairs (CASSANDRA-8366) + * Fix commitlog getPendingTasks to not increment (CASSANDRA-8856) + * Fix parallelism adjustment in range and secondary index queries + when the first fetch does not satisfy the limit (CASSANDRA-8856) + * Check if the filtered sstables is non-empty in STCS (CASSANDRA-8843) + * Upgrade java-driver used for cassandra-stress (CASSANDRA-8842) + * Fix CommitLog.forceRecycleAllSegments() memory access error (CASSANDRA-8812) + * Improve assertions in Memory (CASSANDRA-8792) + * Fix SSTableRewriter cleanup (CASSANDRA-8802) + * Introduce SafeMemory for CompressionMetadata.Writer (CASSANDRA-8758) + * 'nodetool info' prints exception against older node (CASSANDRA-8796) + * Ensure SSTableReader.last corresponds exactly with the file end (CASSANDRA-8750) + * Make SSTableWriter.openEarly more robust and obvious (CASSANDRA-8747) + * Enforce SSTableReader.first/last (CASSANDRA-8744) + * Cleanup SegmentedFile API (CASSANDRA-8749) + * Avoid overlap with early compaction replacement (CASSANDRA-8683) + * Safer Resource Management++ (CASSANDRA-8707) + * Write partition size estimates into a system table (CASSANDRA-7688) + * cqlsh: Fix keys() and full() collection indexes in DESCRIBE output + (CASSANDRA-8154) + * Show progress of streaming in nodetool netstats (CASSANDRA-8886) + * IndexSummaryBuilder utilises offheap memory, and shares data between + each IndexSummary opened from it (CASSANDRA-8757) + * markCompacting only succeeds if the exact SSTableReader instances being + marked are in the live set (CASSANDRA-8689) + * cassandra-stress support for varint (CASSANDRA-8882) + * Fix Adler32 digest for compressed sstables (CASSANDRA-8778) + * Add nodetool statushandoff/statusbackup (CASSANDRA-8912) + * Use stdout for progress and stats in sstableloader (CASSANDRA-8982) + * Correctly identify 2i datadir from older versions (CASSANDRA-9116) +Merged from 2.0: + * Do not attempt to rebuild indexes if no index accepts any column (CASSANDRA-9196) * Don't initiate snitch reconnection for dead states (CASSANDRA-7292) * Fix ArrayIndexOutOfBoundsException in CQLSSTableWriter (CASSANDRA-8978) * Add shutdown gossip state to prevent timeouts during rolling restarts (CASSANDRA-8336) http://git-wip-us.apache.org/repos/asf/cassandra/blob/f18fb044/src/java/org/apache/cassandra/db/index/SecondaryIndex.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/SecondaryIndex.java index 3081016,a83f5dd..5a6aecc --- a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java @@@ -304,11 -290,22 +304,21 @@@ public abstract class SecondaryInde } /** - * Returns true if the provided column name is indexed by this secondary index. - * - * The default implement checks whether the name is one the columnDef name, - * but this should be overriden but subclass if needed. + * Returns true if the provided cell name is indexed by this secondary index. */ - public boolean indexes(ByteBuffer name) + public abstract boolean indexes(CellName name); + + /** ++ * Returns true if the defined column is indexed by this secondary index. ++ * @param column definition of the column to check ++ * @return whether the supplied column is indexed or not ++ */ ++ public boolean indexes(ColumnDefinition column) + { - for (ColumnDefinition columnDef : columnDefs) - { - if (baseCfs.getComparator().compare(columnDef.name, name) == 0) - return true; - } - return false; ++ return columnDefs.contains(column); + } + + /** * This is the primary way to create a secondary index instance for a CF column. * It will validate the index_options before initializing. * http://git-wip-us.apache.org/repos/asf/cassandra/blob/f18fb044/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java index fbbec1d,1db7de6..42e8a93 --- a/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java @@@ -689,22 -581,24 +690,40 @@@ public class SecondaryIndexManage return true; } + static boolean shouldCleanupOldValue(Cell oldCell, Cell newCell) + { + // If any one of name/value/timestamp are different, then we + // should delete from the index. If not, then we can infer that + // at least one of the cells is an ExpiringColumn and that the + // difference is in the expiry time. In this case, we don't want to + // delete the old value from the index as the tombstone we insert + // will just hide the inserted value. + // Completely identical cells (including expiring columns with + // identical ttl & localExpirationTime) will not get this far due + // to the oldCell.equals(newColumn) in StandardUpdater.update + return !oldCell.name().equals(newCell.name()) + || !oldCell.value().equals(newCell.value()) + || oldCell.timestamp() != newCell.timestamp(); + } + + private Set<String> filterByColumn(Set<String> idxNames) + { + Set<SecondaryIndex> indexes = getIndexesByNames(idxNames); + Set<String> filtered = new HashSet<>(idxNames.size()); + for (SecondaryIndex candidate : indexes) + { + for (ColumnDefinition column : baseCfs.metadata.allColumns()) + { - if (candidate.indexes(column.name)) ++ if (candidate.indexes(column)) + { + filtered.add(candidate.getIndexName()); + break; + } + } + } + return filtered; + } + public static interface Updater { /** called when constructing the index against pre-existing data */ http://git-wip-us.apache.org/repos/asf/cassandra/blob/f18fb044/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index ec8ad8a,8f4a18f..a095dba --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@@ -51,58 -40,30 +38,36 @@@ import org.junit.runner.RunWith import org.apache.cassandra.OrderedJUnit4ClassRunner; import org.apache.cassandra.SchemaLoader; import org.apache.cassandra.Util; --import org.apache.cassandra.config.CFMetaData; --import org.apache.cassandra.config.ColumnDefinition; - import org.apache.cassandra.config.DatabaseDescriptor; - import org.apache.cassandra.config.IndexType; --import org.apache.cassandra.config.Schema; ++import org.apache.cassandra.config.*; +import org.apache.cassandra.cql3.Operator; import org.apache.cassandra.db.columniterator.IdentityQueryFilter; - import org.apache.cassandra.db.composites.CellName; - import org.apache.cassandra.db.composites.CellNameType; - import org.apache.cassandra.db.composites.CellNames; - import org.apache.cassandra.db.composites.Composites; - import org.apache.cassandra.db.filter.ColumnSlice; - import org.apache.cassandra.db.filter.IDiskAtomFilter; - import org.apache.cassandra.db.filter.NamesQueryFilter; - import org.apache.cassandra.db.filter.QueryFilter; - import org.apache.cassandra.db.filter.SliceQueryFilter; ++import org.apache.cassandra.db.composites.*; + import org.apache.cassandra.db.filter.*; + import org.apache.cassandra.db.index.PerRowSecondaryIndexTest; import org.apache.cassandra.db.index.SecondaryIndex; -import org.apache.cassandra.db.marshal.CompositeType; import org.apache.cassandra.db.marshal.LexicalUUIDType; import org.apache.cassandra.db.marshal.LongType; - import org.apache.cassandra.dht.Bounds; - import org.apache.cassandra.dht.ExcludingBounds; - import org.apache.cassandra.dht.IPartitioner; - import org.apache.cassandra.dht.IncludingExcludingBounds; - import org.apache.cassandra.dht.Range; - import org.apache.cassandra.io.sstable.Component; - import org.apache.cassandra.io.sstable.Descriptor; - import org.apache.cassandra.io.sstable.SSTableDeletingTask; - import org.apache.cassandra.io.sstable.SSTableReader; - import org.apache.cassandra.io.sstable.SSTableSimpleWriter; - import org.apache.cassandra.io.sstable.SSTableWriter; + import org.apache.cassandra.dht.*; -import org.apache.cassandra.exceptions.ConfigurationException; + import org.apache.cassandra.io.sstable.*; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.service.ActiveRepairService; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.thrift.*; -import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.Pair; -import org.apache.cassandra.utils.WrappedRunnable; - -import static org.junit.Assert.*; -import static org.apache.cassandra.Util.*; +import org.apache.cassandra.thrift.SlicePredicate; +import org.apache.cassandra.thrift.SliceRange; +import org.apache.cassandra.thrift.ThriftValidation; - import org.apache.cassandra.utils.ByteBufferUtil; - import org.apache.cassandra.utils.FBUtilities; - import org.apache.cassandra.utils.Pair; - import org.apache.cassandra.utils.WrappedRunnable; ++import org.apache.cassandra.utils.*; + +import static org.apache.cassandra.Util.cellname; +import static org.apache.cassandra.Util.column; +import static org.apache.cassandra.Util.dk; +import static org.apache.cassandra.Util.rp; import static org.apache.cassandra.utils.ByteBufferUtil.bytes; -import static org.apache.commons.lang3.ArrayUtils.EMPTY_BYTE_ARRAY; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; @RunWith(OrderedJUnit4ClassRunner.class) public class ColumnFamilyStoreTest extends SchemaLoader @@@ -2199,4 -2173,30 +2164,30 @@@ }); System.err.println("Row key: " + rowKey + " Cols: " + transformed); } + + @Test + public void testRebuildSecondaryIndex() throws IOException + { - RowMutation rm; ++ Mutation rm; + - rm = new RowMutation("PerRowSecondaryIndex", ByteBufferUtil.bytes("k1")); - rm.add("Indexed1", ByteBufferUtil.bytes("indexed"), ByteBufferUtil.bytes("foo"), 1); ++ rm = new Mutation("PerRowSecondaryIndex", ByteBufferUtil.bytes("k1")); ++ rm.add("Indexed1", cellname("indexed"), ByteBufferUtil.bytes("foo"), 1); + rm.apply(); + assertTrue(Arrays.equals("k1".getBytes(), PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_KEY.array())); + + Keyspace.open("PerRowSecondaryIndex").getColumnFamilyStore("Indexed1").forceBlockingFlush(); + + PerRowSecondaryIndexTest.TestIndex.reset(); + + ColumnFamilyStore.rebuildSecondaryIndex("PerRowSecondaryIndex", "Indexed1", PerRowSecondaryIndexTest.TestIndex.class.getSimpleName()); + assertTrue(Arrays.equals("k1".getBytes(), PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_KEY.array())); + + PerRowSecondaryIndexTest.TestIndex.reset(); + + PerRowSecondaryIndexTest.TestIndex.ACTIVE = false; + ColumnFamilyStore.rebuildSecondaryIndex("PerRowSecondaryIndex", "Indexed1", PerRowSecondaryIndexTest.TestIndex.class.getSimpleName()); + assertNull(PerRowSecondaryIndexTest.TestIndex.LAST_INDEXED_KEY); + + PerRowSecondaryIndexTest.TestIndex.reset(); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/f18fb044/test/unit/org/apache/cassandra/db/index/PerRowSecondaryIndexTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/index/PerRowSecondaryIndexTest.java index 6aeee67,6a6956c..9dbd3be --- a/test/unit/org/apache/cassandra/db/index/PerRowSecondaryIndexTest.java +++ b/test/unit/org/apache/cassandra/db/index/PerRowSecondaryIndexTest.java @@@ -29,22 -27,15 +29,23 @@@ import org.junit.Before import org.junit.Test; import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.Util; ++import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.QueryProcessor; +import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.columniterator.IdentityQueryFilter; +import org.apache.cassandra.db.composites.CellName; +import org.apache.cassandra.db.filter.ExtendedFilter; import org.apache.cassandra.db.filter.QueryFilter; +import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.concurrent.OpOrder; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.junit.Assert.*; public class PerRowSecondaryIndexTest extends SchemaLoader { @@@ -156,6 -127,12 +159,12 @@@ } @Override - public boolean indexes(ByteBuffer name) ++ public boolean indexes(ColumnDefinition column) + { + return ACTIVE; + } + + @Override public void index(ByteBuffer rowKey, ColumnFamily cf) { QueryFilter filter = QueryFilter.getIdentityFilter(DatabaseDescriptor.getPartitioner().decorateKey(rowKey),
