Give more control over building to 2i impls
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/7f5580b2 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/7f5580b2 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/7f5580b2 Branch: refs/heads/trunk Commit: 7f5580b2241425fd524336d60374f1d759ad93cc Parents: 51b1a1c Author: Sam Tunnicliffe <[email protected]> Authored: Mon Sep 14 11:12:02 2015 +0100 Committer: Sam Tunnicliffe <[email protected]> Committed: Tue Sep 15 11:19:43 2015 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + src/java/org/apache/cassandra/index/Index.java | 13 ++++ .../cassandra/index/SecondaryIndexManager.java | 19 ++++-- .../index/internal/CassandraIndex.java | 6 ++ .../org/apache/cassandra/index/StubIndex.java | 5 ++ .../index/internal/CustomCassandraIndex.java | 5 ++ .../index/internal/CustomIndexTest.java | 67 +++++++++++++++++++- 7 files changed, 109 insertions(+), 7 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/7f5580b2/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 1a1ddeb..cf982dd 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 3.0.0-rc1 + * Give index implementations more control over rebuild operations (CASSANDRA-10312) * Update index file format (CASSANDRA-10314) * Add "shadowable" row tombstones to deal with mv timestamp issues (CASSANDRA-10261) * CFS.loadNewSSTables() broken for pre-3.0 sstables http://git-wip-us.apache.org/repos/asf/cassandra/blob/7f5580b2/src/java/org/apache/cassandra/index/Index.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/Index.java b/src/java/org/apache/cassandra/index/Index.java index 8f126fe..5dc44a4 100644 --- a/src/java/org/apache/cassandra/index/Index.java +++ b/src/java/org/apache/cassandra/index/Index.java @@ -178,6 +178,19 @@ public interface Index */ public Callable<?> getTruncateTask(long truncatedAt); + /** + * Return true if this index can be built or rebuilt when the index manager determines it is necessary. Returning + * false enables the index implementation (or some other component) to control if and when SSTable data is + * incorporated into the index. + * + * This is called by SecondaryIndexManager in buildIndexBlocking, buildAllIndexesBlocking & rebuildIndexesBlocking + * where a return value of false causes the index to be exluded from the set of those which will process the + * SSTable data. + * @return if the index should be included in the set which processes SSTable data, false otherwise. + */ + public boolean shouldBuildBlocking(); + + /* * Index selection */ http://git-wip-us.apache.org/repos/asf/cassandra/blob/7f5580b2/src/java/org/apache/cassandra/index/SecondaryIndexManager.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java index a916bd2..ff4567b 100644 --- a/src/java/org/apache/cassandra/index/SecondaryIndexManager.java +++ b/src/java/org/apache/cassandra/index/SecondaryIndexManager.java @@ -209,7 +209,8 @@ public class SecondaryIndexManager implements IndexRegistry public void rebuildIndexesBlocking(Collection<SSTableReader> sstables, Set<String> indexNames) { Set<Index> toRebuild = indexes.values().stream() - .filter(indexer -> indexNames.contains(indexer.getIndexName())) + .filter(index -> indexNames.contains(index.getIndexName())) + .filter(Index::shouldBuildBlocking) .collect(Collectors.toSet()); if (toRebuild.isEmpty()) { @@ -226,17 +227,23 @@ public class SecondaryIndexManager implements IndexRegistry public void buildAllIndexesBlocking(Collection<SSTableReader> sstables) { - buildIndexesBlocking(sstables, ImmutableSet.copyOf(indexes.values())); + buildIndexesBlocking(sstables, indexes.values() + .stream() + .filter(Index::shouldBuildBlocking) + .collect(Collectors.toSet())); } // For convenience, may be called directly from Index impls public void buildIndexBlocking(Index index) { - try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(View.select(SSTableSet.CANONICAL)); - Refs<SSTableReader> sstables = viewFragment.refs) + if (index.shouldBuildBlocking()) { - buildIndexesBlocking(sstables, Collections.singleton(index)); - markIndexBuilt(index.getIndexName()); + try (ColumnFamilyStore.RefViewFragment viewFragment = baseCfs.selectAndReference(View.select(SSTableSet.CANONICAL)); + Refs<SSTableReader> sstables = viewFragment.refs) + { + buildIndexesBlocking(sstables, Collections.singleton(index)); + markIndexBuilt(index.getIndexName()); + } } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/7f5580b2/src/java/org/apache/cassandra/index/internal/CassandraIndex.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java index 1e2deee..89c072c 100644 --- a/src/java/org/apache/cassandra/index/internal/CassandraIndex.java +++ b/src/java/org/apache/cassandra/index/internal/CassandraIndex.java @@ -215,6 +215,12 @@ public abstract class CassandraIndex implements Index }; } + public boolean shouldBuildBlocking() + { + // built-in indexes are always included in builds initiated from SecondaryIndexManager + return true; + } + public boolean indexes(PartitionColumns columns) { // if we have indexes on the partition key or clustering columns, return true http://git-wip-us.apache.org/repos/asf/cassandra/blob/7f5580b2/test/unit/org/apache/cassandra/index/StubIndex.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/index/StubIndex.java b/test/unit/org/apache/cassandra/index/StubIndex.java index b170c24..0d3cf47 100644 --- a/test/unit/org/apache/cassandra/index/StubIndex.java +++ b/test/unit/org/apache/cassandra/index/StubIndex.java @@ -65,6 +65,11 @@ public class StubIndex implements Index return false; } + public boolean shouldBuildBlocking() + { + return false; + } + public boolean supportsExpression(ColumnDefinition column, Operator operator) { return operator == Operator.EQ; http://git-wip-us.apache.org/repos/asf/cassandra/blob/7f5580b2/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java index 9d9c0b3..0a62d9b 100644 --- a/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java +++ b/test/unit/org/apache/cassandra/index/internal/CustomCassandraIndex.java @@ -163,6 +163,11 @@ public class CustomCassandraIndex implements Index }; } + public boolean shouldBuildBlocking() + { + return true; + } + public boolean indexes(PartitionColumns columns) { // if we have indexes on the partition key or clustering columns, return true http://git-wip-us.apache.org/repos/asf/cassandra/blob/7f5580b2/test/unit/org/apache/cassandra/index/internal/CustomIndexTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/index/internal/CustomIndexTest.java b/test/unit/org/apache/cassandra/index/internal/CustomIndexTest.java index 3cc7987..255af4b 100644 --- a/test/unit/org/apache/cassandra/index/internal/CustomIndexTest.java +++ b/test/unit/org/apache/cassandra/index/internal/CustomIndexTest.java @@ -1,8 +1,16 @@ package org.apache.cassandra.index.internal; -import org.apache.cassandra.cql3.CQLTester; import org.junit.Test; +import org.apache.cassandra.cql3.CQLTester; +import org.apache.cassandra.db.ColumnFamilyStore; +import org.apache.cassandra.index.SecondaryIndexManager; +import org.apache.cassandra.index.StubIndex; +import org.apache.cassandra.schema.IndexMetadata; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + public class CustomIndexTest extends CQLTester { @Test @@ -17,4 +25,61 @@ public class CustomIndexTest extends CQLTester execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 1, 0, 1); execute("INSERT INTO %s (a, b, c, d) VALUES (?, ?, ?, ?)", 0, 2, 0, 0); } + + @Test + public void indexControlsIfIncludedInBuildOnNewSSTables() throws Throwable + { + createTable("CREATE TABLE %s (a int, b int, PRIMARY KEY (a))"); + String toInclude = "include"; + String toExclude = "exclude"; + createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(b) USING '%s'", + toInclude, IndexIncludedInBuild.class.getName())); + createIndex(String.format("CREATE CUSTOM INDEX %s ON %%s(b) USING '%s'", + toExclude, IndexExcludedFromBuild.class.getName())); + + execute("INSERT INTO %s (a, b) VALUES (?, ?)", 0, 0); + execute("INSERT INTO %s (a, b) VALUES (?, ?)", 1, 1); + execute("INSERT INTO %s (a, b) VALUES (?, ?)", 2, 2); + flush(); + + SecondaryIndexManager indexManager = getCurrentColumnFamilyStore().indexManager; + IndexIncludedInBuild included = (IndexIncludedInBuild)indexManager.getIndexByName(toInclude); + included.reset(); + assertTrue(included.rowsInserted.isEmpty()); + + IndexExcludedFromBuild excluded = (IndexExcludedFromBuild)indexManager.getIndexByName(toExclude); + excluded.reset(); + assertTrue(excluded.rowsInserted.isEmpty()); + + indexManager.buildAllIndexesBlocking(getCurrentColumnFamilyStore().getLiveSSTables()); + + assertEquals(3, included.rowsInserted.size()); + assertTrue(excluded.rowsInserted.isEmpty()); + } + + public static final class IndexIncludedInBuild extends StubIndex + { + public IndexIncludedInBuild(ColumnFamilyStore baseCfs, IndexMetadata metadata) + { + super(baseCfs, metadata); + } + + public boolean shouldBuildBlocking() + { + return true; + } + } + + public static final class IndexExcludedFromBuild extends StubIndex + { + public IndexExcludedFromBuild(ColumnFamilyStore baseCfs, IndexMetadata metadata) + { + super(baseCfs, metadata); + } + + public boolean shouldBuildBlocking() + { + return false; + } + } }
