Repository: cassandra Updated Branches: refs/heads/cassandra-2.0 75ff51e12 -> 9269cb83c
Pick unused generations when loading new sstables patch by Tyler Hobbs; reviewed by Jonathan Ellis for CASSANDRA-6514 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/9269cb83 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/9269cb83 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/9269cb83 Branch: refs/heads/cassandra-2.0 Commit: 9269cb83ce0bd45eff0f42611d5b5cd4415cba31 Parents: 75ff51e Author: Tyler Hobbs <[email protected]> Authored: Tue Mar 18 18:03:06 2014 -0500 Committer: Tyler Hobbs <[email protected]> Committed: Tue Mar 18 18:03:06 2014 -0500 ---------------------------------------------------------------------- CHANGES.txt | 2 + .../apache/cassandra/db/ColumnFamilyStore.java | 26 +++++-- .../cassandra/db/ColumnFamilyStoreTest.java | 79 ++++++++++++++++++++ 3 files changed, 101 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/9269cb83/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 9caca38..4741475 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,6 @@ 2.0.7 + * Lower chances for losing new SSTables during nodetool refresh and + ColumnFamilyStore.loadNewSSTables (CASSANDRA-6514) * Add support for DELETE ... IF EXISTS to CQL3 (CASSANDRA-5708) * Update hadoop_cql3_word_count example (CASSANDRA-6793) * Fix handling of RejectedExecution in sync Thrift server (CASSANDRA-6788) http://git-wip-us.apache.org/repos/asf/cassandra/blob/9269cb83/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index eed44a4..5c3eb19 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -652,12 +652,20 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean continue; } - Descriptor newDescriptor = new Descriptor(descriptor.version, - descriptor.directory, - descriptor.ksname, - descriptor.cfname, - fileIndexGenerator.incrementAndGet(), - false); + // Increment the generation until we find a filename that doesn't exist. This is needed because the new + // SSTables that are being loaded might already use these generation numbers. + Descriptor newDescriptor; + do + { + newDescriptor = new Descriptor(descriptor.version, + descriptor.directory, + descriptor.ksname, + descriptor.cfname, + fileIndexGenerator.incrementAndGet(), + false); + } + while (new File(newDescriptor.filenameFor(Component.DATA)).exists()); + logger.info("Renaming new SSTable {} to {}", descriptor, newDescriptor); SSTableWriter.rename(descriptor, newDescriptor, entry.getValue()); @@ -2431,4 +2439,10 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean Pair<ReplayPosition, Long> truncationRecord = SystemKeyspace.getTruncationRecords().get(metadata.cfId); return truncationRecord == null ? Long.MIN_VALUE : truncationRecord.right; } + + @VisibleForTesting + void resetFileIndexGenerator() + { + fileIndexGenerator.set(0); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/9269cb83/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index 2edf6a8..5fc006b 100644 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@ -29,6 +29,8 @@ import java.util.concurrent.Future; import com.google.common.base.Function; import com.google.common.collect.Iterables; import com.google.common.collect.Sets; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.util.FileUtils; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.StringUtils; import org.junit.Test; @@ -1744,6 +1746,83 @@ public class ColumnFamilyStoreTest extends SchemaLoader assert sstables.containsKey(sstable1.descriptor); } + @Test + public void testLoadNewSSTablesAvoidsOverwrites() throws Throwable + { + String ks = "Keyspace1"; + String cf = "Standard1"; + ColumnFamilyStore cfs = Keyspace.open(ks).getColumnFamilyStore(cf); + cfs.truncateBlocking(); + SSTableDeletingTask.waitForDeletions(); + + final CFMetaData cfmeta = Schema.instance.getCFMetaData(ks, cf); + Directories dir = Directories.create(ks, cf); + + // clear old SSTables (probably left by CFS.clearUnsafe() calls in other tests) + for (Map.Entry<Descriptor, Set<Component>> entry : dir.sstableLister().list().entrySet()) + { + for (Component component : entry.getValue()) + { + FileUtils.delete(entry.getKey().filenameFor(component)); + } + } + + // sanity check + int existingSSTables = dir.sstableLister().list().keySet().size(); + assert existingSSTables == 0 : String.format("%d SSTables unexpectedly exist", existingSSTables); + + ByteBuffer key = bytes("key"); + + SSTableSimpleWriter writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), + cfmeta, StorageService.getPartitioner()); + writer.newRow(key); + writer.addColumn(bytes("col"), bytes("val"), 1); + writer.close(); + + writer = new SSTableSimpleWriter(dir.getDirectoryForNewSSTables(), + cfmeta, StorageService.getPartitioner()); + writer.newRow(key); + writer.addColumn(bytes("col"), bytes("val"), 1); + writer.close(); + + Set<Integer> generations = new HashSet<>(); + for (Descriptor descriptor : dir.sstableLister().list().keySet()) + generations.add(descriptor.generation); + + // we should have two generations: [1, 2] + assertEquals(2, generations.size()); + assertTrue(generations.contains(1)); + assertTrue(generations.contains(2)); + + assertEquals(0, cfs.getSSTables().size()); + + // start the generation counter at 1 again (other tests have incremented it already) + cfs.resetFileIndexGenerator(); + + boolean incrementalBackupsEnabled = DatabaseDescriptor.isIncrementalBackupsEnabled(); + try + { + // avoid duplicate hardlinks to incremental backups + DatabaseDescriptor.setIncrementalBackupsEnabled(false); + cfs.loadNewSSTables(); + } + finally + { + DatabaseDescriptor.setIncrementalBackupsEnabled(incrementalBackupsEnabled); + } + + assertEquals(2, cfs.getSSTables().size()); + generations = new HashSet<>(); + for (Descriptor descriptor : dir.sstableLister().list().keySet()) + generations.add(descriptor.generation); + + // normally they would get renamed to generations 1 and 2, but since those filenames already exist, + // they get skipped and we end up with generations 3 and 4 + assertEquals(2, generations.size()); + assertTrue(generations.contains(3)); + assertTrue(generations.contains(4)); + } + private ColumnFamilyStore prepareMultiRangeSlicesTest(int valueSize, boolean flush) throws Throwable { String keyspaceName = "Keyspace1";
