Merge branch 'cassandra-3.0' into cassandra-3.11
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c169d491 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c169d491 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c169d491 Branch: refs/heads/cassandra-3.11 Commit: c169d491ea46abeb3ab33fbae061fd73940db6f1 Parents: f77b663 090f418 Author: Jeff Jirsa <jji...@apple.com> Authored: Wed Dec 6 21:41:53 2017 -0800 Committer: Jeff Jirsa <jji...@apple.com> Committed: Wed Dec 6 21:42:56 2017 -0800 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../db/compaction/CompactionManager.java | 24 +++--- .../org/apache/cassandra/db/CleanupTest.java | 80 ++++++++++++++++++++ 3 files changed, 93 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/c169d491/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 8a7158d,9638886..3c6565c --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,14 -1,6 +1,15 @@@ +3.11.2 + * Remove OpenJDK log warning (CASSANDRA-13916) + * Prevent compaction strategies from looping indefinitely (CASSANDRA-14079) + * Cache disk boundaries (CASSANDRA-13215) + * Add asm jar to build.xml for maven builds (CASSANDRA-11193) + * Round buffer size to powers of 2 for the chunk cache (CASSANDRA-13897) + * Update jackson JSON jars (CASSANDRA-13949) + * Avoid locks when checking LCS fanout and if we should defrag (CASSANDRA-13930) +Merged from 3.0: 3.0.16 + * Fix cleanup on keyspace with no replicas (CASSANDRA-13526) - * Fix updating base table rows with TTL not removing materialized view entries (CASSANDRA-14071) + * Fix updating base table rows with TTL not removing view entries (CASSANDRA-14071) * Reduce garbage created by DynamicSnitch (CASSANDRA-14091) * More frequent commitlog chained markers (CASSANDRA-13987) * Fix serialized size of DataLimits (CASSANDRA-14057) http://git-wip-us.apache.org/repos/asf/cassandra/blob/c169d491/src/java/org/apache/cassandra/db/compaction/CompactionManager.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/CompactionManager.java index 0a2b461,fdda562..3351736 --- a/src/java/org/apache/cassandra/db/compaction/CompactionManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionManager.java @@@ -817,61 -651,6 +813,61 @@@ public class CompactionManager implemen FBUtilities.waitOnFutures(futures); } + public void forceUserDefinedCleanup(String dataFiles) + { + String[] filenames = dataFiles.split(","); + HashMap<ColumnFamilyStore, Descriptor> descriptors = Maps.newHashMap(); + + for (String filename : filenames) + { + // extract keyspace and columnfamily name from filename + Descriptor desc = Descriptor.fromFilename(filename.trim()); + if (Schema.instance.getCFMetaData(desc) == null) + { + logger.warn("Schema does not exist for file {}. Skipping.", filename); + continue; + } + // group by keyspace/columnfamily + ColumnFamilyStore cfs = Keyspace.open(desc.ksname).getColumnFamilyStore(desc.cfname); + desc = cfs.getDirectories().find(new File(filename.trim()).getName()); + if (desc != null) + descriptors.put(cfs, desc); + } + ++ if (!StorageService.instance.isJoined()) ++ { ++ logger.error("Cleanup cannot run before a node has joined the ring"); ++ return; ++ } ++ + for (Map.Entry<ColumnFamilyStore,Descriptor> entry : descriptors.entrySet()) + { + ColumnFamilyStore cfs = entry.getKey(); + Keyspace keyspace = cfs.keyspace; + Collection<Range<Token>> ranges = StorageService.instance.getLocalRanges(keyspace.getName()); + boolean hasIndexes = cfs.indexManager.hasIndexes(); + SSTableReader sstable = lookupSSTable(cfs, entry.getValue()); + - if (ranges.isEmpty()) - { - logger.error("Cleanup cannot run before a node has joined the ring"); - return; - } - + if (sstable == null) + { + logger.warn("Will not clean {}, it is not an active sstable", entry.getValue()); + } + else + { + CleanupStrategy cleanupStrategy = CleanupStrategy.get(cfs, ranges, FBUtilities.nowInSeconds()); + try (LifecycleTransaction txn = cfs.getTracker().tryModify(sstable, OperationType.CLEANUP)) + { + doCleanupOne(cfs, txn, cleanupStrategy, ranges, hasIndexes); + } + catch (IOException e) + { + logger.error("forceUserDefinedCleanup failed: {}", e.getLocalizedMessage()); + } + } + } + } + + public Future<?> submitUserDefined(final ColumnFamilyStore cfs, final Collection<Descriptor> dataFiles, final int gcBefore) { Runnable runnable = new WrappedRunnable() http://git-wip-us.apache.org/repos/asf/cassandra/blob/c169d491/test/unit/org/apache/cassandra/db/CleanupTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/CleanupTest.java index abd5a04,99030c5..80e9b37 --- a/test/unit/org/apache/cassandra/db/CleanupTest.java +++ b/test/unit/org/apache/cassandra/db/CleanupTest.java @@@ -172,34 -204,38 +204,81 @@@ public class CleanupTes assertEquals(0, Util.getAll(Util.cmd(cfs).build()).size()); } - + @Test + public void testCleanupWithNoTokenRange() throws Exception + { ++ testCleanupWithNoTokenRange(false); ++ } ++ ++ @Test ++ public void testUserDefinedCleanupWithNoTokenRange() throws Exception ++ { ++ testCleanupWithNoTokenRange(true); ++ } ++ ++ private void testCleanupWithNoTokenRange(boolean isUserDefined) throws Exception ++ { + + TokenMetadata tmd = StorageService.instance.getTokenMetadata(); + tmd.clearUnsafe(); + tmd.updateHostId(UUID.randomUUID(), InetAddress.getByName("127.0.0.1")); + byte[] tk1 = {2}; + tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1")); + + + Keyspace keyspace = Keyspace.open(KEYSPACE2); + keyspace.setMetadata(KeyspaceMetadata.create(KEYSPACE2, KeyspaceParams.nts("DC1", 1))); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD2); + + // insert data and verify we get it back w/ range query + fillCF(cfs, "val", LOOPS); + assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size()); + + // remove replication on DC1 + keyspace.setMetadata(KeyspaceMetadata.create(KEYSPACE2, KeyspaceParams.nts("DC1", 0))); + + // clear token range for localhost on DC1 - - CompactionManager.instance.performCleanup(cfs, 2); ++ if (isUserDefined) ++ { ++ for (SSTableReader r : cfs.getLiveSSTables()) ++ CompactionManager.instance.forceUserDefinedCleanup(r.getFilename()); ++ } ++ else ++ { ++ CompactionManager.instance.performCleanup(cfs, 2); ++ } + assertEquals(0, Util.getAll(Util.cmd(cfs).build()).size()); + assertTrue(cfs.getLiveSSTables().isEmpty()); + } + @Test + public void testuserDefinedCleanupWithNewToken() throws ExecutionException, InterruptedException, UnknownHostException + { + StorageService.instance.getTokenMetadata().clearUnsafe(); + + Keyspace keyspace = Keyspace.open(KEYSPACE1); + ColumnFamilyStore cfs = keyspace.getColumnFamilyStore(CF_STANDARD1); + + // insert data and verify we get it back w/ range query + fillCF(cfs, "val", LOOPS); + + assertEquals(LOOPS, Util.getAll(Util.cmd(cfs).build()).size()); + TokenMetadata tmd = StorageService.instance.getTokenMetadata(); + + byte[] tk1 = new byte[1], tk2 = new byte[1]; + tk1[0] = 2; + tk2[0] = 1; + tmd.updateNormalToken(new BytesToken(tk1), InetAddress.getByName("127.0.0.1")); + tmd.updateNormalToken(new BytesToken(tk2), InetAddress.getByName("127.0.0.2")); + + for(SSTableReader r: cfs.getLiveSSTables()) + CompactionManager.instance.forceUserDefinedCleanup(r.getFilename()); + + assertEquals(0, Util.getAll(Util.cmd(cfs).build()).size()); + } + + @Test public void testNeedsCleanup() throws Exception { // setup --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
