Merge branch 'cassandra-2.0' into cassandra-2.1
Conflicts:
CHANGES.txt
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/26ff1507
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/26ff1507
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/26ff1507
Branch: refs/heads/trunk
Commit: 26ff15070eb67779050975a9ae07053f4b5e5b04
Parents: 7a6a509 f53aacb
Author: Marcus Eriksson <[email protected]>
Authored: Mon Aug 17 08:30:17 2015 +0200
Committer: Marcus Eriksson <[email protected]>
Committed: Mon Aug 17 08:30:17 2015 +0200
----------------------------------------------------------------------
CHANGES.txt | 1 +
.../apache/cassandra/db/ColumnFamilyStore.java | 55 +++++++++++++++++---
.../DateTieredCompactionStrategy.java | 13 ++++-
.../DateTieredCompactionStrategyTest.java | 1 +
4 files changed, 60 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/cassandra/blob/26ff1507/CHANGES.txt
----------------------------------------------------------------------
diff --cc CHANGES.txt
index 5a76978,905445e..4626899
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@@ -1,27 -1,5 +1,28 @@@
-2.0.17
+2.1.9
+ * (cqlsh) Allow encoding to be set through command line (CASSANDRA-10004)
+ * Add new JMX methods to change local compaction strategy (CASSANDRA-9965)
+ * Write hints for paxos commits (CASSANDRA-7342)
+ * (cqlsh) Fix timestamps before 1970 on Windows, always
+ use UTC for timestamp display (CASSANDRA-10000)
+ * (cqlsh) Avoid overwriting new config file with old config
+ when both exist (CASSANDRA-9777)
+ * Release snapshot selfRef when doing snapshot repair (CASSANDRA-9998)
+ * Cannot replace token does not exist - DN node removed as Fat Client
(CASSANDRA-9871)
+ * Fix handling of enable/disable autocompaction (CASSANDRA-9899)
+ * Commit log segment recycling is disabled by default (CASSANDRA-9896)
+ * Add consistency level to tracing ouput (CASSANDRA-9827)
+ * Fix MarshalException when upgrading superColumn family (CASSANDRA-9582)
+ * Fix broken logging for "empty" flushes in Memtable (CASSANDRA-9837)
+ * Handle corrupt files on startup (CASSANDRA-9686)
+ * Fix clientutil jar and tests (CASSANDRA-9760)
+ * (cqlsh) Allow the SSL protocol version to be specified through the
+ config file or environment variables (CASSANDRA-9544)
+ * Remove repair snapshot leftover on startup (CASSANDRA-7357)
+ * Use random nodes for batch log when only 2 racks (CASSANDRA-8735)
+ * Ensure atomicity inside thrift and stream session (CASSANDRA-7757)
+ * Fix nodetool info error when the node is not joined (CASSANDRA-9031)
+Merged from 2.0:
+ * Make getFullyExpiredSSTables less expensive (CASSANDRA-9882)
* Add tool to find why expired sstables are not getting dropped
(CASSANDRA-10015)
* Remove erroneous pending HH tasks from tpstats/jmx (CASSANDRA-9129)
* Don't cast expected bf size to an int (CASSANDRA-9959)
http://git-wip-us.apache.org/repos/asf/cassandra/blob/26ff1507/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
----------------------------------------------------------------------
diff --cc src/java/org/apache/cassandra/db/ColumnFamilyStore.java
index 7364528,eb688f7..25b3e57
--- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
+++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
@@@ -1367,20 -1025,59 +1367,59 @@@ public class ColumnFamilyStore implemen
// a normal compaction won't ever have an empty sstables list, but we
create a skeleton
// compaction controller for streaming, and that passes an empty list.
- if (sstables.isEmpty())
+ if (!sstables.iterator().hasNext())
return ImmutableSet.of();
- DataTracker.SSTableIntervalTree tree = data.getView().intervalTree;
-
- Set<SSTableReader> results = null;
- for (SSTableReader sstable : sstables)
- List<SSTableReader> sortedByFirst = new ArrayList<>(sstables);
++ List<SSTableReader> sortedByFirst = Lists.newArrayList(sstables);
+ Collections.sort(sortedByFirst, new Comparator<SSTableReader>()
+ {
+ @Override
+ public int compare(SSTableReader o1, SSTableReader o2)
+ {
+ return o1.first.compareTo(o2.first);
+ }
+ });
+ List<Interval<RowPosition, SSTableReader>> intervals = new
ArrayList<>();
+ DecoratedKey first = null, last = null;
+ /*
+ normalize the intervals covered by the sstables
+ assume we have sstables like this (brackets representing first/last
key in the sstable);
+ [ ] [ ] [ ] [ ]
+ [ ] [ ]
+ then we can, instead of searching the interval tree 6 times,
normalize the intervals and
+ only query the tree 2 times, for these intervals;
+ [ ] [ ]
+ */
+ for (SSTableReader sstable : sortedByFirst)
{
- Set<SSTableReader> overlaps =
ImmutableSet.copyOf(tree.search(Interval.<RowPosition,
SSTableReader>create(sstable.first, sstable.last)));
- results = results == null ? overlaps : Sets.union(results,
overlaps).immutableCopy();
+ if (first == null)
+ {
+ first = sstable.first;
+ last = sstable.last;
+ }
+ else
+ {
+ if (sstable.first.compareTo(last) <= 0) // we do overlap
+ {
+ if (sstable.last.compareTo(last) > 0)
+ last = sstable.last;
+ }
+ else
+ {
+ intervals.add(Interval.<RowPosition,
SSTableReader>create(first, last));
+ first = sstable.first;
+ last = sstable.last;
+ }
+ }
}
- results = Sets.difference(results, ImmutableSet.copyOf(sstables));
+ intervals.add(Interval.<RowPosition, SSTableReader>create(first,
last));
+ DataTracker.SSTableIntervalTree tree = data.getView().intervalTree;
+ Set<SSTableReader> results = new HashSet<>();
+
+ for (Interval<RowPosition, SSTableReader> interval : intervals)
+ results.addAll(tree.search(interval));
- return results;
+ return Sets.difference(results, ImmutableSet.copyOf(sstables));
}
/**
http://git-wip-us.apache.org/repos/asf/cassandra/blob/26ff1507/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
----------------------------------------------------------------------
diff --cc
src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index dec0cef,fea4995..41c304b
---
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@@ -35,9 -36,10 +36,11 @@@ public class DateTieredCompactionStrate
{
private static final Logger logger =
LoggerFactory.getLogger(DateTieredCompactionStrategy.class);
- protected DateTieredCompactionStrategyOptions options;
+ private final DateTieredCompactionStrategyOptions options;
protected volatile int estimatedRemainingTasks;
+ private final Set<SSTableReader> sstables = new HashSet<>();
+ @VisibleForTesting
+ long lastExpiredCheck;
public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map<String,
String> options)
{
@@@ -76,13 -81,19 +79,19 @@@
*/
private List<SSTableReader> getNextBackgroundSSTables(final int gcBefore)
{
- if (!isEnabled() || cfs.getSSTables().isEmpty())
+ if (cfs.getSSTables().isEmpty())
return Collections.emptyList();
- Set<SSTableReader> uncompacting = cfs.getUncompactingSSTables();
+ Set<SSTableReader> uncompacting = Sets.intersection(sstables,
cfs.getUncompactingSSTables());
- // Find fully expired SSTables. Those will be included no matter what.
- Set<SSTableReader> expired =
CompactionController.getFullyExpiredSSTables(cfs, uncompacting,
cfs.getOverlappingSSTables(uncompacting), gcBefore);
+ Set<SSTableReader> expired = Collections.emptySet();
+ // we only check for expired sstables every 10 minutes due to it
being an expensive operation
+ if (System.currentTimeMillis() - lastExpiredCheck >
TimeUnit.MINUTES.toMillis(10))
+ {
+ // Find fully expired SSTables. Those will be included no matter
what.
+ expired = CompactionController.getFullyExpiredSSTables(cfs,
uncompacting, cfs.getOverlappingSSTables(uncompacting), gcBefore);
+ lastExpiredCheck = System.currentTimeMillis();
+ }
Set<SSTableReader> candidates =
Sets.newHashSet(filterSuspectSSTables(uncompacting));
List<SSTableReader> compactionCandidates = new
ArrayList<>(getNextNonExpiredSSTables(Sets.difference(candidates, expired),
gcBefore));
http://git-wip-us.apache.org/repos/asf/cassandra/blob/26ff1507/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
----------------------------------------------------------------------
diff --cc
test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
index 14e22f0,0084a16..cea835f
---
a/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
+++
b/test/unit/org/apache/cassandra/db/compaction/DateTieredCompactionStrategyTest.java
@@@ -306,10 -306,9 +306,11 @@@ public class DateTieredCompactionStrate
options.put(DateTieredCompactionStrategyOptions.TIMESTAMP_RESOLUTION_KEY,
"MILLISECONDS");
options.put(DateTieredCompactionStrategyOptions.MAX_SSTABLE_AGE_KEY,
Double.toString((1d / (24 * 60 * 60))));
DateTieredCompactionStrategy dtcs = new
DateTieredCompactionStrategy(cfs, options);
+ for (SSTableReader sstable : cfs.getSSTables())
+ dtcs.addSSTable(sstable);
dtcs.startup();
assertNull(dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis() / 1000)));
+ dtcs.lastExpiredCheck = 0;
Thread.sleep(2000);
AbstractCompactionTask t = dtcs.getNextBackgroundTask((int)
(System.currentTimeMillis()/1000));
assertNotNull(t);
