cassandra git commit: Do size tiered compaction in date tiered compaction windows

[marcuse]

Repository: cassandra
Updated Branches:
  refs/heads/cassandra-2.1 78810f255 -> cedcf07c5


Do size tiered compaction in date tiered compaction windows

Patch by marcuse; reviewed by Jeff Jirsa for CASSANDRA-10276


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/cedcf07c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/cedcf07c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/cedcf07c

Branch: refs/heads/cassandra-2.1
Commit: cedcf07c542235815c023b66f151ad8c7aa9ba9a
Parents: 78810f2
Author: Marcus Eriksson 
Authored: Mon Sep 7 10:39:15 2015 +0200
Committer: Marcus Eriksson 
Committed: Wed Oct 28 08:40:12 2015 +0100

--
 CHANGES.txt |  1 +
 .../DateTieredCompactionStrategy.java   | 46 ++--
 .../DateTieredCompactionStrategyTest.java   | 57 ++--
 3 files changed, 72 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/cassandra/blob/cedcf07c/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index 2ca3b43..5b46eac 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.12
+ * Do STCS in DTCS windows (CASSANDRA-10276)
  * Don't try to get ancestors from half-renamed sstables (CASSANDRA-10501)
  * Avoid repetition of JVM_OPTS in debian package (CASSANDRA-10251)
  * Fix potential NPE from handling result of SIM.highestSelectivityIndex 
(CASSANDRA-10550)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/cedcf07c/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --git 
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java 
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index a8e2aff..ece596f 100644
--- 
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ 
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -18,7 +18,6 @@
 package org.apache.cassandra.db.compaction;
 
 import java.util.*;
-import java.util.concurrent.TimeUnit;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
@@ -40,6 +39,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 protected volatile int estimatedRemainingTasks;
 private final Set sstables = new HashSet<>();
 private long lastExpiredCheck;
+private final SizeTieredCompactionStrategyOptions stcsOptions;
 
 public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
 {
@@ -54,6 +54,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 else
 logger.debug("Enabling tombstone compactions for DTCS");
 
+this.stcsOptions = new SizeTieredCompactionStrategyOptions(options);
 }
 
 @Override
@@ -137,7 +138,8 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy

cfs.getMinimumCompactionThreshold(),

cfs.getMaximumCompactionThreshold(),
now,
-   options.baseTime);
+   options.baseTime,
+   stcsOptions);
 if (!mostInteresting.isEmpty())
 return mostInteresting;
 return null;
@@ -328,7 +330,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 for (List bucket : tasks)
 {
 if (bucket.size() >= cfs.getMinimumCompactionThreshold())
-n += Math.ceil((double)bucket.size() / 
cfs.getMaximumCompactionThreshold());
+n += getSTCSBuckets(bucket, stcsOptions).size();
 }
 estimatedRemainingTasks = n;
 }
@@ -341,7 +343,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
  * @return a bucket (list) of sstables to compact.
  */
 @VisibleForTesting
-static List newestBucket(List buckets, 
int minThreshold, int maxThreshold, long now, long baseTime)
+static List newestBucket(List buckets, 
int minThreshold, int maxThreshold, long now, long baseTime, 
SizeTieredCompactionStrategyOptions stcsOptions)
 {
 // If the "incoming window" has at least minThreshold SSTables, choose 
that one.
 // For any other bucket, at least 2 SSTables is enough.
@@ -349,23 +351,31 @@ 

[1/2] cassandra git commit: Do size tiered compaction in date tiered compaction windows

[marcuse]

Repository: cassandra
Updated Branches:
  refs/heads/cassandra-2.2 40cef770d -> 5a356a706


Do size tiered compaction in date tiered compaction windows

Patch by marcuse; reviewed by Jeff Jirsa for CASSANDRA-10276


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/cedcf07c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/cedcf07c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/cedcf07c

Branch: refs/heads/cassandra-2.2
Commit: cedcf07c542235815c023b66f151ad8c7aa9ba9a
Parents: 78810f2
Author: Marcus Eriksson 
Authored: Mon Sep 7 10:39:15 2015 +0200
Committer: Marcus Eriksson 
Committed: Wed Oct 28 08:40:12 2015 +0100

--
 CHANGES.txt |  1 +
 .../DateTieredCompactionStrategy.java   | 46 ++--
 .../DateTieredCompactionStrategyTest.java   | 57 ++--
 3 files changed, 72 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/cassandra/blob/cedcf07c/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index 2ca3b43..5b46eac 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.12
+ * Do STCS in DTCS windows (CASSANDRA-10276)
  * Don't try to get ancestors from half-renamed sstables (CASSANDRA-10501)
  * Avoid repetition of JVM_OPTS in debian package (CASSANDRA-10251)
  * Fix potential NPE from handling result of SIM.highestSelectivityIndex 
(CASSANDRA-10550)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/cedcf07c/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --git 
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java 
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index a8e2aff..ece596f 100644
--- 
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ 
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -18,7 +18,6 @@
 package org.apache.cassandra.db.compaction;
 
 import java.util.*;
-import java.util.concurrent.TimeUnit;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
@@ -40,6 +39,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 protected volatile int estimatedRemainingTasks;
 private final Set sstables = new HashSet<>();
 private long lastExpiredCheck;
+private final SizeTieredCompactionStrategyOptions stcsOptions;
 
 public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
 {
@@ -54,6 +54,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 else
 logger.debug("Enabling tombstone compactions for DTCS");
 
+this.stcsOptions = new SizeTieredCompactionStrategyOptions(options);
 }
 
 @Override
@@ -137,7 +138,8 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy

cfs.getMinimumCompactionThreshold(),

cfs.getMaximumCompactionThreshold(),
now,
-   options.baseTime);
+   options.baseTime,
+   stcsOptions);
 if (!mostInteresting.isEmpty())
 return mostInteresting;
 return null;
@@ -328,7 +330,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 for (List bucket : tasks)
 {
 if (bucket.size() >= cfs.getMinimumCompactionThreshold())
-n += Math.ceil((double)bucket.size() / 
cfs.getMaximumCompactionThreshold());
+n += getSTCSBuckets(bucket, stcsOptions).size();
 }
 estimatedRemainingTasks = n;
 }
@@ -341,7 +343,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
  * @return a bucket (list) of sstables to compact.
  */
 @VisibleForTesting
-static List newestBucket(List buckets, 
int minThreshold, int maxThreshold, long now, long baseTime)
+static List newestBucket(List buckets, 
int minThreshold, int maxThreshold, long now, long baseTime, 
SizeTieredCompactionStrategyOptions stcsOptions)
 {
 // If the "incoming window" has at least minThreshold SSTables, choose 
that one.
 // For any other bucket, at least 2 SSTables is enough.
@@ -349,23 +351,31 @@ 

[1/3] cassandra git commit: Do size tiered compaction in date tiered compaction windows

[marcuse]

Repository: cassandra
Updated Branches:
  refs/heads/cassandra-3.0 d856d3d7b -> 27ca4915e


Do size tiered compaction in date tiered compaction windows

Patch by marcuse; reviewed by Jeff Jirsa for CASSANDRA-10276


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/cedcf07c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/cedcf07c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/cedcf07c

Branch: refs/heads/cassandra-3.0
Commit: cedcf07c542235815c023b66f151ad8c7aa9ba9a
Parents: 78810f2
Author: Marcus Eriksson 
Authored: Mon Sep 7 10:39:15 2015 +0200
Committer: Marcus Eriksson 
Committed: Wed Oct 28 08:40:12 2015 +0100

--
 CHANGES.txt |  1 +
 .../DateTieredCompactionStrategy.java   | 46 ++--
 .../DateTieredCompactionStrategyTest.java   | 57 ++--
 3 files changed, 72 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/cassandra/blob/cedcf07c/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index 2ca3b43..5b46eac 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.12
+ * Do STCS in DTCS windows (CASSANDRA-10276)
  * Don't try to get ancestors from half-renamed sstables (CASSANDRA-10501)
  * Avoid repetition of JVM_OPTS in debian package (CASSANDRA-10251)
  * Fix potential NPE from handling result of SIM.highestSelectivityIndex 
(CASSANDRA-10550)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/cedcf07c/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --git 
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java 
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index a8e2aff..ece596f 100644
--- 
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ 
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -18,7 +18,6 @@
 package org.apache.cassandra.db.compaction;
 
 import java.util.*;
-import java.util.concurrent.TimeUnit;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
@@ -40,6 +39,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 protected volatile int estimatedRemainingTasks;
 private final Set sstables = new HashSet<>();
 private long lastExpiredCheck;
+private final SizeTieredCompactionStrategyOptions stcsOptions;
 
 public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
 {
@@ -54,6 +54,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 else
 logger.debug("Enabling tombstone compactions for DTCS");
 
+this.stcsOptions = new SizeTieredCompactionStrategyOptions(options);
 }
 
 @Override
@@ -137,7 +138,8 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy

cfs.getMinimumCompactionThreshold(),

cfs.getMaximumCompactionThreshold(),
now,
-   options.baseTime);
+   options.baseTime,
+   stcsOptions);
 if (!mostInteresting.isEmpty())
 return mostInteresting;
 return null;
@@ -328,7 +330,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 for (List bucket : tasks)
 {
 if (bucket.size() >= cfs.getMinimumCompactionThreshold())
-n += Math.ceil((double)bucket.size() / 
cfs.getMaximumCompactionThreshold());
+n += getSTCSBuckets(bucket, stcsOptions).size();
 }
 estimatedRemainingTasks = n;
 }
@@ -341,7 +343,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
  * @return a bucket (list) of sstables to compact.
  */
 @VisibleForTesting
-static List newestBucket(List buckets, 
int minThreshold, int maxThreshold, long now, long baseTime)
+static List newestBucket(List buckets, 
int minThreshold, int maxThreshold, long now, long baseTime, 
SizeTieredCompactionStrategyOptions stcsOptions)
 {
 // If the "incoming window" has at least minThreshold SSTables, choose 
that one.
 // For any other bucket, at least 2 SSTables is enough.
@@ -349,23 +351,31 @@ 

[1/4] cassandra git commit: Do size tiered compaction in date tiered compaction windows

[marcuse]

Repository: cassandra
Updated Branches:
  refs/heads/trunk 7cedeca9a -> 5995de3b6


Do size tiered compaction in date tiered compaction windows

Patch by marcuse; reviewed by Jeff Jirsa for CASSANDRA-10276


Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo
Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/cedcf07c
Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/cedcf07c
Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/cedcf07c

Branch: refs/heads/trunk
Commit: cedcf07c542235815c023b66f151ad8c7aa9ba9a
Parents: 78810f2
Author: Marcus Eriksson 
Authored: Mon Sep 7 10:39:15 2015 +0200
Committer: Marcus Eriksson 
Committed: Wed Oct 28 08:40:12 2015 +0100

--
 CHANGES.txt |  1 +
 .../DateTieredCompactionStrategy.java   | 46 ++--
 .../DateTieredCompactionStrategyTest.java   | 57 ++--
 3 files changed, 72 insertions(+), 32 deletions(-)
--


http://git-wip-us.apache.org/repos/asf/cassandra/blob/cedcf07c/CHANGES.txt
--
diff --git a/CHANGES.txt b/CHANGES.txt
index 2ca3b43..5b46eac 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
 2.1.12
+ * Do STCS in DTCS windows (CASSANDRA-10276)
  * Don't try to get ancestors from half-renamed sstables (CASSANDRA-10501)
  * Avoid repetition of JVM_OPTS in debian package (CASSANDRA-10251)
  * Fix potential NPE from handling result of SIM.highestSelectivityIndex 
(CASSANDRA-10550)

http://git-wip-us.apache.org/repos/asf/cassandra/blob/cedcf07c/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
--
diff --git 
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java 
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
index a8e2aff..ece596f 100644
--- 
a/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
+++ 
b/src/java/org/apache/cassandra/db/compaction/DateTieredCompactionStrategy.java
@@ -18,7 +18,6 @@
 package org.apache.cassandra.db.compaction;
 
 import java.util.*;
-import java.util.concurrent.TimeUnit;
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Predicate;
@@ -40,6 +39,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 protected volatile int estimatedRemainingTasks;
 private final Set sstables = new HashSet<>();
 private long lastExpiredCheck;
+private final SizeTieredCompactionStrategyOptions stcsOptions;
 
 public DateTieredCompactionStrategy(ColumnFamilyStore cfs, Map options)
 {
@@ -54,6 +54,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 else
 logger.debug("Enabling tombstone compactions for DTCS");
 
+this.stcsOptions = new SizeTieredCompactionStrategyOptions(options);
 }
 
 @Override
@@ -137,7 +138,8 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy

cfs.getMinimumCompactionThreshold(),

cfs.getMaximumCompactionThreshold(),
now,
-   options.baseTime);
+   options.baseTime,
+   stcsOptions);
 if (!mostInteresting.isEmpty())
 return mostInteresting;
 return null;
@@ -328,7 +330,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
 for (List bucket : tasks)
 {
 if (bucket.size() >= cfs.getMinimumCompactionThreshold())
-n += Math.ceil((double)bucket.size() / 
cfs.getMaximumCompactionThreshold());
+n += getSTCSBuckets(bucket, stcsOptions).size();
 }
 estimatedRemainingTasks = n;
 }
@@ -341,7 +343,7 @@ public class DateTieredCompactionStrategy extends 
AbstractCompactionStrategy
  * @return a bucket (list) of sstables to compact.
  */
 @VisibleForTesting
-static List newestBucket(List buckets, 
int minThreshold, int maxThreshold, long now, long baseTime)
+static List newestBucket(List buckets, 
int minThreshold, int maxThreshold, long now, long baseTime, 
SizeTieredCompactionStrategyOptions stcsOptions)
 {
 // If the "incoming window" has at least minThreshold SSTables, choose 
that one.
 // For any other bucket, at least 2 SSTables is enough.
@@ -349,23 +351,31 @@ public class