Merge branch 'cassandra-2.1' into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/21a915cd Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/21a915cd Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/21a915cd Branch: refs/heads/trunk Commit: 21a915cd7148d03e9bbe32d68395d7b04dcfc55a Parents: 5ff69f2 9beeba3 Author: Aleksey Yeschenko <[email protected]> Authored: Tue May 12 21:06:54 2015 +0300 Committer: Aleksey Yeschenko <[email protected]> Committed: Tue May 12 21:06:54 2015 +0300 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cassandra/db/ArrayBackedSortedColumns.java | 14 +- .../apache/cassandra/db/AtomicBTreeColumns.java | 5 + .../org/apache/cassandra/db/ColumnFamily.java | 5 + .../apache/cassandra/db/ColumnFamilyStore.java | 2 +- .../cassandra/db/filter/ColumnCounter.java | 48 ++++-- .../cassandra/db/filter/SliceQueryFilter.java | 34 ++-- .../SliceQueryFilterWithTombstonesTest.java | 166 +++++++++++++++++++ 8 files changed, 236 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/CHANGES.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java index f7d7ec1,c53832b..1beb982 --- a/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java +++ b/src/java/org/apache/cassandra/db/ArrayBackedSortedColumns.java @@@ -283,10 -282,10 +283,16 @@@ public class ArrayBackedSortedColumns e public void maybeAppendColumn(Cell cell, DeletionInfo.InOrderTester tester, int gcBefore) { if (cell.getLocalDeletionTime() >= gcBefore && !tester.isDeleted(cell)) -- { -- internalAdd(cell); -- sortedSize++; -- } ++ appendColumn(cell); ++ } ++ ++ /** ++ * Adds a cell, assuming that it sorts *strictly after* the current-last cell in the array. ++ */ ++ public void appendColumn(Cell cell) ++ { ++ internalAdd(cell); ++ sortedSize++; } public void addColumn(Cell cell) http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/AtomicBTreeColumns.java index 0b1a58c,47f0b85..9ef0c14 --- a/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java +++ b/src/java/org/apache/cassandra/db/AtomicBTreeColumns.java @@@ -319,6 -310,6 +319,11 @@@ public class AtomicBTreeColumns extend throw new UnsupportedOperationException(); } ++ public void appendColumn(Cell cell) ++ { ++ throw new UnsupportedOperationException(); ++ } ++ public void addAll(ColumnFamily cf) { throw new UnsupportedOperationException(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/ColumnFamily.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/ColumnFamily.java index 88ab9e4,c9a008f..9caf20b --- a/src/java/org/apache/cassandra/db/ColumnFamily.java +++ b/src/java/org/apache/cassandra/db/ColumnFamily.java @@@ -205,6 -203,6 +205,11 @@@ public abstract class ColumnFamily impl public abstract void maybeAppendColumn(Cell cell, DeletionInfo.InOrderTester tester, int gcBefore); /** ++ * Appends a cell. Requires that the cell to add is sorted strictly after the last cell in the container. ++ */ ++ public abstract void appendColumn(Cell cell); ++ ++ /** * Adds all the columns of a given column map to this column map. * This is equivalent to: * <code> http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/filter/ColumnCounter.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/filter/ColumnCounter.java index 86cfc40,8be26e1..43555bc --- a/src/java/org/apache/cassandra/db/filter/ColumnCounter.java +++ b/src/java/org/apache/cassandra/db/filter/ColumnCounter.java @@@ -37,17 -37,17 +37,23 @@@ public class ColumnCounte this.timestamp = timestamp; } -- public void count(Cell cell, DeletionInfo.InOrderTester tester) ++ /** ++ * @return true if the cell counted as a live cell or a valid tombstone; false if it got immediately discarded for ++ * being shadowed by a range- or a partition tombstone ++ */ ++ public boolean count(Cell cell, DeletionInfo.InOrderTester tester) { - if (!isLive(cell, tester, timestamp)) - ignored++; - else + // The cell is shadowed by a higher-level deletion, and won't be retained. + // For the purposes of this counter, we don't care if it's a tombstone or not. + if (tester.isDeleted(cell)) - return; ++ return false; + + if (cell.isLive(timestamp)) live++; - } + else + tombstones++; + - protected static boolean isLive(Cell cell, DeletionInfo.InOrderTester tester, long timestamp) - { - return cell.isLive(timestamp) && !tester.isDeleted(cell); ++ return true; } public int live() @@@ -96,18 -96,21 +102,22 @@@ assert toGroup == 0 || type != null; } -- public void count(Cell cell, DeletionInfo.InOrderTester tester) ++ @Override ++ public boolean count(Cell cell, DeletionInfo.InOrderTester tester) { - if (!isLive(cell, tester, timestamp)) + if (tester.isDeleted(cell)) - return; ++ return false; + + if (!cell.isLive(timestamp)) { - ignored++; - return; + tombstones++; - return; ++ return true; } if (toGroup == 0) { live = 1; -- return; ++ return true; } CellName current = cell.name(); @@@ -129,7 -132,7 +139,7 @@@ } if (isSameGroup) -- return; ++ return true; // We want to count the static group as 1 (CQL) row only if it's the only // group in the partition. So, since we have already counted it at this point, @@@ -137,12 -140,12 +147,14 @@@ if (previous.isStatic()) { previous = current; -- return; ++ return true; } } live++; previous = current; ++ ++ return true; } } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/21a915cd/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java index 396fc06,1195d4c..d914f51 --- a/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java +++ b/src/java/org/apache/cassandra/db/filter/SliceQueryFilter.java @@@ -202,65 -203,60 +202,68 @@@ public class SliceQueryFilter implement while (reducedColumns.hasNext()) { Cell cell = reducedColumns.next(); + if (logger.isTraceEnabled()) - logger.trace(String.format("collecting %s of %s: %s", - columnCounter.live(), count, cell.getString(container.getComparator()))); + logger.trace("collecting {} of {}: {}", columnCounter.live(), count, cell.getString(container.getComparator())); - columnCounter.count(cell, tester); + // An expired tombstone will be immediately discarded in memory, and needn't be counted. - if (cell.getLocalDeletionTime() < gcBefore) ++ // Neither should be any cell shadowed by a range- or a partition tombstone. ++ if (cell.getLocalDeletionTime() < gcBefore || !columnCounter.count(cell, tester)) + continue; - columnCounter.count(cell, tester); - if (columnCounter.live() > count) break; - if (respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneFailureThreshold()) + if (respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneFailureThreshold()) { - Tracing.trace("Scanned over {} tombstones; query aborted (see tombstone_failure_threshold)", - DatabaseDescriptor.getTombstoneFailureThreshold()); - logger.error("Scanned over {} tombstones in {}.{}; query aborted (see tombstone_failure_threshold)", - DatabaseDescriptor.getTombstoneFailureThreshold(), - container.metadata().ksName, - container.metadata().cfName); - throw new TombstoneOverwhelmingException(); + Tracing.trace("Scanned over {} tombstones; query aborted (see tombstone_failure_threshold); slices={}", + DatabaseDescriptor.getTombstoneFailureThreshold(), getSlicesInfo(container)); + - throw new TombstoneOverwhelmingException(columnCounter.ignored(), ++ throw new TombstoneOverwhelmingException(columnCounter.tombstones(), + count, + container.metadata().ksName, + container.metadata().cfName, + container.getComparator().getString(cell.name()), + getSlicesInfo(container)); } -- container.maybeAppendColumn(cell, tester, gcBefore); ++ container.appendColumn(cell); } - boolean warnTombstones = logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.ignored() > DatabaseDescriptor.getTombstoneWarnThreshold(); - Tracing.trace("Read {} live and {} tombstone cells", columnCounter.live(), columnCounter.tombstones()); - if (logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold()) ++ boolean warnTombstones = logger.isWarnEnabled() && respectTombstoneThresholds() && columnCounter.tombstones() > DatabaseDescriptor.getTombstoneWarnThreshold(); + if (warnTombstones) { - String msg = String.format("Read %d live and %d tombstoned cells in %s.%s for key: %1.512s (see tombstone_warn_threshold). %d columns were requested, slices=%1.512s", - StringBuilder sb = new StringBuilder(); - CellNameType type = container.metadata().comparator; - - for (ColumnSlice sl : slices) - { - assert sl != null; - - sb.append('['); - sb.append(type.getString(sl.start)); - sb.append('-'); - sb.append(type.getString(sl.finish)); - sb.append(']'); - } - + String msg = String.format("Read %d live and %d tombstone cells in %s.%s for key: %1.512s (see tombstone_warn_threshold). %d columns were requested, slices=%1.512s", columnCounter.live(), - columnCounter.ignored(), + columnCounter.tombstones(), container.metadata().ksName, container.metadata().cfName, container.metadata().getKeyValidator().getString(key.getKey()), count, - sb); + getSlicesInfo(container)); logger.warn(msg); } - Tracing.trace("Read {} live and {} tombstoned cells{}", ++ Tracing.trace("Read {} live and {} tombstone cells{}", + columnCounter.live(), - columnCounter.ignored(), ++ columnCounter.tombstones(), + warnTombstones ? " (see tombstone_warn_threshold)" : ""); + } + + private String getSlicesInfo(ColumnFamily container) + { + StringBuilder sb = new StringBuilder(); + CellNameType type = container.metadata().comparator; + for (ColumnSlice sl : slices) + { + assert sl != null; + + sb.append('['); + sb.append(type.getString(sl.start)); + sb.append('-'); + sb.append(type.getString(sl.finish)); + sb.append(']'); + } + return sb.toString(); } protected boolean respectTombstoneThresholds()
