Updated Branches: refs/heads/cassandra-1.2 5f6261096 -> 3f66fbfc6 refs/heads/cassandra-2.0 f15681b67 -> 8e8259050 refs/heads/trunk f15d6d591 -> c1d7291c8
add snapshot space used to cfstats patch by Mikhail Stepura; reviewed by jbellis for CASSANDRA-6231 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/e178ff45 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/e178ff45 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/e178ff45 Branch: refs/heads/trunk Commit: e178ff45c0510c56257c26da2dc8d082ba301522 Parents: f15d6d5 Author: Jonathan Ellis <jbel...@apache.org> Authored: Tue Nov 26 14:42:05 2013 -0600 Committer: Jonathan Ellis <jbel...@apache.org> Committed: Tue Nov 26 14:49:17 2013 -0600 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../apache/cassandra/db/ColumnFamilyStore.java | 5 ++ .../cassandra/db/ColumnFamilyStoreMBean.java | 5 ++ .../org/apache/cassandra/db/Directories.java | 90 +++++++++++++++++++- .../cassandra/metrics/ColumnFamilyMetrics.java | 15 +++- .../org/apache/cassandra/tools/NodeCmd.java | 1 + .../org/apache/cassandra/tools/NodeProbe.java | 1 + 7 files changed, 114 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 7bb1fa0..00797e6 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -18,6 +18,7 @@ 2.0.4 * Fix setting last compacted key in the wrong level for LCS (CASSANDRA-6284) * Add sub-ms precision formats to the timestamp parser (CASSANDRA-6395) + * Add snapshot space used to cfstats (CASSANDRA-6231) Merged from 1.2: * Fix thundering herd on endpoint cache invalidation (CASSANDRA-6345) * cqlsh: quote single quotes in strings inside collections (CASSANDRA-6172) http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/db/ColumnFamilyStore.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 7037635..ccc15ab 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -2356,4 +2356,9 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean Pair<ReplayPosition, Long> truncationRecord = SystemKeyspace.getTruncationRecords().get(metadata.cfId); return truncationRecord == null ? Long.MIN_VALUE : truncationRecord.right; } + + public long trueSnapshotsSize() + { + return directories.trueSnapshotsSize(); + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java index 1ca922b..fc1a7b1 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java @@ -344,4 +344,9 @@ public interface ColumnFamilyStoreMBean * @return ratio */ public double getDroppableTombstoneRatio(); + + /** + * @return the size of SSTables in "snapshots" subdirectory which aren't live anymore + */ + public long trueSnapshotsSize(); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/db/Directories.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Directories.java b/src/java/org/apache/cassandra/db/Directories.java index 9795a27..ea5c2f4 100644 --- a/src/java/org/apache/cassandra/db/Directories.java +++ b/src/java/org/apache/cassandra/db/Directories.java @@ -17,16 +17,25 @@ */ package org.apache.cassandra.db; +import static com.google.common.collect.Sets.newHashSet; + import java.io.File; import java.io.FileFilter; import java.io.IOError; import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.*; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicLong; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.google.common.collect.ImmutableSet.Builder; import com.google.common.primitives.Longs; import com.google.common.util.concurrent.Uninterruptibles; @@ -35,7 +44,6 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.cassandra.config.*; -import org.apache.cassandra.db.compaction.LeveledManifest; import org.apache.cassandra.io.FSError; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.util.FileUtils; @@ -332,7 +340,7 @@ public class Directories private FileFilter getFilter() { // Note: the prefix needs to include cfname + separator to distinguish between a cfs and it's secondary indexes - final String sstablePrefix = keyspacename + Component.separator + cfname + Component.separator; + final String sstablePrefix = getSSTablePrefix(); return new FileFilter() { // This function always return false since accepts adds to the components map @@ -401,6 +409,37 @@ public class Directories } throw new RuntimeException("Snapshot " + snapshotName + " doesn't exist"); } + + public long trueSnapshotsSize() + { + long result = 0L; + for (File dir : sstableDirectories) + result += getTrueAllocatedSizeIn(new File(dir, join(SNAPSHOT_SUBDIR))); + return result; + } + + private String getSSTablePrefix() + { + return keyspacename + Component.separator + cfname + Component.separator; + } + + public long getTrueAllocatedSizeIn(File input) + { + if (!input.isDirectory()) + return 0; + + TrueFilesSizeVisitor visitor = new TrueFilesSizeVisitor(); + try + { + Files.walkFileTree(input.toPath(), visitor); + } + catch (IOException e) + { + logger.error("Could not calculate the size of {}. {}", input, e); + } + + return visitor.getAllocatedSize(); + } private static File getOrCreate(File base, String... subdirs) { @@ -436,4 +475,51 @@ public class Directories for (int i = 0; i < locations.length; ++i) dataFileLocations[i] = new DataDirectory(new File(locations[i])); } + + private class TrueFilesSizeVisitor extends SimpleFileVisitor<Path> + { + private final AtomicLong size = new AtomicLong(0); + private final Set<String> visited = newHashSet(); //count each file only once + private final Set<String> alive; + private final String prefix = getSSTablePrefix(); + + public TrueFilesSizeVisitor() + { + super(); + Builder<String> builder = ImmutableSet.builder(); + for (File file: sstableLister().listFiles()) + builder.add(file.getName()); + alive = builder.build(); + } + + private boolean isAcceptable(Path file) + { + String fileName = file.toFile().getName(); + return fileName.startsWith(prefix) + && !visited.contains(fileName) + && !alive.contains(fileName); + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException + { + if (isAcceptable(file)) + { + size.addAndGet(attrs.size()); + visited.add(file.toFile().getName()); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException + { + return FileVisitResult.CONTINUE; + } + + public long getAllocatedSize() + { + return size.get(); + } + } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java index aabf373..038e27e 100644 --- a/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java +++ b/src/java/org/apache/cassandra/metrics/ColumnFamilyMetrics.java @@ -82,6 +82,8 @@ public class ColumnFamilyMetrics public final Histogram tombstoneScannedHistogram; /** Live cells scanned in queries on this CF */ public final Histogram liveScannedHistogram; + /** Disk space used by snapshot files which */ + public final Gauge<Long> trueSnapshotsSize; public final Timer coordinatorReadLatency; public final Timer coordinatorScanLatency; @@ -238,7 +240,7 @@ public class ColumnFamilyMetrics public Long value() { long count = 0L; - for (SSTableReader sstable: cfs.getSSTables()) + for (SSTableReader sstable : cfs.getSSTables()) count += sstable.getRecentBloomFilterFalsePositiveCount(); return count; } @@ -249,7 +251,7 @@ public class ColumnFamilyMetrics { long falseCount = 0L; long trueCount = 0L; - for (SSTableReader sstable: cfs.getSSTables()) + for (SSTableReader sstable : cfs.getSSTables()) { falseCount += sstable.getBloomFilterFalsePositiveCount(); trueCount += sstable.getBloomFilterTruePositiveCount(); @@ -308,6 +310,14 @@ public class ColumnFamilyMetrics liveScannedHistogram = Metrics.newHistogram(factory.createMetricName("LiveScannedHistogram"), true); coordinatorReadLatency = Metrics.newTimer(factory.createMetricName("CoordinatorReadLatency"), TimeUnit.MICROSECONDS, TimeUnit.SECONDS); coordinatorScanLatency = Metrics.newTimer(factory.createMetricName("CoordinatorScanLatency"), TimeUnit.MICROSECONDS, TimeUnit.SECONDS); + + trueSnapshotsSize = Metrics.newGauge(factory.createMetricName("SnapshotsSize"), new Gauge<Long>() + { + public Long value() + { + return cfs.trueSnapshotsSize(); + } + }); } public void updateSSTableIterated(int count) @@ -349,6 +359,7 @@ public class ColumnFamilyMetrics Metrics.defaultRegistry().removeMetric(factory.createMetricName("LiveScannedHistogram")); Metrics.defaultRegistry().removeMetric(factory.createMetricName("CoordinatorReadLatency")); Metrics.defaultRegistry().removeMetric(factory.createMetricName("CoordinatorScanLatency")); + Metrics.defaultRegistry().removeMetric(factory.createMetricName("SnapshotsSize")); } class ColumnFamilyMetricNameFactory implements MetricNameFactory http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/tools/NodeCmd.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeCmd.java b/src/java/org/apache/cassandra/tools/NodeCmd.java index d415f06..a6c0010 100644 --- a/src/java/org/apache/cassandra/tools/NodeCmd.java +++ b/src/java/org/apache/cassandra/tools/NodeCmd.java @@ -885,6 +885,7 @@ public class NodeCmd } outs.println("\t\tSpace used (live), bytes: " + probe.getColumnFamilyMetric(keyspaceName, cfName, "LiveDiskSpaceUsed")); outs.println("\t\tSpace used (total), bytes: " + probe.getColumnFamilyMetric(keyspaceName, cfName, "TotalDiskSpaceUsed")); + outs.println("\t\tSpace used by snapshots (total), bytes: " + probe.getColumnFamilyMetric(keyspaceName, cfName, "SnapshotsSize")); outs.println("\t\tSSTable Compression Ratio: " + probe.getColumnFamilyMetric(keyspaceName, cfName, "CompressionRatio")); outs.println("\t\tMemtable cell count: " + probe.getColumnFamilyMetric(keyspaceName, cfName, "MemtableColumnsCount")); outs.println("\t\tMemtable data size, bytes: " + probe.getColumnFamilyMetric(keyspaceName, cfName, "MemtableDataSize")); http://git-wip-us.apache.org/repos/asf/cassandra/blob/e178ff45/src/java/org/apache/cassandra/tools/NodeProbe.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/tools/NodeProbe.java b/src/java/org/apache/cassandra/tools/NodeProbe.java index 48d2013..2489de6 100644 --- a/src/java/org/apache/cassandra/tools/NodeProbe.java +++ b/src/java/org/apache/cassandra/tools/NodeProbe.java @@ -935,6 +935,7 @@ public class NodeProbe case "PendingTasks": case "RecentBloomFilterFalsePositives": case "RecentBloomFilterFalseRatio": + case "SnapshotsSize": return JMX.newMBeanProxy(mbeanServerConn, new ObjectName("org.apache.cassandra.metrics:type=ColumnFamily,keyspace=" + ks + ",scope=" + cf + ",name=" + metricName), JmxReporter.GaugeMBean.class).getValue();
