This is an automated email from the ASF dual-hosted git repository. marcuse pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
commit 3f6f7be63faf2d07b274caf25a2fa1dcbde14187 Author: Marcus Eriksson <[email protected]> AuthorDate: Wed Jun 30 16:34:09 2021 +0200 Display bytes per level in tablestats for LCS tables Patch by marcuse; reviewed by Brandon Williams for CASSANDRA-16779 --- CHANGES.txt | 1 + .../org/apache/cassandra/db/ColumnFamilyStore.java | 5 +++ .../cassandra/db/ColumnFamilyStoreMBean.java | 6 +++ .../db/compaction/CompactionStrategyManager.java | 38 +++++++++++++++++ .../db/compaction/LeveledCompactionStrategy.java | 5 +++ .../db/compaction/LeveledGenerations.java | 8 ++++ .../cassandra/db/compaction/LeveledManifest.java | 5 +++ .../cassandra/tools/nodetool/stats/StatsTable.java | 1 + .../tools/nodetool/stats/TableStatsHolder.java | 12 ++++++ .../tools/nodetool/stats/TableStatsPrinter.java | 6 ++- .../compaction/LeveledCompactionStrategyTest.java | 47 ++++++++++++++++++++++ 11 files changed, 133 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index dcfacb4..d2a0cff 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.1 + * Display bytes per level in tablestats for LCS tables (CASSANDRA-16799) * Add isolated flush timer to CommitLogMetrics and ensure writes correspond to single WaitingOnCommit data points (CASSANDRA-16701) * Add a system property to set hostId if not yet initialized (CASSANDRA-14582) * GossiperTest.testHasVersion3Nodes didn't take into account trunk version changes, fixed to rely on latest version (CASSANDRA-16651) diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java index 113a916..df9f763 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStore.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStore.java @@ -2638,6 +2638,11 @@ public class ColumnFamilyStore implements ColumnFamilyStoreMBean return compactionStrategyManager.getSSTableCountPerLevel(); } + public long[] getPerLevelSizeBytes() + { + return compactionStrategyManager.getPerLevelSizeBytes(); + } + public int getLevelFanoutSize() { return compactionStrategyManager.getLevelFanoutSize(); diff --git a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java index 0360e34..f06b65a 100644 --- a/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java +++ b/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java @@ -203,6 +203,12 @@ public interface ColumnFamilyStoreMBean public int[] getSSTableCountPerLevel(); /** + * @return total size on disk for each level. null unless leveled compaction is used. + * array index corresponds to level(int[0] is for level 0, ...). + */ + public long[] getPerLevelSizeBytes(); + + /** * @return sstable fanout size for level compaction strategy. */ public int getLevelFanoutSize(); diff --git a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java index deece30..d7d3ba5 100644 --- a/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java +++ b/src/java/org/apache/cassandra/db/compaction/CompactionStrategyManager.java @@ -587,6 +587,29 @@ public class CompactionStrategyManager implements INotificationConsumer return null; } + public long[] getPerLevelSizeBytes() + { + readLock.lock(); + try + { + if (repaired.first() instanceof LeveledCompactionStrategy) + { + long [] res = new long[LeveledGenerations.MAX_LEVEL_COUNT]; + for (AbstractCompactionStrategy strategy : getAllStrategies()) + { + long[] repairedCountPerLevel = ((LeveledCompactionStrategy) strategy).getAllLevelSizeBytes(); + res = sumArrays(res, repairedCountPerLevel); + } + return res; + } + return null; + } + finally + { + readLock.unlock(); + } + } + static int[] sumArrays(int[] a, int[] b) { int[] res = new int[Math.max(a.length, b.length)]; @@ -602,6 +625,21 @@ public class CompactionStrategyManager implements INotificationConsumer return res; } + static long[] sumArrays(long[] a, long[] b) + { + long[] res = new long[Math.max(a.length, b.length)]; + for (int i = 0; i < res.length; i++) + { + if (i < a.length && i < b.length) + res[i] = a[i] + b[i]; + else if (i < a.length) + res[i] = a[i]; + else + res[i] = b[i]; + } + return res; + } + /** * Should only be called holding the readLock */ diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java index dd7c9df..6faca63 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledCompactionStrategy.java @@ -109,6 +109,11 @@ public class LeveledCompactionStrategy extends AbstractCompactionStrategy return manifest.getAllLevelSize(); } + public long[] getAllLevelSizeBytes() + { + return manifest.getAllLevelSizeBytes(); + } + @Override public void startup() { diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java b/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java index 64027f2..2cc1b1a 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledGenerations.java @@ -226,6 +226,14 @@ class LeveledGenerations return counts; } + long[] getAllLevelSizeBytes() + { + long[] sums = new long[levelCount()]; + for (int i = 0; i < sums.length; i++) + sums[i] = get(i).stream().map(SSTableReader::onDiskLength).reduce(0L, Long::sum); + return sums; + } + Set<SSTableReader> allSSTables() { ImmutableSet.Builder<SSTableReader> builder = ImmutableSet.builder(); diff --git a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java index 7c865c7..8b636fc 100644 --- a/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java +++ b/src/java/org/apache/cassandra/db/compaction/LeveledManifest.java @@ -395,6 +395,11 @@ public class LeveledManifest return generations.getAllLevelSize(); } + public synchronized long[] getAllLevelSizeBytes() + { + return generations.getAllLevelSizeBytes(); + } + @VisibleForTesting public synchronized int remove(SSTableReader reader) { diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java index d897eab..6f5d904 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java +++ b/src/java/org/apache/cassandra/tools/nodetool/stats/StatsTable.java @@ -69,5 +69,6 @@ public class StatsTable public long maximumTombstonesPerSliceLastFiveMinutes; public String droppedMutations; public List<String> sstablesInEachLevel = new ArrayList<>(); + public List<String> sstableBytesInEachLevel = new ArrayList<>(); public Boolean isInCorrectLocation = null; // null: option not active } diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java index b1685e6..b66c802 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java +++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsHolder.java @@ -118,6 +118,7 @@ public class TableStatsHolder implements StatsHolder { Map<String, Object> mpTable = new HashMap<>(); mpTable.put("sstables_in_each_level", table.sstablesInEachLevel); + mpTable.put("sstable_bytes_in_each_level", table.sstableBytesInEachLevel); mpTable.put("space_used_live", table.spaceUsedLive); mpTable.put("space_used_total", table.spaceUsedTotal); mpTable.put("space_used_by_snapshots_total", table.spaceUsedBySnapshotsTotal); @@ -235,6 +236,17 @@ public class TableStatsHolder implements StatsHolder } } + long[] leveledSSTablesBytes = table.getPerLevelSizeBytes(); + if (leveledSSTablesBytes != null) + { + statsTable.isLeveledSstable = true; + for (int level = 0; level < leveledSSTablesBytes.length; level++) + { + long size = leveledSSTablesBytes[level]; + statsTable.sstableBytesInEachLevel.add(format(size, humanReadable)); + } + } + if (locationCheck) statsTable.isInCorrectLocation = !table.hasMisplacedSSTables(); diff --git a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java index f4188e4..50a22d6 100644 --- a/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java +++ b/src/java/org/apache/cassandra/tools/nodetool/stats/TableStatsPrinter.java @@ -79,8 +79,12 @@ public class TableStatsPrinter<T extends StatsHolder> out.println(indent + "SSTable count: " + table.sstableCount); out.println(indent + "Old SSTable count: " + table.oldSSTableCount); if (table.isLeveledSstable) + { out.println(indent + "SSTables in each level: [" + String.join(", ", - table.sstablesInEachLevel) + "]"); + table.sstablesInEachLevel) + "]"); + out.println(indent + "SSTable bytes in each level: [" + String.join(", ", + table.sstableBytesInEachLevel) + "]"); + } out.println(indent + "Space used (live): " + table.spaceUsedLive); out.println(indent + "Space used (total): " + table.spaceUsedTotal); diff --git a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java index 2d6835b..68a9ac8 100644 --- a/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java +++ b/test/unit/org/apache/cassandra/db/compaction/LeveledCompactionStrategyTest.java @@ -771,6 +771,53 @@ public class LeveledCompactionStrategyTest } return newLevels; } + @Test + public void testPerLevelSizeBytes() throws IOException + { + byte [] b = new byte[100]; + new Random().nextBytes(b); + ByteBuffer value = ByteBuffer.wrap(b); + int rows = 5; + int columns = 5; + + cfs.disableAutoCompaction(); + for (int r = 0; r < rows; r++) + { + UpdateBuilder update = UpdateBuilder.create(cfs.metadata(), String.valueOf(r)); + for (int c = 0; c < columns; c++) + update.newRow("column" + c).add("val", value); + update.applyUnsafe(); + } + cfs.forceBlockingFlush(); + + SSTableReader sstable = cfs.getLiveSSTables().iterator().next(); + long [] levelSizes = cfs.getPerLevelSizeBytes(); + for (int i = 0; i < levelSizes.length; i++) + { + if (i != 0) + assertEquals(0, levelSizes[i]); + else + assertEquals(sstable.onDiskLength(), levelSizes[i]); + } + + assertEquals(sstable.onDiskLength(), cfs.getPerLevelSizeBytes()[0]); + + LeveledCompactionStrategy strategy = (LeveledCompactionStrategy) ( cfs.getCompactionStrategyManager()).getStrategies().get(1).get(0); + strategy.manifest.remove(sstable); + sstable.descriptor.getMetadataSerializer().mutateLevel(sstable.descriptor, 2); + sstable.reloadSSTableMetadata(); + strategy.manifest.addSSTables(Collections.singleton(sstable)); + + levelSizes = cfs.getPerLevelSizeBytes(); + for (int i = 0; i < levelSizes.length; i++) + { + if (i != 2) + assertEquals(0, levelSizes[i]); + else + assertEquals(sstable.onDiskLength(), levelSizes[i]); + } + + } /** * brute-force checks if the new sstables can be added to the correct level in manifest --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
