This is an automated email from the ASF dual-hosted git repository. smiklosovic pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push: new 006ec71f63 Add duration and count of partition keys to sstablemetadata 006ec71f63 is described below commit 006ec71f63f55532b58060a24e80ad496de51b0f Author: timothytu12 <timmy.tu2...@gmail.com> AuthorDate: Wed Jul 5 13:00:33 2023 -0400 Add duration and count of partition keys to sstablemetadata patch by Timothy Tu; reviewed by Stefan Miklosovic and Brandon Williams for CASSANDRA-18639 --- CHANGES.txt | 1 + .../managing/tools/sstable/sstablemetadata.adoc | 46 ++++++++++------- .../cassandra/tools/SSTableMetadataViewer.java | 58 ++++++++++++++++------ 3 files changed, 74 insertions(+), 31 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index bf53e8aa08..69b78213e1 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 5.0 + * Add duration and count of partition keys to sstablemetadata (CASSANDRA-18639) * Remove deprecated compaction_tombstone_warning_threshold and compaction_large_partition_warning_threshold from yaml (CASSANDRA-18626) * Enhance nodetool compactionstats with additional metrics (CASSANDRA-18305) * Added support for type VECTOR<type, dimension> (CASSANDRA-18504) diff --git a/doc/modules/cassandra/pages/managing/tools/sstable/sstablemetadata.adoc b/doc/modules/cassandra/pages/managing/tools/sstable/sstablemetadata.adoc index 0516bef3e7..202cd3a286 100644 --- a/doc/modules/cassandra/pages/managing/tools/sstable/sstablemetadata.adoc +++ b/doc/modules/cassandra/pages/managing/tools/sstable/sstablemetadata.adoc @@ -16,7 +16,11 @@ sstablemetadata <options> <sstable filename(s)> [cols=",",] |=== +|--colors |Use ANSI color sequences |--gc_grace_seconds <arg> |The gc_grace_seconds to use when calculating +|--scan |Full sstable scan for additional details. Only available in 3.0+ sstables. Defaults: false +|--timestamp_unit <arg> |Time unit that cell timestamps are written with +|--unicode |Use unicode to draw histograms and progress bars droppable tombstones |=== @@ -28,29 +32,34 @@ necessary, find the__Data.db file(s) using sstableutil. Example: .... -sstableutil keyspace1 standard1 | grep Data +$ sstableutil keyspace1 standard1 | grep Data /var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big-Data.db -sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big-Data.db +$ sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big-Data.db SSTable: /var/lib/cassandra/data/keyspace1/standard1-f6845640a6cb11e8b6836d2c86545d91/mc-1-big Partitioner: org.apache.cassandra.dht.Murmur3Partitioner Bloom Filter FP chance: 0.010000 -Minimum timestamp: 1535025576141000 -Maximum timestamp: 1535025604309000 -SSTable min local deletion time: 2147483647 -SSTable max local deletion time: 2147483647 +Minimum timestamp: 06/28/2023 15:15:04 (1687979704767589) +Maximum timestamp: 06/28/2023 15:15:09 (1687979709601420) +Duration: Days: 0 Hours: 0 Minutes: 0 Seconds: 4 +SSTable min local deletion time: 08/17/292278994 02:12:55 (9223372036854775807) +SSTable max local deletion time: 08/17/292278994 02:12:55 (9223372036854775807) Compressor: org.apache.cassandra.io.compress.LZ4Compressor +Compression ratio: 0.553999337227618 TTL min: 86400 TTL max: 86400 First token: -9223004712949498654 (key=39373333373831303130) Last token: 9222554117157811897 (key=4f3438394e39374d3730) +Covered clusterings: ALL Estimated droppable tombstones: 0.9188263888888889 SSTable Level: 0 Repaired at: 0 +Originating host id: badfcb83-a283-402b-818e-78ede00aa3c4 +Pending repair: -- Replay positions covered: {CommitLogPosition(segmentId=1535025390651, position=226400)=CommitLogPosition(segmentId=1535025390651, position=6849139)} -totalColumnsSet: 100000 -totalRows: 20000 +Total Column Cells: 72918 +Total Rows: 24306 Estimated tombstone drop times: 1535039100: 80390 1535039160: 5645 @@ -214,6 +223,7 @@ KeyType: org.apache.cassandra.db.marshal.BytesType ClusteringTypes: [org.apache.cassandra.db.marshal.UTF8Type] StaticColumns: {C3:org.apache.cassandra.db.marshal.BytesType, C4:org.apache.cassandra.db.marshal.BytesType, C0:org.apache.cassandra.db.marshal.BytesType, C1:org.apache.cassandra.db.marshal.BytesType, C2:org.apache.cassandra.db.marshal.BytesType} RegularColumns: {} +IsTransient: false .... == Specify gc grace seconds @@ -231,25 +241,25 @@ ref: https://issues.apache.org/jira/browse/CASSANDRA-12208 Example: .... -sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated tombstone drop times" -A4 +$ sstablemetadata /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated tombstone drop times" -A4 Estimated tombstone drop times: 1536599100: 1 1536599640: 1 1536599700: 2 -echo $(date +%s) +$ echo $(date +%s) 1536602005 # if gc_grace_seconds was configured at 100, all of the tombstones would be currently droppable -sstablemetadata --gc_grace_seconds 100 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones" +$ sstablemetadata --gc_grace_seconds 100 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones" Estimated droppable tombstones: 4.0E-5 # if gc_grace_seconds was configured at 4700, some of the tombstones would be currently droppable -sstablemetadata --gc_grace_seconds 4700 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones" +$ sstablemetadata --gc_grace_seconds 4700 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones" Estimated droppable tombstones: 9.61111111111111E-6 # if gc_grace_seconds was configured at 100, none of the tombstones would be currently droppable -sstablemetadata --gc_grace_seconds 5000 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones" +$ sstablemetadata --gc_grace_seconds 5000 /var/lib/cassandra/data/keyspace1/standard1-41b52700b4ed11e896476d2c86545d91/mc-12-big-Data.db | grep "Estimated droppable tombstones" Estimated droppable tombstones: 0.0 .... @@ -268,7 +278,9 @@ in reads; defined in the table definition timestamp of any entry in this sstable, in epoch microseconds |Maximum timestamp |maximum timestamp of any entry in this sstable, in epoch -microseconds +microseconds +|Duration |Difference between Maximum timestamp +and Minimum timestamp |SSTable min local deletion time |minimum timestamp of deletion date, based on TTL, in epoch seconds |SSTable max local deletion @@ -291,9 +303,9 @@ this sstable was marked as repaired via sstablerepairedset, in epoch milliseconds |Replay positions covered |the interval of time and commitlog positions related to this sstable -|totalColumnsSet |number of cells in the -table -|totalRows |number of rows in the table +|Total Columns Cells |number of cells in the table +|Total Rows |number of rows in the table +|Total Partitions |total number of partitions |Estimated tombstone drop times |approximate number of rows that will expire, ordered by epoch seconds diff --git a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java index 1d9dd91bc1..2047111cac 100644 --- a/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java +++ b/src/java/org/apache/cassandra/tools/SSTableMetadataViewer.java @@ -71,6 +71,8 @@ import static org.apache.cassandra.tools.Util.WHITE; import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; import static org.apache.commons.lang3.time.DurationFormatUtils.formatDurationWords; +import org.apache.cassandra.io.sstable.KeyIterator; + /** * Shows the contents of sstable metadata */ @@ -83,7 +85,7 @@ public class SSTableMetadataViewer private static final String GCGS_KEY = "g"; private static final String TIMESTAMP_UNIT = "t"; private static final String SCAN = "s"; - private static Comparator<ValuedByteBuffer> VCOMP = Comparator.comparingLong(ValuedByteBuffer::getValue).reversed(); + private static final Comparator<ValuedByteBuffer> VCOMP = Comparator.comparingLong(ValuedByteBuffer::getValue).reversed(); static { @@ -321,6 +323,20 @@ public class SSTableMetadataViewer CompactionMetadata compaction = statsComponent.compactionMetadata(); SerializationHeader.Component header = statsComponent.serializationHeader(); Class<? extends ICompressor> compressorClass = null; + + TableMetadata metadata = Util.metadataFromSSTable(descriptor); + SSTableReader sstable = SSTableReader.openNoValidation(null, descriptor, TableMetadataRef.forOfflineTools(metadata)); + int count = 0; + + try (KeyIterator iter = sstable.keyIterator()) + { + while (iter.hasNext()) + { + iter.next(); + count += 1; + } + } + try (CompressionMetadata compression = CompressionInfoComponent.loadIfExists(descriptor)) { compressorClass = compression != null ? compression.compressor().getClass() : null; @@ -338,10 +354,12 @@ public class SSTableMetadataViewer } if (stats != null) { - field("Minimum timestamp", stats.minTimestamp, toDateString(stats.minTimestamp, tsUnit)); - field("Maximum timestamp", stats.maxTimestamp, toDateString(stats.maxTimestamp, tsUnit)); - field("SSTable min local deletion time", stats.minLocalDeletionTime, deletion(stats.minLocalDeletionTime)); - field("SSTable max local deletion time", stats.maxLocalDeletionTime, deletion(stats.maxLocalDeletionTime)); + TimeUnit tsUnit = TimeUnit.MICROSECONDS; + field("Minimum timestamp", toDateString(stats.minTimestamp, tsUnit), Long.toString(stats.minTimestamp)); + field("Maximum timestamp", toDateString(stats.maxTimestamp, tsUnit), Long.toString(stats.maxTimestamp)); + field("Duration", durationString(stats.maxTimestamp - stats.minTimestamp)); + field("SSTable min local deletion time", deletion(stats.minLocalDeletionTime), Long.toString(stats.minLocalDeletionTime)); + field("SSTable max local deletion time", deletion(stats.maxLocalDeletionTime), Long.toString(stats.maxLocalDeletionTime)); field("Compressor", compressorClass != null ? compressorClass.getName() : "-"); if (compressorClass != null) field("Compression ratio", stats.compressionRatio); @@ -363,8 +381,8 @@ public class SSTableMetadataViewer field("Originating host id", stats.originatingHostId); field("Pending repair", stats.pendingRepair); field("Replay positions covered", stats.commitLogIntervals); - field("totalColumnsSet", stats.totalColumnsSet); - field("totalRows", stats.totalRows); + field("Total Column Cells", stats.totalColumnsSet); + field("Total Rows", stats.totalRows); field("Estimated tombstone drop times", ""); TermHistogram estDropped = new TermHistogram(stats.estimatedTombstoneDropTime, @@ -376,6 +394,7 @@ public class SSTableMetadataViewer String::valueOf); estDropped.printHistogram(out, color, unicode); field("Partition Size", ""); + field("Total Partitions", count); TermHistogram rowSize = new TermHistogram(stats.estimatedPartitionSize, "Size (bytes)", offset -> String.format("%d %s", @@ -410,15 +429,16 @@ public class SSTableMetadataViewer field("EncodingStats minTTL", encodingStats.minTTL, toDurationString(encodingStats.minTTL, TimeUnit.SECONDS)); - field("EncodingStats minLocalDeletionTime", encodingStats.minLocalDeletionTime, - toDateString(encodingStats.minLocalDeletionTime, TimeUnit.SECONDS)); - field("EncodingStats minTimestamp", encodingStats.minTimestamp, - toDateString(encodingStats.minTimestamp, tsUnit)); + field("EncodingStats minLocalDeletionTime", toDateString(encodingStats.minLocalDeletionTime, + TimeUnit.SECONDS), Long.toString(encodingStats.minLocalDeletionTime)); + field("EncodingStats minTimestamp", toDateString(encodingStats.minTimestamp, tsUnit), + Long.toString(encodingStats.minTimestamp)); field("KeyType", keyType.toString()); field("ClusteringTypes", clusteringTypes.toString()); field("StaticColumns", FBUtilities.toString(statics)); field("RegularColumns", FBUtilities.toString(regulars)); - field("IsTransient", stats.isTransient); + if (stats != null) + field("IsTransient", stats.isTransient); } } @@ -442,10 +462,20 @@ public class SSTableMetadataViewer if (color) sb.append(WHITE); sb.append(" ("); sb.append(comment); - sb.append(")"); + sb.append(')'); if (color) sb.append(RESET); } - this.out.println(sb.toString()); + this.out.println(sb); + } + + public static String durationString(Long value) + { + long seconds = TimeUnit.MICROSECONDS.toSeconds(value); + long day = TimeUnit.SECONDS.toDays(seconds); + long hours = TimeUnit.SECONDS.toHours(seconds) - (day * 24); + long minute = TimeUnit.SECONDS.toMinutes(seconds) - (TimeUnit.SECONDS.toHours(seconds) * 60); + long second = TimeUnit.SECONDS.toSeconds(seconds) - (TimeUnit.SECONDS.toMinutes(seconds) * 60); + return "Days: " + day + " Hours: " + hours + " Minutes: " + minute + " Seconds: " + second; } private static void printUsage() --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org