Author: xedin
Date: Tue Nov 8 20:04:17 2011
New Revision: 1199427
URL: http://svn.apache.org/viewvc?rev=1199427&view=rev
Log:
Report compression ratio in CFSMBean
patch by vijay; reviewed by xedin for CASSANDRA-3393
Modified:
cassandra/branches/cassandra-1.0/CHANGES.txt
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java
cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
Modified: cassandra/branches/cassandra-1.0/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/CHANGES.txt?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0/CHANGES.txt Tue Nov 8 20:04:17 2011
@@ -13,7 +13,7 @@ Merged from 0.8:
* Make counter shard merging thread safe (CASSANDRA-3178)
* Revert CASSANDRA-2855
* Fix bug preventing the use of efficient cross-DC writes (CASSANDRA-3472)
-
+ * Report compression ratio in CFSMBean (CASSANDRA-3393)
1.0.2
* "defragment" rows for name-based queries under STCS (CASSANDRA-2503)
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Tue Nov 8 20:04:17 2011
@@ -1784,6 +1784,12 @@ public class ColumnFamilyStore implement
return data.getEstimatedColumnCountHistogram();
}
+ @Override
+ public double getCompressionRatio()
+ {
+ return data.getCompressionRatio();
+ }
+
/** true if this CFS contains secondary index data */
public boolean isIndex()
{
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
Tue Nov 8 20:04:17 2011
@@ -213,6 +213,7 @@ public interface ColumnFamilyStoreMBean
public long[] getEstimatedRowSizeHistogram();
public long[] getEstimatedColumnCountHistogram();
+ public double getCompressionRatio();
/**
* Returns a list of the names of the built column indexes for current
store
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/DataTracker.java
Tue Nov 8 20:04:17 2011
@@ -364,6 +364,21 @@ public class DataTracker
return histogram;
}
+ public double getCompressionRatio()
+ {
+ double sum = 0;
+ int total = 0;
+ for (SSTableReader sstable : getSSTables())
+ {
+ if (sstable.getCompressionRatio() != Double.MIN_VALUE)
+ {
+ sum += sstable.getCompressionRatio();
+ total++;
+ }
+ }
+ return total != 0 ? (double)sum/total: 0;
+ }
+
public long getMinRowSize()
{
long min = 0;
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/compress/CompressedSequentialWriter.java
Tue Nov 8 20:04:17 2011
@@ -23,14 +23,15 @@ import java.io.IOException;
import java.util.zip.CRC32;
import java.util.zip.Checksum;
+import org.apache.cassandra.io.sstable.SSTableMetadata.Collector;
import org.apache.cassandra.io.util.FileMark;
import org.apache.cassandra.io.util.SequentialWriter;
public class CompressedSequentialWriter extends SequentialWriter
{
- public static SequentialWriter open(String dataFilePath, String
indexFilePath, boolean skipIOCache, CompressionParameters parameters) throws
IOException
+ public static SequentialWriter open(String dataFilePath, String
indexFilePath, boolean skipIOCache, CompressionParameters parameters, Collector
sstableMetadataCollector) throws IOException
{
- return new CompressedSequentialWriter(new File(dataFilePath),
indexFilePath, skipIOCache, parameters);
+ return new CompressedSequentialWriter(new File(dataFilePath),
indexFilePath, skipIOCache, parameters, sstableMetadataCollector);
}
// holds offset in the file where current chunk should be written
@@ -49,7 +50,11 @@ public class CompressedSequentialWriter
private final Checksum checksum = new CRC32();
- public CompressedSequentialWriter(File file, String indexFilePath, boolean
skipIOCache, CompressionParameters parameters) throws IOException
+ private long originalSize = 0, compressedSize = 0;
+
+ private Collector sstableMetadataCollector;
+
+ public CompressedSequentialWriter(File file, String indexFilePath, boolean
skipIOCache, CompressionParameters parameters, Collector
sstableMetadataCollector) throws IOException
{
super(file, parameters.chunkLength(), skipIOCache);
this.compressor = parameters.sstableCompressor;
@@ -60,6 +65,7 @@ public class CompressedSequentialWriter
/* Index File (-CompressionInfo.db component) and it's header */
metadataWriter = new CompressionMetadata.Writer(indexFilePath);
metadataWriter.writeHeader(parameters);
+ this.sstableMetadataCollector = sstableMetadataCollector;
}
@Override
@@ -82,6 +88,9 @@ public class CompressedSequentialWriter
// compressing data with buffer re-use
int compressedLength = compressor.compress(buffer, 0,
validBufferBytes, compressed, 0);
+ originalSize += validBufferBytes;
+ compressedSize += compressedLength;
+
// update checksum
checksum.update(buffer, 0, validBufferBytes);
@@ -179,7 +188,7 @@ public class CompressedSequentialWriter
return; // already closed
super.close();
-
+ sstableMetadataCollector.addCompressionRatio(compressedSize,
originalSize);
metadataWriter.finalizeHeader(current, chunkCount);
metadataWriter.close();
}
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
Tue Nov 8 20:04:17 2011
@@ -48,7 +48,7 @@ public class Descriptor
// f (0.7.0): switched bloom filter implementations in data component
// g (0.8): tracks flushed-at context in metadata component
// h (1.0): tracks max client timestamp in metadata component
- public static final String CURRENT_VERSION = "h";
+ public static final String CURRENT_VERSION = "i";
public final File directory;
public final String version;
@@ -112,6 +112,11 @@ public class Descriptor
return version.compareTo("g") >= 0;
}
+ public boolean hasCompressionRatio()
+ {
+ return version.compareTo("i") >= 0;
+ }
+
public String filenameFor(Component component)
{
return filenameFor(component.name());
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableMetadata.java
Tue Nov 8 20:04:17 2011
@@ -53,6 +53,7 @@ public class SSTableMetadata
protected final EstimatedHistogram estimatedColumnCount;
protected final ReplayPosition replayPosition;
protected final long maxTimestamp;
+ protected final double compressionRatio;
public static final SSTableMetadataSerializer serializer = new
SSTableMetadataSerializer();
private SSTableMetadata()
@@ -63,15 +64,16 @@ public class SSTableMetadata
// when there is no max timestamp recorded, default to max long
private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram
columnCounts, ReplayPosition replayPosition)
{
- this(rowSizes, columnCounts, replayPosition, Long.MAX_VALUE);
+ this(rowSizes, columnCounts, replayPosition, Long.MAX_VALUE,
Double.MIN_VALUE);
}
- private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram
columnCounts, ReplayPosition replayPosition, long maxTimestamp)
+ private SSTableMetadata(EstimatedHistogram rowSizes, EstimatedHistogram
columnCounts, ReplayPosition replayPosition, long maxTimestamp, double cr)
{
this.estimatedRowSize = rowSizes;
this.estimatedColumnCount = columnCounts;
this.replayPosition = replayPosition;
this.maxTimestamp = maxTimestamp;
+ this.compressionRatio = cr;
}
public static SSTableMetadata createDefaultInstance()
@@ -104,6 +106,11 @@ public class SSTableMetadata
return maxTimestamp;
}
+ public double getCompressionRatio()
+ {
+ return compressionRatio;
+ }
+
static EstimatedHistogram defaultColumnCountHistogram()
{
// EH of 114 can track a max value of 2395318855, i.e., > 2B columns
@@ -122,6 +129,7 @@ public class SSTableMetadata
protected EstimatedHistogram estimatedColumnCount;
protected ReplayPosition replayPosition;
protected long maxTimestamp;
+ protected double compressionRatio;
private Collector()
{
@@ -129,6 +137,7 @@ public class SSTableMetadata
this.estimatedColumnCount = defaultColumnCountHistogram();
this.replayPosition = ReplayPosition.NONE;
this.maxTimestamp = Long.MIN_VALUE;
+ this.compressionRatio = Double.MIN_VALUE;
}
public void addRowSize(long rowSize)
@@ -141,6 +150,15 @@ public class SSTableMetadata
estimatedColumnCount.add(columnCount);
}
+ /**
+ * Ratio is compressed/uncompressed and it is
+ * if you have 1.x then compression isn't helping
+ */
+ public void addCompressionRatio(long compressed, long uncompressed)
+ {
+ compressionRatio = (double) compressed/uncompressed;
+ }
+
public void updateMaxTimestamp(long potentialMax)
{
maxTimestamp = Math.max(maxTimestamp, potentialMax);
@@ -148,7 +166,7 @@ public class SSTableMetadata
public SSTableMetadata finalizeMetadata()
{
- return new SSTableMetadata(estimatedRowSize, estimatedColumnCount,
replayPosition, maxTimestamp);
+ return new SSTableMetadata(estimatedRowSize, estimatedColumnCount,
replayPosition, maxTimestamp, compressionRatio);
}
public Collector estimatedRowSize(EstimatedHistogram estimatedRowSize)
@@ -180,6 +198,7 @@ public class SSTableMetadata
EstimatedHistogram.serializer.serialize(sstableStats.getEstimatedColumnCount(),
dos);
ReplayPosition.serializer.serialize(sstableStats.getReplayPosition(), dos);
dos.writeLong(sstableStats.getMaxTimestamp());
+ dos.writeDouble(sstableStats.getCompressionRatio());
}
public SSTableMetadata deserialize(Descriptor descriptor) throws
IOException
@@ -205,8 +224,14 @@ public class SSTableMetadata
ReplayPosition replayPosition = descriptor.hasReplayPosition()
?
ReplayPosition.serializer.deserialize(dis)
: ReplayPosition.NONE;
-
- return new SSTableMetadata(rowSizes, columnCounts,
replayPosition);
+ long maxTimestamp = Long.MAX_VALUE;
+ double compressionRatio = Double.MIN_VALUE;
+ if (descriptor.hasCompressionRatio())
+ {
+ maxTimestamp = dis.readLong();
+ compressionRatio = dis.readDouble();
+ }
+ return new SSTableMetadata(rowSizes, columnCounts,
replayPosition, maxTimestamp, compressionRatio);
}
finally
{
@@ -220,7 +245,7 @@ public class SSTableMetadata
EstimatedHistogram columnCounts =
EstimatedHistogram.serializer.deserialize(dis);
ReplayPosition replayPosition =
ReplayPosition.serializer.deserialize(dis);
long maxTimestamp = dis.readLong();
- return new SSTableMetadata(rowSizes, columnCounts, replayPosition,
maxTimestamp);
+ return new SSTableMetadata(rowSizes, columnCounts, replayPosition,
maxTimestamp, Double.MIN_VALUE);
}
public long serializedSize(SSTableMetadata object)
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Tue Nov 8 20:04:17 2011
@@ -894,6 +894,11 @@ public class SSTableReader extends SSTab
return sstableMetadata.getEstimatedColumnCount();
}
+ public double getCompressionRatio()
+ {
+ return sstableMetadata.getCompressionRatio();
+ }
+
public ReplayPosition getReplayPosition()
{
return sstableMetadata.getReplayPosition();
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
Tue Nov 8 20:04:17 2011
@@ -87,7 +87,8 @@ public class SSTableWriter extends SSTab
dataFile = CompressedSequentialWriter.open(getFilename(),
descriptor.filenameFor(Component.COMPRESSION_INFO),
true,
-
metadata.compressionParameters());
+
metadata.compressionParameters(),
+
sstableMetadataCollector);
}
else
{
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/tools/NodeCmd.java
Tue Nov 8 20:04:17 2011
@@ -489,6 +489,8 @@ public class NodeCmd
outs.println("\t\tCompacted row minimum size: " +
cfstore.getMinRowSize());
outs.println("\t\tCompacted row maximum size: " +
cfstore.getMaxRowSize());
outs.println("\t\tCompacted row mean size: " +
cfstore.getMeanRowSize());
+ if (cfstore.getCompressionRatio() != 0)
+ outs.println("\t\tSSTable Compression Ratio: " +
cfstore.getCompressionRatio());
outs.println("");
}
Modified:
cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java?rev=1199427&r1=1199426&r2=1199427&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
(original)
+++
cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/compress/CompressedRandomAccessReaderTest.java
Tue Nov 8 20:04:17 2011
@@ -24,6 +24,7 @@ import java.util.concurrent.Callable;
import org.junit.Test;
+import org.apache.cassandra.io.sstable.SSTableMetadata;
import org.apache.cassandra.io.util.*;
import static org.junit.Assert.assertEquals;
@@ -54,8 +55,9 @@ public class CompressedRandomAccessReade
try
{
+ SSTableMetadata.Collector sstableMetadataCollector =
SSTableMetadata.createCollector().replayPosition(null);
SequentialWriter writer = compressed
- ? new CompressedSequentialWriter(f, filename + ".metadata",
false, new CompressionParameters(SnappyCompressor.instance))
+ ? new CompressedSequentialWriter(f, filename + ".metadata",
false, new CompressionParameters(SnappyCompressor.instance),
sstableMetadataCollector)
: new SequentialWriter(f,
CompressionParameters.DEFAULT_CHUNK_LENGTH, false);
writer.write("The quick ".getBytes());
@@ -103,8 +105,8 @@ public class CompressedRandomAccessReade
File metadata = new File(file.getPath() + ".meta");
metadata.deleteOnExit();
-
- SequentialWriter writer = new CompressedSequentialWriter(file,
metadata.getPath(), false, new
CompressionParameters(SnappyCompressor.instance));
+ SSTableMetadata.Collector sstableMetadataCollector =
SSTableMetadata.createCollector().replayPosition(null);
+ SequentialWriter writer = new CompressedSequentialWriter(file,
metadata.getPath(), false, new
CompressionParameters(SnappyCompressor.instance), sstableMetadataCollector);
writer.write(CONTENT.getBytes());
writer.close();
