Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java?rev=1074329&r1=1074328&r2=1074329&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java Thu Feb 24 22:33:03 2011 @@ -441,7 +441,7 @@ public class NodeCmd { ColumnFamilyStoreMBean store = this.probe.getCfsProxy(keySpace, columnFamily); // default is 90 offsets - long[] offsets = new EstimatedHistogram(90).getBucketOffsets(); + long[] offsets = new EstimatedHistogram().getBucketOffsets(); long[] rrlh = store.getRecentReadLatencyHistogramMicros(); long[] rwlh = store.getRecentWriteLatencyHistogramMicros();
Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java?rev=1074329&r1=1074328&r2=1074329&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/tools/SSTableImport.java Thu Feb 24 22:33:03 2011 @@ -367,7 +367,7 @@ public class SSTableImport */ private static JsonParser getParser(String fileName) throws IOException { - return factory.createJsonParser(new File(fileName)); + return factory.createJsonParser(new File(fileName)).configure(JsonParser.Feature.INTERN_FIELD_NAMES, false); } /** Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java?rev=1074329&r1=1074328&r2=1074329&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/utils/EstimatedHistogram.java Thu Feb 24 22:33:03 2011 @@ -31,23 +31,24 @@ import org.apache.cassandra.io.ICompactS public class EstimatedHistogram { + public static EstimatedHistogramSerializer serializer = new EstimatedHistogramSerializer(); /** * The series of values to which the counts in `buckets` correspond: - * 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 15, 18, 22, etc. + * 1, 2, 3, 4, 5, 6, 7, 8, 10, 12, 14, 17, 20, etc. * Thus, a `buckets` of [0, 0, 1, 10] would mean we had seen one value of 3 and 10 values of 4. * * The series starts at 1 and grows by 1.2 each time (rounding and removing duplicates). It goes from 1 * to around 36M by default (creating 90+1 buckets), which will give us timing resolution from microseconds to * 36 seconds, with less precision as the numbers get larger. + * + * Each bucket represents values from (previous bucket offset, current offset]. */ private long[] bucketOffsets; - private int numBuckets; + // buckets is one element longer than bucketOffsets -- the last element is values greater than the last offset final AtomicLongArray buckets; - public static EstimatedHistogramSerializer serializer = new EstimatedHistogramSerializer(); - public EstimatedHistogram() { this(90); @@ -56,7 +57,7 @@ public class EstimatedHistogram public EstimatedHistogram(int bucketCount) { makeOffsets(bucketCount); - buckets = new AtomicLongArray(numBuckets); + buckets = new AtomicLongArray(bucketOffsets.length + 1); } public EstimatedHistogram(long[] offsets, long[] bucketData) @@ -64,7 +65,6 @@ public class EstimatedHistogram assert bucketData.length == offsets.length +1; bucketOffsets = offsets; buckets = new AtomicLongArray(bucketData); - numBuckets = bucketData.length; } private void makeOffsets(int size) @@ -72,7 +72,7 @@ public class EstimatedHistogram bucketOffsets = new long[size]; long last = 1; bucketOffsets[0] = last; - for(int i = 1; i < size; i++) + for (int i = 1; i < size; i++) { long next = Math.round(last * 1.2); if (next == last) @@ -80,58 +80,79 @@ public class EstimatedHistogram bucketOffsets[i] = next; last = next; } - numBuckets = bucketOffsets.length + 1; } + /** + * @return the histogram values corresponding to each bucket index + */ public long[] getBucketOffsets() { return bucketOffsets; } - + + /** + * Increments the count of the bucket closest to n, rounding UP. + * @param n + */ public void add(long n) { int index = Arrays.binarySearch(bucketOffsets, n); if (index < 0) { - //inexact match, find closest bucket + // inexact match, take the first bucket higher than n index = -index - 1; } - else - { - //exact match, so we want the next highest one - index += 1; - } + // else exact match; we're good buckets.incrementAndGet(index); } - public long[] get(boolean reset) + /** + * @return the count in the given bucket + */ + long get(int bucket) + { + return buckets.get(bucket); + } + + /** + * @param reset: zero out buckets afterwards if true + * @return a long[] containing the current histogram buckets + */ + public long[] getBuckets(boolean reset) { - long[] rv = new long[numBuckets]; - for (int i = 0; i < numBuckets; i++) + long[] rv = new long[buckets.length()]; + for (int i = 0; i < buckets.length(); i++) rv[i] = buckets.get(i); if (reset) - for (int i = 0; i < numBuckets; i++) + for (int i = 0; i < buckets.length(); i++) buckets.set(i, 0L); return rv; } + /** + * @return the smallest value that could have been added to this histogram + */ public long min() { - for (int i = 0; i < numBuckets; i++) + for (int i = 0; i < buckets.length(); i++) { if (buckets.get(i) > 0) - return bucketOffsets[i == 0 ? 0 : i - 1]; + return i == 0 ? 0 : 1 + bucketOffsets[i - 1]; } return 0; } + /** + * @return the largest value that could have been added to this histogram. If the histogram + * overflowed, returns Long.MAX_VALUE. + */ public long max() { - int lastBucket = numBuckets - 1; + int lastBucket = buckets.length() - 1; if (buckets.get(lastBucket) > 0) - throw new IllegalStateException("Unable to compute ceiling for max when all buckets are full"); + return Long.MAX_VALUE; for (int i = lastBucket - 1; i >= 0; i--) { @@ -141,20 +162,33 @@ public class EstimatedHistogram return 0; } - public long median() + /** + * @return the mean histogram value (average of bucket offsets, weighted by count) + * @throws IllegalStateException if any values were greater than the largest bucket threshold + */ + public long mean() { - long max = 0; - long median = 0; - for (int i = 0; i < numBuckets; i++) + int lastBucket = buckets.length() - 1; + if (buckets.get(lastBucket) > 0) + throw new IllegalStateException("Unable to compute ceiling for max when histogram overflowed"); + + long elements = 0; + long sum = 0; + for (int i = 0; i < lastBucket; i++) { - if (max < 1 || buckets.get(i) > max) - { - max = buckets.get(i); - if (max > 0) - median = bucketOffsets[i == 0 ? 0 : i - 1]; - } + elements += buckets.get(i); + sum += buckets.get(i) * bucketOffsets[i]; } - return median; + + return (long) Math.ceil((double) sum / elements); + } + + /** + * @return true if this histogram has overflowed -- that is, a value larger than our largest bucket could bound was added + */ + public boolean isOverflowed() + { + return buckets.get(buckets.length() - 1) > 0; } public static class EstimatedHistogramSerializer implements ICompactSerializer2<EstimatedHistogram> @@ -162,7 +196,7 @@ public class EstimatedHistogram public void serialize(EstimatedHistogram eh, DataOutput dos) throws IOException { long[] offsets = eh.getBucketOffsets(); - long[] buckets = eh.get(false); + long[] buckets = eh.getBuckets(false); dos.writeInt(buckets.length); for (int i = 0; i < buckets.length; i++) { Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/LatencyTracker.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/LatencyTracker.java?rev=1074329&r1=1074328&r2=1074329&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/utils/LatencyTracker.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/utils/LatencyTracker.java Thu Feb 24 22:33:03 2011 @@ -76,11 +76,11 @@ public class LatencyTracker public long[] getTotalLatencyHistogramMicros() { - return totalHistogram.get(false); + return totalHistogram.getBuckets(false); } public long[] getRecentLatencyHistogramMicros() { - return recentHistogram.get(true); + return recentHistogram.getBuckets(true); } } Modified: cassandra/trunk/test/conf/cassandra.yaml URL: http://svn.apache.org/viewvc/cassandra/trunk/test/conf/cassandra.yaml?rev=1074329&r1=1074328&r2=1074329&view=diff ============================================================================== --- cassandra/trunk/test/conf/cassandra.yaml (original) +++ cassandra/trunk/test/conf/cassandra.yaml Thu Feb 24 22:33:03 2011 @@ -189,6 +189,13 @@ keyspaces: rows_cached: 0 keys_cached: 0 + - name: Super5 + column_type: Super + compare_with: TimeUUIDType + compare_subcolumns_with: BytesType + rows_cached: 0 + keys_cached: 0 + - name: Keyspace5 replica_placement_strategy: org.apache.cassandra.locator.SimpleStrategy replication_factor: 2 Modified: cassandra/trunk/test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java?rev=1074329&r1=1074328&r2=1074329&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/io/LazilyCompactedRowTest.java Thu Feb 24 22:33:03 2011 @@ -45,6 +45,8 @@ import org.apache.cassandra.io.util.Data import org.apache.cassandra.io.util.MappedFileDataInput; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.UUIDGen; import org.junit.Test; @@ -277,6 +279,28 @@ public class LazilyCompactedRowTest exte assertDigest(cfs, Integer.MAX_VALUE, true); } + @Test + public void testTwoRowSuperColumn() throws IOException, ExecutionException, InterruptedException + { + CompactionManager.instance.disableAutoCompaction(); + + Table table = Table.open("Keyspace4"); + ColumnFamilyStore cfs = table.getColumnFamilyStore("Super5"); + + ByteBuffer key = ByteBufferUtil.bytes("k"); + RowMutation rm = new RowMutation("Keyspace4", key); + ByteBuffer scKey = ByteBuffer.wrap(UUIDGen.decompose(UUIDGen.makeType1UUIDFromHost(FBUtilities.getLocalAddress()))); + rm.add(new QueryPath("Super5", scKey , ByteBufferUtil.bytes("c")), ByteBufferUtil.EMPTY_BYTE_BUFFER, 0); + rm.apply(); + cfs.forceBlockingFlush(); + + rm.apply(); + cfs.forceBlockingFlush(); + + assertBytes(cfs, Integer.MAX_VALUE, true); + } + + private static class LazyCompactionIterator extends CompactionIterator { private final ColumnFamilyStore cfStore; Modified: cassandra/trunk/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java?rev=1074329&r1=1074328&r2=1074329&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/utils/EstimatedHistogramTest.java Thu Feb 24 22:33:03 2011 @@ -26,28 +26,49 @@ import static org.junit.Assert.*; public class EstimatedHistogramTest { @Test - public void testFindingCorrectBuckets() + public void testSimple() { + // 0 and 1 map to the same, first bucket EstimatedHistogram histogram = new EstimatedHistogram(); + histogram.add(0); + assertEquals(1, histogram.get(0)); + histogram.add(1); + assertEquals(2, histogram.get(0)); + } - histogram.add(0L); - assertEquals(1, histogram.get(false)[0]); + @Test + public void testOverflow() + { + EstimatedHistogram histogram = new EstimatedHistogram(1); + histogram.add(100); + assert histogram.isOverflowed(); + assertEquals(Long.MAX_VALUE, histogram.max()); + } - histogram.add(23282687); - assertEquals(1, histogram.get(false)[histogram.buckets.length() - 2]); + @Test + public void testMinMax() + { + EstimatedHistogram histogram = new EstimatedHistogram(); + histogram.add(16); + assertEquals(15, histogram.min()); + assertEquals(17, histogram.max()); + } - histogram.add(1); - assertEquals(1, histogram.get(false)[1]); + @Test + public void testFindingCorrectBuckets() + { + EstimatedHistogram histogram = new EstimatedHistogram(); + histogram.add(23282687); + assert !histogram.isOverflowed(); + assertEquals(1, histogram.getBuckets(false)[histogram.buckets.length() - 2]); histogram.add(9); - assertEquals(1, histogram.get(false)[8]); + assertEquals(1, histogram.getBuckets(false)[8]); histogram.add(20); histogram.add(21); histogram.add(22); - assertEquals(3, histogram.get(false)[13]); - assertEquals(1, histogram.min()); - assertEquals(25109160, histogram.max()); - assertEquals(20, histogram.median()); + assertEquals(2, histogram.getBuckets(false)[13]); + assertEquals(5021848, histogram.mean()); } }
