Modified: cassandra/trunk/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/dht/ByteOrderedPartitioner.java Tue Nov 23 16:53:59 2010 @@ -19,6 +19,9 @@ package org.apache.cassandra.dht; import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.List; +import java.util.Map; public class ByteOrderedPartitioner extends AbstractByteOrderedPartitioner { @@ -28,4 +31,6 @@ public class ByteOrderedPartitioner exte return MINIMUM; return new BytesToken(key); } + + public Map<Token, Float> describeOwnership(List<Token> sortedTokens){ throw new UnsupportedOperationException(); } }
Modified: cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/dht/CollatingOrderPreservingPartitioner.java Tue Nov 23 16:53:59 2010 @@ -21,7 +21,10 @@ package org.apache.cassandra.dht; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; import java.text.Collator; +import java.util.HashMap; +import java.util.List; import java.util.Locale; +import java.util.Map; import org.apache.cassandra.utils.FBUtilities; @@ -45,4 +48,6 @@ public class CollatingOrderPreservingPar } return new BytesToken(ByteBuffer.wrap(collator.getCollationKey(skey).toByteArray())); } + + public Map<Token, Float> describeOwnership(List<Token> sortedTokens){ throw new UnsupportedOperationException(); } } Modified: cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/dht/IPartitioner.java Tue Nov 23 16:53:59 2010 @@ -19,6 +19,8 @@ package org.apache.cassandra.dht; import java.nio.ByteBuffer; +import java.util.List; +import java.util.Map; import org.apache.cassandra.db.DecoratedKey; @@ -73,4 +75,13 @@ public interface IPartitioner<T extends * it generates. */ public boolean preservesOrder(); + + /** + * Calculate the deltas between tokens in the ring in order to compare + * relative sizes. + * + * @param sortedTokens a sorted List of Tokens + * @return the mapping from 'token' to 'percentage of the ring owned by that token'. + */ + public Map<Token, Float> describeOwnership(List<Token> sortedTokens); } Modified: cassandra/trunk/src/java/org/apache/cassandra/dht/LocalPartitioner.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/LocalPartitioner.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/dht/LocalPartitioner.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/dht/LocalPartitioner.java Tue Nov 23 16:53:59 2010 @@ -20,6 +20,7 @@ package org.apache.cassandra.dht; import java.nio.ByteBuffer; +import java.util.*; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.db.marshal.AbstractType; @@ -74,4 +75,9 @@ public class LocalPartitioner implements { return true; } + + public Map<Token, Float> describeOwnership(List<Token> sortedTokens) + { + return Collections.singletonMap((Token)getMinimumToken(), new Float(1.0)); + } } Modified: cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/dht/OrderPreservingPartitioner.java Tue Nov 23 16:53:59 2010 @@ -21,11 +21,14 @@ package org.apache.cassandra.dht; import java.math.BigInteger; import java.nio.ByteBuffer; import java.nio.charset.CharacterCodingException; -import java.util.Random; +import java.util.*; import com.google.common.base.Charsets; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.DecoratedKey; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; @@ -157,4 +160,41 @@ public class OrderPreservingPartitioner } return new StringToken(skey); } + + public Map<Token, Float> describeOwnership(List<Token> sortedTokens) + { + // alltokens will contain the count and be returned, sorted_ranges is shorthand for token<->token math. + Map<Token, Float> alltokens = new HashMap<Token, Float>(); + List<Range> sorted_ranges = new ArrayList<Range>(); + + // this initializes the counts to 0 and calcs the ranges in order. + Token last_t = sortedTokens.get(sortedTokens.size()-1); + for (Token node : sortedTokens) + { + alltokens.put(node, new Float(0.0)); + sorted_ranges.add(new Range(last_t, node)); + last_t = node; + } + + for(String ks : DatabaseDescriptor.getTables()) + { + for (CFMetaData cfmd : DatabaseDescriptor.getKSMetaData(ks).cfMetaData().values()) + { + for (Range r : sorted_ranges) + { + // Looping over every KS:CF:Range, get the splits size and add it to the count + alltokens.put(r.right, alltokens.get(r.right) + StorageService.instance.getSplits(ks, cfmd.cfName, r, 1).size()); + } + } + } + + // Sum every count up and divide count/total for the fractional ownership. + Float total = new Float(0.0); + for (Float f : alltokens.values()) { total += f; } + for (Map.Entry<Token, Float> row : alltokens.entrySet()) { + alltokens.put(row.getKey(), row.getValue() / total); + } + + return alltokens; + } } Modified: cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/dht/RandomPartitioner.java Tue Nov 23 16:53:59 2010 @@ -20,9 +20,10 @@ package org.apache.cassandra.dht; import static com.google.common.base.Charsets.UTF_8; +import java.math.BigDecimal; import java.math.BigInteger; import java.nio.ByteBuffer; -import java.util.Arrays; +import java.util.*; import org.apache.cassandra.db.DecoratedKey; import org.apache.cassandra.utils.FBUtilities; @@ -127,4 +128,36 @@ public class RandomPartitioner implement return MINIMUM; return new BigIntegerToken(FBUtilities.md5hash(key)); } + + public Map<Token, Float> describeOwnership(List<Token> sortedTokens) + { + Map<Token, Float> ownerships = new HashMap<Token, Float>(); + Iterator i = sortedTokens.iterator(); + + // 0-case + if (!i.hasNext()) { throw new RuntimeException("No nodes present in the cluster. How did you call this?"); } + // 1-case + if (sortedTokens.size() == 1) { + ownerships.put((Token)i.next(), new Float(1.0)); + } + // n-case + else { + // NOTE: All divisions must take place in BigDecimals, and all modulo operators must take place in BigIntegers. + final BigInteger ri = new BigInteger("2").pow(127); // (used for addition later) + final BigDecimal r = new BigDecimal(ri); // The entire range, 2**127 + Token start = (Token)i.next(); BigInteger ti = ((BigIntegerToken)start).token; // The first token and its value + Token t; BigInteger tim1 = ti; // The last token and its value (after loop) + while (i.hasNext()) { + t = (Token)i.next(); ti = ((BigIntegerToken)t).token; // The next token and its value + float x = new BigDecimal(ti.subtract(tim1)).divide(r).floatValue(); // %age = T(i) - T(i-1) / R + ownerships.put(t, x); // save (T(i) -> %age) + tim1 = ti; // -> advance loop + } + // The start token's range extends backward to the last token, which is why both were saved + // above. The simple calculation for this is: T(start) - T(end) + r % r / r. + // (In the 1-case, this produces 0% instead of 100%.) + ownerships.put(start, new BigDecimal(((BigIntegerToken)start).token.subtract(ti).add(ri).mod(ri)).divide(r).floatValue()); + } + return ownerships; + } } Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTable.java Tue Nov 23 16:53:59 2010 @@ -23,6 +23,7 @@ import java.io.File; import java.io.FilenameFilter; import java.io.IOError; import java.io.IOException; +import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -59,12 +60,10 @@ public abstract class SSTable public static final String COMPONENT_FILTER = Component.Type.FILTER.repr; public static final String COMPONENT_STATS = Component.Type.STATS.repr; - public static final String COMPONENT_COMPACTED = Component.Type.COMPACTED_MARKER.repr; - public static final String TEMPFILE_MARKER = "tmp"; public final Descriptor descriptor; - public final Set<Component> components; + protected final Set<Component> components; public final CFMetaData metadata; public final IPartitioner partitioner; @@ -94,7 +93,10 @@ public abstract class SSTable protected SSTable(Descriptor descriptor, Set<Component> components, CFMetaData metadata, IPartitioner partitioner, EstimatedHistogram rowSizes, EstimatedHistogram columnCounts) { this.descriptor = descriptor; - this.components = components; + Set<Component> dataComponents = new HashSet<Component>(components); + for (Component component : components) + assert component.type != Component.Type.COMPACTED_MARKER; + this.components = Collections.unmodifiableSet(dataComponents); this.metadata = metadata; this.partitioner = partitioner; estimatedRowSize = rowSizes; @@ -122,11 +124,8 @@ public abstract class SSTable * * @return true if the file was deleted */ - public static boolean conditionalDelete(Descriptor desc, Set<Component> components) + public static boolean delete(Descriptor desc, Set<Component> components) { - if (!components.contains(Component.COMPACTED_MARKER) && !desc.temporary) - // not compacted or temporary - return false; try { // remove the DATA component first if it exists @@ -139,8 +138,7 @@ public abstract class SSTable FileUtils.deleteWithConfirm(desc.filenameFor(component)); } // remove the COMPACTED_MARKER component last if it exists - if (components.contains(Component.COMPACTED_MARKER)) - FileUtils.deleteWithConfirm(desc.filenameFor(Component.COMPACTED_MARKER)); + FileUtils.delete(desc.filenameFor(Component.COMPACTED_MARKER)); } catch (IOException e) { Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableDeletingReference.java Tue Nov 23 16:53:59 2010 @@ -90,9 +90,9 @@ public class SSTableDeletingReference ex return; } } - // let the remainder be cleaned up by conditionalDelete + // let the remainder be cleaned up by delete components.remove(Component.DATA); - SSTable.conditionalDelete(desc, components); + SSTable.delete(desc, components); tracker.spaceReclaimed(size); } } Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableReader.java Tue Nov 23 16:53:59 2010 @@ -55,10 +55,8 @@ import org.apache.cassandra.io.util.Buff import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.SegmentedFile; import org.apache.cassandra.service.StorageService; -import org.apache.cassandra.utils.BloomFilter; -import org.apache.cassandra.utils.EstimatedHistogram; -import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.utils.Pair; +import org.apache.cassandra.utils.*; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -524,7 +522,6 @@ public class SSTableReader extends SSTab { throw new IOError(e); } - components.add(Component.COMPACTED_MARKER); phantomReference.deleteOnCleanup(); } @@ -597,6 +594,16 @@ public class SSTableReader extends SSTab return in.readLong(); } + public void createLinks(String snapshotDirectoryPath) throws IOException + { + for (Component component : components) + { + File sourceFile = new File(descriptor.filenameFor(component)); + File targetLink = new File(snapshotDirectoryPath, sourceFile.getName()); + CLibrary.createHardLink(sourceFile, targetLink); + } + } + /** * Conditionally use the deprecated 'IPartitioner.convertFromDiskFormat' method. */ Modified: cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java Tue Nov 23 16:53:59 2010 @@ -25,6 +25,7 @@ import java.io.FileOutputStream; import java.io.IOError; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.HashSet; import java.util.Set; @@ -65,16 +66,15 @@ public class SSTableWriter extends SSTab public SSTableWriter(String filename, long keyCount, CFMetaData metadata, IPartitioner partitioner) throws IOException { - super(Descriptor.fromFilename(filename), new HashSet<Component>(), metadata, partitioner, SSTable.defaultRowHistogram(), SSTable.defaultColumnHistogram()); + super(Descriptor.fromFilename(filename), + new HashSet<Component>(Arrays.asList(Component.DATA, Component.FILTER, Component.PRIMARY_INDEX, Component.STATS)), + metadata, + partitioner, + SSTable.defaultRowHistogram(), + SSTable.defaultColumnHistogram()); iwriter = new IndexWriter(descriptor, partitioner, keyCount); dbuilder = SegmentedFile.getBuilder(DatabaseDescriptor.getDiskAccessMode()); dataFile = new BufferedRandomAccessFile(getFilename(), "rw", DatabaseDescriptor.getInMemoryCompactionLimit()); - - // the set of required components - components.add(Component.DATA); - components.add(Component.FILTER); - components.add(Component.PRIMARY_INDEX); - components.add(Component.STATS); } public void mark() Modified: cassandra/trunk/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/locator/AbstractReplicationStrategy.java Tue Nov 23 16:53:59 2010 @@ -140,8 +140,6 @@ public abstract class AbstractReplicatio { Multimap<InetAddress, InetAddress> map = HashMultimap.create(targets.size(), 1); - IEndpointSnitch endpointSnitch = DatabaseDescriptor.getEndpointSnitch(); - // first, add the live endpoints for (InetAddress ep : targets) { @@ -168,7 +166,7 @@ public abstract class AbstractReplicatio InetAddress destination = map.isEmpty() ? localAddress - : endpointSnitch.getSortedListByProximity(localAddress, map.keySet()).get(0); + : snitch.getSortedListByProximity(localAddress, map.keySet()).get(0); map.put(destination, ep); } Modified: cassandra/trunk/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/locator/NetworkTopologyStrategy.java Tue Nov 23 16:53:59 2010 @@ -29,10 +29,13 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import com.google.common.collect.Multimap; +import org.apache.commons.lang.StringUtils; + import org.apache.cassandra.config.ConfigurationException; import org.apache.cassandra.dht.Token; import org.apache.cassandra.service.*; import org.apache.cassandra.thrift.ConsistencyLevel; +import org.apache.cassandra.utils.FBUtilities; /** * This Replication Strategy takes a property file that gives the intended @@ -62,12 +65,11 @@ public class NetworkTopologyStrategy ext if (configOptions != null) { for (Entry entry : configOptions.entrySet()) - { newDatacenters.put((String) entry.getKey(), Integer.parseInt((String) entry.getValue())); - } } datacenters = Collections.unmodifiableMap(newDatacenters); + logger.debug("Configured datacenter replicas are {}", FBUtilities.toString(datacenters)); } public List<InetAddress> calculateNaturalEndpoints(Token searchToken, TokenMetadata tokenMetadata) @@ -116,6 +118,9 @@ public class NetworkTopologyStrategy ext if (dcEndpoints.size() < dcReplicas) throw new IllegalStateException(String.format("datacenter (%s) has no more endpoints, (%s) replicas still needed", dcName, dcReplicas - dcEndpoints.size())); + if (logger.isDebugEnabled()) + logger.debug("{} endpoints in datacenter {} for token {} ", + new Object[] { StringUtils.join(dcEndpoints, ","), dcName, searchToken}); endpoints.addAll(dcEndpoints); } Modified: cassandra/trunk/src/java/org/apache/cassandra/service/AntiEntropyService.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/AntiEntropyService.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/service/AntiEntropyService.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/service/AntiEntropyService.java Tue Nov 23 16:53:59 2010 @@ -216,7 +216,7 @@ public class AntiEntropyService for (Differencer differencer : differencers) { logger.info("Queueing comparison " + differencer); - StageManager.getStage(Stage.ANTIENTROPY).execute(differencer); + StageManager.getStage(Stage.ANTI_ENTROPY).execute(differencer); } } @@ -406,7 +406,7 @@ public class AntiEntropyService for (MerkleTree.RowHash minrow : minrows) range.addHash(minrow); - StageManager.getStage(Stage.ANTIENTROPY).submit(this); + StageManager.getStage(Stage.ANTI_ENTROPY).submit(this); logger.debug("Validated " + validated + " rows into AEService tree for " + request); } Modified: cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/service/StorageService.java Tue Nov 23 16:53:59 2010 @@ -181,8 +181,8 @@ public class StorageService implements I put(Verb.STREAM_REQUEST, Stage.STREAM); put(Verb.RANGE_SLICE, Stage.READ); put(Verb.BOOTSTRAP_TOKEN, Stage.MISC); - put(Verb.TREE_REQUEST, Stage.ANTIENTROPY); - put(Verb.TREE_RESPONSE, Stage.ANTIENTROPY); + put(Verb.TREE_REQUEST, Stage.ANTI_ENTROPY); + put(Verb.TREE_RESPONSE, Stage.ANTI_ENTROPY); put(Verb.GOSSIP_DIGEST_ACK, Stage.GOSSIP); put(Verb.GOSSIP_DIGEST_ACK2, Stage.GOSSIP); put(Verb.GOSSIP_DIGEST_SYN, Stage.GOSSIP); @@ -2058,4 +2058,11 @@ public class StorageService implements I logger_.debug("cache saves completed"); } + public Map<Token, Float> getOwnership() + { + List<Token> sortedTokens = new ArrayList<Token>(getTokenToEndpointMap().keySet()); + Collections.sort(sortedTokens); + return partitioner_.describeOwnership(sortedTokens); + } + } Modified: cassandra/trunk/src/java/org/apache/cassandra/service/StorageServiceMBean.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/service/StorageServiceMBean.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/service/StorageServiceMBean.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/service/StorageServiceMBean.java Tue Nov 23 16:53:59 2010 @@ -29,6 +29,7 @@ import java.util.concurrent.ExecutionExc import java.util.concurrent.TimeoutException; import org.apache.cassandra.config.ConfigurationException; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; import org.apache.cassandra.thrift.UnavailableException; @@ -263,4 +264,10 @@ public interface StorageServiceMBean /** save row and key caches */ public void saveCaches() throws ExecutionException, InterruptedException; + + /** + * given a list of tokens (representing the nodes in the cluster), returns + * a mapping from "token -> %age of cluster owned by that token" + */ + public Map<Token, Float> getOwnership(); } Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/tools/NodeCmd.java Tue Nov 23 16:53:59 2010 @@ -25,6 +25,7 @@ import java.io.IOException; import java.io.PrintStream; import java.lang.management.MemoryUsage; import java.net.InetAddress; +import java.text.DecimalFormat; import java.util.*; import java.util.Map.Entry; import java.util.concurrent.ExecutionException; @@ -34,6 +35,7 @@ import org.apache.commons.cli.*; import org.apache.cassandra.cache.JMXInstrumentedCacheMBean; import org.apache.cassandra.concurrent.IExecutorMBean; +import org.apache.cassandra.config.DatabaseDescriptor; import org.apache.cassandra.db.ColumnFamilyStoreMBean; import org.apache.cassandra.dht.Token; import org.apache.cassandra.net.MessagingServiceMBean; @@ -95,23 +97,33 @@ public class NodeCmd { Collection<String> leavingNodes = probe.getLeavingNodes(); Map<String, String> loadMap = probe.getLoadMap(); - outs.printf("%-16s%-7s%-8s%-16s%-44s\n", "Address", "Status", "State", "Load", "Token"); + outs.printf("%-16s%-7s%-8s%-16s%-8s%-44s\n", "Address", "Status", "State", "Load", "Owns", "Token"); // show pre-wrap token twice so you can always read a node's range as // (previous line token, current line token] if (sortedTokens.size() > 1) - outs.printf("%-14s%-11s%-14s%-43s\n", "", "", "", sortedTokens.get(sortedTokens.size() - 1)); + outs.printf("%-16s%-7s%-8s%-16s%-8s%-44s\n", "", "", "", "", "", sortedTokens.get(sortedTokens.size() - 1)); + + // Calculate per-token ownership of the ring + Map<Token, Float> ownerships = probe.getOwnership(); for (Token token : sortedTokens) { String primaryEndpoint = tokenToEndpoint.get(token); String status = liveNodes.contains(primaryEndpoint) ? "Up" - : deadNodes.contains(primaryEndpoint) ? "Down" : "?"; + : deadNodes.contains(primaryEndpoint) + ? "Down" + : "?"; String state = joiningNodes.contains(primaryEndpoint) ? "Joining" - : leavingNodes.contains(primaryEndpoint) ? "Leaving" : "Normal"; - String load = loadMap.containsKey(primaryEndpoint) ? loadMap.get(primaryEndpoint) : "?"; - outs.printf("%-16s%-7s%-8s%-16s%-44s\n", primaryEndpoint, status, state, load, token); + : leavingNodes.contains(primaryEndpoint) + ? "Leaving" + : "Normal"; + String load = loadMap.containsKey(primaryEndpoint) + ? loadMap.get(primaryEndpoint) + : "?"; + String owns = new DecimalFormat("##0.00%").format(ownerships.get(token)); + outs.printf("%-16s%-7s%-8s%-16s%-8s%-44s\n", primaryEndpoint, status, state, load, owns, token); } } Modified: cassandra/trunk/src/java/org/apache/cassandra/tools/NodeProbe.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/tools/NodeProbe.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/tools/NodeProbe.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/tools/NodeProbe.java Tue Nov 23 16:53:59 2010 @@ -24,6 +24,7 @@ import java.lang.management.ManagementFa import java.lang.management.MemoryMXBean; import java.lang.management.MemoryUsage; import java.lang.management.RuntimeMXBean; +import java.lang.reflect.Constructor; import java.net.InetAddress; import java.nio.ByteBuffer; import java.util.*; @@ -44,12 +45,17 @@ import org.apache.cassandra.cache.JMXIns import org.apache.cassandra.concurrent.IExecutorMBean; import org.apache.cassandra.config.ConfigurationException; import org.apache.cassandra.db.ColumnFamilyStoreMBean; +import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.IEndpointSnitch; +import org.apache.cassandra.locator.TokenMetadata; import org.apache.cassandra.net.MessagingServiceMBean; +import org.apache.cassandra.service.StorageService; import org.apache.cassandra.service.StorageServiceMBean; import org.apache.cassandra.streaming.StreamingService; import org.apache.cassandra.streaming.StreamingServiceMBean; import org.apache.cassandra.thrift.UnavailableException; +import org.apache.cassandra.utils.FBUtilities; import static com.google.common.base.Charsets.UTF_8; @@ -197,6 +203,11 @@ public class NodeProbe return ssProxy.getLoadMap(); } + public Map<Token, Float> getOwnership() + { + return ssProxy.getOwnership(); + } + public Iterator<Map.Entry<String, ColumnFamilyStoreMBean>> getColumnFamilyStoreMBeanProxies() { try Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/CLibrary.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/CLibrary.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/utils/CLibrary.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/utils/CLibrary.java Tue Nov 23 16:53:59 2010 @@ -50,6 +50,10 @@ public final class CLibrary { logger.info("Unable to link C library. Native methods will be disabled."); } + catch (NoSuchMethodError e) + { + logger.warn("Obsolete version of JNA present; unable to register C library. Upgrade to JNA 3.2.7 or later"); + } } private static native int mlockall(int flags) throws LastErrorException; Modified: cassandra/trunk/test/system/test_thrift_server.py URL: http://svn.apache.org/viewvc/cassandra/trunk/test/system/test_thrift_server.py?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/test/system/test_thrift_server.py (original) +++ cassandra/trunk/test/system/test_thrift_server.py Tue Nov 23 16:53:59 2010 @@ -1474,6 +1474,31 @@ class TestMutations(ThriftTester): except NotFoundException: assert True, 'column did not exist' + def test_get_range_slice_after_deletion(self): + _set_keyspace('Keyspace2') + key = 'key1' + # three supercoluns, each with "col1" subcolumn + for i in range(1,4): + client.insert(key, ColumnParent('Super3', 'sc%d' % i), Column('col1', 'val1', 0), ConsistencyLevel.ONE) + + cp = ColumnParent('Super3') + predicate = SlicePredicate(slice_range=SliceRange('sc1', 'sc3', False, count=1)) + k_range = KeyRange(start_key=key, end_key=key, count=1) + + # validate count=1 restricts to 1 supercolumn + result = client.get_range_slices(cp, predicate, k_range, ConsistencyLevel.ONE) + assert len(result[0].columns) == 1 + + # remove sc1; add back subcolumn to override tombstone + client.remove(key, ColumnPath('Super3', 'sc1'), 1, ConsistencyLevel.ONE) + result = client.get_range_slices(cp, predicate, k_range, ConsistencyLevel.ONE) + assert len(result[0].columns) == 1 + client.insert(key, ColumnParent('Super3', 'sc1'), Column('col1', 'val1', 2), ConsistencyLevel.ONE) + result = client.get_range_slices(cp, predicate, k_range, ConsistencyLevel.ONE) + assert len(result[0].columns) == 1, result[0].columns + assert result[0].columns[0].super_column.name == 'sc1' + + class TestTruncate(ThriftTester): def test_truncate(self): _set_keyspace('Keyspace1') Modified: cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/cli/CliTest.java Tue Nov 23 16:53:59 2010 @@ -35,66 +35,76 @@ public class CliTest extends CleanupHelp { // please add new statements here so they could be auto-runned by this test. private String[] statements = { - "use TestKeySpace", - "create column family CF1 with comparator=UTF8Type and column_metadata=[{ column_name:world, validation_class:IntegerType, index_type:0, index_name:IdxName }, { column_name:world2, validation_class:LongType, index_type:0, index_name:LongIdxName}]", - "set CF1[hello][world] = 123848374878933948398384", - "get CF1[hello][world]", - "set CF1[hello][world2] = 15", - "get CF1 where world2 = long(15)", - "set CF1['hello'][time_spent_uuid] = timeuuid(a8098c1a-f86e-11da-bd1a-00112444be1e)", - "create column family CF2 with comparator=IntegerType", - "set CF2['key'][98349387493847748398334] = 'some text'", - "get CF2['key'][98349387493847748398334]", - "set CF2['key'][98349387493] = 'some text other'", - "get CF2['key'][98349387493]", - "create column family CF3 with comparator=UTF8Type and column_metadata=[{column_name:'big world', validation_class:LongType}]", - "set CF3['hello']['big world'] = 3748", - "get CF3['hello']['big world']", - "list CF3", - "list CF3[:]", - "list CF3[h:]", - "list CF3 limit 10", - "list CF3[h:] limit 10", - "create column family CF4 with comparator=IntegerType and column_metadata=[{column_name:9999, validation_class:LongType}]", - "set CF4['hello'][9999] = 1234", - "get CF4['hello'][9999]", - "get CF4['hello'][9999] as Long", - "get CF4['hello'][9999] as Bytes", - "set CF4['hello'][9999] = Long(1234)", - "get CF4['hello'][9999]", - "get CF4['hello'][9999] as Long", - "del CF4['hello'][9999]", - "get CF4['hello'][9999]", - "create column family SCF1 with column_type=Super and comparator=IntegerType and subcomparator=LongType and column_metadata=[{column_name:9999, validation_class:LongType}]", - "set SCF1['hello'][1][9999] = 1234", - "get SCF1['hello'][1][9999]", - "get SCF1['hello'][1][9999] as Long", - "get SCF1['hello'][1][9999] as Bytes", - "set SCF1['hello'][1][9999] = Long(1234)", - "get SCF1['hello'][1][9999]", - "get SCF1['hello'][1][9999] as Long", - "del SCF1['hello'][1][9999]", - "get SCF1['hello'][1][9999]", - "set SCF1['hello'][1][9999] = Long(1234)", - "del SCF1['hello'][9999]", - "get SCF1['hello'][1][9999]", - "truncate CF1", - "update keyspace TestKeySpace with placement_strategy='org.apache.cassandra.locator.LocalStrategy'", - "update keyspace TestKeySpace with replication_factor=1 and strategy_options=[{DC1:3, DC2:4, DC5:1}]", - "assume CF1 comparator as utf8", - "assume CF1 sub_comparator as integer", - "assume CF1 validator as lexicaluuid", - "assume CF1 keys as timeuuid", - "create column family CF7", - "set CF7[1][timeuuid()] = utf8(test1)", - "set CF7[2][lexicaluuid()] = utf8('hello world!')", - "set CF7[3][lexicaluuid(550e8400-e29b-41d4-a716-446655440000)] = utf8(test2)", - "set CF7[key2][timeuuid()] = utf8(test3)", - "assume CF7 comparator as lexicaluuid", - "assume CF7 keys as utf8", - "list CF7", - "get CF7[3]", - "get CF7[3][lexicaluuid(550e8400-e29b-41d4-a716-446655440000)]" + "use TestKeySpace;", + "create column family CF1 with comparator=UTF8Type and column_metadata=[{ column_name:world, validation_class:IntegerType, index_type:0, index_name:IdxName }, { column_name:world2, validation_class:LongType, index_type:0, index_name:LongIdxName}];", + "set CF1[hello][world] = 123848374878933948398384;", + "get CF1[hello][world];", + "set CF1[hello][world2] = 15;", + "get CF1 where world2 = long(15);", + "set CF1['hello'][time_spent_uuid] = timeuuid(a8098c1a-f86e-11da-bd1a-00112444be1e);", + "create column family CF2 with comparator=IntegerType;", + "set CF2['key'][98349387493847748398334] = 'some text';", + "get CF2['key'][98349387493847748398334];", + "set CF2['key'][98349387493] = 'some text other';", + "get CF2['key'][98349387493];", + "create column family CF3 with comparator=UTF8Type and column_metadata=[{column_name:'big world', validation_class:LongType}];", + "set CF3['hello']['big world'] = 3748;", + "get CF3['hello']['big world'];", + "list CF3;", + "list CF3[:];", + "list CF3[h:];", + "list CF3 limit 10;", + "list CF3[h:] limit 10;", + "create column family CF4 with comparator=IntegerType and column_metadata=[{column_name:9999, validation_class:LongType}];", + "set CF4['hello'][9999] = 1234;", + "get CF4['hello'][9999];", + "get CF4['hello'][9999] as Long;", + "get CF4['hello'][9999] as Bytes;", + "set CF4['hello'][9999] = Long(1234);", + "get CF4['hello'][9999];", + "get CF4['hello'][9999] as Long;", + "del CF4['hello'][9999];", + "get CF4['hello'][9999];", + "create column family SCF1 with column_type=Super and comparator=IntegerType and subcomparator=LongType and column_metadata=[{column_name:9999, validation_class:LongType}];", + "set SCF1['hello'][1][9999] = 1234;", + "get SCF1['hello'][1][9999];", + "get SCF1['hello'][1][9999] as Long;", + "get SCF1['hello'][1][9999] as Bytes;", + "set SCF1['hello'][1][9999] = Long(1234);", + "get SCF1['hello'][1][9999];", + "get SCF1['hello'][1][9999] as Long;", + "del SCF1['hello'][1][9999];", + "get SCF1['hello'][1][9999];", + "set SCF1['hello'][1][9999] = Long(1234);", + "del SCF1['hello'][9999];", + "get SCF1['hello'][1][9999];", + "truncate CF1;", + "update keyspace TestKeySpace with placement_strategy='org.apache.cassandra.locator.LocalStrategy';", + "update keyspace TestKeySpace with replication_factor=1 and strategy_options=[{DC1:3, DC2:4, DC5:1}];", + "assume CF1 comparator as utf8;", + "assume CF1 sub_comparator as integer;", + "assume CF1 validator as lexicaluuid;", + "assume CF1 keys as timeuuid;", + "create column family CF7;", + "set CF7[1][timeuuid()] = utf8(test1);", + "set CF7[2][lexicaluuid()] = utf8('hello world!');", + "set CF7[3][lexicaluuid(550e8400-e29b-41d4-a716-446655440000)] = utf8(test2);", + "set CF7[key2][timeuuid()] = utf8(test3);", + "assume CF7 comparator as lexicaluuid;", + "assume CF7 keys as utf8;", + "list CF7;", + "get CF7[3];", + "get CF7[3][lexicaluuid(550e8400-e29b-41d4-a716-446655440000)];", + "get sCf1['hello'][1][9999];", + "set sCf1['hello'][1][9999] = 938;", + "list sCf1;", + "del SCF1['hello'][1][9999];", + "assume sCf1 comparator as utf8;", + "create column family CF8;", + "drop column family cF8;", + "create keyspace TESTIN;", + "drop keyspace tesTIN;", + "use TestKEYSpace;", }; @Test @@ -115,8 +125,8 @@ public class CliTest extends CleanupHelp // re-creating keyspace for tests // dropping in case it exists e.g. could be left from previous run - CliMain.processStatement("drop keyspace TestKeySpace"); - CliMain.processStatement("create keyspace TestKeySpace"); + CliMain.processStatement("drop keyspace TestKeySpace;"); + CliMain.processStatement("create keyspace TestKeySpace;"); for (String statement : statements) { Modified: cassandra/trunk/test/unit/org/apache/cassandra/service/AntiEntropyServiceTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/service/AntiEntropyServiceTest.java?rev=1038200&r1=1038199&r2=1038200&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/service/AntiEntropyServiceTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/service/AntiEntropyServiceTest.java Tue Nov 23 16:53:59 2010 @@ -19,7 +19,6 @@ package org.apache.cassandra.service; import java.net.InetAddress; -import java.nio.ByteBuffer; import java.util.*; import java.util.concurrent.Callable; import java.util.concurrent.TimeUnit; @@ -47,7 +46,6 @@ import org.apache.cassandra.locator.Toke import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.MerkleTree; -import static com.google.common.base.Charsets.UTF_8; import static org.apache.cassandra.service.AntiEntropyService.*; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; @@ -244,7 +242,7 @@ public class AntiEntropyServiceTest exte void flushAES() throws Exception { - final ThreadPoolExecutor stage = StageManager.getStage(Stage.ANTIENTROPY); + final ThreadPoolExecutor stage = StageManager.getStage(Stage.ANTI_ENTROPY); final Callable noop = new Callable<Object>() { public Boolean call()
