Author: xedin
Date: Mon Aug 15 21:01:39 2011
New Revision: 1158008
URL: http://svn.apache.org/viewvc?rev=1158008&view=rev
Log:
Add 'load new SSTables' functionality to JMX and corresponding "refresh"
command to the nodetool
patch by Pavel Yaskevich; reviewed by Brandon Williams for CASSANDRA-2991
Modified:
cassandra/branches/cassandra-0.8/CHANGES.txt
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/StorageService.java
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/StorageServiceMBean.java
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/tools/NodeCmd.java
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/tools/NodeProbe.java
Modified: cassandra/branches/cassandra-0.8/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/CHANGES.txt?rev=1158008&r1=1158007&r2=1158008&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.8/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.8/CHANGES.txt Mon Aug 15 21:01:39 2011
@@ -7,7 +7,8 @@
in a commitlog segment (CASSANDRA-3021)
* fix cassandra.bat when CASSANDRA_HOME contains spaces (CASSANDRA-2952)
* fix to SSTableSimpleUnsortedWriter bufferSize calculation (CASSANDRA-3027)
-
+ * add a 'load new SSTables' functionality to JMX and corresponding "refresh"
+ command to the nodetool (CASSANDRA-2991)
0.8.4
* include files-to-be-streamed in StreamInSession.getSources (CASSANDRA-2972)
Modified:
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1158008&r1=1158007&r2=1158008&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Mon Aug 15 21:01:39 2011
@@ -274,22 +274,10 @@ public class ColumnFamilyStore implement
List<SSTableReader> sstables = new ArrayList<SSTableReader>();
for (Map.Entry<Descriptor,Set<Component>> sstableFiles :
files(table.name, columnFamilyName, false, false).entrySet())
{
- SSTableReader sstable;
- try
- {
- sstable = SSTableReader.open(sstableFiles.getKey(),
sstableFiles.getValue(), savedKeys, data, metadata, this.partitioner);
- }
- catch (FileNotFoundException ex)
- {
- logger.error("Missing sstable component in " + sstableFiles +
"; skipped because of " + ex.getMessage());
- continue;
- }
- catch (IOException ex)
- {
- logger.error("Corrupt sstable " + sstableFiles + "; skipped",
ex);
- continue;
- }
- sstables.add(sstable);
+ SSTableReader reader = openSSTableReader(sstableFiles, savedKeys,
data, metadata, partitioner);
+
+ if (reader != null) // if == null, logger errors where already
fired
+ sstables.add(reader);
}
data.addSSTables(sstables);
@@ -465,7 +453,99 @@ public class ColumnFamilyStore implement
return new ColumnFamilyStore(table, columnFamily, partitioner, value,
metadata);
}
-
+
+ /**
+ * See #{@code StorageService.loadNewSSTables(String, String)} for more
info
+ *
+ * @param ksName The keyspace name
+ * @param cfName The columnFamily name
+ */
+ public static synchronized void loadNewSSTables(String ksName, String
cfName)
+ {
+ /** ks/cf existence checks will be done by open and getCFS methods for
us */
+ Table table = Table.open(ksName);
+ table.getColumnFamilyStore(cfName).loadNewSSTables();
+ }
+
+ /**
+ * #{@inheritDoc}
+ */
+ public synchronized void loadNewSSTables()
+ {
+ logger.info("Loading new SSTables for " + table.name + "/" +
columnFamily + "...");
+
+ // current view over ColumnFamilyStore
+ DataTracker.View view = data.getView();
+ // descriptors of currently registered SSTables
+ Set<Descriptor> currentDescriptors = new HashSet<Descriptor>();
+ // going to hold new SSTable view of the CFS containing old and new
SSTables
+ Set<SSTableReader> sstables = new HashSet<SSTableReader>();
+ Set<DecoratedKey> savedKeys = keyCache.readSaved();
+ // get the max generation number, to prevent generation conflicts
+ int generation = 0;
+
+ for (SSTableReader reader : view.sstables)
+ {
+ sstables.add(reader); // first of all, add old SSTables
+ currentDescriptors.add(reader.descriptor);
+
+ if (reader.descriptor.generation > generation)
+ generation = reader.descriptor.generation;
+ }
+
+
+ SSTableReader reader;
+ // set to true if we have at least one new SSTable to load
+ boolean atLeastOneNew = false;
+
+ for (Map.Entry<Descriptor, Set<Component>> rawSSTable :
files(table.name, columnFamily, false, false).entrySet())
+ {
+ Descriptor descriptor = rawSSTable.getKey();
+
+ if (currentDescriptors.contains(descriptor))
+ continue; // old (initialized) SSTable found, skipping
+
+ if (!descriptor.cfname.equals(columnFamily))
+ continue;
+
+ if (descriptor.isFromTheFuture())
+ throw new RuntimeException(String.format("Can't open sstables
from the future! Current version %s, found file: %s",
+
Descriptor.CURRENT_VERSION,
+ descriptor));
+
+ logger.info("Initializing new SSTable {}", rawSSTable);
+ reader = openSSTableReader(rawSSTable, savedKeys, data, metadata,
partitioner);
+
+ if (reader == null)
+ continue; // something wrong with SSTable, skipping
+
+ sstables.add(reader);
+
+ if (descriptor.generation > generation)
+ generation = descriptor.generation;
+
+ if (!atLeastOneNew) // set flag only once
+ atLeastOneNew = true;
+ }
+
+ if (!atLeastOneNew)
+ {
+ logger.info("No new SSTables where found for " + table.name + "/"
+ columnFamily);
+ return;
+ }
+
+ logger.info("Loading new SSTable Set for " + table.name + "/" +
columnFamily + ": " + sstables);
+ data.addSSTables(sstables); // this will call updateCacheSizes() for us
+
+ logger.info("Requesting a full secondary index re-build for " +
table.name + "/" + columnFamily);
+ buildSecondaryIndexes(sstables, getIndexedColumns());
+
+ logger.info("Setting up new generation: " + generation);
+ fileIndexGenerator.set(generation);
+
+ logger.info("Done loading load new SSTables for " + table.name + "/" +
columnFamily);
+ }
+
/**
* Removes unnecessary files from the cf directory at startup: these
include temp files, orphans, zero-length files
* and compacted sstables. Files that cannot be recognized will be ignored.
@@ -2224,4 +2304,28 @@ public class ColumnFamilyStore implement
{
return data.getMemtablesPendingFlush();
}
+
+ private static SSTableReader openSSTableReader(Map.Entry<Descriptor,
Set<Component>> rawSSTable,
+ Set<DecoratedKey> savedKeys,
+ DataTracker tracker,
+ CFMetaData metadata,
+ IPartitioner partitioner)
+ {
+ SSTableReader reader = null;
+
+ try
+ {
+ reader = SSTableReader.open(rawSSTable.getKey(),
rawSSTable.getValue(), savedKeys, tracker, metadata, partitioner);
+ }
+ catch (FileNotFoundException ex)
+ {
+ logger.error("Missing sstable component in " + rawSSTable + ";
skipped because of " + ex.getMessage());
+ }
+ catch (IOException ex)
+ {
+ logger.error("Corrupt sstable " + rawSSTable + "; skipped", ex);
+ }
+
+ return reader;
+ }
}
Modified:
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java?rev=1158008&r1=1158007&r2=1158008&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
(original)
+++
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/db/ColumnFamilyStoreMBean.java
Mon Aug 15 21:01:39 2011
@@ -234,4 +234,10 @@ public interface ColumnFamilyStoreMBean
public int getKeyCacheSavePeriodInSeconds();
public void setKeyCacheSavePeriodInSeconds(int kcspis);
+
+ /**
+ * Scan through Keyspace/ColumnFamily's data directory
+ * determine which SSTables should be loaded and load them
+ */
+ public void loadNewSSTables();
}
Modified:
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/StorageService.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/StorageService.java?rev=1158008&r1=1158007&r2=1158008&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/StorageService.java
(original)
+++
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/StorageService.java
Mon Aug 15 21:01:39 2011
@@ -2615,4 +2615,12 @@ public class StorageService implements I
{
return AbstractCassandraDaemon.exceptions.get();
}
+
+ /**
+ * #{@inheritDoc}
+ */
+ public void loadNewSSTables(String ksName, String cfName)
+ {
+ ColumnFamilyStore.loadNewSSTables(ksName, cfName);
+ }
}
Modified:
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/StorageServiceMBean.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/StorageServiceMBean.java?rev=1158008&r1=1158007&r2=1158008&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/StorageServiceMBean.java
(original)
+++
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/service/StorageServiceMBean.java
Mon Aug 15 21:01:39 2011
@@ -321,4 +321,12 @@ public interface StorageServiceMBean
public void setCompactionThroughputMbPerSec(int value);
public void bulkLoad(String directory);
+
+ /**
+ * Load new SSTables to the given keyspace/columnFamily
+ *
+ * @param ksName The parent keyspace name
+ * @param cfName The ColumnFamily name where SSTables belong
+ */
+ public void loadNewSSTables(String ksName, String cfName);
}
Modified:
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/tools/NodeCmd.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/tools/NodeCmd.java?rev=1158008&r1=1158007&r2=1158008&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/tools/NodeCmd.java
(original)
+++
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/tools/NodeCmd.java
Mon Aug 15 21:01:39 2011
@@ -79,7 +79,8 @@ public class NodeCmd
DECOMMISSION, MOVE, REMOVETOKEN, REPAIR, CLEANUP, COMPACT, SCRUB,
SETCACHECAPACITY, GETCOMPACTIONTHRESHOLD, SETCOMPACTIONTHRESHOLD,
NETSTATS, CFHISTOGRAMS,
COMPACTIONSTATS, DISABLEGOSSIP, ENABLEGOSSIP, INVALIDATEKEYCACHE,
INVALIDATEROWCACHE,
- DISABLETHRIFT, ENABLETHRIFT, STATUSTHRIFT, JOIN,
SETCOMPACTIONTHROUGHPUT, GETENDPOINTS
+ DISABLETHRIFT, ENABLETHRIFT, STATUSTHRIFT, JOIN,
SETCOMPACTIONTHROUGHPUT, GETENDPOINTS,
+ REFRESH
}
@@ -125,6 +126,7 @@ public class NodeCmd
addCmdHelp(header, "invalidaterowcache [keyspace] [cfnames]",
"Invalidate the key cache of one or more column family");
addCmdHelp(header, "getcompactionthreshold <keyspace> <cfname>",
"Print min and max compaction thresholds for a given column family");
addCmdHelp(header, "cfhistograms <keyspace> <cfname>", "Print
statistic histograms for a given column family");
+ addCmdHelp(header, "refresh <keyspace> <cf-name>", "Load newly placed
SSTables to the system without restart.");
// Three args
addCmdHelp(header, "getendpoints <keyspace> <cf> <key>", "Print the
end points that owns the key");
@@ -699,6 +701,11 @@ public class NodeCmd
nodeCmd.printEndPoints(arguments[0], arguments[1],
arguments[2], System.out);
break;
+ case REFRESH:
+ if (arguments.length != 2) { badUse("load_new_sstables
requires ks and cf args"); }
+ probe.loadNewSSTables(arguments[0], arguments[1]);
+ break;
+
default :
throw new RuntimeException("Unreachable code.");
Modified:
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/tools/NodeProbe.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/tools/NodeProbe.java?rev=1158008&r1=1158007&r2=1158008&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/tools/NodeProbe.java
(original)
+++
cassandra/branches/cassandra-0.8/src/java/org/apache/cassandra/tools/NodeProbe.java
Mon Aug 15 21:01:39 2011
@@ -591,6 +591,11 @@ public class NodeProbe
{
return msProxy.getDroppedMessages();
}
+
+ public void loadNewSSTables(String ksName, String cfName)
+ {
+ ssProxy.loadNewSSTables(ksName, cfName);
+ }
}
class ColumnFamilyStoreMBeanIterator implements Iterator<Map.Entry<String,
ColumnFamilyStoreMBean>>
