Author: jbellis
Date: Fri Dec 2 21:00:43 2011
New Revision: 1209689
URL: http://svn.apache.org/viewvc?rev=1209689&view=rev
Log:
Optimize componentsFor method for compaction and startup time
patch by jbellis and Eric Parusel; reviewed by slebresne for CASSANDRA-3532
Modified:
cassandra/branches/cassandra-1.0/CHANGES.txt
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/Memtable.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Component.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/streaming/IncomingStreamReader.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/utils/FBUtilities.java
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/utils/WrappedRunnable.java
cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java
Modified: cassandra/branches/cassandra-1.0/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/CHANGES.txt?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
--- cassandra/branches/cassandra-1.0/CHANGES.txt (original)
+++ cassandra/branches/cassandra-1.0/CHANGES.txt Fri Dec 2 21:00:43 2011
@@ -10,6 +10,8 @@
* fix race between cf flush and its 2ndary indexes flush (CASSANDRA-3547)
* fix potential race in AES when a repair fails (CASSANDRA-3548)
* fix default value validation usage in CLI SET command (CASSANDRA-3553)
+ * Optimize componentsFor method for compaction and startup time
+ (CASSANDRA-3532)
Merged from 0.8:
* use cannonical host for local node in nodetool info (CASSANDRA-3556)
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/Memtable.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/Memtable.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/Memtable.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/Memtable.java
Fri Dec 2 21:00:43 2011
@@ -40,6 +40,7 @@ import org.apache.cassandra.db.filter.Sl
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.SSTableWriter;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.SlabAllocator;
import org.apache.cassandra.utils.WrappedRunnable;
import org.github.jamm.MemoryMeter;
@@ -268,9 +269,10 @@ public class Memtable
ssTable = writer.closeAndOpenReader();
}
- finally
+ catch (Exception e)
{
- writer.cleanupIfNecessary();
+ writer.abort();
+ throw FBUtilities.unchecked(e);
}
logger.info(String.format("Completed flushing %s (%d bytes)",
ssTable.getFilename(), new
File(ssTable.getFilename()).length()));
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionManager.java
Fri Dec 2 21:00:43 2011
@@ -632,10 +632,14 @@ public class CompactionManager implement
if (writer.getFilePointer() > 0)
newSstable = writer.closeAndOpenReader(sstable.maxDataAge);
}
- finally
+ catch (Exception e)
{
if (writer != null)
- writer.cleanupIfNecessary();
+ writer.abort();
+ throw FBUtilities.unchecked(e);
+ }
+ finally
+ {
FileUtils.closeQuietly(dataFile);
FileUtils.closeQuietly(indexFile);
@@ -759,12 +763,16 @@ public class CompactionManager implement
if (writer != null)
newSstable = writer.closeAndOpenReader(sstable.maxDataAge);
}
+ catch (Exception e)
+ {
+ if (writer != null)
+ writer.abort();
+ throw FBUtilities.unchecked(e);
+ }
finally
{
scanner.close();
executor.finishCompaction(ci);
- if (writer != null)
- writer.cleanupIfNecessary();
executor.finishCompaction(ci);
}
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/db/compaction/CompactionTask.java
Fri Dec 2 21:00:43 2011
@@ -36,6 +36,7 @@ import org.apache.cassandra.io.sstable.S
import org.apache.cassandra.io.sstable.SSTableReader;
import org.apache.cassandra.io.sstable.SSTableWriter;
import org.apache.cassandra.utils.CloseableIterator;
+import org.apache.cassandra.utils.FBUtilities;
public class CompactionTask extends AbstractCompactionTask
{
@@ -183,13 +184,17 @@ public class CompactionTask extends Abst
}
}
}
+ catch (Exception e)
+ {
+ for (SSTableWriter writer : writers)
+ writer.abort();
+ throw FBUtilities.unchecked(e);
+ }
finally
{
iter.close();
if (collector != null)
collector.finishCompaction(ci);
- for (SSTableWriter writer : writers)
- writer.cleanupIfNecessary();
}
cfs.replaceCompactedSSTables(toCompact, sstables);
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Component.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Component.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Component.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Component.java
Fri Dec 2 21:00:43 2011
@@ -53,8 +53,6 @@ public class Component
COMPRESSION_INFO("CompressionInfo.db"),
// statistical metadata about the content of the sstable
STATS("Statistics.db"),
- // a bitmap secondary index: many of these may exist per sstable
- BITMAP_INDEX("Bitidx.db"),
// holds sha1 sum of the data file (to be checked by sha1sum)
DIGEST("Digest.sha1");
@@ -103,25 +101,11 @@ public class Component
*/
public String name()
{
- switch(type)
- {
- case DATA:
- case PRIMARY_INDEX:
- case FILTER:
- case COMPACTED_MARKER:
- case COMPRESSION_INFO:
- case STATS:
- case DIGEST:
- return type.repr;
- case BITMAP_INDEX:
- return String.format("%d%c%s", id, separator, type.repr);
- }
- throw new IllegalStateException();
+ return type.repr;
}
/**
* Filename of the form
"<ksname>/<cfname>-[tmp-][<version>-]<gen>-<component>",
- * where <component> is of the form "[<id>-]<component>".
* @return A Descriptor for the SSTable, and a Component for this
particular file.
* TODO move descriptor into Component field
*/
@@ -130,15 +114,7 @@ public class Component
Pair<Descriptor,String> path = Descriptor.fromFilename(directory,
name);
// parse the component suffix
- String repr = path.right;
- int id = -1;
- int separatorPos = repr.indexOf(separator);
- if (separatorPos != -1)
- {
- id = Integer.parseInt(repr.substring(0, separatorPos));
- repr = repr.substring(separatorPos+1, repr.length());
- }
- Type type = Type.fromRepresentation(repr);
+ Type type = Type.fromRepresentation(path.right);
// build (or retrieve singleton for) the component object
Component component;
switch(type)
@@ -150,9 +126,6 @@ public class Component
case COMPRESSION_INFO: component = Component.COMPRESSION_INFO;
break;
case STATS: component = Component.STATS;
break;
case DIGEST: component = Component.DIGEST;
break;
- case BITMAP_INDEX:
- component = new Component(type, id);
- break;
default:
throw new IllegalStateException();
}
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/Descriptor.java
Fri Dec 2 21:00:43 2011
@@ -79,21 +79,6 @@ public class Descriptor
public final boolean hasCompressionRatio;
public final boolean hasPartitioner;
- public enum TempState
- {
- LIVE,
- TEMP,
- ANY;
-
- boolean isMatch(Descriptor descriptor)
- {
- assert descriptor != null;
- if (TempState.ANY == this)
- return true;
- return (TempState.TEMP == this) ? descriptor.temporary :
!descriptor.temporary;
- }
- }
-
/**
* A descriptor that assumes CURRENT_VERSION.
*/
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTable.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTable.java
Fri Dec 2 21:00:43 2011
@@ -25,6 +25,8 @@ import java.io.IOException;
import java.util.*;
import com.google.common.collect.Ordering;
+import com.google.common.collect.Sets;
+
import org.apache.cassandra.db.DecoratedKey;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -192,26 +194,15 @@ public abstract class SSTable
/**
* Discovers existing components for the descriptor. Slow: only intended
for use outside the critical path.
*/
- static Set<Component> componentsFor(final Descriptor desc, final
Descriptor.TempState matchState)
+ static Set<Component> componentsFor(final Descriptor desc)
{
- final Set<Component> components = new HashSet<Component>();
- final String sstableFilePrefix = desc.cfname + Component.separator;
-
- desc.directory.listFiles(new FileFilter()
+ Set<Component> components =
Sets.newHashSetWithExpectedSize(Component.TYPES.size());
+ for (Component.Type componentType : Component.TYPES)
{
- public boolean accept(File file)
- {
- if (file.isDirectory() ||
!file.getName().startsWith(sstableFilePrefix))
- return false;
-
- Pair<Descriptor, Component> component =
tryComponentFromFilename(file.getParentFile(), file.getName());
-
- if (component != null && component.left.equals(desc) &&
(matchState.isMatch(component.left)))
- components.add(component.right);
-
- return false;
- }
- });
+ Component component = new Component(componentType);
+ if (new File(desc.filenameFor(component)).exists())
+ components.add(component);
+ }
return components;
}
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableReader.java
Fri Dec 2 21:00:43 2011
@@ -115,7 +115,7 @@ public class SSTableReader extends SSTab
public static SSTableReader open(Descriptor desc, CFMetaData metadata)
throws IOException
{
- return open(desc, componentsFor(desc, Descriptor.TempState.LIVE),
metadata, StorageService.getPartitioner());
+ return open(desc, componentsFor(desc), metadata,
StorageService.getPartitioner());
}
public static SSTableReader open(Descriptor descriptor, Set<Component>
components, CFMetaData metadata, IPartitioner partitioner) throws IOException
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java
Fri Dec 2 21:00:43 2011
@@ -277,16 +277,17 @@ public class SSTableWriter extends SSTab
}
/**
- * Attempt to close the index writer and data file before deleting all
temp components for the sstable
+ * After failure, attempt to close the index writer and data file before
deleting all temp components for the sstable
*/
- public void cleanupIfNecessary()
+ public void abort()
{
+ assert descriptor.temporary;
FileUtils.closeQuietly(iwriter);
FileUtils.closeQuietly(dataFile);
try
{
- Set<Component> components = SSTable.componentsFor(descriptor,
Descriptor.TempState.TEMP);
+ Set<Component> components = SSTable.componentsFor(descriptor);
if (!components.isEmpty())
SSTable.delete(descriptor, components);
}
@@ -452,6 +453,7 @@ public class SSTableWriter extends SSTab
indexFile.resetAndTruncate(mark);
}
+ @Override
public String toString()
{
return "IndexWriter(" + desc + ")";
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/streaming/IncomingStreamReader.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/streaming/IncomingStreamReader.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/streaming/IncomingStreamReader.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/streaming/IncomingStreamReader.java
Fri Dec 2 21:00:43 2011
@@ -28,7 +28,6 @@ import org.apache.cassandra.db.ColumnFam
import org.apache.cassandra.db.ColumnFamilyStore;
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.Table;
-import org.apache.cassandra.db.compaction.AbstractCompactedRow;
import org.apache.cassandra.db.compaction.CompactionController;
import org.apache.cassandra.db.compaction.PrecompactedRow;
import org.apache.cassandra.io.IColumnSerializer;
@@ -37,6 +36,7 @@ import org.apache.cassandra.io.util.File
import org.apache.cassandra.service.StorageService;
import org.apache.cassandra.utils.ByteBufferUtil;
import org.apache.cassandra.utils.BytesReadTracker;
+import org.apache.cassandra.utils.FBUtilities;
import org.apache.cassandra.utils.Pair;
import com.ning.compress.lzf.LZFInputStream;
@@ -155,9 +155,10 @@ public class IncomingStreamReader
}
return writer.closeAndOpenReader();
}
- finally
+ catch (Exception e)
{
- writer.cleanupIfNecessary();
+ writer.abort();
+ throw FBUtilities.unchecked(e);
}
}
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/utils/FBUtilities.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/utils/FBUtilities.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/utils/FBUtilities.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/utils/FBUtilities.java
Fri Dec 2 21:00:43 2011
@@ -683,4 +683,9 @@ public class FBUtilities
buffer.getData().length, buffer.getLength(),
size, object);
return buffer.getData();
}
+
+ public static RuntimeException unchecked(Exception e)
+ {
+ return e instanceof RuntimeException ? (RuntimeException) e : new
RuntimeException(e);
+ }
}
Modified:
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/utils/WrappedRunnable.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/utils/WrappedRunnable.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/utils/WrappedRunnable.java
(original)
+++
cassandra/branches/cassandra-1.0/src/java/org/apache/cassandra/utils/WrappedRunnable.java
Fri Dec 2 21:00:43 2011
@@ -31,10 +31,7 @@ public abstract class WrappedRunnable im
}
catch (Exception e)
{
- if (e instanceof RuntimeException)
- throw (RuntimeException) e;
- else
- throw new RuntimeException(e);
+ throw FBUtilities.unchecked(e);
}
}
Modified:
cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java?rev=1209689&r1=1209688&r2=1209689&view=diff
==============================================================================
---
cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java
(original)
+++
cassandra/branches/cassandra-1.0/test/unit/org/apache/cassandra/io/sstable/SSTableTest.java
Fri Dec 2 21:00:43 2011
@@ -23,6 +23,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.*;
+import com.google.common.collect.Sets;
import org.junit.Test;
import org.apache.cassandra.CleanupHelper;
@@ -75,12 +76,10 @@ public class SSTableTest extends Cleanup
ssTable = SSTableReader.open(ssTable.descriptor); // read the index
from disk
verifyMany(ssTable, map);
- Set<Component> live = SSTable.componentsFor(ssTable.descriptor,
Descriptor.TempState.LIVE);
- assert !live.isEmpty() : "SSTable has live components";
- Set<Component> all = SSTable.componentsFor(ssTable.descriptor,
Descriptor.TempState.ANY);
- assert live.equals(all) : "live components same as all components";
- all.removeAll(live);
- assert all.isEmpty() : "SSTable has no temp components";
+ Set<Component> live = SSTable.componentsFor(ssTable.descriptor);
+ assert !live.isEmpty() : "SSTable has no live components";
+ Set<Component> temp =
SSTable.componentsFor(ssTable.descriptor.asTemporary(true));
+ assert temp.isEmpty() : "SSTable has unexpected temp components";
}
private void verifyMany(SSTableReader sstable, Map<ByteBuffer, ByteBuffer>
map) throws IOException
