Author: jbellis
Date: Mon Apr 25 17:29:08 2011
New Revision: 1096531
URL: http://svn.apache.org/viewvc?rev=1096531&view=rev
Log:
force GC to reclaim disk space on flush, if necessary
patch by jbellis; reviewed by slebresne for CASSANDRA-2404
Modified:
cassandra/branches/cassandra-0.7/CHANGES.txt
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/BinaryMemtable.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/streaming/StreamIn.java
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/DefsTest.java
Modified: cassandra/branches/cassandra-0.7/CHANGES.txt
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/CHANGES.txt?rev=1096531&r1=1096530&r2=1096531&view=diff
==============================================================================
--- cassandra/branches/cassandra-0.7/CHANGES.txt (original)
+++ cassandra/branches/cassandra-0.7/CHANGES.txt Mon Apr 25 17:29:08 2011
@@ -1,3 +1,7 @@
+0.7.6
+ * force GC to reclaim disk space on flush, if necessary (CASSANDRA-2404)
+
+
0.7.5
* improvements/fixes to PIG driver (CASSANDRA-1618, CASSANDRA-2387,
CASSANDRA-2465, CASSANDRA-2484)
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/BinaryMemtable.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/BinaryMemtable.java?rev=1096531&r1=1096530&r2=1096531&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/BinaryMemtable.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/BinaryMemtable.java
Mon Apr 25 17:29:08 2011
@@ -125,7 +125,7 @@ public class BinaryMemtable implements I
private SSTableReader writeSortedContents(List<DecoratedKey> sortedKeys)
throws IOException
{
logger.info("Writing " + this);
- SSTableWriter writer = cfs.createFlushWriter(sortedKeys.size());
+ SSTableWriter writer = cfs.createFlushWriter(sortedKeys.size(),
DatabaseDescriptor.getBMTThreshold());
for (DecoratedKey key : sortedKeys)
{
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=1096531&r1=1096530&r2=1096531&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Mon Apr 25 17:29:08 2011
@@ -657,22 +657,16 @@ public class ColumnFamilyStore implement
return columnFamily;
}
- public String getFlushPath()
- {
- return getFlushPath(Descriptor.CURRENT_VERSION);
- }
-
/*
* @return a temporary file name for an sstable.
* When the sstable object is closed, it will be renamed to a non-temporary
* format, so incomplete sstables can be recognized and removed on startup.
*/
- public String getFlushPath(String version)
+ public String getFlushPath(long estimatedSize, String version)
{
- long guessedSize = 2L * memsize.value() * 1024*1024; // 2* adds room
for keys, column indexes
- String location =
DatabaseDescriptor.getDataFileLocationForTable(table.name, guessedSize);
+ String location = table.getDataFileLocation(estimatedSize);
if (location == null)
- throw new RuntimeException("Insufficient disk space to flush");
+ throw new RuntimeException("Insufficient disk space to flush " +
estimatedSize + " bytes");
return getTempSSTablePath(location, version);
}
@@ -2221,9 +2215,9 @@ public class ColumnFamilyStore implement
}
}
- public SSTableWriter createFlushWriter(long estimatedRows) throws
IOException
+ public SSTableWriter createFlushWriter(long estimatedRows, long
estimatedSize) throws IOException
{
- return new SSTableWriter(getFlushPath(), estimatedRows, metadata,
partitioner);
+ return new SSTableWriter(getFlushPath(estimatedSize,
Descriptor.CURRENT_VERSION), estimatedRows, metadata, partitioner);
}
public SSTableWriter createCompactionWriter(long estimatedRows, String
location) throws IOException
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java?rev=1096531&r1=1096530&r2=1096531&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Memtable.java
Mon Apr 25 17:29:08 2011
@@ -29,7 +29,6 @@ import java.util.concurrent.ConcurrentNa
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicLong;
import com.google.common.collect.Iterators;
@@ -93,7 +92,7 @@ public class Memtable implements Compara
{
return currentThroughput.get();
}
-
+
public long getCurrentOperations()
{
return currentOperations.get();
@@ -156,7 +155,15 @@ public class Memtable implements Compara
private SSTableReader writeSortedContents() throws IOException
{
logger.info("Writing " + this);
- SSTableWriter writer = cfs.createFlushWriter(columnFamilies.size());
+
+ long keySize = 0;
+ for (DecoratedKey key : columnFamilies.keySet())
+ keySize += key.key.remaining();
+ long estimatedSize = (long) ((keySize // index entries
+ + keySize // keys in data file
+ + currentThroughput.get()) // data
+ * 1.2); // bloom filter and row index
overhead
+ SSTableWriter writer = cfs.createFlushWriter(columnFamilies.size(),
estimatedSize);
for (Map.Entry<DecoratedKey, ColumnFamily> entry :
columnFamilies.entrySet())
writer.append(entry.getKey(), entry.getValue());
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/streaming/StreamIn.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/streaming/StreamIn.java?rev=1096531&r1=1096530&r2=1096531&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/streaming/StreamIn.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/streaming/StreamIn.java
Mon Apr 25 17:29:08 2011
@@ -69,7 +69,7 @@ public class StreamIn
// new local sstable
Table table = Table.open(remotedesc.ksname);
ColumnFamilyStore cfStore =
table.getColumnFamilyStore(remotedesc.cfname);
- Descriptor localdesc =
Descriptor.fromFilename(cfStore.getFlushPath(remote.desc.version));
+ Descriptor localdesc =
Descriptor.fromFilename(cfStore.getFlushPath(remote.size, remote.desc.version));
return new PendingFile(localdesc, remote);
}
Modified:
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/DefsTest.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/DefsTest.java?rev=1096531&r1=1096530&r2=1096531&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/DefsTest.java
(original)
+++
cassandra/branches/cassandra-0.7/test/unit/org/apache/cassandra/db/DefsTest.java
Mon Apr 25 17:29:08 2011
@@ -300,7 +300,6 @@ public class DefsTest extends CleanupHel
ColumnFamilyStore store =
Table.open(cfm.tableName).getColumnFamilyStore(cfm.cfName);
assert store != null;
store.forceBlockingFlush();
- store.getFlushPath();
assert DefsTable.getFiles(cfm.tableName, cfm.cfName).size() > 0;
new DropColumnFamily(ks.name, cfm.cfName).apply();
