Author: jbellis
Date: Tue Sep 29 03:42:42 2009
New Revision: 819826
URL: http://svn.apache.org/viewvc?rev=819826&view=rev
Log:
merge from 0.4 branch
Modified:
incubator/cassandra/trunk/ (props changed)
incubator/cassandra/trunk/CHANGES.txt
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java
(props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Column.java
(props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java
(props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java
(props changed)
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/SuperColumn.java
(props changed)
incubator/cassandra/trunk/src/java/org/ (props changed)
incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLogHeader.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
incubator/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
incubator/cassandra/trunk/test/unit/org/ (props changed)
Propchange: incubator/cassandra/trunk/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 29 03:42:42 2009
@@ -1,2 +1,2 @@
/incubator/cassandra/branches/cassandra-0.3:774578-796573
-/incubator/cassandra/branches/cassandra-0.4:810145-819661
+/incubator/cassandra/branches/cassandra-0.4:810145-819824
Modified: incubator/cassandra/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/CHANGES.txt?rev=819826&r1=819825&r2=819826&view=diff
==============================================================================
--- incubator/cassandra/trunk/CHANGES.txt (original)
+++ incubator/cassandra/trunk/CHANGES.txt Tue Sep 29 03:42:42 2009
@@ -1,3 +1,12 @@
+0.4.1
+ * Fix FlushPeriod columnfamily configuration regression
+ (CASSANDRA-455)
+ * Fix long column name support (CASSANDRA-460)
+ * Fix for serializing a row that only contains tombstones
+ (CASSANDRA-458)
+ * Fix for discarding unneeded commitlog segments (CASSANDRA-459)
+
+
0.4.0
* fix get_key_range problems when a node is down (CASSANDRA-440)
and add UnavailableException to more Thrift methods
Propchange:
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 29 03:42:42 2009
@@ -1,3 +1,3 @@
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/Cassandra.java:774578-796573
-/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-819661
+/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Cassandra.java:810145-819824
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Cassandra.java:749219-768588
Propchange:
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Column.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 29 03:42:42 2009
@@ -1,4 +1,4 @@
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/column_t.java:774578-792198
-/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-819661
+/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/Column.java:810145-819824
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/Column.java:749219-794428
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/column_t.java:749219-768588
Propchange:
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 29 03:42:42 2009
@@ -1,3 +1,3 @@
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:774578-796573
-/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-819661
+/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:810145-819824
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/InvalidRequestException.java:749219-768588
Propchange:
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 29 03:42:42 2009
@@ -1,3 +1,3 @@
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:774578-796573
-/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-819661
+/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:810145-819824
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/NotFoundException.java:749219-768588
Propchange:
incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/SuperColumn.java
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 29 03:42:42 2009
@@ -1,4 +1,4 @@
/incubator/cassandra/branches/cassandra-0.3/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:774578-792198
-/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-819661
+/incubator/cassandra/branches/cassandra-0.4/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:810145-819824
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/SuperColumn.java:749219-794428
/incubator/cassandra/trunk/interface/gen-java/org/apache/cassandra/service/superColumn_t.java:749219-768588
Propchange: incubator/cassandra/trunk/src/java/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 29 03:42:42 2009
@@ -1,3 +1,3 @@
/incubator/cassandra/branches/cassandra-0.3/src/java/org:774578-796573
-/incubator/cassandra/branches/cassandra-0.4/src/java/org:810145-819661
+/incubator/cassandra/branches/cassandra-0.4/src/java/org:810145-819824
/incubator/cassandra/trunk/src/java/org:749219-769885
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java?rev=819826&r1=819825&r2=819826&view=diff
==============================================================================
---
incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
(original)
+++
incubator/cassandra/trunk/src/java/org/apache/cassandra/config/DatabaseDescriptor.java
Tue Sep 29 03:42:42 2009
@@ -538,12 +538,14 @@
CFMetaData data = new CFMetaData();
data.columnType = "Standard";
data.comparator = new UTF8Type();
+ data.flushPeriodInMinutes = 1;
systemMetadata.put(SystemTable.LOCATION_CF, data);
data = new CFMetaData();
data.columnType = "Super";
data.comparator = new UTF8Type();
data.subcolumnComparator = new BytesType();
+ data.flushPeriodInMinutes = 10;
systemMetadata.put(HintedHandOffManager.HINTS_CF, data);
tableToCFMetaDataMap_.put(Table.SYSTEM_TABLE, systemMetadata);
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java?rev=819826&r1=819825&r2=819826&view=diff
==============================================================================
---
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
(original)
+++
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/ColumnFamilyStore.java
Tue Sep 29 03:42:42 2009
@@ -165,6 +165,8 @@
void onStart() throws IOException
{
+ if (logger_.isDebugEnabled())
+ logger_.debug("Starting CFS " + columnFamily_);
// scan for data files corresponding to this CF
List<File> sstableFiles = new ArrayList<File>();
String[] dataFileDirectories =
DatabaseDescriptor.getAllDataFileLocationsForTable(table_);
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java?rev=819826&r1=819825&r2=819826&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLog.java
Tue Sep 29 03:42:42 2009
@@ -35,8 +35,6 @@
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReentrantLock;
-import org.apache.commons.lang.StringUtils;
-
/*
* Commit Log tracks every write operation into the system. The aim
* of the commit log is to be able to successfully recover data that was
@@ -96,6 +94,15 @@
{
return (position != -1L);
}
+
+ @Override
+ public String toString()
+ {
+ return "CommitLogContext(" +
+ "file='" + file + '\'' +
+ ", position=" + position +
+ ')';
+ }
}
public static class CommitLogFileComparator implements Comparator<String>
@@ -359,7 +366,7 @@
for (ColumnFamily columnFamily : rm.getColumnFamilies())
{
int id = table.getColumnFamilyId(columnFamily.name());
- if (!clHeader_.isDirty(id) || (clHeader_.isDirty(id) &&
clHeader_.getPosition(id) == 0))
+ if (!clHeader_.isDirty(id))
{
clHeader_.turnOn(id, logWriter_.getFilePointer());
seekAndWriteCommitLogHeader(clHeader_.toByteArray());
@@ -455,79 +462,85 @@
*/
private void discardCompletedSegments(CommitLog.CommitLogContext cLogCtx,
int id) throws IOException
{
+ if (logger_.isDebugEnabled())
+ logger_.debug("discard completed log segments for " + cLogCtx + ",
column family " + id + ". CFIDs are " +
Table.TableMetadata.getColumnFamilyIDString());
/* retrieve the commit log header associated with the file in the
context */
- CommitLogHeader commitLogHeader = clHeaders_.get(cLogCtx.file);
- if(commitLogHeader == null )
+ if (clHeaders_.get(cLogCtx.file) == null)
{
- if( logFile_.equals(cLogCtx.file) )
+ if (logFile_.equals(cLogCtx.file))
{
/* this means we are dealing with the current commit log. */
- commitLogHeader = clHeader_;
clHeaders_.put(cLogCtx.file, clHeader_);
}
else
+ {
+ logger_.error("Unknown commitlog file " + cLogCtx.file);
return;
+ }
}
/*
* log replay assumes that we only have to look at entries past the
last
* flush position, so verify that this flush happens after the last.
- * (Currently Memtables are flushed on a single thread so this should
be fine.)
*/
- assert cLogCtx.position >= commitLogHeader.getPosition(id);
+ assert cLogCtx.position >=
clHeaders_.get(cLogCtx.file).getPosition(id);
- commitLogHeader.turnOff(id);
/* Sort the commit logs based on creation time */
List<String> oldFiles = new ArrayList<String>(clHeaders_.keySet());
Collections.sort(oldFiles, new CommitLogFileComparator());
- List<String> listOfDeletedFiles = new ArrayList<String>();
+
/*
* Loop through all the commit log files in the history. Now process
* all files that are older than the one in the context. For each of
- * these files the header needs to modified by performing a bitwise &
- * of the header with the header of the file in the context. If we
- * encounter the file in the context in our list of old commit log
files
- * then we update the header and write it back to the commit log.
+ * these files the header needs to modified by resetting the dirty
+ * bit corresponding to the flushed CF.
*/
for (String oldFile : oldFiles)
{
+ CommitLogHeader header = clHeaders_.get(oldFile);
if (oldFile.equals(cLogCtx.file))
{
- /*
- * We need to turn on again. This is because we always keep
- * the bit turned on and the position indicates from where the
- * commit log needs to be read. When a flush occurs we turn off
- * perform & operation and then turn on with the new position.
- */
- commitLogHeader.turnOn(id, cLogCtx.position);
- seekAndWriteCommitLogHeader(commitLogHeader.toByteArray());
- break;
- }
- else
- {
- CommitLogHeader oldCommitLogHeader = clHeaders_.get(oldFile);
- oldCommitLogHeader.and(commitLogHeader);
- if (oldCommitLogHeader.isSafeToDelete())
- {
- logger_.info("Deleting obsolete commit log:" + oldFile);
- FileUtils.deleteAsync(oldFile);
- listOfDeletedFiles.add(oldFile);
+ // we can't just mark the segment where the flush happened
clean,
+ // since there may have been writes to it between when the
flush
+ // started and when it finished. so mark the flush position as
+ // the replay point for this CF, instead.
+ if (logger_.isDebugEnabled())
+ logger_.debug("Marking replay position " +
cLogCtx.position + " on commit log " + oldFile);
+ header.turnOn(id, cLogCtx.position);
+ if (oldFile.equals(logFile_))
+ {
+ seekAndWriteCommitLogHeader(header.toByteArray());
}
else
{
- BufferedRandomAccessFile logWriter =
CommitLog.createWriter(oldFile);
- writeCommitLogHeader(logWriter,
oldCommitLogHeader.toByteArray());
- logWriter.close();
+ writeOldCommitLogHeader(oldFile, header);
}
+ break;
}
- }
- for ( String deletedFile : listOfDeletedFiles)
- {
- clHeaders_.remove(deletedFile);
+ header.turnOff(id);
+ if (header.isSafeToDelete())
+ {
+ logger_.info("Deleting obsolete commit log:" + oldFile);
+ FileUtils.deleteAsync(oldFile);
+ clHeaders_.remove(oldFile);
+ }
+ else
+ {
+ if (logger_.isDebugEnabled())
+ logger_.debug("Not safe to delete commit log " + oldFile +
"; dirty is " + header.dirtyString());
+ writeOldCommitLogHeader(oldFile, header);
+ }
}
}
+ private void writeOldCommitLogHeader(String oldFile, CommitLogHeader
header) throws IOException
+ {
+ BufferedRandomAccessFile logWriter = CommitLog.createWriter(oldFile);
+ writeCommitLogHeader(logWriter, header.toByteArray());
+ logWriter.close();
+ }
+
private boolean maybeRollLog() throws IOException
{
if (logWriter_.length() >= SEGMENT_SIZE)
@@ -541,10 +554,7 @@
logWriter_ = CommitLog.createWriter(logFile_);
/* squirrel away the old commit log header */
clHeaders_.put(oldLogFile, new CommitLogHeader(clHeader_));
- // we leave the old 'dirty' bits alone, so we can test for
- // whether it's safe to remove a given log segment by and-ing its
dirty
- // with the current one.
- clHeader_.zeroPositions();
+ clHeader_.clear();
writeCommitLogHeader(logWriter_, clHeader_.toByteArray());
return true;
}
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLogHeader.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLogHeader.java?rev=819826&r1=819825&r2=819826&view=diff
==============================================================================
---
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLogHeader.java
(original)
+++
incubator/cassandra/trunk/src/java/org/apache/cassandra/db/CommitLogHeader.java
Tue Sep 29 03:42:42 2009
@@ -20,6 +20,7 @@
import java.io.*;
import java.util.BitSet;
+import java.util.Arrays;
import org.apache.cassandra.io.ICompactSerializer;
import org.apache.cassandra.io.DataInputBuffer;
@@ -35,17 +36,6 @@
return serializer;
}
- public static BitSet and(byte[] bytes1, byte[] bytes2) throws IOException
- {
- DataInputBuffer bufIn = new DataInputBuffer();
- bufIn.reset(bytes1, 0, bytes1.length);
- CommitLogHeader header1 = serializer.deserialize(bufIn);
- bufIn.reset(bytes2, 0, bytes2.length);
- CommitLogHeader header2 = serializer.deserialize(bufIn);
- header1.and(header2);
- return header1.dirty;
- }
-
static int getLowestPosition(CommitLogHeader clHeader)
{
int minPosition = Integer.MAX_VALUE;
@@ -116,17 +106,12 @@
return dirty.isEmpty();
}
- void zeroPositions()
+ void clear()
{
- int size = lastFlushedAt.length;
- lastFlushedAt = new int[size];
+ dirty.clear();
+ Arrays.fill(lastFlushedAt, 0);
}
-
- void and(CommitLogHeader commitLogHeader)
- {
- dirty.and(commitLogHeader.dirty);
- }
-
+
byte[] toByteArray() throws IOException
{
ByteArrayOutputStream bos = new ByteArrayOutputStream();
@@ -154,6 +139,19 @@
return sb.toString();
}
+ public String dirtyString()
+ {
+ StringBuilder sb = new StringBuilder();
+ for (int i = 0; i < dirty.length(); i++)
+ {
+ if (dirty.get(i))
+ {
+ sb.append(i).append(", ");
+ }
+ }
+ return sb.toString();
+ }
+
static class CommitLogHeaderSerializer implements
ICompactSerializer<CommitLogHeader>
{
public void serialize(CommitLogHeader clHeader, DataOutputStream dos)
throws IOException
Modified: incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java?rev=819826&r1=819825&r2=819826&view=diff
==============================================================================
--- incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java
(original)
+++ incubator/cassandra/trunk/src/java/org/apache/cassandra/db/Table.java Tue
Sep 29 03:42:42 2009
@@ -140,24 +140,18 @@
public String toString()
{
- StringBuilder sb = new StringBuilder("");
- Set<String> cfNames = cfIdMap_.keySet();
-
- for ( String cfName : cfNames )
- {
- sb.append(cfName);
- sb.append("---->");
- sb.append(cfIdMap_.get(cfName));
- sb.append(System.getProperty("line.separator"));
- }
-
- return sb.toString();
+ return "TableMetadata(" + FBUtilities.mapToString(cfIdMap_) + ")";
}
public static int getColumnFamilyCount()
{
return idCfMap_.size();
}
+
+ public static String getColumnFamilyIDString()
+ {
+ return FBUtilities.mapToString(tableMetadataMap_);
+ }
}
/**
Modified:
incubator/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
URL:
http://svn.apache.org/viewvc/incubator/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java?rev=819826&r1=819825&r2=819826&view=diff
==============================================================================
---
incubator/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
(original)
+++
incubator/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java
Tue Sep 29 03:42:42 2009
@@ -25,11 +25,7 @@
import java.security.MessageDigest;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Date;
-import java.util.List;
-import java.util.StringTokenizer;
+import java.util.*;
import java.util.zip.DataFormatException;
import java.util.zip.Deflater;
import java.util.zip.Inflater;
@@ -412,4 +408,16 @@
}
return new String(chars);
}
+
+ public static String mapToString(Map<?,?> map)
+ {
+ StringBuilder sb = new StringBuilder("");
+
+ for (Map.Entry entry : map.entrySet())
+ {
+ sb.append(entry.getKey()).append(":
").append(entry.getValue()).append(", ");
+ }
+
+ return sb.append("}").toString();
+ }
}
Propchange: incubator/cassandra/trunk/test/unit/org/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Tue Sep 29 03:42:42 2009
@@ -1,3 +1,3 @@
/incubator/cassandra/branches/cassandra-0.3/test/unit/org:774578-796573
-/incubator/cassandra/branches/cassandra-0.4/test/unit/org:810145-819661
+/incubator/cassandra/branches/cassandra-0.4/test/unit/org:810145-819824
/incubator/cassandra/trunk/test/unit/org:749219-768583
