Author: slebresne
Date: Mon Apr 18 21:44:00 2011
New Revision: 1094780
URL: http://svn.apache.org/viewvc?rev=1094780&view=rev
Log:
Make scrub validate column fields
patch by slebresne; reviewed by jbellis for CASSANDRA-2460
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Column.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamily.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/DeletedColumn.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ExpiringColumn.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/IColumn.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Column.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Column.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Column.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/Column.java
Mon Apr 18 21:44:00 2011
@@ -26,7 +26,9 @@ import java.util.Collection;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MarshalException;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -232,5 +234,19 @@ public class Column implements IColumn
{
return !isMarkedForDelete();
}
+
+ protected void validateName(CFMetaData metadata) throws MarshalException
+ {
+ AbstractType nameValidator = metadata.cfType == ColumnFamilyType.Super
? metadata.subcolumnComparator : metadata.comparator;
+ nameValidator.validate(name());
+ }
+
+ public void validateFields(CFMetaData metadata) throws MarshalException
+ {
+ validateName(metadata);
+ AbstractType valueValidator = metadata.getValueValidator(name());
+ if (valueValidator != null)
+ valueValidator.validate(value());
+ }
}
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamily.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamily.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamily.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ColumnFamily.java
Mon Apr 18 21:44:00 2011
@@ -35,6 +35,7 @@ import org.apache.cassandra.config.CFMet
import org.apache.cassandra.config.DatabaseDescriptor;
import org.apache.cassandra.db.filter.QueryPath;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MarshalException;
import org.apache.cassandra.io.IColumnSerializer;
import org.apache.cassandra.io.util.IIterableColumns;
import org.apache.cassandra.utils.FBUtilities;
@@ -424,4 +425,18 @@ public class ColumnFamily implements ICo
remove(column.name());
addColumn(column.deepCopy());
}
+
+ /**
+ * Goes over all columns and check the fields are valid (as far as we can
+ * tell).
+ * This is used to detect corruption after deserialization.
+ */
+ public void validateColumnFields() throws MarshalException
+ {
+ CFMetaData metadata = metadata();
+ for (IColumn column : getSortedColumns())
+ {
+ column.validateFields(metadata);
+ }
+ }
}
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/DeletedColumn.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/DeletedColumn.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/DeletedColumn.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/DeletedColumn.java
Mon Apr 18 21:44:00 2011
@@ -23,6 +23,8 @@ import java.nio.ByteBuffer;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CFMetaData;
+import org.apache.cassandra.db.marshal.MarshalException;
import org.apache.cassandra.utils.ByteBufferUtil;
public class DeletedColumn extends Column
@@ -62,4 +64,14 @@ public class DeletedColumn extends Colum
{
return new DeletedColumn(ByteBufferUtil.clone(name),
ByteBufferUtil.clone(value), timestamp);
}
+
+ @Override
+ public void validateFields(CFMetaData metadata) throws MarshalException
+ {
+ validateName(metadata);
+ if (value().remaining() != 4)
+ throw new MarshalException("A tombstone value should be 4 bytes
long");
+ if (getLocalDeletionTime() < 0)
+ throw new MarshalException("The local deletion time should not be
negative");
+ }
}
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ExpiringColumn.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ExpiringColumn.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ExpiringColumn.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/ExpiringColumn.java
Mon Apr 18 21:44:00 2011
@@ -24,7 +24,9 @@ import java.security.MessageDigest;
import org.apache.log4j.Logger;
+import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MarshalException;
import org.apache.cassandra.io.util.DataOutputBuffer;
import org.apache.cassandra.utils.ByteBufferUtil;
@@ -135,4 +137,14 @@ public class ExpiringColumn extends Colu
throw new IllegalStateException("column is not marked for delete");
}
}
+
+ @Override
+ public void validateFields(CFMetaData metadata) throws MarshalException
+ {
+ super.validateFields(metadata);
+ if (timeToLive <= 0)
+ throw new MarshalException("A column TTL should be > 0");
+ if (localExpirationTime < 0)
+ throw new MarshalException("The local expiration time should not
be negative");
+ }
}
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/IColumn.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/IColumn.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/IColumn.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/IColumn.java
Mon Apr 18 21:44:00 2011
@@ -22,7 +22,9 @@ import java.nio.ByteBuffer;
import java.security.MessageDigest;
import java.util.Collection;
+import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MarshalException;
import org.apache.cassandra.utils.FBUtilities;
public interface IColumn
@@ -45,6 +47,7 @@ public interface IColumn
public void updateDigest(MessageDigest digest);
public int getLocalDeletionTime(); // for tombstone GC, so int is
sufficient granularity
public String getString(AbstractType comparator);
+ public void validateFields(CFMetaData metadata) throws MarshalException;
/** clones the column, making copies of any underlying byte buffers */
IColumn deepCopy();
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/db/SuperColumn.java
Mon Apr 18 21:44:00 2011
@@ -32,7 +32,9 @@ import java.util.concurrent.atomic.Atomi
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import org.apache.cassandra.config.CFMetaData;
import org.apache.cassandra.db.marshal.AbstractType;
+import org.apache.cassandra.db.marshal.MarshalException;
import org.apache.cassandra.io.IColumnSerializer;
import org.apache.cassandra.io.util.ColumnSortedMap;
import org.apache.cassandra.io.util.DataOutputBuffer;
@@ -306,6 +308,15 @@ public class SuperColumn implements ICol
{
throw new UnsupportedOperationException("This operation is unsupported
on super columns.");
}
+
+ public void validateFields(CFMetaData metadata) throws MarshalException
+ {
+ metadata.comparator.validate(name());
+ for (IColumn column : getSubColumns())
+ {
+ column.validateFields(metadata);
+ }
+ }
}
class SuperColumnSerializer implements IColumnSerializer
Modified:
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
URL:
http://svn.apache.org/viewvc/cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java?rev=1094780&r1=1094779&r2=1094780&view=diff
==============================================================================
---
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
(original)
+++
cassandra/branches/cassandra-0.7/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java
Mon Apr 18 21:44:00 2011
@@ -34,6 +34,7 @@ import org.apache.cassandra.db.ColumnFam
import org.apache.cassandra.db.DecoratedKey;
import org.apache.cassandra.db.IColumn;
import org.apache.cassandra.db.columniterator.IColumnIterator;
+import org.apache.cassandra.db.marshal.MarshalException;
import org.apache.cassandra.io.util.BufferedRandomAccessFile;
import org.apache.cassandra.utils.Filter;
@@ -55,6 +56,8 @@ public class SSTableIdentityIterator imp
// Used by lazilyCompactedRow, so that we see the same things when
deserializing the first and second time
private final int expireBefore;
+ private final boolean validateColumns;
+
/**
* Used to iterate through the columns of a row.
* @param sstable SSTable we are reading ffrom.
@@ -70,7 +73,17 @@ public class SSTableIdentityIterator imp
this(sstable, file, key, dataStart, dataSize, false);
}
- public SSTableIdentityIterator(SSTableReader sstable,
BufferedRandomAccessFile file, DecoratedKey key, long dataStart, long dataSize,
boolean deserializeRowHeader)
+ /**
+ * Used to iterate through the columns of a row.
+ * @param sstable SSTable we are reading ffrom.
+ * @param file Reading using this file.
+ * @param key Key of this row.
+ * @param dataStart Data for this row starts at this pos.
+ * @param dataSize length of row data
+ * @param checkData if true, do its best to deserialize and check the
coherence of row data
+ * @throws IOException
+ */
+ public SSTableIdentityIterator(SSTableReader sstable,
BufferedRandomAccessFile file, DecoratedKey key, long dataStart, long dataSize,
boolean checkData)
throws IOException
{
this.sstable = sstable;
@@ -79,12 +92,13 @@ public class SSTableIdentityIterator imp
this.dataStart = dataStart;
this.dataSize = dataSize;
this.expireBefore = (int)(System.currentTimeMillis() / 1000);
+ this.validateColumns = checkData;
finishedAt = dataStart + dataSize;
try
{
file.seek(this.dataStart);
- if (deserializeRowHeader)
+ if (checkData)
{
try
{
@@ -141,12 +155,19 @@ public class SSTableIdentityIterator imp
{
try
{
- return sstable.getColumnSerializer().deserialize(file,
expireBefore);
+ IColumn column = sstable.getColumnSerializer().deserialize(file,
expireBefore);
+ if (validateColumns)
+ column.validateFields(sstable.metadata);
+ return column;
}
catch (IOException e)
{
throw new IOError(e);
}
+ catch (MarshalException e)
+ {
+ throw new IOError(new IOException("Error validating row " + key,
e));
+ }
}
public void remove()
@@ -178,6 +199,17 @@ public class SSTableIdentityIterator imp
file.seek(columnPosition - 4); // seek to before column count int
ColumnFamily cf = columnFamily.cloneMeShallow();
ColumnFamily.serializer().deserializeColumns(file, cf);
+ if (validateColumns)
+ {
+ try
+ {
+ cf.validateColumnFields();
+ }
+ catch (MarshalException e)
+ {
+ throw new IOException("Error validating row " + key, e);
+ }
+ }
return cf;
}
