Modified: cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java?rev=960123&r1=960122&r2=960123&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/thrift/ThriftValidation.java Fri Jul 2 22:11:04 2010 @@ -20,28 +20,28 @@ package org.apache.cassandra.thrift; * */ -import java.util.Comparator; import java.util.Arrays; -import org.apache.commons.lang.ArrayUtils; +import java.util.Comparator; -import org.apache.cassandra.db.KeyspaceNotDefinedException; -import org.apache.cassandra.db.ColumnFamily; -import org.apache.cassandra.db.IColumn; -import org.apache.cassandra.db.ColumnFamilyType; -import org.apache.cassandra.db.IClock; -import org.apache.cassandra.db.TimestampClock; -import org.apache.cassandra.db.marshal.AbstractType; -import org.apache.cassandra.db.marshal.MarshalException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.marshal.AbstractType; +import org.apache.cassandra.db.marshal.MarshalException; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.RandomPartitioner; import org.apache.cassandra.dht.Token; +import org.apache.cassandra.locator.DatacenterShardStrategy; +import org.apache.cassandra.service.ColumnValidator; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.FBUtilities; public class ThriftValidation { + private static final Logger logger = LoggerFactory.getLogger(DatacenterShardStrategy.class); + static void validateKey(byte[] key) throws InvalidRequestException { if (key == null || key.length == 0) @@ -302,10 +302,31 @@ public class ThriftValidation validateColumns(keyspace, cfName, scName, predicate.column_names); } + public static void runExternalColumnVerifier(String keyspace, ColumnParent column_parent, Column column) throws InvalidRequestException + { + try + { + ColumnValidator validator = null; + validator = DatabaseDescriptor.getColumnValidator(keyspace, column_parent.column_family, column.name); + if (validator != null) + validator.validate(keyspace, column_parent, column); + } + catch (MarshalException me) + { + String msg = String.format("[%s][%s][md5(byte[])=%s] = [md5(byte[])=%s] failed validation (%s)", + keyspace, column_parent.getColumn_family(), + FBUtilities.hexHash("MD5", column.name), + FBUtilities.hexHash("MD5", column.value), + me.getMessage()); + throw new InvalidRequestException(msg); //why doesn't IRE except a caused_by argument? + } + } + public static void validateColumn(String keyspace, ColumnParent column_parent, Column column) throws InvalidRequestException { validateTtl(column); validateColumns(keyspace, column_parent, Arrays.asList(column.name)); + runExternalColumnVerifier(keyspace, column_parent, column); } public static void validatePredicate(String keyspace, ColumnParent column_parent, SlicePredicate predicate)
Added: cassandra/trunk/src/java/org/apache/cassandra/utils/ByteArrayKey.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/ByteArrayKey.java?rev=960123&view=auto ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/utils/ByteArrayKey.java (added) +++ cassandra/trunk/src/java/org/apache/cassandra/utils/ByteArrayKey.java Fri Jul 2 22:11:04 2010 @@ -0,0 +1,38 @@ +package org.apache.cassandra.utils; + +import java.util.Arrays; + +/** + * A wrapper class for making a byte[] suitable for use as keys (i.e. hashCode/equals) + */ +public class ByteArrayKey +{ + private final byte[] bytes; + + public ByteArrayKey(byte[] bytes) + { + this.bytes = bytes; + } + + @Override + public int hashCode() + { + return Arrays.hashCode(this.bytes); + } + + @Override + public boolean equals(Object obj) + { + if (obj == this) + { + return true; + } + else if (obj == null || obj.getClass() != getClass()) + { + return false; + } + + return Arrays.equals(this.bytes, ((ByteArrayKey) obj).bytes); + } +} + Modified: cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java URL: http://svn.apache.org/viewvc/cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java?rev=960123&r1=960122&r2=960123&view=diff ============================================================================== --- cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java (original) +++ cassandra/trunk/src/java/org/apache/cassandra/utils/FBUtilities.java Fri Jul 2 22:11:04 2010 @@ -229,6 +229,11 @@ public class FBUtilities return hash.abs(); } + public static String hexHash(String type, byte[]... data) + { + return bytesToHex(hash(type, data)); + } + public static byte[] hash(String type, byte[]... data) { byte[] result; Modified: cassandra/trunk/test/system/test_thrift_server.py URL: http://svn.apache.org/viewvc/cassandra/trunk/test/system/test_thrift_server.py?rev=960123&r1=960122&r2=960123&view=diff ============================================================================== --- cassandra/trunk/test/system/test_thrift_server.py (original) +++ cassandra/trunk/test/system/test_thrift_server.py Fri Jul 2 22:11:04 2010 @@ -167,8 +167,8 @@ def _verify_super(supercf='Super1', key= def _expect_exception(fn, type_): try: r = fn() - except type_: - pass + except type_, t: + return t else: raise Exception('expected %s; got %s' % (type_.__name__, r)) @@ -1090,11 +1090,45 @@ class TestMutations(ThriftTester): client.describe_keyspace('RenameKeyspace') _expect_exception(get_second_ks, NotFoundException) + def test_column_validators(self): + ks = 'Keyspace1' + _set_keyspace(ks) + cd = ColumnDef('col', 'org.apache.cassandra.service.ExampleColumnValidator', None, None) + cf = CfDef('Keyspace1', 'ValidatorColumnFamily', column_metadata=[cd]) + client.system_add_column_family(cf) + dks = client.describe_keyspace(ks) + assert 'ValidatorColumnFamily' in dks + + cp = ColumnParent('ValidatorColumnFamily') + col0 = Column('col', 'valuegood', Clock(0)) + col1 = Column('col', 'valuebad', Clock(0)) + client.insert('key0', cp, col0, ConsistencyLevel.ONE) + e = _expect_exception(lambda: client.insert('key1', cp, col1, ConsistencyLevel.ONE), InvalidRequestException) + assert e.why.find("failed validation") >= 0 + assert e.why.find("column.value.length is even") >= 0 + + def test_super_column_validators(self): + ks = 'Keyspace1' + _set_keyspace(ks) + cd = ColumnDef('col', 'org.apache.cassandra.service.ExampleColumnValidator', None, None) + cf = CfDef('Keyspace1', 'SuperValidatorColumnFamily', 'Super', column_metadata=[cd]) + client.system_add_column_family(cf) + dks = client.describe_keyspace('Keyspace1') + assert 'SuperValidatorColumnFamily' in dks + + cp = ColumnParent('SuperValidatorColumnFamily', 'a subcolumn') + col0 = Column('col', 'valuegood', Clock(0)) + col1 = Column('col', 'valuebad', Clock(0)) + client.insert('key0', cp, col0, ConsistencyLevel.ONE) + e = _expect_exception(lambda: client.insert('key1', cp, col1, ConsistencyLevel.ONE), InvalidRequestException) + assert e.why.find("failed validation") >= 0 + assert e.why.find("column.value.length is even") >= 0 + def test_system_column_family_operations(self): - """ Test cf (add, drop, rename) operations """ _set_keyspace('Keyspace1') # create - newcf = CfDef('Keyspace1', 'NewColumnFamily') + cd = ColumnDef('ValidationColumn', 'randomclass', None, None) + newcf = CfDef('Keyspace1', 'NewColumnFamily', column_metadata=[cd]) client.system_add_column_family(newcf) ks1 = client.describe_keyspace('Keyspace1') assert 'NewColumnFamily' in ks1 @@ -1113,11 +1147,11 @@ class TestMutations(ThriftTester): assert 'Standard1' in ks1 def test_system_super_column_family_operations(self): - """test cf (add, drop, rename) operations""" _set_keyspace('Keyspace1') # create - newcf = CfDef('Keyspace1', 'NewSuperColumnFamily', 'Super') + cd = ColumnDef('ValidationColumn', 'randomclass', None, None) + newcf = CfDef('Keyspace1', 'NewSuperColumnFamily', 'Super', column_metadata=[cd]) client.system_add_column_family(newcf) ks1 = client.describe_keyspace('Keyspace1') assert 'NewSuperColumnFamily' in ks1 @@ -1134,7 +1168,7 @@ class TestMutations(ThriftTester): assert 'RenameSuperColumnFamily' not in ks1 assert 'NewSuperColumnFamily' not in ks1 assert 'Standard1' in ks1 - + def test_insert_ttl(self): """ Test simple insertion of a column with ttl """ _set_keyspace('Keyspace1') Modified: cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java URL: http://svn.apache.org/viewvc/cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java?rev=960123&r1=960122&r2=960123&view=diff ============================================================================== --- cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java (original) +++ cassandra/trunk/test/unit/org/apache/cassandra/db/DefsTest.java Fri Jul 2 22:11:04 2010 @@ -17,45 +17,32 @@ */ package org.apache.cassandra.db; +import org.apache.cassandra.utils.ByteArrayKey; + +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; +import java.io.DataOutputStream; +import java.io.IOException; +import java.util.*; +import java.util.concurrent.ExecutionException; + +import org.junit.Test; import org.apache.cassandra.CleanupHelper; import org.apache.cassandra.Util; - -import org.apache.cassandra.config.CFMetaData; -import org.apache.cassandra.config.ConfigurationException; -import org.apache.cassandra.config.DatabaseDescriptor; -import org.apache.cassandra.config.KSMetaData; +import org.apache.cassandra.config.*; import org.apache.cassandra.db.clock.TimestampReconciler; import org.apache.cassandra.db.filter.QueryFilter; import org.apache.cassandra.db.filter.QueryPath; import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.migration.AddColumnFamily; -import org.apache.cassandra.db.migration.AddKeyspace; -import org.apache.cassandra.db.migration.DropColumnFamily; -import org.apache.cassandra.db.migration.DropKeyspace; -import org.apache.cassandra.db.migration.Migration; -import org.apache.cassandra.db.migration.RenameColumnFamily; -import org.apache.cassandra.db.migration.RenameKeyspace; +import org.apache.cassandra.db.marshal.UTF8Type; +import org.apache.cassandra.db.migration.*; +import org.apache.cassandra.dht.BytesToken; import org.apache.cassandra.locator.RackUnawareStrategy; +import org.apache.cassandra.utils.ByteArrayKey; import org.apache.cassandra.utils.FBUtilities; -import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.utils.UUIDGen; -import org.junit.Test; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.DataOutputStream; -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.List; -import java.util.SortedSet; -import java.util.TreeSet; -import java.util.UUID; -import java.util.concurrent.ExecutionException; - public class DefsTest extends CleanupHelper { @Test @@ -78,7 +65,7 @@ public class DefsTest extends CleanupHel @Test public void addNewCfToBogusTable() throws InterruptedException { - CFMetaData newCf = new CFMetaData("MadeUpKeyspace", "NewCF", ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "new cf", 0, false, 1.0, 0); + CFMetaData newCf = new CFMetaData("MadeUpKeyspace", "NewCF", ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "new cf", 0, false, 1.0, 0, Collections.<ByteArrayKey, ColumnDefinition>emptyMap()); try { new AddColumnFamily(newCf).apply(); @@ -103,7 +90,7 @@ public class DefsTest extends CleanupHel assert DatabaseDescriptor.getDefsVersion().equals(prior); // add a cf. - CFMetaData newCf1 = new CFMetaData("Keyspace1", "MigrationCf_1", ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "Migration CF ", 0, false, 1.0, 0); + CFMetaData newCf1 = new CFMetaData("Keyspace1", "MigrationCf_1", ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "Migration CF ", 0, false, 1.0, 0, Collections.<ByteArrayKey, ColumnDefinition>emptyMap()); Migration m1 = new AddColumnFamily(newCf1); m1.apply(); UUID ver1 = m1.getVersion(); @@ -162,7 +149,7 @@ public class DefsTest extends CleanupHel final String cf = "BrandNewCf"; KSMetaData original = DatabaseDescriptor.getTableDefinition(ks); - CFMetaData newCf = new CFMetaData(original.name, cf, ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "A New Column Family", 0, false, 1.0, 0); + CFMetaData newCf = new CFMetaData(original.name, cf, ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "A New Column Family", 0, false, 1.0, 0, Collections.<ByteArrayKey, ColumnDefinition>emptyMap()); assert !DatabaseDescriptor.getTableDefinition(ks).cfMetaData().containsKey(newCf.cfName); new AddColumnFamily(newCf).apply(); @@ -185,6 +172,43 @@ public class DefsTest extends CleanupHel } @Test + public void testCanAddColumnDefinitionsInColumnMetaData() throws Exception + { + String ks = "Keyspace1"; + String cf = "ValidatorColumnFamily"; + KSMetaData original = DatabaseDescriptor.getTableDefinition(ks); + + Map<ByteArrayKey, ColumnDefinition> column_metadata = new HashMap<ByteArrayKey, ColumnDefinition>(); + + ColumnDefinition cd0 = new ColumnDefinition(); + cd0.name = "TestColumn1".getBytes("UTF8"); + cd0.validation_class = "random class one"; + cd0.index_name = null; + cd0.index_type = null; + + ColumnDefinition cd1 = new ColumnDefinition(); + cd1.name = "*".getBytes("UTF8"); + cd1.validation_class = "random class two"; + cd1.index_name = "some name"; + cd1.index_type = "some type"; + + column_metadata.put(new ByteArrayKey(cd0.name), cd0); + column_metadata.put(new ByteArrayKey(cd1.name), cd1); + + CFMetaData newCf = new CFMetaData(original.name, cf, ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "A New Column Family", 0, false, 1.0, 0, column_metadata); + assert !DatabaseDescriptor.getTableDefinition(ks).cfMetaData().containsKey(newCf.cfName); + new AddColumnFamily(newCf).apply(); + + assert DatabaseDescriptor.getTableDefinition(ks).cfMetaData().containsKey(newCf.cfName); + assert DatabaseDescriptor.getTableDefinition(ks).cfMetaData().get(newCf.cfName).equals(newCf); + + ColumnFamilyStore store = Table.open(ks).getColumnFamilyStore(cf); + assert store != null; + store.forceBlockingFlush(); + } + + + @Test public void dropCf() throws ConfigurationException, IOException, ExecutionException, InterruptedException { DecoratedKey dk = Util.dk("dropCf"); @@ -276,7 +300,7 @@ public class DefsTest extends CleanupHel public void addNewKS() throws ConfigurationException, IOException, ExecutionException, InterruptedException { DecoratedKey dk = Util.dk("key0"); - CFMetaData newCf = new CFMetaData("NewKeyspace1", "AddedStandard1", ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "A new cf for a new ks", 0, false, 1.0, 0); + CFMetaData newCf = new CFMetaData("NewKeyspace1", "AddedStandard1", ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "A new cf for a new ks", 0, false, 1.0, 0, Collections.<ByteArrayKey, ColumnDefinition>emptyMap()); KSMetaData newKs = new KSMetaData(newCf.tableName, RackUnawareStrategy.class, 5, newCf); new AddKeyspace(newKs).apply(); @@ -432,7 +456,7 @@ public class DefsTest extends CleanupHel new AddKeyspace(newKs).apply(); assert DatabaseDescriptor.getTableDefinition("EmptyKeyspace") != null; - CFMetaData newCf = new CFMetaData("EmptyKeyspace", "AddedLater", ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "A new CF to add to an empty KS", 0, false, 1.0, 0); + CFMetaData newCf = new CFMetaData("EmptyKeyspace", "AddedLater", ColumnFamilyType.Standard, ClockType.Timestamp, UTF8Type.instance, null, new TimestampReconciler(), "A new CF to add to an empty KS", 0, false, 1.0, 0, Collections.<ByteArrayKey, ColumnDefinition>emptyMap()); //should not exist until apply assert !DatabaseDescriptor.getTableDefinition(newKs.name).cfMetaData().containsKey(newCf.cfName);
