Support for frozen collections Patch by Tyler Hobbs; reviewed by Sylvain Lebresne for CASSANDRA-7859
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/ee55f361 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/ee55f361 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/ee55f361 Branch: refs/heads/trunk Commit: ee55f361b76f9ce7dd2a21a0ff4e80da931c77d2 Parents: 0337620 Author: Tyler Hobbs <ty...@datastax.com> Authored: Tue Nov 11 12:40:48 2014 -0600 Committer: Tyler Hobbs <ty...@datastax.com> Committed: Tue Nov 11 12:40:48 2014 -0600 ---------------------------------------------------------------------- CHANGES.txt | 1 + bin/cqlsh | 18 + pylib/cqlshlib/cql3handling.py | 19 +- .../apache/cassandra/cql3/AbstractMarker.java | 2 +- src/java/org/apache/cassandra/cql3/CQL3Row.java | 2 +- .../org/apache/cassandra/cql3/CQL3Type.java | 158 ++-- .../apache/cassandra/cql3/ColumnCondition.java | 275 ++++--- .../org/apache/cassandra/cql3/Constants.java | 3 +- src/java/org/apache/cassandra/cql3/Cql.g | 14 +- src/java/org/apache/cassandra/cql3/Lists.java | 66 +- src/java/org/apache/cassandra/cql3/Maps.java | 62 +- .../org/apache/cassandra/cql3/Operation.java | 16 +- src/java/org/apache/cassandra/cql3/Sets.java | 80 +- src/java/org/apache/cassandra/cql3/Term.java | 6 + src/java/org/apache/cassandra/cql3/Tuples.java | 13 +- .../apache/cassandra/cql3/UntypedResultSet.java | 6 +- .../apache/cassandra/cql3/UpdateParameters.java | 2 +- .../org/apache/cassandra/cql3/UserTypes.java | 3 +- .../cql3/statements/AlterTableStatement.java | 39 +- .../cql3/statements/AlterTypeStatement.java | 18 +- .../cql3/statements/CreateIndexStatement.java | 24 +- .../cql3/statements/CreateTableStatement.java | 28 +- .../cql3/statements/DeleteStatement.java | 2 +- .../cql3/statements/DropTypeStatement.java | 6 +- .../cassandra/cql3/statements/IndexTarget.java | 23 +- .../cassandra/cql3/statements/Restriction.java | 6 + .../cql3/statements/SelectStatement.java | 107 ++- .../statements/SingleColumnRestriction.java | 24 + .../org/apache/cassandra/db/CFRowAdder.java | 4 +- .../db/composites/AbstractCellNameType.java | 4 +- .../cassandra/db/composites/CellNameType.java | 2 +- .../composites/CompoundSparseCellNameType.java | 5 +- .../cassandra/db/filter/ExtendedFilter.java | 47 +- .../cassandra/db/index/SecondaryIndex.java | 6 +- .../db/index/SecondaryIndexManager.java | 32 + .../db/index/SecondaryIndexSearcher.java | 2 + .../db/index/composites/CompositesIndex.java | 4 +- .../CompositesIndexOnCollectionValue.java | 2 +- .../cassandra/db/marshal/AbstractType.java | 18 + .../cassandra/db/marshal/CollectionType.java | 113 ++- .../db/marshal/ColumnToCollectionType.java | 2 +- .../apache/cassandra/db/marshal/FrozenType.java | 62 ++ .../apache/cassandra/db/marshal/ListType.java | 77 +- .../apache/cassandra/db/marshal/MapType.java | 105 ++- .../apache/cassandra/db/marshal/SetType.java | 69 +- .../apache/cassandra/db/marshal/TupleType.java | 9 +- .../apache/cassandra/db/marshal/TypeParser.java | 34 +- .../apache/cassandra/db/marshal/UserType.java | 2 +- .../apache/cassandra/hadoop/pig/CqlStorage.java | 8 +- .../serializers/CollectionSerializer.java | 24 +- .../cassandra/serializers/ListSerializer.java | 36 +- .../cassandra/serializers/MapSerializer.java | 38 +- .../apache/cassandra/transport/DataType.java | 16 +- .../org/apache/cassandra/cql3/CQLTester.java | 84 +- .../cassandra/cql3/ColumnConditionTest.java | 28 +- .../cassandra/cql3/FrozenCollectionsTest.java | 791 +++++++++++++++++++ .../apache/cassandra/cql3/TupleTypeTest.java | 44 +- .../db/marshal/CollectionTypeTest.java | 22 +- .../cassandra/transport/SerDeserTest.java | 13 +- .../cassandra/stress/generate/values/Lists.java | 2 +- .../cassandra/stress/generate/values/Sets.java | 2 +- 61 files changed, 2139 insertions(+), 591 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index b1b5df8..5b63f48 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 2.1.3 + * Support for frozen collections (CASSANDRA-7859) * Fix overflow on histogram computation (CASSANDRA-8028) * Have paxos reuse the timestamp generation of normal queries (CASSANDRA-7801) Merged from 2.0: http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/bin/cqlsh ---------------------------------------------------------------------- diff --git a/bin/cqlsh b/bin/cqlsh index 6ace914..f105273 100755 --- a/bin/cqlsh +++ b/bin/cqlsh @@ -486,6 +486,24 @@ def auto_format_udts(): cassandra.cqltypes.UserType.make_udt_class = classmethod(new_make_udt_class) +class FrozenType(cassandra.cqltypes._ParameterizedType): + """ + Needed until the bundled python driver adds FrozenType. + """ + typename = "frozen" + num_subtypes = 1 + + @classmethod + def deserialize_safe(cls, byts, protocol_version): + subtype, = cls.subtypes + return subtype.from_binary(byts) + + @classmethod + def serialize_safe(cls, val, protocol_version): + subtype, = cls.subtypes + return subtype.to_binary(val, protocol_version) + + class Shell(cmd.Cmd): custom_prompt = os.getenv('CQLSH_PROMPT', '') if custom_prompt is not '': http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/pylib/cqlshlib/cql3handling.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py index b1179ca..dbd3709 100644 --- a/pylib/cqlshlib/cql3handling.py +++ b/pylib/cqlshlib/cql3handling.py @@ -255,13 +255,20 @@ JUNK ::= /([ \t\r\f\v]+|(--|[/][/])[^\n\r]*([\n\r]|$)|[/][*].*?[*][/])/ ; <userType> ::= utname=<cfOrKsName> ; -<storageType> ::= <simpleStorageType> | <collectionType> | <userType> ; +<storageType> ::= <simpleStorageType> | <collectionType> | <frozenCollectionType> | <userType> ; +# Note: autocomplete for frozen collection types does not handle nesting past depth 1 properly, +# but that's a lot of work to fix for little benefit. <collectionType> ::= "map" "<" <simpleStorageType> "," ( <simpleStorageType> | <userType> ) ">" | "list" "<" ( <simpleStorageType> | <userType> ) ">" | "set" "<" ( <simpleStorageType> | <userType> ) ">" ; +<frozenCollectionType> ::= "frozen" "<" "map" "<" <storageType> "," <storageType> ">" ">" + | "frozen" "<" "list" "<" <storageType> ">" ">" + | "frozen" "<" "set" "<" <storageType> ">" ">" + ; + <columnFamilyName> ::= ( ksname=<cfOrKsName> dot="." )? cfname=<cfOrKsName> ; <userTypeName> ::= ( ksname=<cfOrKsName> dot="." )? utname=<cfOrKsName> ; @@ -906,11 +913,11 @@ syntax_rules += r''' <cfamOrdering> ::= [ordercol]=<cident> ( "ASC" | "DESC" ) ; -<singleKeyCfSpec> ::= [newcolname]=<cident> <simpleStorageType> "PRIMARY" "KEY" +<singleKeyCfSpec> ::= [newcolname]=<cident> <storageType> "PRIMARY" "KEY" ( "," [newcolname]=<cident> <storageType> )* ; -<compositeKeyCfSpec> ::= [newcolname]=<cident> <simpleStorageType> +<compositeKeyCfSpec> ::= [newcolname]=<cident> <storageType> "," [newcolname]=<cident> <storageType> ( "static" )? ( "," [newcolname]=<cident> <storageType> ( "static" )? )* "," "PRIMARY" k="KEY" p="(" ( partkey=<pkDef> | [pkey]=<cident> ) @@ -986,7 +993,11 @@ def create_cf_composite_primary_key_comma_completer(ctxt, cass): syntax_rules += r''' <createIndexStatement> ::= "CREATE" "CUSTOM"? "INDEX" ("IF" "NOT" "EXISTS")? indexname=<identifier>? "ON" - cf=<columnFamilyName> ( "(" col=<cident> ")" | "(" "KEYS" "(" col=<cident> ")" ")") + cf=<columnFamilyName> "(" ( + col=<cident> | + "keys(" col=<cident> ")" | + "fullCollection(" col=<cident> ")" + ) ")" ( "USING" <stringLiteral> ( "WITH" "OPTIONS" "=" <mapLiteral> )? )? ; http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/AbstractMarker.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/AbstractMarker.java b/src/java/org/apache/cassandra/cql3/AbstractMarker.java index 10a4dff..d18790c 100644 --- a/src/java/org/apache/cassandra/cql3/AbstractMarker.java +++ b/src/java/org/apache/cassandra/cql3/AbstractMarker.java @@ -99,7 +99,7 @@ public abstract class AbstractMarker extends Term.NonTerminal private static ColumnSpecification makeInReceiver(ColumnSpecification receiver) { ColumnIdentifier inName = new ColumnIdentifier("in(" + receiver.name + ")", true); - return new ColumnSpecification(receiver.ksName, receiver.cfName, inName, ListType.getInstance(receiver.type)); + return new ColumnSpecification(receiver.ksName, receiver.cfName, inName, ListType.getInstance(receiver.type, false)); } @Override http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/CQL3Row.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/CQL3Row.java b/src/java/org/apache/cassandra/cql3/CQL3Row.java index 6fa2b64..e3e76d1 100644 --- a/src/java/org/apache/cassandra/cql3/CQL3Row.java +++ b/src/java/org/apache/cassandra/cql3/CQL3Row.java @@ -27,7 +27,7 @@ public interface CQL3Row { public ByteBuffer getClusteringColumn(int i); public Cell getColumn(ColumnIdentifier name); - public List<Cell> getCollection(ColumnIdentifier name); + public List<Cell> getMultiCellColumn(ColumnIdentifier name); public interface Builder { http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/CQL3Type.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/CQL3Type.java b/src/java/org/apache/cassandra/cql3/CQL3Type.java index 6d55285..b656de8 100644 --- a/src/java/org/apache/cassandra/cql3/CQL3Type.java +++ b/src/java/org/apache/cassandra/cql3/CQL3Type.java @@ -26,9 +26,13 @@ import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.exceptions.SyntaxException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; public interface CQL3Type { + static final Logger logger = LoggerFactory.getLogger(CQL3Type.class); + public boolean isCollection(); public AbstractType<?> getType(); @@ -160,17 +164,30 @@ public interface CQL3Type @Override public String toString() { + boolean isFrozen = !this.type.isMultiCell(); + StringBuilder sb = new StringBuilder(isFrozen ? "frozen<" : ""); switch (type.kind) { case LIST: - return "list<" + ((ListType)type).elements.asCQL3Type() + ">"; + AbstractType<?> listType = ((ListType)type).getElementsType(); + sb.append("list<").append(listType.asCQL3Type()); + break; case SET: - return "set<" + ((SetType)type).elements.asCQL3Type() + ">"; + AbstractType<?> setType = ((SetType)type).getElementsType(); + sb.append("set<").append(setType.asCQL3Type()); + break; case MAP: - MapType mt = (MapType)type; - return "map<" + mt.keys.asCQL3Type() + ", " + mt.values.asCQL3Type() + ">"; + AbstractType<?> keysType = ((MapType)type).getKeysType(); + AbstractType<?> valuesType = ((MapType)type).getValuesType(); + sb.append("map<").append(keysType.asCQL3Type()).append(", ").append(valuesType.asCQL3Type()); + break; + default: + throw new AssertionError(); } - throw new AssertionError(); + sb.append(">"); + if (isFrozen) + sb.append(">"); + return sb.toString(); } } @@ -284,7 +301,9 @@ public interface CQL3Type // actual type used, so Raw is a "not yet prepared" CQL3Type. public abstract class Raw { - protected boolean frozen; + protected boolean frozen = false; + + protected abstract boolean supportsFreezing(); public boolean isCollection() { @@ -296,10 +315,10 @@ public interface CQL3Type return false; } - public Raw freeze() + public void freeze() throws InvalidRequestException { - frozen = true; - return this; + String message = String.format("frozen<> is only allowed on collections, tuples, and user-defined types (got %s)", this); + throw new InvalidRequestException(message); } public abstract CQL3Type prepare(String keyspace) throws InvalidRequestException; @@ -314,53 +333,30 @@ public interface CQL3Type return new RawUT(name); } - public static Raw map(CQL3Type.Raw t1, CQL3Type.Raw t2) throws InvalidRequestException + public static Raw map(CQL3Type.Raw t1, CQL3Type.Raw t2) { - if (t1.isCollection() || t2.isCollection()) - throw new InvalidRequestException("map type cannot contain another collection"); - if (t1.isCounter() || t2.isCounter()) - throw new InvalidRequestException("counters are not allowed inside a collection"); - return new RawCollection(CollectionType.Kind.MAP, t1, t2); } - public static Raw list(CQL3Type.Raw t) throws InvalidRequestException + public static Raw list(CQL3Type.Raw t) { - if (t.isCollection()) - throw new InvalidRequestException("list type cannot contain another collection"); - if (t.isCounter()) - throw new InvalidRequestException("counters are not allowed inside a collection"); - return new RawCollection(CollectionType.Kind.LIST, null, t); } - public static Raw set(CQL3Type.Raw t) throws InvalidRequestException + public static Raw set(CQL3Type.Raw t) { - if (t.isCollection()) - throw new InvalidRequestException("set type cannot contain another collection"); - if (t.isCounter()) - throw new InvalidRequestException("counters are not allowed inside a collection"); - return new RawCollection(CollectionType.Kind.SET, null, t); } - public static Raw tuple(List<CQL3Type.Raw> ts) throws InvalidRequestException + public static Raw tuple(List<CQL3Type.Raw> ts) { - for (int i = 0; i < ts.size(); i++) - if (ts.get(i) != null && ts.get(i).isCounter()) - throw new InvalidRequestException("counters are not allowed inside tuples"); - return new RawTuple(ts); } public static Raw frozen(CQL3Type.Raw t) throws InvalidRequestException { - if (t instanceof RawUT) - return ((RawUT)t).freeze(); - if (t instanceof RawTuple) - return ((RawTuple)t).freeze(); - - throw new InvalidRequestException("frozen<> is only currently only allowed on User-Defined and tuple types"); + t.freeze(); + return t; } private static class RawType extends Raw @@ -377,6 +373,11 @@ public interface CQL3Type return type; } + protected boolean supportsFreezing() + { + return false; + } + public boolean isCounter() { return type == Native.COUNTER; @@ -402,12 +403,18 @@ public interface CQL3Type this.values = values; } - public Raw freeze() + public void freeze() throws InvalidRequestException { - if (keys != null) + if (keys != null && keys.supportsFreezing()) keys.freeze(); - values.freeze(); - return super.freeze(); + if (values != null && values.supportsFreezing()) + values.freeze(); + frozen = true; + } + + protected boolean supportsFreezing() + { + return true; } public boolean isCollection() @@ -417,11 +424,28 @@ public interface CQL3Type public CQL3Type prepare(String keyspace) throws InvalidRequestException { + assert values != null : "Got null values type for a collection"; + + if (!frozen && values.supportsFreezing() && !values.frozen) + throw new InvalidRequestException("Non-frozen collections are not allowed inside collections: " + this); + if (values.isCounter()) + throw new InvalidRequestException("Counters are not allowed inside collections: " + this); + + if (keys != null) + { + if (!frozen && keys.supportsFreezing() && !keys.frozen) + throw new InvalidRequestException("Non-frozen collections are not allowed inside collections: " + this); + } + switch (kind) { - case LIST: return new Collection(ListType.getInstance(values.prepare(keyspace).getType())); - case SET: return new Collection(SetType.getInstance(values.prepare(keyspace).getType())); - case MAP: return new Collection(MapType.getInstance(keys.prepare(keyspace).getType(), values.prepare(keyspace).getType())); + case LIST: + return new Collection(ListType.getInstance(values.prepare(keyspace).getType(), !frozen)); + case SET: + return new Collection(SetType.getInstance(values.prepare(keyspace).getType(), !frozen)); + case MAP: + assert keys != null : "Got null keys type for a collection"; + return new Collection(MapType.getInstance(keys.prepare(keyspace).getType(), values.prepare(keyspace).getType(), !frozen)); } throw new AssertionError(); } @@ -429,11 +453,13 @@ public interface CQL3Type @Override public String toString() { + String start = frozen? "frozen<" : ""; + String end = frozen ? ">" : ""; switch (kind) { - case LIST: return "list<" + values + ">"; - case SET: return "set<" + values + ">"; - case MAP: return "map<" + keys + ", " + values + ">"; + case LIST: return start + "list<" + values + ">" + end; + case SET: return start + "set<" + values + ">" + end; + case MAP: return start + "map<" + keys + ", " + values + ">" + end; } throw new AssertionError(); } @@ -448,6 +474,11 @@ public interface CQL3Type this.name = name; } + public void freeze() + { + frozen = true; + } + public CQL3Type prepare(String keyspace) throws InvalidRequestException { if (name.hasKeyspace()) @@ -477,7 +508,7 @@ public interface CQL3Type return new UserDefined(name.toString(), type); } - public boolean isUDT() + protected boolean supportsFreezing() { return true; } @@ -498,12 +529,9 @@ public interface CQL3Type this.types = types; } - public Raw freeze() + protected boolean supportsFreezing() { - for (CQL3Type.Raw t : types) - if (t != null) - t.freeze(); - return super.freeze(); + return true; } public boolean isCollection() @@ -511,15 +539,29 @@ public interface CQL3Type return false; } - public CQL3Type prepare(String keyspace) throws InvalidRequestException + public void freeze() throws InvalidRequestException { - List<AbstractType<?>> ts = new ArrayList<>(types.size()); for (CQL3Type.Raw t : types) - ts.add(t.prepare(keyspace).getType()); + { + if (t.supportsFreezing()) + t.freeze(); + } + frozen = true; + } + public CQL3Type prepare(String keyspace) throws InvalidRequestException + { if (!frozen) - throw new InvalidRequestException("Non-frozen tuples are not supported, please use frozen<>"); + freeze(); + List<AbstractType<?>> ts = new ArrayList<>(types.size()); + for (CQL3Type.Raw t : types) + { + if (t.isCounter()) + throw new InvalidRequestException("Counters are not allowed inside tuples"); + + ts.add(t.prepare(keyspace).getType()); + } return new Tuple(new TupleType(ts)); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/ColumnCondition.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/ColumnCondition.java b/src/java/org/apache/cassandra/cql3/ColumnCondition.java index 25cb07d..1a8e5a3 100644 --- a/src/java/org/apache/cassandra/cql3/ColumnCondition.java +++ b/src/java/org/apache/cassandra/cql3/ColumnCondition.java @@ -20,7 +20,6 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; import java.util.*; -import com.google.common.base.Objects; import com.google.common.base.Predicate; import com.google.common.collect.Iterators; import static com.google.common.collect.Lists.newArrayList; @@ -33,12 +32,13 @@ import org.apache.cassandra.db.composites.Composite; import org.apache.cassandra.db.filter.ColumnSlice; import org.apache.cassandra.db.marshal.*; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.transport.Server; import org.apache.cassandra.utils.ByteBufferUtil; import org.slf4j.Logger; import org.slf4j.LoggerFactory; /** - * A CQL3 condition. + * A CQL3 condition on the value of a column or collection element. For example, "UPDATE .. IF a = 0". */ public class ColumnCondition { @@ -235,12 +235,6 @@ public class ColumnCondition CellName name = current.metadata().comparator.create(rowPrefix, column); return isSatisfiedByValue(value, current.getColumn(name), column.type, operator, now); } - - @Override - public int hashCode() - { - return Objects.hashCode(column, value, operator); - } } /** @@ -275,12 +269,6 @@ public class ColumnCondition } return false; } - - @Override - public int hashCode() - { - return Objects.hashCode(column, inValues, operator); - } } /** A condition on an element of a collection column. IN operators are not supported here, see ElementAccessInBound. */ @@ -305,16 +293,37 @@ public class ColumnCondition if (column.type instanceof MapType) { - Cell cell = current.getColumn(current.metadata().comparator.create(rowPrefix, column, collectionElement)); - return isSatisfiedByValue(value, cell, ((MapType) column.type).values, operator, now); + MapType mapType = (MapType) column.type; + if (column.type.isMultiCell()) + { + Cell cell = current.getColumn(current.metadata().comparator.create(rowPrefix, column, collectionElement)); + return isSatisfiedByValue(value, cell, mapType.getValuesType(), operator, now); + } + else + { + Cell cell = current.getColumn(current.metadata().comparator.create(rowPrefix, column)); + ByteBuffer mapElementValue = cell.isLive(now) ? mapType.getSerializer().getSerializedValue(cell.value(), collectionElement, mapType.getKeysType()) + : null; + return compareWithOperator(operator, mapType.getValuesType(), value, mapElementValue); + } } // sets don't have element access, so it's a list - assert column.type instanceof ListType; - ByteBuffer columnValue = getListItem( - collectionColumns(current.metadata().comparator.create(rowPrefix, column), current, now), - getListIndex(collectionElement)); - return compareWithOperator(operator, ((ListType)column.type).elements, value, columnValue); + ListType listType = (ListType) column.type; + if (column.type.isMultiCell()) + { + ByteBuffer columnValue = getListItem( + collectionColumns(current.metadata().comparator.create(rowPrefix, column), current, now), + getListIndex(collectionElement)); + return compareWithOperator(operator, listType.getElementsType(), value, columnValue); + } + else + { + Cell cell = current.getColumn(current.metadata().comparator.create(rowPrefix, column)); + ByteBuffer listElementValue = cell.isLive(now) ? listType.getSerializer().getElement(cell.value(), getListIndex(collectionElement)) + : null; + return compareWithOperator(operator, listType.getElementsType(), value, listElementValue); + } } static int getListIndex(ByteBuffer collectionElement) throws InvalidRequestException @@ -338,12 +347,6 @@ public class ColumnCondition { return collectionElement; } - - @Override - public int hashCode() - { - return Objects.hashCode(column, collectionElement, value, operator); - } } static class ElementAccessInBound extends Bound @@ -375,47 +378,86 @@ public class ColumnCondition CellNameType nameType = current.metadata().comparator; if (column.type instanceof MapType) { - CellName name = nameType.create(rowPrefix, column, collectionElement); - Cell item = current.getColumn(name); - AbstractType<?> valueType = ((MapType) column.type).values; - for (ByteBuffer value : inValues) + MapType mapType = (MapType) column.type; + AbstractType<?> valueType = mapType.getValuesType(); + if (column.type.isMultiCell()) { - if (isSatisfiedByValue(value, item, valueType, Operator.EQ, now)) - return true; + CellName name = nameType.create(rowPrefix, column, collectionElement); + Cell item = current.getColumn(name); + for (ByteBuffer value : inValues) + { + if (isSatisfiedByValue(value, item, valueType, Operator.EQ, now)) + return true; + } + return false; + } + else + { + Cell cell = current.getColumn(nameType.create(rowPrefix, column)); + ByteBuffer mapElementValue = null; + if (cell != null && cell.isLive(now)) + mapElementValue = mapType.getSerializer().getSerializedValue(cell.value(), collectionElement, mapType.getKeysType()); + for (ByteBuffer value : inValues) + { + if (value == null) + { + if (mapElementValue == null) + return true; + continue; + } + if (valueType.compare(value, mapElementValue) == 0) + return true; + } + return false; } - return false; } - assert column.type instanceof ListType; - ByteBuffer columnValue = ElementAccessBound.getListItem( - collectionColumns(nameType.create(rowPrefix, column), current, now), - ElementAccessBound.getListIndex(collectionElement)); + ListType listType = (ListType) column.type; + AbstractType<?> elementsType = listType.getElementsType(); + if (column.type.isMultiCell()) + { + ByteBuffer columnValue = ElementAccessBound.getListItem( + collectionColumns(nameType.create(rowPrefix, column), current, now), + ElementAccessBound.getListIndex(collectionElement)); - AbstractType<?> valueType = ((ListType) column.type).elements; - for (ByteBuffer value : inValues) + for (ByteBuffer value : inValues) + { + if (compareWithOperator(Operator.EQ, elementsType, value, columnValue)) + return true; + } + } + else { - if (compareWithOperator(Operator.EQ, valueType, value, columnValue)) - return true; + Cell cell = current.getColumn(nameType.create(rowPrefix, column)); + ByteBuffer listElementValue = null; + if (cell != null && cell.isLive(now)) + listElementValue = listType.getSerializer().getElement(cell.value(), ElementAccessBound.getListIndex(collectionElement)); + + for (ByteBuffer value : inValues) + { + if (value == null) + { + if (listElementValue == null) + return true; + continue; + } + if (elementsType.compare(value, listElementValue) == 0) + return true; + } } return false; } - - @Override - public int hashCode() - { - return Objects.hashCode(column, collectionElement, inValues, operator); - } } /** A condition on an entire collection column. IN operators are not supported here, see CollectionInBound. */ static class CollectionBound extends Bound { - public final Term.Terminal value; + private final Term.Terminal value; private CollectionBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException { super(condition.column, condition.operator); - assert column.type instanceof CollectionType && condition.collectionElement == null; + assert column.type.isCollection() && condition.collectionElement == null; assert !condition.operator.equals(Operator.IN); this.value = condition.value.bind(options); } @@ -424,18 +466,44 @@ public class ColumnCondition { CollectionType type = (CollectionType)column.type; - Iterator<Cell> iter = collectionColumns(current.metadata().comparator.create(rowPrefix, column), current, now); + if (type.isMultiCell()) + { + Iterator<Cell> iter = collectionColumns(current.metadata().comparator.create(rowPrefix, column), current, now); + if (value == null) + { + if (operator.equals(Operator.EQ)) + return !iter.hasNext(); + else if (operator.equals(Operator.NEQ)) + return iter.hasNext(); + else + throw new InvalidRequestException(String.format("Invalid comparison with null for operator \"%s\"", operator)); + } + + return valueAppliesTo(type, iter, value, operator); + } + + // frozen collections + Cell cell = current.getColumn(current.metadata().comparator.create(rowPrefix, column)); if (value == null) { - if (operator.equals(Operator.EQ)) - return !iter.hasNext(); - else if (operator.equals(Operator.NEQ)) - return iter.hasNext(); + if (operator == Operator.EQ) + return cell == null || !cell.isLive(now); + else if (operator == Operator.NEQ) + return cell != null && cell.isLive(now); else throw new InvalidRequestException(String.format("Invalid comparison with null for operator \"%s\"", operator)); } - return valueAppliesTo(type, iter, value, operator); + // make sure we use v3 serialization format for comparison + ByteBuffer conditionValue; + if (type.kind == CollectionType.Kind.LIST) + conditionValue = ((Lists.Value) value).getWithProtocolVersion(Server.VERSION_3); + else if (type.kind == CollectionType.Kind.SET) + conditionValue = ((Sets.Value) value).getWithProtocolVersion(Server.VERSION_3); + else + conditionValue = ((Maps.Value) value).getWithProtocolVersion(Server.VERSION_3); + + return compareWithOperator(operator, type, conditionValue, cell.value()); } static boolean valueAppliesTo(CollectionType type, Iterator<Cell> iter, Term.Terminal value, Operator operator) @@ -495,15 +563,15 @@ public class ColumnCondition static boolean listAppliesTo(ListType type, Iterator<Cell> iter, List<ByteBuffer> elements, Operator operator) { - return setOrListAppliesTo(type.elements, iter, elements.iterator(), operator, false); + return setOrListAppliesTo(type.getElementsType(), iter, elements.iterator(), operator, false); } static boolean setAppliesTo(SetType type, Iterator<Cell> iter, Set<ByteBuffer> elements, Operator operator) { ArrayList<ByteBuffer> sortedElements = new ArrayList<>(elements.size()); sortedElements.addAll(elements); - Collections.sort(sortedElements, type.elements); - return setOrListAppliesTo(type.elements, iter, sortedElements.iterator(), operator, true); + Collections.sort(sortedElements, type.getElementsType()); + return setOrListAppliesTo(type.getElementsType(), iter, sortedElements.iterator(), operator, true); } static boolean mapAppliesTo(MapType type, Iterator<Cell> iter, Map<ByteBuffer, ByteBuffer> elements, Operator operator) @@ -518,12 +586,12 @@ public class ColumnCondition Cell c = iter.next(); // compare the keys - int comparison = type.keys.compare(c.name().collectionElement(), conditionEntry.getKey()); + int comparison = type.getKeysType().compare(c.name().collectionElement(), conditionEntry.getKey()); if (comparison != 0) return evaluateComparisonWithOperator(comparison, operator); // compare the values - comparison = type.values.compare(c.value(), conditionEntry.getValue()); + comparison = type.getValuesType().compare(c.value(), conditionEntry.getValue()); if (comparison != 0) return evaluateComparisonWithOperator(comparison, operator); } @@ -534,33 +602,11 @@ public class ColumnCondition // they're equal return operator == Operator.EQ || operator == Operator.LTE || operator == Operator.GTE; } - - @Override - public int hashCode() - { - Object val = null; - if (value != null) - { - switch (((CollectionType)column.type).kind) - { - case LIST: - val = ((Lists.Value)value).elements.hashCode(); - break; - case SET: - val = ((Sets.Value)value).elements.hashCode(); - break; - case MAP: - val = ((Maps.Value)value).map.hashCode(); - break; - } - } - return Objects.hashCode(column, val); - } } public static class CollectionInBound extends Bound { - public final List<Term.Terminal> inValues; + private final List<Term.Terminal> inValues; private CollectionInBound(ColumnCondition condition, QueryOptions options) throws InvalidRequestException { @@ -575,7 +621,7 @@ public class ColumnCondition Lists.Marker inValuesMarker = (Lists.Marker) condition.value; if (column.type instanceof ListType) { - ListType deserializer = ListType.getInstance(collectionType.valueComparator()); + ListType deserializer = ListType.getInstance(collectionType.valueComparator(), false); for (ByteBuffer buffer : inValuesMarker.bind(options).elements) { if (buffer == null) @@ -586,7 +632,7 @@ public class ColumnCondition } else if (column.type instanceof MapType) { - MapType deserializer = MapType.getInstance(collectionType.nameComparator(), collectionType.valueComparator()); + MapType deserializer = MapType.getInstance(collectionType.nameComparator(), collectionType.valueComparator(), false); for (ByteBuffer buffer : inValuesMarker.bind(options).elements) { if (buffer == null) @@ -597,7 +643,7 @@ public class ColumnCondition } else if (column.type instanceof SetType) { - SetType deserializer = SetType.getInstance(collectionType.valueComparator()); + SetType deserializer = SetType.getInstance(collectionType.valueComparator(), false); for (ByteBuffer buffer : inValuesMarker.bind(options).elements) { if (buffer == null) @@ -618,45 +664,34 @@ public class ColumnCondition { CollectionType type = (CollectionType)column.type; CellName name = current.metadata().comparator.create(rowPrefix, column); - - // copy iterator contents so that we can properly reuse them for each comparison with an IN value - List<Cell> cells = newArrayList(collectionColumns(name, current, now)); - for (Term.Terminal value : inValues) + if (type.isMultiCell()) { - if (CollectionBound.valueAppliesTo(type, cells.iterator(), value, Operator.EQ)) - return true; + // copy iterator contents so that we can properly reuse them for each comparison with an IN value + List<Cell> cells = newArrayList(collectionColumns(name, current, now)); + for (Term.Terminal value : inValues) + { + if (CollectionBound.valueAppliesTo(type, cells.iterator(), value, Operator.EQ)) + return true; + } + return false; } - return false; - } - - @Override - public int hashCode() - { - List<Collection<ByteBuffer>> inValueBuffers = new ArrayList<>(inValues.size()); - switch (((CollectionType)column.type).kind) + else { - case LIST: - for (Term.Terminal term : inValues) - inValueBuffers.add(term == null ? null : ((Lists.Value)term).elements); - break; - case SET: - for (Term.Terminal term : inValues) - inValueBuffers.add(term == null ? null : ((Sets.Value)term).elements); - break; - case MAP: - for (Term.Terminal term : inValues) + Cell cell = current.getColumn(name); + for (Term.Terminal value : inValues) + { + if (value == null) { - if (term != null) - { - inValueBuffers.add(((Maps.Value)term).map.keySet()); - inValueBuffers.add(((Maps.Value)term).map.values()); - } - else - inValueBuffers.add(null); + if (cell == null || !cell.isLive(now)) + return true; } - break; + else if (type.compare(((Term.CollectionTerminal)value).getWithProtocolVersion(Server.VERSION_3), cell.value()) == 0) + { + return true; + } + } + return false; } - return Objects.hashCode(column, inValueBuffers, operator); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/Constants.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/Constants.java b/src/java/org/apache/cassandra/cql3/Constants.java index a8f0120..01fbdf0 100644 --- a/src/java/org/apache/cassandra/cql3/Constants.java +++ b/src/java/org/apache/cassandra/cql3/Constants.java @@ -18,7 +18,6 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; -import java.util.List; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -368,7 +367,7 @@ public abstract class Constants public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { CellName cname = cf.getComparator().create(prefix, column); - if (column.type.isCollection()) + if (column.type.isMultiCell()) cf.addAtom(params.makeRangeTombstone(cname.slice())); else cf.addColumn(params.makeTombstone(cname)); http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/Cql.g ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/Cql.g b/src/java/org/apache/cassandra/cql3/Cql.g index b1c598b..06ba81c 100644 --- a/src/java/org/apache/cassandra/cql3/Cql.g +++ b/src/java/org/apache/cassandra/cql3/Cql.g @@ -575,8 +575,9 @@ createIndexStatement returns [CreateIndexStatement expr] ; indexIdent returns [IndexTarget.Raw id] - : c=cident { $id = IndexTarget.Raw.of(c); } + : c=cident { $id = IndexTarget.Raw.valuesOf(c); } | K_KEYS '(' c=cident ')' { $id = IndexTarget.Raw.keysOf(c); } + | K_FULL '(' c=cident ')' { $id = IndexTarget.Raw.fullCollection(c); } ; @@ -1148,21 +1149,21 @@ native_type returns [CQL3Type t] collection_type returns [CQL3Type.Raw pt] : K_MAP '<' t1=comparatorType ',' t2=comparatorType '>' - { try { + { // if we can't parse either t1 or t2, antlr will "recover" and we may have t1 or t2 null. if (t1 != null && t2 != null) $pt = CQL3Type.Raw.map(t1, t2); - } catch (InvalidRequestException e) { addRecognitionError(e.getMessage()); } } + } | K_LIST '<' t=comparatorType '>' - { try { if (t != null) $pt = CQL3Type.Raw.list(t); } catch (InvalidRequestException e) { addRecognitionError(e.getMessage()); } } + { if (t != null) $pt = CQL3Type.Raw.list(t); } | K_SET '<' t=comparatorType '>' - { try { if (t != null) $pt = CQL3Type.Raw.set(t); } catch (InvalidRequestException e) { addRecognitionError(e.getMessage()); } } + { if (t != null) $pt = CQL3Type.Raw.set(t); } ; tuple_type returns [CQL3Type.Raw t] : K_TUPLE '<' { List<CQL3Type.Raw> types = new ArrayList<>(); } t1=comparatorType { types.add(t1); } (',' tn=comparatorType { types.add(tn); })* - '>' { try { $t = CQL3Type.Raw.tuple(types); } catch (InvalidRequestException e) { addRecognitionError(e.getMessage()); }} + '>' { $t = CQL3Type.Raw.tuple(types); } ; username @@ -1227,6 +1228,7 @@ K_WHERE: W H E R E; K_AND: A N D; K_KEY: K E Y; K_KEYS: K E Y S; +K_FULL: F U L L; K_INSERT: I N S E R T; K_UPDATE: U P D A T E; K_WITH: W I T H; http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/Lists.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/Lists.java b/src/java/org/apache/cassandra/cql3/Lists.java index 9d22364..d5174d1 100644 --- a/src/java/org/apache/cassandra/cql3/Lists.java +++ b/src/java/org/apache/cassandra/cql3/Lists.java @@ -19,8 +19,6 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collections; -import java.util.Iterator; import java.util.List; import java.util.concurrent.atomic.AtomicReference; @@ -34,6 +32,7 @@ import org.apache.cassandra.db.marshal.ListType; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.serializers.CollectionSerializer; import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.transport.Server; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.UUIDGen; @@ -56,7 +55,7 @@ public abstract class Lists public static ColumnSpecification valueSpecOf(ColumnSpecification column) { - return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((ListType)column.type).elements); + return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((ListType)column.type).getElementsType()); } public static class Literal implements Term.Raw @@ -124,7 +123,7 @@ public abstract class Lists } } - public static class Value extends Term.MultiItemTerminal + public static class Value extends Term.MultiItemTerminal implements Term.CollectionTerminal { public final List<ByteBuffer> elements; @@ -143,7 +142,7 @@ public abstract class Lists List<ByteBuffer> elements = new ArrayList<ByteBuffer>(l.size()); for (Object element : l) // elements can be null in lists that represent a set of IN values - elements.add(element == null ? null : type.elements.decompose(element)); + elements.add(element == null ? null : type.getElementsType().decompose(element)); return new Value(elements); } catch (MarshalException e) @@ -154,7 +153,12 @@ public abstract class Lists public ByteBuffer get(QueryOptions options) { - return CollectionSerializer.pack(elements, elements.size(), options.getProtocolVersion()); + return getWithProtocolVersion(options.getProtocolVersion()); + } + + public ByteBuffer getWithProtocolVersion(int protocolVersion) + { + return CollectionSerializer.pack(elements, elements.size(), protocolVersion); } public boolean equals(ListType lt, Value v) @@ -163,7 +167,7 @@ public abstract class Lists return false; for (int i = 0; i < elements.size(); i++) - if (lt.elements.compare(elements.get(i), v.elements.get(i)) != 0) + if (lt.getElementsType().compare(elements.get(i), v.elements.get(i)) != 0) return false; return true; @@ -292,9 +296,12 @@ public abstract class Lists public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { - // delete + append - CellName name = cf.getComparator().create(prefix, column); - cf.addAtom(params.makeTombstoneForOverwrite(name.slice())); + if (column.type.isMultiCell()) + { + // delete + append + CellName name = cf.getComparator().create(prefix, column); + cf.addAtom(params.makeTombstoneForOverwrite(name.slice())); + } Appender.doAppend(t, cf, prefix, column, params); } } @@ -324,6 +331,9 @@ public abstract class Lists public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { + // we should not get here for frozen lists + assert column.type.isMultiCell() : "Attempted to set an individual element on a frozen list"; + ByteBuffer index = idx.bindAndGet(params.options); ByteBuffer value = t.bindAndGet(params.options); @@ -362,23 +372,36 @@ public abstract class Lists public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { + assert column.type.isMultiCell() : "Attempted to append to a frozen list"; doAppend(t, cf, prefix, column, params); } static void doAppend(Term t, ColumnFamily cf, Composite prefix, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException { Term.Terminal value = t.bind(params.options); - // If we append null, do nothing. Note that for Setter, we've - // already removed the previous value so we're good here too - if (value == null) - return; - - assert value instanceof Lists.Value; - List<ByteBuffer> toAdd = ((Lists.Value)value).elements; - for (int i = 0; i < toAdd.size(); i++) + Lists.Value listValue = (Lists.Value)value; + if (column.type.isMultiCell()) { - ByteBuffer uuid = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes()); - cf.addColumn(params.makeColumn(cf.getComparator().create(prefix, column, uuid), toAdd.get(i))); + // If we append null, do nothing. Note that for Setter, we've + // already removed the previous value so we're good here too + if (value == null) + return; + + List<ByteBuffer> toAdd = listValue.elements; + for (int i = 0; i < toAdd.size(); i++) + { + ByteBuffer uuid = ByteBuffer.wrap(UUIDGen.getTimeUUIDBytes()); + cf.addColumn(params.makeColumn(cf.getComparator().create(prefix, column, uuid), toAdd.get(i))); + } + } + else + { + // for frozen lists, we're overwriting the whole cell value + CellName name = cf.getComparator().create(prefix, column); + if (value == null) + cf.addAtom(params.makeTombstone(name)); + else + cf.addColumn(params.makeColumn(name, listValue.getWithProtocolVersion(Server.CURRENT_VERSION))); } } } @@ -392,6 +415,7 @@ public abstract class Lists public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { + assert column.type.isMultiCell() : "Attempted to prepend to a frozen list"; Term.Terminal value = t.bind(params.options); if (value == null) return; @@ -424,6 +448,7 @@ public abstract class Lists public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { + assert column.type.isMultiCell() : "Attempted to delete from a frozen list"; List<Cell> existingList = params.getPrefetchedList(rowKey, column.name); // We want to call bind before possibly returning to reject queries where the value provided is not a list. Term.Terminal value = t.bind(params.options); @@ -464,6 +489,7 @@ public abstract class Lists public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { + assert column.type.isMultiCell() : "Attempted to delete an item by index from a frozen list"; Term.Terminal index = t.bind(params.options); if (index == null) throw new InvalidRequestException("Invalid null value for list index"); http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/Maps.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/Maps.java b/src/java/org/apache/cassandra/cql3/Maps.java index ce0ba2a..5b58833 100644 --- a/src/java/org/apache/cassandra/cql3/Maps.java +++ b/src/java/org/apache/cassandra/cql3/Maps.java @@ -19,7 +19,6 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; import java.util.ArrayList; -import java.util.Collections; import java.util.Comparator; import java.util.HashMap; import java.util.Iterator; @@ -36,6 +35,7 @@ import org.apache.cassandra.db.marshal.MapType; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.serializers.CollectionSerializer; import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.transport.Server; import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; @@ -48,12 +48,12 @@ public abstract class Maps public static ColumnSpecification keySpecOf(ColumnSpecification column) { - return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("key(" + column.name + ")", true), ((MapType)column.type).keys); + return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("key(" + column.name + ")", true), ((MapType)column.type).getKeysType()); } public static ColumnSpecification valueSpecOf(ColumnSpecification column) { - return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((MapType)column.type).values); + return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((MapType)column.type).getValuesType()); } public static class Literal implements Term.Raw @@ -86,7 +86,7 @@ public abstract class Maps values.put(k, v); } - DelayedValue value = new DelayedValue(((MapType)receiver.type).keys, values); + DelayedValue value = new DelayedValue(((MapType)receiver.type).getKeysType(), values); return allTerminal ? value.bind(QueryOptions.DEFAULT) : value; } @@ -134,7 +134,7 @@ public abstract class Maps } } - public static class Value extends Term.Terminal + public static class Value extends Term.Terminal implements Term.CollectionTerminal { public final Map<ByteBuffer, ByteBuffer> map; @@ -152,7 +152,7 @@ public abstract class Maps Map<?, ?> m = (Map<?, ?>)type.getSerializer().deserializeForNativeProtocol(value, version); Map<ByteBuffer, ByteBuffer> map = new LinkedHashMap<ByteBuffer, ByteBuffer>(m.size()); for (Map.Entry<?, ?> entry : m.entrySet()) - map.put(type.keys.decompose(entry.getKey()), type.values.decompose(entry.getValue())); + map.put(type.getKeysType().decompose(entry.getKey()), type.getValuesType().decompose(entry.getValue())); return new Value(map); } catch (MarshalException e) @@ -163,13 +163,18 @@ public abstract class Maps public ByteBuffer get(QueryOptions options) { - List<ByteBuffer> buffers = new ArrayList<ByteBuffer>(2 * map.size()); + return getWithProtocolVersion(options.getProtocolVersion()); + } + + public ByteBuffer getWithProtocolVersion(int protocolVersion) + { + List<ByteBuffer> buffers = new ArrayList<>(2 * map.size()); for (Map.Entry<ByteBuffer, ByteBuffer> entry : map.entrySet()) { buffers.add(entry.getKey()); buffers.add(entry.getValue()); } - return CollectionSerializer.pack(buffers, map.size(), options.getProtocolVersion()); + return CollectionSerializer.pack(buffers, map.size(), protocolVersion); } public boolean equals(MapType mt, Value v) @@ -184,7 +189,7 @@ public abstract class Maps { Map.Entry<ByteBuffer, ByteBuffer> thisEntry = thisIter.next(); Map.Entry<ByteBuffer, ByteBuffer> thatEntry = thatIter.next(); - if (mt.keys.compare(thisEntry.getKey(), thatEntry.getKey()) != 0 || mt.values.compare(thisEntry.getValue(), thatEntry.getValue()) != 0) + if (mt.getKeysType().compare(thisEntry.getKey(), thatEntry.getKey()) != 0 || mt.getValuesType().compare(thisEntry.getValue(), thatEntry.getValue()) != 0) return false; } @@ -266,9 +271,12 @@ public abstract class Maps public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { - // delete + put - CellName name = cf.getComparator().create(prefix, column); - cf.addAtom(params.makeTombstoneForOverwrite(name.slice())); + if (column.type.isMultiCell()) + { + // delete + put + CellName name = cf.getComparator().create(prefix, column); + cf.addAtom(params.makeTombstoneForOverwrite(name.slice())); + } Putter.doPut(t, cf, prefix, column, params); } } @@ -292,6 +300,7 @@ public abstract class Maps public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { + assert column.type.isMultiCell() : "Attempted to set a value for a single key on a frozen map"; ByteBuffer key = k.bindAndGet(params.options); ByteBuffer value = t.bindAndGet(params.options); if (key == null) @@ -325,21 +334,33 @@ public abstract class Maps public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { + assert column.type.isMultiCell() : "Attempted to add items to a frozen map"; doPut(t, cf, prefix, column, params); } static void doPut(Term t, ColumnFamily cf, Composite prefix, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException { Term.Terminal value = t.bind(params.options); - if (value == null) - return; - assert value instanceof Maps.Value; - - Map<ByteBuffer, ByteBuffer> toAdd = ((Maps.Value)value).map; - for (Map.Entry<ByteBuffer, ByteBuffer> entry : toAdd.entrySet()) + Maps.Value mapValue = (Maps.Value) value; + if (column.type.isMultiCell()) + { + if (value == null) + return; + + for (Map.Entry<ByteBuffer, ByteBuffer> entry : mapValue.map.entrySet()) + { + CellName cellName = cf.getComparator().create(prefix, column, entry.getKey()); + cf.addColumn(params.makeColumn(cellName, entry.getValue())); + } + } + else { - CellName cellName = cf.getComparator().create(prefix, column, entry.getKey()); - cf.addColumn(params.makeColumn(cellName, entry.getValue())); + // for frozen maps, we're overwriting the whole cell + CellName cellName = cf.getComparator().create(prefix, column); + if (value == null) + cf.addAtom(params.makeTombstone(cellName)); + else + cf.addColumn(params.makeColumn(cellName, mapValue.getWithProtocolVersion(Server.CURRENT_VERSION))); } } } @@ -353,6 +374,7 @@ public abstract class Maps public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { + assert column.type.isMultiCell() : "Attempted to delete a single key in a frozen map"; Term.Terminal key = t.bind(params.options); if (key == null) throw new InvalidRequestException("Invalid null map key"); http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/Operation.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/Operation.java b/src/java/org/apache/cassandra/cql3/Operation.java index 32b6a12..816acb2 100644 --- a/src/java/org/apache/cassandra/cql3/Operation.java +++ b/src/java/org/apache/cassandra/cql3/Operation.java @@ -163,7 +163,7 @@ public abstract class Operation if (receiver.type instanceof CounterColumnType) throw new InvalidRequestException(String.format("Cannot set the value of counter column %s (counters can only be incremented/decremented, not set)", receiver.name)); - if (!(receiver.type instanceof CollectionType)) + if (!(receiver.type.isCollection())) return new Constants.Setter(receiver, v); switch (((CollectionType)receiver.type).kind) @@ -206,6 +206,8 @@ public abstract class Operation { if (!(receiver.type instanceof CollectionType)) throw new InvalidRequestException(String.format("Invalid operation (%s) for non collection column %s", toString(receiver), receiver.name)); + else if (!(receiver.type.isMultiCell())) + throw new InvalidRequestException(String.format("Invalid operation (%s) for frozen collection column %s", toString(receiver), receiver.name)); switch (((CollectionType)receiver.type).kind) { @@ -255,6 +257,8 @@ public abstract class Operation throw new InvalidRequestException(String.format("Invalid operation (%s) for non counter column %s", toString(receiver), receiver.name)); return new Constants.Adder(receiver, v); } + else if (!(receiver.type.isMultiCell())) + throw new InvalidRequestException(String.format("Invalid operation (%s) for frozen collection column %s", toString(receiver), receiver.name)); switch (((CollectionType)receiver.type).kind) { @@ -296,6 +300,8 @@ public abstract class Operation throw new InvalidRequestException(String.format("Invalid operation (%s) for non counter column %s", toString(receiver), receiver.name)); return new Constants.Substracter(receiver, value.prepare(keyspace, receiver)); } + else if (!(receiver.type.isMultiCell())) + throw new InvalidRequestException(String.format("Invalid operation (%s) for frozen collection column %s", toString(receiver), receiver.name)); switch (((CollectionType)receiver.type).kind) { @@ -308,7 +314,7 @@ public abstract class Operation ColumnSpecification vr = new ColumnSpecification(receiver.ksName, receiver.cfName, receiver.name, - SetType.getInstance(((MapType)receiver.type).keys)); + SetType.getInstance(((MapType)receiver.type).getKeysType(), false)); return new Sets.Discarder(receiver, value.prepare(keyspace, vr)); } throw new AssertionError(); @@ -340,6 +346,8 @@ public abstract class Operation if (!(receiver.type instanceof ListType)) throw new InvalidRequestException(String.format("Invalid operation (%s) for non list column %s", toString(receiver), receiver.name)); + else if (!(receiver.type.isMultiCell())) + throw new InvalidRequestException(String.format("Invalid operation (%s) for frozen list column %s", toString(receiver), receiver.name)); return new Lists.Prepender(receiver, v); } @@ -394,8 +402,10 @@ public abstract class Operation public Operation prepare(String keyspace, ColumnDefinition receiver) throws InvalidRequestException { - if (!(receiver.type instanceof CollectionType)) + if (!(receiver.type.isCollection())) throw new InvalidRequestException(String.format("Invalid deletion operation for non collection column %s", receiver.name)); + else if (!(receiver.type.isMultiCell())) + throw new InvalidRequestException(String.format("Invalid deletion operation for frozen collection column %s", receiver.name)); switch (((CollectionType)receiver.type).kind) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/Sets.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/Sets.java b/src/java/org/apache/cassandra/cql3/Sets.java index 315d7d3..c5a8e05 100644 --- a/src/java/org/apache/cassandra/cql3/Sets.java +++ b/src/java/org/apache/cassandra/cql3/Sets.java @@ -18,15 +18,7 @@ package org.apache.cassandra.cql3; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Comparator; -import java.util.HashSet; -import java.util.Iterator; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; -import java.util.TreeSet; +import java.util.*; import com.google.common.base.Joiner; @@ -34,11 +26,13 @@ import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.ColumnFamily; import org.apache.cassandra.db.composites.CellName; import org.apache.cassandra.db.composites.Composite; +import org.apache.cassandra.db.marshal.AbstractType; import org.apache.cassandra.db.marshal.MapType; import org.apache.cassandra.db.marshal.SetType; import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.serializers.CollectionSerializer; import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.transport.Server; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.FBUtilities; @@ -51,7 +45,7 @@ public abstract class Sets public static ColumnSpecification valueSpecOf(ColumnSpecification column) { - return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((SetType)column.type).elements); + return new ColumnSpecification(column.ksName, column.cfName, new ColumnIdentifier("value(" + column.name + ")", true), ((SetType)column.type).getElementsType()); } public static class Literal implements Term.Raw @@ -87,7 +81,7 @@ public abstract class Sets values.add(t); } - DelayedValue value = new DelayedValue(((SetType)receiver.type).elements, values); + DelayedValue value = new DelayedValue(((SetType)receiver.type).getElementsType(), values); return allTerminal ? value.bind(QueryOptions.DEFAULT) : value; } @@ -97,7 +91,7 @@ public abstract class Sets { // We've parsed empty maps as a set literal to break the ambiguity so // handle that case now - if (receiver.type instanceof MapType && elements.isEmpty()) + if ((receiver.type instanceof MapType) && elements.isEmpty()) return; throw new InvalidRequestException(String.format("Invalid set literal for %s of type %s", receiver.name, receiver.type.asCQL3Type())); @@ -131,11 +125,11 @@ public abstract class Sets } } - public static class Value extends Term.Terminal + public static class Value extends Term.Terminal implements Term.CollectionTerminal { - public final Set<ByteBuffer> elements; + public final SortedSet<ByteBuffer> elements; - public Value(Set<ByteBuffer> elements) + public Value(SortedSet<ByteBuffer> elements) { this.elements = elements; } @@ -147,9 +141,9 @@ public abstract class Sets // Collections have this small hack that validate cannot be called on a serialized object, // but compose does the validation (so we're fine). Set<?> s = (Set<?>)type.getSerializer().deserializeForNativeProtocol(value, version); - Set<ByteBuffer> elements = new LinkedHashSet<ByteBuffer>(s.size()); + SortedSet<ByteBuffer> elements = new TreeSet<ByteBuffer>(type.getElementsType()); for (Object element : s) - elements.add(type.elements.decompose(element)); + elements.add(type.getElementsType().decompose(element)); return new Value(elements); } catch (MarshalException e) @@ -160,7 +154,12 @@ public abstract class Sets public ByteBuffer get(QueryOptions options) { - return CollectionSerializer.pack(new ArrayList<ByteBuffer>(elements), elements.size(), options.getProtocolVersion()); + return getWithProtocolVersion(options.getProtocolVersion()); + } + + public ByteBuffer getWithProtocolVersion(int protocolVersion) + { + return CollectionSerializer.pack(new ArrayList<>(elements), elements.size(), protocolVersion); } public boolean equals(SetType st, Value v) @@ -170,8 +169,9 @@ public abstract class Sets Iterator<ByteBuffer> thisIter = elements.iterator(); Iterator<ByteBuffer> thatIter = v.elements.iterator(); + AbstractType elementsType = st.getElementsType(); while (thisIter.hasNext()) - if (st.elements.compare(thisIter.next(), thatIter.next()) != 0) + if (elementsType.compare(thisIter.next(), thatIter.next()) != 0) return false; return true; @@ -202,7 +202,7 @@ public abstract class Sets public Value bind(QueryOptions options) throws InvalidRequestException { - Set<ByteBuffer> buffers = new TreeSet<ByteBuffer>(comparator); + SortedSet<ByteBuffer> buffers = new TreeSet<>(comparator); for (Term t : elements) { ByteBuffer bytes = t.bindAndGet(options); @@ -246,9 +246,12 @@ public abstract class Sets public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { - // delete + add - CellName name = cf.getComparator().create(prefix, column); - cf.addAtom(params.makeTombstoneForOverwrite(name.slice())); + if (column.type.isMultiCell()) + { + // delete + add + CellName name = cf.getComparator().create(prefix, column); + cf.addAtom(params.makeTombstoneForOverwrite(name.slice())); + } Adder.doAdd(t, cf, prefix, column, params); } } @@ -262,22 +265,35 @@ public abstract class Sets public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { + assert column.type.isMultiCell() : "Attempted to add items to a frozen set"; + doAdd(t, cf, prefix, column, params); } static void doAdd(Term t, ColumnFamily cf, Composite prefix, ColumnDefinition column, UpdateParameters params) throws InvalidRequestException { Term.Terminal value = t.bind(params.options); - if (value == null) - return; - - assert value instanceof Sets.Value : value; + Sets.Value setValue = (Sets.Value)value; + if (column.type.isMultiCell()) + { + if (value == null) + return; - Set<ByteBuffer> toAdd = ((Sets.Value)value).elements; - for (ByteBuffer bb : toAdd) + Set<ByteBuffer> toAdd = setValue.elements; + for (ByteBuffer bb : toAdd) + { + CellName cellName = cf.getComparator().create(prefix, column, bb); + cf.addColumn(params.makeColumn(cellName, ByteBufferUtil.EMPTY_BYTE_BUFFER)); + } + } + else { - CellName cellName = cf.getComparator().create(prefix, column, bb); - cf.addColumn(params.makeColumn(cellName, ByteBufferUtil.EMPTY_BYTE_BUFFER)); + // for frozen sets, we're overwriting the whole cell + CellName cellName = cf.getComparator().create(prefix, column); + if (value == null) + cf.addAtom(params.makeTombstone(cellName)); + else + cf.addColumn(params.makeColumn(cellName, ((Value) value).getWithProtocolVersion(Server.CURRENT_VERSION))); } } } @@ -292,6 +308,8 @@ public abstract class Sets public void execute(ByteBuffer rowKey, ColumnFamily cf, Composite prefix, UpdateParameters params) throws InvalidRequestException { + assert column.type.isMultiCell() : "Attempted to remove items from a frozen set"; + Term.Terminal value = t.bind(params.options); if (value == null) return; http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/Term.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/Term.java b/src/java/org/apache/cassandra/cql3/Term.java index e5206c8..1587df1 100644 --- a/src/java/org/apache/cassandra/cql3/Term.java +++ b/src/java/org/apache/cassandra/cql3/Term.java @@ -138,6 +138,12 @@ public interface Term public abstract List<ByteBuffer> getElements(); } + public interface CollectionTerminal + { + /** Gets the value of the collection when serialized with the given protocol version format */ + public ByteBuffer getWithProtocolVersion(int protocolVersion); + } + /** * A non terminal term, i.e. a term that can only be reduce to a byte buffer * at execution time. http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/Tuples.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/Tuples.java b/src/java/org/apache/cassandra/cql3/Tuples.java index 883cc60..4f1f141 100644 --- a/src/java/org/apache/cassandra/cql3/Tuples.java +++ b/src/java/org/apache/cassandra/cql3/Tuples.java @@ -248,13 +248,13 @@ public class Tuples // but the deserialization does the validation (so we're fine). List<?> l = (List<?>)type.getSerializer().deserializeForNativeProtocol(value, options.getProtocolVersion()); - assert type.elements instanceof TupleType; - TupleType tupleType = (TupleType) type.elements; + assert type.getElementsType() instanceof TupleType; + TupleType tupleType = (TupleType) type.getElementsType(); // type.split(bytes) List<List<ByteBuffer>> elements = new ArrayList<>(l.size()); for (Object element : l) - elements.add(Arrays.asList(tupleType.split(type.elements.decompose(element)))); + elements.add(Arrays.asList(tupleType.split(type.getElementsType().decompose(element)))); return new InValue(elements); } catch (MarshalException e) @@ -337,15 +337,16 @@ public class Tuples if (i < receivers.size() - 1) inName.append(","); - if (receiver.type instanceof CollectionType) - throw new InvalidRequestException("Collection columns do not support IN relations"); + if (receiver.type.isCollection() && receiver.type.isMultiCell()) + throw new InvalidRequestException("Non-frozen collection columns do not support IN relations"); + types.add(receiver.type); } inName.append(')'); ColumnIdentifier identifier = new ColumnIdentifier(inName.toString(), true); TupleType type = new TupleType(types); - return new ColumnSpecification(receivers.get(0).ksName, receivers.get(0).cfName, identifier, ListType.getInstance(type)); + return new ColumnSpecification(receivers.get(0).ksName, receivers.get(0).cfName, identifier, ListType.getInstance(type, false)); } public AbstractMarker prepare(String keyspace, List<? extends ColumnSpecification> receivers) throws InvalidRequestException http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/UntypedResultSet.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java index 42d0cb8..fef70fb 100644 --- a/src/java/org/apache/cassandra/cql3/UntypedResultSet.java +++ b/src/java/org/apache/cassandra/cql3/UntypedResultSet.java @@ -267,19 +267,19 @@ public abstract class UntypedResultSet implements Iterable<UntypedResultSet.Row> public <T> Set<T> getSet(String column, AbstractType<T> type) { ByteBuffer raw = data.get(column); - return raw == null ? null : SetType.getInstance(type).compose(raw); + return raw == null ? null : SetType.getInstance(type, true).compose(raw); } public <T> List<T> getList(String column, AbstractType<T> type) { ByteBuffer raw = data.get(column); - return raw == null ? null : ListType.getInstance(type).compose(raw); + return raw == null ? null : ListType.getInstance(type, true).compose(raw); } public <K, V> Map<K, V> getMap(String column, AbstractType<K> keyType, AbstractType<V> valueType) { ByteBuffer raw = data.get(column); - return raw == null ? null : MapType.getInstance(keyType, valueType).compose(raw); + return raw == null ? null : MapType.getInstance(keyType, valueType, true).compose(raw); } public List<ColumnSpecification> getColumns() http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/UpdateParameters.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/UpdateParameters.java b/src/java/org/apache/cassandra/cql3/UpdateParameters.java index 62ec09c..74c3214 100644 --- a/src/java/org/apache/cassandra/cql3/UpdateParameters.java +++ b/src/java/org/apache/cassandra/cql3/UpdateParameters.java @@ -97,6 +97,6 @@ public class UpdateParameters return Collections.emptyList(); CQL3Row row = prefetchedLists.get(rowKey); - return row == null ? Collections.<Cell>emptyList() : row.getCollection(cql3ColumnName); + return row == null ? Collections.<Cell>emptyList() : row.getMultiCellColumn(cql3ColumnName); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/ee55f361/src/java/org/apache/cassandra/cql3/UserTypes.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/UserTypes.java b/src/java/org/apache/cassandra/cql3/UserTypes.java index 9d66c16..c92bc49 100644 --- a/src/java/org/apache/cassandra/cql3/UserTypes.java +++ b/src/java/org/apache/cassandra/cql3/UserTypes.java @@ -24,6 +24,7 @@ import org.apache.cassandra.db.marshal.CollectionType; import org.apache.cassandra.db.marshal.UserType; import org.apache.cassandra.db.marshal.UTF8Type; import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.transport.Server; /** * Static helper methods and classes for user types. @@ -171,7 +172,7 @@ public abstract class UserTypes buffers[i] = values.get(i).bindAndGet(options); // Inside UDT values, we must force the serialization of collections to v3 whatever protocol // version is in use since we're going to store directly that serialized value. - if (version < 3 && type.fieldType(i).isCollection() && buffers[i] != null) + if (version < Server.VERSION_3 && type.fieldType(i).isCollection() && buffers[i] != null) buffers[i] = ((CollectionType)type.fieldType(i)).getSerializer().reserializeToV3(buffers[i]); } return buffers;