Updated Branches: refs/heads/trunk a7b80d78a -> b505b90a8
Add custom secondary index support to CQL3 patch by Aleksey Yeschenko; reviewed by Sylvain Lebresne for CASSANDRA-5484 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/64c0d1e7 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/64c0d1e7 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/64c0d1e7 Branch: refs/heads/trunk Commit: 64c0d1e7d508367c91da8f5fa835635e0524a37d Parents: 1405912 Author: Aleksey Yeschenko <[email protected]> Authored: Fri May 3 17:59:33 2013 +0300 Committer: Aleksey Yeschenko <[email protected]> Committed: Fri May 3 17:59:33 2013 +0300 ---------------------------------------------------------------------- CHANGES.txt | 1 + NEWS.txt | 10 ++ bin/cqlsh | 2 +- doc/cql3/CQL.textile | 9 ++- pylib/cqlshlib/cql3handling.py | 7 +- src/java/org/apache/cassandra/cql3/Cql.g | 11 ++- .../org/apache/cassandra/cql3/IndexPropDefs.java | 68 ++++++++++ .../org/apache/cassandra/cql3/QueryProcessor.java | 2 +- .../cql3/statements/CreateIndexStatement.java | 96 ++++++++------ 9 files changed, 157 insertions(+), 49 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/64c0d1e7/CHANGES.txt ---------------------------------------------------------------------- diff --git a/CHANGES.txt b/CHANGES.txt index 7429401..21be695 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -19,6 +19,7 @@ * Disallow renaming columns one at a time for thrift table in CQL3 (CASSANDRA-5531) * cqlsh: add CLUSTERING ORDER BY support to DESCRIBE (CASSANDRA-5528) + * Add custom secondary index support to CQL3 (CASSANDRA-5484) Merged from 1.1 * Add retry mechanism to OTC for non-droppable_verbs (CASSANDRA-5393) * Use allocator information to improve memtable memory usage estimate http://git-wip-us.apache.org/repos/asf/cassandra/blob/64c0d1e7/NEWS.txt ---------------------------------------------------------------------- diff --git a/NEWS.txt b/NEWS.txt index 5321338..905e7de 100644 --- a/NEWS.txt +++ b/NEWS.txt @@ -8,6 +8,16 @@ upgrade, just in case you need to roll back to the previous version. (Cassandra version X + 1 will always be able to read data files created by version X, but the inverse is not necessarily the case.) +1.2.5 +===== + +Features +-------- + - Custom secondary index support has been added to CQL3. Refer to + CQL3 documentation (http://cassandra.apache.org/doc/cql3/CQL.html) + for details and examples. + + 1.2.4 ===== http://git-wip-us.apache.org/repos/asf/cassandra/blob/64c0d1e7/bin/cqlsh ---------------------------------------------------------------------- diff --git a/bin/cqlsh b/bin/cqlsh index 853e3fd..aa894b6 100755 --- a/bin/cqlsh +++ b/bin/cqlsh @@ -32,7 +32,7 @@ exit 1 from __future__ import with_statement description = "CQL Shell for Apache Cassandra" -version = "3.0.0" +version = "3.0.1" from StringIO import StringIO from itertools import groupby http://git-wip-us.apache.org/repos/asf/cassandra/blob/64c0d1e7/doc/cql3/CQL.textile ---------------------------------------------------------------------- diff --git a/doc/cql3/CQL.textile b/doc/cql3/CQL.textile index 83ed82a..00ac447 100644 --- a/doc/cql3/CQL.textile +++ b/doc/cql3/CQL.textile @@ -391,13 +391,15 @@ h3(#createIndexStmt). CREATE INDEX __Syntax:__ -bc(syntax). <create-index-stmt> ::= CREATE INDEX <identifier>? ON <tablename> '(' <identifier> ')' +bc(syntax). <create-index-stmt> ::= CREATE ( CUSTOM )? INDEX <identifier>? ON <tablename> '(' <identifier> ')' + ( WITH <properties> )? __Sample:__ bc(sample). CREATE INDEX userIndex ON NerdMovies (user); CREATE INDEX ON Mutants (abilityId); +CREATE CUSTOM INDEX ON users (email) WITH options = {'class': 'path.to.the.IndexClass'}; The @CREATE INDEX@ statement is used to create a new (automatic) secondary index for a given (existing) column in a given table. A name for the index itself can be specified before the @ON@ keyword, if desired. If data already exists for the column, it will be indexed during the execution of this statement. After the index is created, new data for the column is indexed automatically at insertion time. @@ -991,6 +993,7 @@ CQL distinguishes between _reserved_ and _non-reserved_ keywords. Reserved keywo | @GRANT@ | yes | | @IN@ | yes | | @INDEX@ | yes | +| @CUSTOM@ | no | | @INSERT@ | yes | | @INT@ | no | | @INTO@ | yes | @@ -1045,6 +1048,10 @@ h2(#changes). Changes The following describes the addition/changes brought for each version of CQL. +h3. 3.0.3 + +* Support for custom "secondary indexes":#createIndexStmt has been added. + h3. 3.0.2 * Type validation for the "constants":#constants has been fixed. For instance, the implementation used to allow @'2'@ as a valid value for an @int@ column (interpreting it has the equivalent of @2@), or @42@ as a valid @blob@ value (in which case @42@ was interpreted as an hexadecimal representation of the blob). This is no longer the case, type validation of constants is now more strict. See the "data types":#types section for details on which constant is allowed for which type. http://git-wip-us.apache.org/repos/asf/cassandra/blob/64c0d1e7/pylib/cqlshlib/cql3handling.py ---------------------------------------------------------------------- diff --git a/pylib/cqlshlib/cql3handling.py b/pylib/cqlshlib/cql3handling.py index 15f7c54..5f4db2c 100644 --- a/pylib/cqlshlib/cql3handling.py +++ b/pylib/cqlshlib/cql3handling.py @@ -49,11 +49,11 @@ class Cql3ParsingRuleSet(CqlParsingRuleSet): 'keyspace', 'schema', 'columnfamily', 'table', 'index', 'on', 'drop', 'primary', 'into', 'values', 'timestamp', 'ttl', 'alter', 'add', 'type', 'compact', 'storage', 'order', 'by', 'asc', 'desc', 'clustering', - 'token', 'writetime', 'map', 'list', 'to' + 'token', 'writetime', 'map', 'list', 'to', 'custom' )) unreserved_keywords = set(( - 'key', 'clustering', 'ttl', 'compact', 'storage', 'type', 'values' + 'key', 'clustering', 'ttl', 'compact', 'storage', 'type', 'values', 'custom' )) columnfamily_options = ( @@ -1202,8 +1202,9 @@ def create_cf_composite_primary_key_comma_completer(ctxt, cass): return [','] syntax_rules += r''' -<createIndexStatement> ::= "CREATE" "INDEX" indexname=<identifier>? "ON" +<createIndexStatement> ::= "CREATE" "CUSTOM"? "INDEX" indexname=<identifier>? "ON" cf=<columnFamilyName> "(" col=<cident> ")" + ( "WITH" "options = {'class': " <stringLiteral> "}" )? ; ''' http://git-wip-us.apache.org/repos/asf/cassandra/blob/64c0d1e7/src/java/org/apache/cassandra/cql3/Cql.g ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/Cql.g b/src/java/org/apache/cassandra/cql3/Cql.g index a91e529..34e88a1 100644 --- a/src/java/org/apache/cassandra/cql3/Cql.g +++ b/src/java/org/apache/cassandra/cql3/Cql.g @@ -454,8 +454,13 @@ cfamOrdering[CreateColumnFamilyStatement.RawStatement expr] * CREATE INDEX [indexName] ON columnFamily (columnName); */ createIndexStatement returns [CreateIndexStatement expr] - : K_CREATE K_INDEX (idxName=IDENT)? K_ON cf=columnFamilyName '(' id=cident ')' - { $expr = new CreateIndexStatement(cf, $idxName.text, id); } + @init { + boolean isCustom = false; + IndexPropDefs props = new IndexPropDefs(); + } + : K_CREATE (K_CUSTOM { isCustom = true; })? K_INDEX (idxName=IDENT)? K_ON cf=columnFamilyName '(' id=cident ')' + ( K_WITH properties[props] )? + { $expr = new CreateIndexStatement(cf, $idxName.text, id, isCustom, props); } ; /** @@ -873,6 +878,7 @@ unreserved_function_keyword returns [String str] | K_SUPERUSER | K_NOSUPERUSER | K_PASSWORD + | K_CUSTOM ) { $str = $k.text; } | t=native_type { $str = t.toString(); } ; @@ -906,6 +912,7 @@ K_KEYSPACES: K E Y S P A C E S; K_COLUMNFAMILY:( C O L U M N F A M I L Y | T A B L E ); K_INDEX: I N D E X; +K_CUSTOM: C U S T O M; K_ON: O N; K_TO: T O; K_DROP: D R O P; http://git-wip-us.apache.org/repos/asf/cassandra/blob/64c0d1e7/src/java/org/apache/cassandra/cql3/IndexPropDefs.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/IndexPropDefs.java b/src/java/org/apache/cassandra/cql3/IndexPropDefs.java new file mode 100644 index 0000000..fca12c8 --- /dev/null +++ b/src/java/org/apache/cassandra/cql3/IndexPropDefs.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.cql3; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Map; +import java.util.Set; + +import org.apache.cassandra.db.index.SecondaryIndex; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.exceptions.RequestValidationException; +import org.apache.cassandra.exceptions.SyntaxException; + +public class IndexPropDefs extends PropertyDefinitions +{ + public static final String KW_OPTIONS = "options"; + + public static final Set<String> keywords = new HashSet<String>(); + public static final Set<String> obsoleteKeywords = new HashSet<String>(); + + public static final String INDEX_CLASS_KEY = "class"; + + static + { + keywords.add(KW_OPTIONS); + } + + public void validate(boolean isCustom) throws RequestValidationException + { + validate(keywords, obsoleteKeywords); + if (isCustom && !getOptions().containsKey(SecondaryIndex.CUSTOM_INDEX_OPTION_NAME)) + throw new InvalidRequestException(String.format("Custom index requires '%s' option to be specified", INDEX_CLASS_KEY)); + if (!isCustom && !getOptions().isEmpty()) + throw new InvalidRequestException(String.format("Only custom indexes can currently be parametrized")); + } + + public Map<String, String> getOptions() throws SyntaxException + { + Map<String, String> options = getMap(KW_OPTIONS); + + if (options == null) + return Collections.emptyMap(); + + if (!options.isEmpty() && options.containsKey(INDEX_CLASS_KEY)) + { + options.put(SecondaryIndex.CUSTOM_INDEX_OPTION_NAME, options.get(INDEX_CLASS_KEY)); + options.remove(INDEX_CLASS_KEY); + } + + return options; + } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/64c0d1e7/src/java/org/apache/cassandra/cql3/QueryProcessor.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/QueryProcessor.java b/src/java/org/apache/cassandra/cql3/QueryProcessor.java index cd4ff25..4e74fee 100644 --- a/src/java/org/apache/cassandra/cql3/QueryProcessor.java +++ b/src/java/org/apache/cassandra/cql3/QueryProcessor.java @@ -41,7 +41,7 @@ import org.apache.cassandra.utils.SemanticVersion; public class QueryProcessor { - public static final SemanticVersion CQL_VERSION = new SemanticVersion("3.0.1"); + public static final SemanticVersion CQL_VERSION = new SemanticVersion("3.0.3"); private static final Logger logger = LoggerFactory.getLogger(QueryProcessor.class); http://git-wip-us.apache.org/repos/asf/cassandra/blob/64c0d1e7/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java b/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java index 4e0f536..b371c11 100644 --- a/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/CreateIndexStatement.java @@ -27,15 +27,15 @@ import org.slf4j.LoggerFactory; import org.apache.cassandra.auth.Permission; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; -import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.config.Schema; +import org.apache.cassandra.db.index.SecondaryIndex; +import org.apache.cassandra.exceptions.*; import org.apache.cassandra.cql3.*; import org.apache.cassandra.db.index.composites.CompositesIndex; import org.apache.cassandra.db.marshal.CompositeType; -import org.apache.cassandra.exceptions.UnauthorizedException; import org.apache.cassandra.service.ClientState; import org.apache.cassandra.service.MigrationManager; import org.apache.cassandra.thrift.IndexType; -import org.apache.cassandra.exceptions.InvalidRequestException; import org.apache.cassandra.thrift.ThriftValidation; import org.apache.cassandra.transport.messages.ResultMessage; @@ -46,12 +46,16 @@ public class CreateIndexStatement extends SchemaAlteringStatement private final String indexName; private final ColumnIdentifier columnName; + private final boolean isCustom; + private final IndexPropDefs props; - public CreateIndexStatement(CFName name, String indexName, ColumnIdentifier columnName) + public CreateIndexStatement(CFName name, String indexName, ColumnIdentifier columnName, boolean isCustom, IndexPropDefs props) { super(name); this.indexName = indexName; this.columnName = columnName; + this.isCustom = isCustom; + this.props = props; } public void checkAccess(ClientState state) throws UnauthorizedException, InvalidRequestException @@ -59,59 +63,69 @@ public class CreateIndexStatement extends SchemaAlteringStatement state.hasColumnFamilyAccess(keyspace(), columnFamily(), Permission.ALTER); } - public void announceMigration() throws InvalidRequestException, ConfigurationException + @Override + public void validate(ClientState state) throws RequestValidationException { - CFMetaData oldCfm = ThriftValidation.validateColumnFamily(keyspace(), columnFamily()); - boolean columnExists = false; - // Mutating oldCfm directly would be bad so cloning. - CFMetaData cfm = oldCfm.clone(); - CFDefinition cfDef = oldCfm.getCfDef(); + CFMetaData cfm = ThriftValidation.validateColumnFamily(keyspace(), columnFamily()); + CFDefinition.Name name = cfm.getCfDef().get(columnName); + + if (name == null) + throw new InvalidRequestException("No column definition found for column " + columnName); - for (ColumnDefinition cd : cfm.getColumn_metadata().values()) + switch (name.kind) { - if (cd.name.equals(columnName.key)) - { + case KEY_ALIAS: + case COLUMN_ALIAS: + throw new InvalidRequestException(String.format("Cannot create index on PRIMARY KEY part %s", columnName)); + case VALUE_ALIAS: + throw new InvalidRequestException(String.format("Cannot create index on column %s of compact CF", columnName)); + case COLUMN_METADATA: + ColumnDefinition cd = cfm.getColumnDefinition(columnName.key); + if (cd.getIndexType() != null) throw new InvalidRequestException("Index already exists"); - if (logger.isDebugEnabled()) - logger.debug("Updating column {} definition for index {}", columnName, indexName); if (cd.getValidator().isCollection()) throw new InvalidRequestException("Indexes on collections are no yet supported"); - if (cfDef.isComposite) - { - CompositeType composite = (CompositeType)cfm.comparator; - Map<String, String> opts = new HashMap<String, String>(); - opts.put(CompositesIndex.PREFIX_SIZE_OPTION, String.valueOf(composite.types.size() - (cfDef.hasCollections ? 2 : 1))); - cd.setIndexType(IndexType.COMPOSITES, opts); - } - else - { - cd.setIndexType(IndexType.KEYS, Collections.<String, String>emptyMap()); - } - cd.setIndexName(indexName); - columnExists = true; + props.validate(isCustom); break; - } + default: + throw new AssertionError(); } - if (!columnExists) + } + + public void announceMigration() throws InvalidRequestException, ConfigurationException + { + logger.debug("Updating column {} definition for index {}", columnName, indexName); + CFMetaData cfm = Schema.instance.getCFMetaData(keyspace(), columnFamily()).clone(); + CFDefinition cfDef = cfm.getCfDef(); + ColumnDefinition cd = cfm.getColumnDefinition(columnName.key); + + if (isCustom) { - CFDefinition.Name name = cfDef.get(columnName); - if (name != null) + try { - switch (name.kind) - { - case KEY_ALIAS: - case COLUMN_ALIAS: - throw new InvalidRequestException(String.format("Cannot create index on PRIMARY KEY part %s", columnName)); - case VALUE_ALIAS: - throw new InvalidRequestException(String.format("Cannot create index on column %s of compact CF", columnName)); - } + cd.setIndexType(IndexType.CUSTOM, props.getOptions()); } - throw new InvalidRequestException("No column definition found for column " + columnName); + catch (SyntaxException e) + { + throw new AssertionError(); // can't happen after validation. + } + } + else if (cfDef.isComposite) + { + CompositeType composite = (CompositeType)cfm.comparator; + Map<String, String> opts = new HashMap<String, String>(); + opts.put(CompositesIndex.PREFIX_SIZE_OPTION, String.valueOf(composite.types.size() - (cfDef.hasCollections ? 2 : 1))); + cd.setIndexType(IndexType.COMPOSITES, opts); + } + else + { + cd.setIndexType(IndexType.KEYS, Collections.<String, String>emptyMap()); } + cd.setIndexName(indexName); cfm.addDefaultIndexNames(); MigrationManager.announceColumnFamilyUpdate(cfm); }
