Merge branch 'cassandra-2.2' into cassandra-3.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/943e732c Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/943e732c Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/943e732c Branch: refs/heads/cassandra-3.0 Commit: 943e732caeb1da59e5e9af5479cd9d75ca9d9030 Parents: 474d3bf dff7b44 Author: Sam Tunnicliffe <s...@beobal.com> Authored: Fri Apr 29 11:50:03 2016 +0100 Committer: Sam Tunnicliffe <s...@beobal.com> Committed: Fri Apr 29 11:55:07 2016 +0100 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../restrictions/PrimaryKeyRestrictionSet.java | 71 +++++++++++++++++++- .../restrictions/StatementRestrictions.java | 34 +++++++--- .../SelectSingleColumnRelationTest.java | 4 ++ 4 files changed, 97 insertions(+), 13 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/943e732c/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 268d011,fb06cd6..3184cce --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,19 -1,5 +1,20 @@@ -2.2.7 +3.0.6 + * Fix paging for range queries where all clustering columns are specified (CASSANDRA-11669) + * Don't require HEAP_NEW_SIZE to be set when using G1 (CASSANDRA-11600) + * Fix sstabledump not showing cells after tombstone marker (CASSANDRA-11654) + * Ignore all LocalStrategy keyspaces for streaming and other related + operations (CASSANDRA-11627) + * Ensure columnfilter covers indexed columns for thrift 2i queries (CASSANDRA-11523) + * Only open one sstable scanner per sstable (CASSANDRA-11412) + * Option to specify ProtocolVersion in cassandra-stress (CASSANDRA-11410) + * ArithmeticException in avgFunctionForDecimal (CASSANDRA-11485) + * LogAwareFileLister should only use OLD sstable files in current folder to determine disk consistency (CASSANDRA-11470) + * Notify indexers of expired rows during compaction (CASSANDRA-11329) + * Properly respond with ProtocolError when a v1/v2 native protocol + header is received (CASSANDRA-11464) + * Validate that num_tokens and initial_token are consistent with one another (CASSANDRA-10120) +Merged from 2.2: + * Restore ability to filter on clustering columns when using a 2i (CASSANDRA-11510) * JSON datetime formatting needs timezone (CASSANDRA-11137) * Fix is_dense recalculation for Thrift-updated tables (CASSANDRA-11502) * Remove unnescessary file existence check during anticompaction (CASSANDRA-11660) http://git-wip-us.apache.org/repos/asf/cassandra/blob/943e732c/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java index 8121858,0c10f13..e063f91 --- a/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/PrimaryKeyRestrictionSet.java @@@ -64,27 -67,48 +64,58 @@@ final class PrimaryKeyRestrictionSet ex */ private boolean contains; + /** + * <code>true</code> if the restrictions corresponding to a partition key, <code>false</code> if it's clustering columns. + */ + private boolean isPartitionKey; + /** + * If restrictions apply to clustering columns, we need to check whether they can be satisfied by an index lookup + * as this affects which other restrictions can legally be specified (if an index is present, we are more lenient + * about what additional filtering can be performed on the results of a lookup - see CASSANDRA-11510). + * + * We don't hold a reference to the SecondaryIndexManager itself as this is not strictly a singleton (although + * we often treat is as one), the field would also require annotation with @Unmetered to avoid blowing up the + * object size (used when calculating the size of prepared statements for caching). Instead, we refer to the + * CFMetaData and retrieve the index manager when necessary. + * + * There are a couple of scenarios where the CFM can be null (and we make sure and test for null when we use it): + * * where an empty set of restrictions are created for use in processing query results - see + * SelectStatement.forSelection + * * where the restrictions apply to partition keys and not clustering columns e.g. + * StatementRestrictions.partitionKeyRestrictions + * * in unit tests (in particular PrimaryKeyRestrictionSetTest which is primarily concerned with the correct + * generation of bounds when secondary indexes are not used). + */ + private final CFMetaData cfm; + - public PrimaryKeyRestrictionSet(CType ctype) + public PrimaryKeyRestrictionSet(ClusteringComparator comparator, boolean isPartitionKey) { - this(ctype, null); ++ this(comparator, isPartitionKey, null); + } + - public PrimaryKeyRestrictionSet(CType ctype, CFMetaData cfm) ++ public PrimaryKeyRestrictionSet(ClusteringComparator comparator, boolean isPartitionKey, CFMetaData cfm) + { - super(ctype); + super(comparator); ++ ++ if (cfm != null) ++ assert !isPartitionKey; ++ + this.cfm = cfm; this.restrictions = new RestrictionSet(); this.eq = true; + this.isPartitionKey = isPartitionKey; } private PrimaryKeyRestrictionSet(PrimaryKeyRestrictionSet primaryKeyRestrictions, - Restriction restriction) throws InvalidRequestException + Restriction restriction) throws InvalidRequestException { - super(primaryKeyRestrictions.ctype); + super(primaryKeyRestrictions.comparator); this.restrictions = primaryKeyRestrictions.restrictions.addRestriction(restriction); + this.isPartitionKey = primaryKeyRestrictions.isPartitionKey; + this.cfm = primaryKeyRestrictions.cfm; - if (!primaryKeyRestrictions.isEmpty()) + if (!primaryKeyRestrictions.isEmpty() && !hasSupportingIndex(restriction)) { ColumnDefinition lastRestrictionStart = primaryKeyRestrictions.restrictions.lastRestriction().getFirstColumn(); ColumnDefinition newRestrictionStart = restriction.getFirstColumn(); @@@ -110,15 -134,12 +141,22 @@@ this.eq = true; } + private List<ByteBuffer> toByteBuffers(SortedSet<? extends ClusteringPrefix> clusterings) + { + // It's currently a tad hard to follow that this is only called for partition key so we should fix that + List<ByteBuffer> l = new ArrayList<>(clusterings.size()); + for (ClusteringPrefix clustering : clusterings) + l.add(CFMetaData.serializePartitionKey(clustering)); + return l; + } + + private boolean hasSupportingIndex(Restriction restriction) + { + return cfm != null - && restriction.hasSupportingIndex(Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfId).indexManager); ++ && restriction.hasSupportingIndex(Keyspace.openAndGetStore(cfm).indexManager); ++ + } + @Override public boolean isSlice() { http://git-wip-us.apache.org/repos/asf/cassandra/blob/943e732c/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java index 763a7be,5b7c58d..942d1f9 --- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java @@@ -125,53 -112,28 +126,58 @@@ public final class StatementRestriction VariableSpecifications boundNames, boolean selectsOnlyStaticColumns, boolean selectACollection, - boolean useFiltering) + boolean useFiltering, + boolean forView) throws InvalidRequestException { - this(type, cfm); ++ this.type = type; + this.cfm = cfm; - this.partitionKeyRestrictions = new PrimaryKeyRestrictionSet(cfm.getKeyValidatorAsCType()); - this.clusteringColumnsRestrictions = new PrimaryKeyRestrictionSet(cfm.comparator, cfm); ++ this.partitionKeyRestrictions = new PrimaryKeyRestrictionSet(cfm.getKeyValidatorAsClusteringComparator(), true); ++ this.clusteringColumnsRestrictions = new PrimaryKeyRestrictionSet(cfm.comparator, false, cfm); + this.nonPrimaryKeyRestrictions = new RestrictionSet(); ++ this.notNullColumns = new HashSet<>(); /* - * WHERE clause. For a given entity, rules are: - EQ relation conflicts with anything else (including a 2nd EQ) - * - Can't have more than one LT(E) relation (resp. GT(E) relation) - IN relation are restricted to row keys - * (for now) and conflicts with anything else (we could allow two IN for the same entity but that doesn't seem - * very useful) - The value_alias cannot be restricted in any way (we don't support wide rows with indexed value - * in CQL so far) + * WHERE clause. For a given entity, rules are: + * - EQ relation conflicts with anything else (including a 2nd EQ) + * - Can't have more than one LT(E) relation (resp. GT(E) relation) + * - IN relation are restricted to row keys (for now) and conflicts with anything else (we could + * allow two IN for the same entity but that doesn't seem very useful) + * - The value_alias cannot be restricted in any way (we don't support wide rows with indexed value + * in CQL so far) */ - for (Relation relation : whereClause) - addRestriction(relation.toRestriction(cfm, boundNames)); + for (Relation relation : whereClause.relations) + { + if (relation.operator() == Operator.IS_NOT) + { + if (!forView) + throw new InvalidRequestException("Unsupported restriction: " + relation); + + for (ColumnDefinition def : relation.toRestriction(cfm, boundNames).getColumnDefs()) + this.notNullColumns.add(def); + } + else + { + addRestriction(relation.toRestriction(cfm, boundNames)); + } + } + + boolean hasQueriableClusteringColumnIndex = false; + boolean hasQueriableIndex = false; - SecondaryIndexManager secondaryIndexManager = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName).indexManager; - boolean hasQueriableClusteringColumnIndex = clusteringColumnsRestrictions.hasSupportingIndex(secondaryIndexManager); - boolean hasQueriableIndex = hasQueriableClusteringColumnIndex - || partitionKeyRestrictions.hasSupportingIndex(secondaryIndexManager) - || nonPrimaryKeyRestrictions.hasSupportingIndex(secondaryIndexManager); + if (type.allowUseOfSecondaryIndices()) + { + ColumnFamilyStore cfs = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfName); + SecondaryIndexManager secondaryIndexManager = cfs.indexManager; + + if (whereClause.containsCustomExpressions()) + processCustomIndexExpressions(whereClause.expressions, boundNames, secondaryIndexManager); + + hasQueriableClusteringColumnIndex = clusteringColumnsRestrictions.hasSupportingIndex(secondaryIndexManager); + hasQueriableIndex = !indexRestrictions.getCustomIndexExpressions().isEmpty() + || hasQueriableClusteringColumnIndex + || partitionKeyRestrictions.hasSupportingIndex(secondaryIndexManager) + || nonPrimaryKeyRestrictions.hasSupportingIndex(secondaryIndexManager); + } // At this point, the select statement if fully constructed, but we still have a few things to validate processPartitionKeyRestrictions(hasQueriableIndex); @@@ -430,130 -301,61 +436,136 @@@ * Processes the clustering column restrictions. * * @param hasQueriableIndex <code>true</code> if some of the queried data are indexed, <code>false</code> otherwise + * @param selectsOnlyStaticColumns <code>true</code> if the selected or modified columns are all statics, + * <code>false</code> otherwise. * @param selectACollection <code>true</code> if the query should return a collection column - * @throws InvalidRequestException if the request is invalid */ private void processClusteringColumnsRestrictions(boolean hasQueriableIndex, - boolean selectACollection) throws InvalidRequestException + boolean selectsOnlyStaticColumns, + boolean selectACollection, + boolean forView) throws InvalidRequestException { - checkFalse(clusteringColumnsRestrictions.isIN() && selectACollection, - "Cannot restrict clustering columns by IN relations when a collection is selected by the query"); - checkFalse(clusteringColumnsRestrictions.isContains() && !hasQueriableIndex, - "Cannot restrict clustering columns by a CONTAINS relation without a secondary index"); + checkFalse(!type.allowClusteringColumnSlices() && clusteringColumnsRestrictions.isSlice(), + "Slice restrictions are not supported on the clustering columns in %s statements", type); - if (hasClusteringColumnsRestriction() && clusteringRestrictionsNeedFiltering()) + if (!type.allowClusteringColumnSlices() + && (!cfm.isCompactTable() || (cfm.isCompactTable() && !hasClusteringColumnsRestriction()))) { - if (hasQueriableIndex) - { - usesSecondaryIndexing = true; - return; - } - - List<ColumnDefinition> clusteringColumns = cfm.clusteringColumns(); - List<ColumnDefinition> restrictedColumns = new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs()); + if (!selectsOnlyStaticColumns && hasUnrestrictedClusteringColumns()) + throw invalidRequest("Some clustering keys are missing: %s", + Joiner.on(", ").join(getUnrestrictedClusteringColumns())); + } + else + { + checkFalse(clusteringColumnsRestrictions.isIN() && selectACollection, + "Cannot restrict clustering columns by IN relations when a collection is selected by the query"); + checkFalse(clusteringColumnsRestrictions.isContains() && !hasQueriableIndex, + "Cannot restrict clustering columns by a CONTAINS relation without a secondary index"); - if (hasClusteringColumnsRestriction()) - for (int i = 0, m = restrictedColumns.size(); i < m; i++) ++ if (hasClusteringColumnsRestriction() && clusteringRestrictionsNeedFiltering()) { - ColumnDefinition clusteringColumn = clusteringColumns.get(i); - ColumnDefinition restrictedColumn = restrictedColumns.get(i); ++ if (hasQueriableIndex || forView) ++ { ++ usesSecondaryIndexing = true; ++ return; ++ } ++ + List<ColumnDefinition> clusteringColumns = cfm.clusteringColumns(); + List<ColumnDefinition> restrictedColumns = new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs()); - if (!clusteringColumn.equals(restrictedColumn)) + for (int i = 0, m = restrictedColumns.size(); i < m; i++) { - throw invalidRequest( - "PRIMARY KEY column \"%s\" cannot be restricted as preceding column \"%s\" is not restricted", - restrictedColumn.name, - clusteringColumn.name); + ColumnDefinition clusteringColumn = clusteringColumns.get(i); + ColumnDefinition restrictedColumn = restrictedColumns.get(i); + + if (!clusteringColumn.equals(restrictedColumn)) + { - checkTrue(hasQueriableIndex || forView, - "PRIMARY KEY column \"%s\" cannot be restricted as preceding column \"%s\" is not restricted", - restrictedColumn.name, - clusteringColumn.name); - - usesSecondaryIndexing = true; // handle gaps and non-keyrange cases. - break; ++ throw invalidRequest( ++ "PRIMARY KEY column \"%s\" cannot be restricted as preceding column \"%s\" is not restricted", ++ restrictedColumn.name, ++ clusteringColumn.name); + } } } } + } - if (clusteringColumnsRestrictions.isContains()) - usesSecondaryIndexing = true; + public final boolean clusteringRestrictionsNeedFiltering() + { + assert clusteringColumnsRestrictions instanceof PrimaryKeyRestrictionSet; + return ((PrimaryKeyRestrictionSet) clusteringColumnsRestrictions).needsFiltering(); } - public List<IndexExpression> getIndexExpressions(SecondaryIndexManager indexManager, - QueryOptions options) throws InvalidRequestException + /** + * Returns the clustering columns that are not restricted. + * @return the clustering columns that are not restricted. + */ + private Collection<ColumnIdentifier> getUnrestrictedClusteringColumns() + { + List<ColumnDefinition> missingClusteringColumns = new ArrayList<>(cfm.clusteringColumns()); + missingClusteringColumns.removeAll(new LinkedList<>(clusteringColumnsRestrictions.getColumnDefs())); + return ColumnDefinition.toIdentifiers(missingClusteringColumns); + } + + /** + * Checks if some clustering columns are not restricted. + * @return <code>true</code> if some clustering columns are not restricted, <code>false</code> otherwise. + */ + private boolean hasUnrestrictedClusteringColumns() + { + return cfm.clusteringColumns().size() != clusteringColumnsRestrictions.size(); + } + + private void processCustomIndexExpressions(List<CustomIndexExpression> expressions, + VariableSpecifications boundNames, + SecondaryIndexManager indexManager) + { + if (!MessagingService.instance().areAllNodesAtLeast30()) + throw new InvalidRequestException("Please upgrade all nodes to at least 3.0 before using custom index expressions"); + + if (expressions.size() > 1) + throw new InvalidRequestException(IndexRestrictions.MULTIPLE_EXPRESSIONS); + + CustomIndexExpression expression = expressions.get(0); + + CFName cfName = expression.targetIndex.getCfName(); + if (cfName.hasKeyspace() + && !expression.targetIndex.getKeyspace().equals(cfm.ksName)) + throw IndexRestrictions.invalidIndex(expression.targetIndex, cfm); + + if (cfName.getColumnFamily() != null && !cfName.getColumnFamily().equals(cfm.cfName)) + throw IndexRestrictions.invalidIndex(expression.targetIndex, cfm); + + if (!cfm.getIndexes().has(expression.targetIndex.getIdx())) + throw IndexRestrictions.indexNotFound(expression.targetIndex, cfm); + + Index index = indexManager.getIndex(cfm.getIndexes().get(expression.targetIndex.getIdx()).get()); + + if (!index.getIndexMetadata().isCustom()) + throw IndexRestrictions.nonCustomIndexInExpression(expression.targetIndex); + + AbstractType<?> expressionType = index.customExpressionValueType(); + if (expressionType == null) + throw IndexRestrictions.customExpressionNotSupported(expression.targetIndex); + + expression.prepareValue(cfm, expressionType, boundNames); + + indexRestrictions.add(expression); + } + + public RowFilter getRowFilter(SecondaryIndexManager indexManager, QueryOptions options) { - if (!usesSecondaryIndexing || indexRestrictions.isEmpty()) - return Collections.emptyList(); + if (indexRestrictions.isEmpty()) + return RowFilter.NONE; + + RowFilter filter = RowFilter.create(); + for (Restrictions restrictions : indexRestrictions.getRestrictions()) + restrictions.addRowFilterTo(filter, indexManager, options); - List<IndexExpression> expressions = new ArrayList<>(); - for (Restrictions restrictions : indexRestrictions) - restrictions.addIndexExpressionTo(expressions, indexManager, options); + for (CustomIndexExpression expression : indexRestrictions.getCustomIndexExpressions()) + expression.addToRowFilter(filter, cfm, options); - return expressions; + return filter; } /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/943e732c/test/unit/org/apache/cassandra/cql3/validation/operations/SelectSingleColumnRelationTest.java ----------------------------------------------------------------------
