[6/8] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.11
http://git-wip-us.apache.org/repos/asf/cassandra/blob/4734ce7d/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java -- diff --cc src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java index 15a090b,84c6958..3b7504f --- a/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java +++ b/src/java/org/apache/cassandra/cql3/restrictions/StatementRestrictions.java @@@ -37,7 -37,7 +38,6 @@@ import org.apache.cassandra.exceptions. import org.apache.cassandra.index.Index; import org.apache.cassandra.index.SecondaryIndexManager; import org.apache.cassandra.net.MessagingService; --import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.btree.BTreeSet; import static org.apache.cassandra.cql3.statements.RequestValidations.checkFalse; @@@ -163,20 -159,12 +163,23 @@@ public final class StatementRestriction for (ColumnDefinition def : relation.toRestriction(cfm, boundNames).getColumnDefs()) this.notNullColumns.add(def); } +else if (relation.isLIKE()) +{ +Restriction restriction = relation.toRestriction(cfm, boundNames); + +if (!type.allowUseOfSecondaryIndices() || !restriction.hasSupportingIndex(secondaryIndexManager)) +throw new InvalidRequestException(String.format("LIKE restriction is only supported on properly " + +"indexed columns. %s is not valid.", + relation.toString())); + +addRestriction(restriction); +} else { - addRestriction(relation.toRestriction(cfm, boundNames)); + if (cfm.isSuper() && cfm.isDense() && !relation.onToken()) + addRestriction(relation.toSuperColumnAdapter().toRestriction(cfm, boundNames)); + else + addRestriction(relation.toRestriction(cfm, boundNames)); } } @@@ -250,11 -237,18 +253,18 @@@ Joiner.on(", ").join(nonPrimaryKeyColumns)); } if (hasQueriableIndex) + { usesSecondaryIndexing = true; - else if (!allowFiltering) + } + else if (!allowFiltering && !cfm.isSuper()) + { throw invalidRequest(StatementRestrictions.REQUIRES_ALLOW_FILTERING_MESSAGE); + } + + checkFalse(clusteringColumnsRestrictions.isEmpty() && cfm.isSuper(), +"Filtering is not supported on SuperColumn tables"); -indexRestrictions.add(nonPrimaryKeyRestrictions); +filterRestrictions.add(nonPrimaryKeyRestrictions); } if (usesSecondaryIndexing) @@@ -824,18 -852,21 +834,29 @@@ public boolean hasAllPKColumnsRestrictedByEqualities() { return !isPartitionKeyRestrictionsOnToken() - && !hasUnrestrictedPartitionKeyComponents() - && (partitionKeyRestrictions.isEQ() || partitionKeyRestrictions.isIN()) - && !hasUnrestrictedClusteringColumns() - && (clusteringColumnsRestrictions.isEQ() || clusteringColumnsRestrictions.isIN()); +&& !partitionKeyRestrictions.hasUnrestrictedPartitionKeyComponents(cfm) +&& (partitionKeyRestrictions.hasOnlyEqualityRestrictions()) +&& !hasUnrestrictedClusteringColumns() +&& (clusteringColumnsRestrictions.hasOnlyEqualityRestrictions()); } +/** + * Checks if one of the restrictions applies to a regular column. + * @return {@code true} if one of the restrictions applies to a regular column, {@code false} otherwise. + */ +public boolean hasRegularColumnsRestrictions() +{ +return hasRegularColumnsRestrictions; +} + + private SuperColumnCompatibility.SuperColumnRestrictions cached; + public SuperColumnCompatibility.SuperColumnRestrictions getSuperColumnRestrictions() + { + assert cfm.isSuper() && cfm.isDense(); + + if (cached == null) -cached = new SuperColumnCompatibility.SuperColumnRestrictions(Iterators.concat(((PrimaryKeyRestrictionSet) clusteringColumnsRestrictions).iterator(), ++cached = new SuperColumnCompatibility.SuperColumnRestrictions(Iterators.concat(clusteringColumnsRestrictions.iterator(), + nonPrimaryKeyRestrictions.iterator())); + return cached; + } }
[6/8] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.11
http://git-wip-us.apache.org/repos/asf/cassandra/blob/3e3d56ec/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java -- diff --cc test/unit/org/apache/cassandra/cql3/GcCompactionTest.java index a31b9a1,000..84a20de mode 100644,00..100644 --- a/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java +++ b/test/unit/org/apache/cassandra/cql3/GcCompactionTest.java @@@ -1,389 -1,0 +1,390 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.cql3; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Collections; +import java.util.HashSet; +import java.util.Set; +import java.util.function.Function; + +import com.google.common.collect.Iterables; +import org.junit.Test; + +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.compaction.CompactionManager; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.io.sstable.ISSTableScanner; +import org.apache.cassandra.io.sstable.format.SSTableReader; +import org.apache.cassandra.utils.FBUtilities; + +public class GcCompactionTest extends CQLTester +{ +static final int KEY_COUNT = 10; +static final int CLUSTERING_COUNT = 20; + +// Test needs synchronous table drop to avoid flushes causing flaky failures + +@Override +protected String createTable(String query) +{ +return super.createTable(KEYSPACE_PER_TEST, query); +} + +@Override +protected UntypedResultSet execute(String query, Object... values) throws Throwable +{ +return executeFormattedQuery(formatQuery(KEYSPACE_PER_TEST, query), values); +} + +@Override +public ColumnFamilyStore getCurrentColumnFamilyStore() +{ +return super.getCurrentColumnFamilyStore(KEYSPACE_PER_TEST); +} + +public void flush() +{ +flush(KEYSPACE_PER_TEST); +} + +@Test +public void testGcCompactionPartitions() throws Throwable +{ +runCompactionTest("CREATE TABLE %s(" + + " key int," + + " column int," + + " data int," + + " extra text," + + " PRIMARY KEY((key, column), data)" + + ") WITH compaction = { 'class' : 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones' : 'row' };" + ); + +} + +@Test +public void testGcCompactionRows() throws Throwable +{ +runCompactionTest("CREATE TABLE %s(" + + " key int," + + " column int," + + " data int," + + " extra text," + + " PRIMARY KEY(key, column)" + + ") WITH compaction = { 'class' : 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones' : 'row' };" + ); + +} + +@Test +public void testGcCompactionRanges() throws Throwable +{ + +runCompactionTest("CREATE TABLE %s(" + + " key int," + + " column int," + + " col2 int," + + " data int," + + " extra text," + + " PRIMARY KEY(key, column, data)" + + ") WITH compaction = { 'class' : 'SizeTieredCompactionStrategy', 'provide_overlapping_tombstones' : 'row' };" + ); +} + +private void runCompactionTest(String tableDef) throws Throwable +{ +createTable(tableDef); + +for (int i = 0; i < KEY_COUNT; ++i) +for (int j = 0; j < CLUSTERING_COUNT; ++j) +execute("INSERT INTO %s (key, column, data, extra) VALUES (?, ?, ?, ?)", i, j, i+j, "" + i + ":" + j); + +Set readers = new HashSet<>(); +ColumnFamilyStore cfs = getCurrentColumnFamilyStore(); + +flush(); +
[6/8] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.11
Merge branch 'cassandra-3.0' into cassandra-3.11 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/031d76c9 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/031d76c9 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/031d76c9 Branch: refs/heads/trunk Commit: 031d76c9b4bcf7a3da735cc15d32b5a88be7aeda Parents: 35dc3b6 e817c83 Author: Jason BrownAuthored: Tue Aug 29 08:36:58 2017 -0700 Committer: Jason Brown Committed: Tue Aug 29 08:36:58 2017 -0700 -- -- - To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org
[6/8] cassandra git commit: Merge branch cassandra-3.0 into cassandra-3.11
Merge branch cassandra-3.0 into cassandra-3.11 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/8d98a992 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/8d98a992 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/8d98a992 Branch: refs/heads/trunk Commit: 8d98a99231f38102d0ef572ffe948d705169aae4 Parents: 07f cf0b6d1 Author: Benjamin LererAuthored: Thu Aug 24 18:21:08 2017 +0200 Committer: Benjamin Lerer Committed: Thu Aug 24 18:22:13 2017 +0200 -- CHANGES.txt | 1 + .../cassandra/db/SerializationHeader.java | 13 +- .../db/rows/AbstractTypeVersionComparator.java | 121 src/java/org/apache/cassandra/db/rows/Row.java | 18 +- src/java/org/apache/cassandra/db/rows/Rows.java | 22 ++- .../io/sstable/format/SSTableReader.java| 20 +- .../org/apache/cassandra/cql3/CQLTester.java| 20 +- .../cql3/validation/entities/UserTypesTest.java | 151 +++ .../rows/AbstractTypeVersionComparatorTest.java | 188 +++ 9 files changed, 533 insertions(+), 21 deletions(-) -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/8d98a992/CHANGES.txt -- diff --cc CHANGES.txt index d39f78c,ba35152..8fb8e2f --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,11 -1,5 +1,12 @@@ -3.0.15 +3.11.1 + * Fix cassandra-stress hang issues when an error during cluster connection happens (CASSANDRA-12938) + * Better bootstrap failure message when blocked by (potential) range movement (CASSANDRA-13744) + * "ignore" option is ignored in sstableloader (CASSANDRA-13721) + * Deadlock in AbstractCommitLogSegmentManager (CASSANDRA-13652) + * Duplicate the buffer before passing it to analyser in SASI operation (CASSANDRA-13512) + * Properly evict pstmts from prepared statements cache (CASSANDRA-13641) +Merged from 3.0: + * Fix the merging of cells with different user type versions (CASSANDRA-13776) * Copy session properties on cqlsh.py do_login (CASSANDRA-13640) * Potential AssertionError during ReadRepair of range tombstone and partition deletions (CASSANDRA-13719) * Don't let stress write warmup data if n=0 (CASSANDRA-13773) http://git-wip-us.apache.org/repos/asf/cassandra/blob/8d98a992/src/java/org/apache/cassandra/db/SerializationHeader.java -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/8d98a992/src/java/org/apache/cassandra/db/rows/Row.java -- http://git-wip-us.apache.org/repos/asf/cassandra/blob/8d98a992/src/java/org/apache/cassandra/db/rows/Rows.java -- diff --cc src/java/org/apache/cassandra/db/rows/Rows.java index a92bdac,09213a4..3331506 --- a/src/java/org/apache/cassandra/db/rows/Rows.java +++ b/src/java/org/apache/cassandra/db/rows/Rows.java @@@ -26,9 -25,7 +26,8 @@@ import org.apache.cassandra.config.CFMe import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; import org.apache.cassandra.db.partitions.PartitionStatisticsCollector; - import org.apache.cassandra.db.rows.Row.Deletion; import org.apache.cassandra.utils.MergeIterator; +import org.apache.cassandra.utils.WrappedInt; /** * Static utilities to work on Row objects. @@@ -330,78 -312,20 +330,96 @@@ public abstract class Row } /** + * Returns a row that is obtained from the given existing row by removing everything that is shadowed by data in + * the update row. In other words, produces the smallest result row such that + * {@code merge(result, update, nowInSec) == merge(existing, update, nowInSec)} after filtering by rangeDeletion. + * + * @param existing source row + * @param update shadowing row + * @param rangeDeletion extra {@code DeletionTime} from covering tombstone + * @param nowInSec the current time in seconds (which plays a role during reconciliation + * because deleted cells always have precedence on timestamp equality and deciding if a + * cell is a live or not depends on the current time due to expiring cells). + */ +public static Row removeShadowedCells(Row existing, Row update, DeletionTime rangeDeletion, int nowInSec) +{ +Row.Builder builder = BTreeRow.sortedBuilder(); +Clustering clustering = existing.clustering(); +builder.newRow(clustering); + +DeletionTime deletion = update.deletion().time(); +if (rangeDeletion.supersedes(deletion)) +deletion = rangeDeletion; + +LivenessInfo existingInfo =
[6/8] cassandra git commit: Merge branch 'cassandra-3.0' into cassandra-3.11
http://git-wip-us.apache.org/repos/asf/cassandra/blob/47a2839b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java -- diff --cc test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java index a6ce08b,000..e9051b4 mode 100644,00..100644 --- a/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java +++ b/test/unit/org/apache/cassandra/index/sasi/SASIIndexTest.java @@@ -1,2552 -1,0 +1,2566 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.index.sasi; + +import java.io.FileWriter; +import java.io.Writer; +import java.nio.ByteBuffer; +import java.nio.file.FileSystems; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.*; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.cassandra.SchemaLoader; +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.index.Index; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.cql3.*; +import org.apache.cassandra.cql3.Term; +import org.apache.cassandra.cql3.statements.IndexTarget; +import org.apache.cassandra.cql3.statements.SelectStatement; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.db.filter.DataLimits; +import org.apache.cassandra.db.filter.RowFilter; +import org.apache.cassandra.db.lifecycle.SSTableSet; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.db.partitions.PartitionUpdate; +import org.apache.cassandra.db.partitions.UnfilteredPartitionIterator; +import org.apache.cassandra.db.rows.*; +import org.apache.cassandra.dht.IPartitioner; +import org.apache.cassandra.dht.Murmur3Partitioner; +import org.apache.cassandra.dht.Range; +import org.apache.cassandra.exceptions.ConfigurationException; +import org.apache.cassandra.exceptions.InvalidRequestException; +import org.apache.cassandra.index.sasi.conf.ColumnIndex; +import org.apache.cassandra.index.sasi.disk.OnDiskIndexBuilder; +import org.apache.cassandra.index.sasi.exceptions.TimeQuotaExceededException; +import org.apache.cassandra.index.sasi.memory.IndexMemtable; +import org.apache.cassandra.index.sasi.plan.QueryController; +import org.apache.cassandra.index.sasi.plan.QueryPlan; +import org.apache.cassandra.io.sstable.SSTable; +import org.apache.cassandra.schema.IndexMetadata; +import org.apache.cassandra.schema.KeyspaceMetadata; +import org.apache.cassandra.schema.KeyspaceParams; +import org.apache.cassandra.schema.Tables; +import org.apache.cassandra.serializers.MarshalException; +import org.apache.cassandra.serializers.TypeSerializer; +import org.apache.cassandra.service.MigrationManager; +import org.apache.cassandra.service.QueryState; +import org.apache.cassandra.thrift.CqlRow; +import org.apache.cassandra.transport.messages.ResultMessage; +import org.apache.cassandra.utils.ByteBufferUtil; +import org.apache.cassandra.utils.FBUtilities; +import org.apache.cassandra.utils.Pair; + +import com.google.common.collect.Lists; +import com.google.common.util.concurrent.Uninterruptibles; + +import junit.framework.Assert; + +import org.junit.*; + +public class SASIIndexTest +{ +private static final IPartitioner PARTITIONER; + +static { +System.setProperty("cassandra.config", "cassandra-murmur.yaml"); +PARTITIONER = Murmur3Partitioner.instance; +} + +private static final String KS_NAME = "sasi"; +private static final String CF_NAME = "test_cf"; +private static final String CLUSTERING_CF_NAME_1 = "clustering_test_cf_1"; +private static final String CLUSTERING_CF_NAME_2 = "clustering_test_cf_2"; +private static final String STATIC_CF_NAME = "static_sasi_test_cf"; +private static final String FTS_CF_NAME =
[6/8] cassandra git commit: Merge branch cassandra-3.0 into cassandra-3.11
http://git-wip-us.apache.org/repos/asf/cassandra/blob/5c9db9af/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java -- diff --cc src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java index 4536036,000..84a742b mode 100644,00..100644 --- a/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java +++ b/src/java/org/apache/cassandra/db/rows/UnfilteredRowIteratorWithLowerBound.java @@@ -1,261 -1,0 +1,263 @@@ +/* + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * + */ +package org.apache.cassandra.db.rows; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.util.Comparator; +import java.util.List; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.filter.ClusteringIndexFilter; +import org.apache.cassandra.db.filter.ColumnFilter; +import org.apache.cassandra.io.sstable.IndexInfo; +import org.apache.cassandra.io.sstable.format.SSTableReader; ++import org.apache.cassandra.io.sstable.format.SSTableReadsListener; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.thrift.ThriftResultsMerger; +import org.apache.cassandra.utils.IteratorWithLowerBound; + +/** + * An unfiltered row iterator with a lower bound retrieved from either the global + * sstable statistics or the row index lower bounds (if available in the cache). + * Before initializing the sstable unfiltered row iterator, we return an empty row + * with the clustering set to the lower bound. The empty row will be filtered out and + * the result is that if we don't need to access this sstable, i.e. due to the LIMIT conditon, + * then we will not. See CASSANDRA-8180 for examples of why this is useful. + */ +public class UnfilteredRowIteratorWithLowerBound extends LazilyInitializedUnfilteredRowIterator implements IteratorWithLowerBound +{ +private final SSTableReader sstable; +private final ClusteringIndexFilter filter; +private final ColumnFilter selectedColumns; +private final boolean isForThrift; +private final int nowInSec; +private final boolean applyThriftTransformation; ++private final SSTableReadsListener listener; +private ClusteringBound lowerBound; +private boolean firstItemRetrieved; + +public UnfilteredRowIteratorWithLowerBound(DecoratedKey partitionKey, + SSTableReader sstable, + ClusteringIndexFilter filter, + ColumnFilter selectedColumns, + boolean isForThrift, + int nowInSec, -boolean applyThriftTransformation) ++ boolean applyThriftTransformation, ++ SSTableReadsListener listener) +{ +super(partitionKey); +this.sstable = sstable; +this.filter = filter; +this.selectedColumns = selectedColumns; +this.isForThrift = isForThrift; +this.nowInSec = nowInSec; +this.applyThriftTransformation = applyThriftTransformation; ++this.listener = listener; +this.lowerBound = null; +this.firstItemRetrieved = false; +} + +public Unfiltered lowerBound() +{ +if (lowerBound != null) +return makeBound(lowerBound); + +// The partition index lower bound is more accurate than the sstable metadata lower bound but it is only +// present if the iterator has already been initialized, which we only do when there are tombstones since in +// this case we cannot use the sstable metadata clustering values +ClusteringBound ret = getPartitionIndexLowerBound(); +return ret != null ? makeBound(ret) : makeBound(getMetadataLowerBound()); +} + +private Unfiltered makeBound(ClusteringBound