Merge branch 'cassandra-2.0' into cassandra-2.1 Conflicts: CHANGES.txt src/java/org/apache/cassandra/cql3/CFDefinition.java src/java/org/apache/cassandra/cql3/ColumnNameBuilder.java src/java/org/apache/cassandra/cql3/statements/ModificationStatement.java src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/cab2b25b Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/cab2b25b Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/cab2b25b Branch: refs/heads/cassandra-2.1 Commit: cab2b25b0a5b2029a9c3e1324d080b4982fbdc50 Parents: 35f173a 0e3d9fc Author: Aleksey Yeschenko <alek...@apache.org> Authored: Mon Nov 24 15:14:14 2014 +0300 Committer: Aleksey Yeschenko <alek...@apache.org> Committed: Mon Nov 24 15:14:14 2014 +0300 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../cql3/statements/UpdateStatement.java | 17 +++- .../cassandra/io/sstable/SSTableWriter.java | 9 ++ .../cql3/IndexedValuesValidationTest.java | 86 ++++++++++++++++++++ 4 files changed, 112 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/cab2b25b/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 313000a,412eb59..9db65e9 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,18 -1,5 +1,19 @@@ -2.0.12: +2.1.3 + * Fix high size calculations for prepared statements (CASSANDRA-8231) + * Centralize shared executors (CASSANDRA-8055) + * Fix filtering for CONTAINS (KEY) relations on frozen collection + clustering columns when the query is restricted to a single + partition (CASSANDRA-8203) + * Do more aggressive entire-sstable TTL expiry checks (CASSANDRA-8243) + * Add more log info if readMeter is null (CASSANDRA-8238) + * add check of the system wall clock time at startup (CASSANDRA-8305) + * Support for frozen collections (CASSANDRA-7859) + * Fix overflow on histogram computation (CASSANDRA-8028) + * Have paxos reuse the timestamp generation of normal queries (CASSANDRA-7801) + * Fix incremental repair not remove parent session on remote (CASSANDRA-8291) + * Improve JBOD disk utilization (CASSANDRA-7386) +Merged from 2.0: + * Validate size of indexed column values (CASSANDRA-8280) * Make LCS split compaction results over all data directories (CASSANDRA-8329) * Fix some failing queries that use multi-column relations on COMPACT STORAGE tables (CASSANDRA-8264) http://git-wip-us.apache.org/repos/asf/cassandra/blob/cab2b25b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java index 2c87173,9d98c84..09f26d6 --- a/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java +++ b/src/java/org/apache/cassandra/cql3/statements/UpdateStatement.java @@@ -22,12 -22,11 +22,12 @@@ import java.util.* import org.apache.cassandra.cql3.*; import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; import org.apache.cassandra.db.*; +import org.apache.cassandra.db.composites.Composite; - import org.apache.cassandra.db.marshal.AbstractType; + import org.apache.cassandra.db.index.SecondaryIndexManager; import org.apache.cassandra.exceptions.*; import org.apache.cassandra.utils.ByteBufferUtil; -import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.Pair; /** @@@ -95,8 -103,22 +95,23 @@@ public class UpdateStatement extends Mo else { for (Operation update : updates) - update.execute(key, cf, builder.copy(), params); + update.execute(key, cf, prefix, params); } + + SecondaryIndexManager indexManager = Keyspace.open(cfm.ksName).getColumnFamilyStore(cfm.cfId).indexManager; + if (indexManager.hasIndexes()) + { - for (Column column : cf) ++ for (Cell cell : cf) + { - if (!indexManager.validate(column)) ++ // Indexed values must be validated by any applicable index. See CASSANDRA-3057/4240/8081 for more details ++ if (!indexManager.validate(cell)) + throw new InvalidRequestException(String.format("Can't index column value of size %d for index %s on %s.%s", - column.value().remaining(), - cfm.getColumnDefinitionFromColumnName(column.name()).getIndexName(), ++ cell.value().remaining(), ++ cfm.getColumnDefinition(cell.name()).getIndexName(), + cfm.ksName, + cfm.cfName)); + } + } } public static class ParsedInsert extends ModificationStatement.Parsed http://git-wip-us.apache.org/repos/asf/cassandra/blob/cab2b25b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/SSTableWriter.java index 0f95a9b,afa066d..53176e3 --- a/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java @@@ -52,18 -33,10 +52,19 @@@ import org.apache.cassandra.db.compacti import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.io.FSWriteError; import org.apache.cassandra.io.compress.CompressedSequentialWriter; -import org.apache.cassandra.io.util.*; +import org.apache.cassandra.io.sstable.metadata.MetadataCollector; +import org.apache.cassandra.io.sstable.metadata.MetadataComponent; +import org.apache.cassandra.io.sstable.metadata.MetadataType; +import org.apache.cassandra.io.sstable.metadata.StatsMetadata; +import org.apache.cassandra.io.util.DataOutputPlus; +import org.apache.cassandra.io.util.DataOutputStreamAndChannel; +import org.apache.cassandra.io.util.FileMark; +import org.apache.cassandra.io.util.FileUtils; +import org.apache.cassandra.io.util.SegmentedFile; +import org.apache.cassandra.io.util.SequentialWriter; import org.apache.cassandra.service.StorageService; import org.apache.cassandra.utils.ByteBufferUtil; + import org.apache.cassandra.utils.FBUtilities; import org.apache.cassandra.utils.FilterFactory; import org.apache.cassandra.utils.IFilter; import org.apache.cassandra.utils.Pair; @@@ -212,6 -182,14 +213,14 @@@ public class SSTableWriter extends SSTa public void append(DecoratedKey decoratedKey, ColumnFamily cf) { - if (decoratedKey.key.remaining() > FBUtilities.MAX_UNSIGNED_SHORT) ++ if (decoratedKey.getKey().remaining() > FBUtilities.MAX_UNSIGNED_SHORT) + { + logger.error("Key size {} exceeds maximum of {}, skipping row", - decoratedKey.key.remaining(), ++ decoratedKey.getKey().remaining(), + FBUtilities.MAX_UNSIGNED_SHORT); + return; + } + long startPosition = beforeAppend(decoratedKey); try { http://git-wip-us.apache.org/repos/asf/cassandra/blob/cab2b25b/test/unit/org/apache/cassandra/cql3/IndexedValuesValidationTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/cql3/IndexedValuesValidationTest.java index 0000000,9c2bc0f..05acf86 mode 000000,100644..100644 --- a/test/unit/org/apache/cassandra/cql3/IndexedValuesValidationTest.java +++ b/test/unit/org/apache/cassandra/cql3/IndexedValuesValidationTest.java @@@ -1,0 -1,124 +1,86 @@@ + /* + * + * * Licensed to the Apache Software Foundation (ASF) under one + * * or more contributor license agreements. See the NOTICE file + * * distributed with this work for additional information + * * regarding copyright ownership. The ASF licenses this file + * * to you under the Apache License, Version 2.0 (the + * * "License"); you may not use this file except in compliance + * * with the License. You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ ++ + package org.apache.cassandra.cql3; + + import java.nio.ByteBuffer; -import java.util.Collections; -import org.junit.BeforeClass; ++ + import org.junit.Test; + -import org.apache.cassandra.SchemaLoader; -import org.apache.cassandra.db.ConsistencyLevel; + import org.apache.cassandra.exceptions.InvalidRequestException; -import org.apache.cassandra.exceptions.RequestExecutionException; -import org.apache.cassandra.exceptions.RequestValidationException; -import org.apache.cassandra.service.ClientState; -import org.apache.cassandra.service.QueryState; -import org.apache.cassandra.utils.MD5Digest; + + import static org.junit.Assert.fail; -import static org.apache.cassandra.cql3.QueryProcessor.process; + -public class IndexedValuesValidationTest ++public class IndexedValuesValidationTest extends CQLTester + { - static ClientState clientState; - static String keyspace = "indexed_value_validation_test"; - - @BeforeClass - public static void setUpClass() throws Throwable - { - SchemaLoader.loadSchema(); - executeSchemaChange("CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': '1'}"); - clientState = ClientState.forInternalCalls(); - } - + // CASSANDRA-8280/8081 + // reject updates with indexed values where value > 64k + @Test + public void testIndexOnCompositeValueOver64k() throws Throwable + { - executeSchemaChange("CREATE TABLE %s.composite_index_table (a int, b int, c blob, PRIMARY KEY (a))"); - executeSchemaChange("CREATE INDEX ON %s.composite_index_table(c)"); - performInsertWithIndexedValueOver64k("INSERT INTO %s.composite_index_table (a, b, c) VALUES (0, 0, ?)"); ++ createTable("CREATE TABLE %s(a int, b int, c blob, PRIMARY KEY (a))"); ++ createIndex("CREATE INDEX ON %s(c)"); ++ performInsertWithIndexedValueOver64k("INSERT INTO %s (a, b, c) VALUES (0, 0, ?)"); + } + + @Test + public void testIndexOnClusteringValueOver64k() throws Throwable + { - executeSchemaChange("CREATE TABLE %s.ck_index_table (a int, b blob, c int, PRIMARY KEY (a, b))"); - executeSchemaChange("CREATE INDEX ON %s.ck_index_table(b)"); - performInsertWithIndexedValueOver64k("INSERT INTO %s.ck_index_table (a, b, c) VALUES (0, ?, 0)"); ++ createTable("CREATE TABLE %s(a int, b blob, c int, PRIMARY KEY (a, b))"); ++ createIndex("CREATE INDEX ON %s(b)"); ++ performInsertWithIndexedValueOver64k("INSERT INTO %s (a, b, c) VALUES (0, ?, 0)"); + } + + @Test + public void testIndexOnPartitionKeyOver64k() throws Throwable + { - executeSchemaChange("CREATE TABLE %s.pk_index_table (a blob, b int, c int, PRIMARY KEY ((a, b)))"); - executeSchemaChange("CREATE INDEX ON %s.pk_index_table(a)"); - performInsertWithIndexedValueOver64k("INSERT INTO %s.pk_index_table (a, b, c) VALUES (?, 0, 0)"); ++ createTable("CREATE TABLE %s(a blob, b int, c int, PRIMARY KEY ((a, b)))"); ++ createIndex("CREATE INDEX ON %s(a)"); ++ performInsertWithIndexedValueOver64k("INSERT INTO %s (a, b, c) VALUES (?, 0, 0)"); + } + + @Test + public void testCompactTableWithValueOver64k() throws Throwable + { - executeSchemaChange("CREATE TABLE %s.compact_table (a int, b blob, PRIMARY KEY (a)) WITH COMPACT STORAGE"); - executeSchemaChange("CREATE INDEX ON %s.compact_table(b)"); - performInsertWithIndexedValueOver64k("INSERT INTO %s.compact_table (a, b) VALUES (0, ?)"); ++ createTable("CREATE TABLE %s(a int, b blob, PRIMARY KEY (a)) WITH COMPACT STORAGE"); ++ createIndex("CREATE INDEX ON %s(b)"); ++ performInsertWithIndexedValueOver64k("INSERT INTO %s (a, b) VALUES (0, ?)"); + } + - private static void performInsertWithIndexedValueOver64k(String insertCQL) throws Exception ++ public void performInsertWithIndexedValueOver64k(String insertCQL) throws Throwable + { + ByteBuffer buf = ByteBuffer.allocate(1024 * 65); + buf.clear(); ++ ++ //read more than 64k + for (int i=0; i<1024 + 1; i++) + buf.put((byte)0); + + try + { - execute(String.format(insertCQL, keyspace), buf); ++ execute(insertCQL, buf); + fail("Expected statement to fail validation"); + } + catch (InvalidRequestException e) + { + // as expected + } + } - - private static void execute(String query, ByteBuffer value) throws RequestValidationException, RequestExecutionException - { - MD5Digest statementId = QueryProcessor.prepare(String.format(query, keyspace), clientState, false).statementId; - CQLStatement statement = QueryProcessor.instance.getPrepared(statementId); - statement.executeInternal(QueryState.forInternalCalls(), - new QueryOptions(ConsistencyLevel.ONE, Collections.singletonList(value))); - } - - private static void executeSchemaChange(String query) throws Throwable - { - try - { - process(String.format(query, keyspace), ConsistencyLevel.ONE); - } - catch (RuntimeException exc) - { - throw exc.getCause(); - } - } + } -