Updated Branches: refs/heads/cassandra-1.1 ec1c64ab5 -> 2cd73633b refs/heads/trunk 781b248f3 -> 67cc0dc4a
merge from 1.1 Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/67cc0dc4 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/67cc0dc4 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/67cc0dc4 Branch: refs/heads/trunk Commit: 67cc0dc4ade15a3693a62cb6ebb85ac4fc9ad328 Parents: 781b248 2cd7363 Author: Jonathan Ellis <[email protected]> Authored: Wed Sep 5 16:39:08 2012 -0500 Committer: Jonathan Ellis <[email protected]> Committed: Wed Sep 5 16:41:05 2012 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + .../AbstractSimplePerColumnSecondaryIndex.java | 6 ++++++ .../apache/cassandra/db/index/SecondaryIndex.java | 10 +++++++++- .../cassandra/db/index/SecondaryIndexManager.java | 7 ++++--- .../apache/cassandra/db/index/keys/KeysIndex.java | 12 ++---------- .../cassandra/db/SecondaryIndexColumnSizeTest.java | 10 ++++++++++ 6 files changed, 32 insertions(+), 14 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/67cc0dc4/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 8125e38,04fa0bf..46222c5 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,56 -1,5 +1,57 @@@ +1.2-dev + * optimize WriteResponse (CASSANDRA-4548) + * new metrics (CASSANDRA-4009) + * redesign KEYS indexes to avoid read-before-write (CASSANDRA-2897) + * debug tracing (CASSANDRA-1123) + * parallelize row cache loading (CASSANDRA-4282) + * Make compaction, flush JBOD-aware (CASSANDRA-4292) + * run local range scans on the read stage (CASSANDRA-3687) + * clean up ioexceptions (CASSANDRA-2116) + * add disk_failure_policy (CASSANDRA-2118) + * Introduce new json format with row level deletion (CASSANDRA-4054) + * remove redundant "name" column from schema_keyspaces (CASSANDRA-4433) + * improve "nodetool ring" handling of multi-dc clusters (CASSANDRA-3047) + * update NTS calculateNaturalEndpoints to be O(N log N) (CASSANDRA-3881) + * add UseCondCardMark XX jvm settings on jdk 1.7 (CASSANDRA-4366) + * split up rpc timeout by operation type (CASSANDRA-2819) + * rewrite key cache save/load to use only sequential i/o (CASSANDRA-3762) + * update MS protocol with a version handshake + broadcast address id + (CASSANDRA-4311) + * multithreaded hint replay (CASSANDRA-4189) + * add inter-node message compression (CASSANDRA-3127) + * remove COPP (CASSANDRA-2479) + * Track tombstone expiration and compact when tombstone content is + higher than a configurable threshold, default 20% (CASSANDRA-3442, 4234) + * update MurmurHash to version 3 (CASSANDRA-2975) + * (CLI) track elapsed time for `delete' operation (CASSANDRA-4060) + * (CLI) jline version is bumped to 1.0 to properly support + 'delete' key function (CASSANDRA-4132) + * Save IndexSummary into new SSTable 'Summary' component (CASSANDRA-2392, 4289) + * Add support for range tombstones (CASSANDRA-3708) + * Improve MessagingService efficiency (CASSANDRA-3617) + * Avoid ID conflicts from concurrent schema changes (CASSANDRA-3794) + * Set thrift HSHA server thread limit to unlimited by default (CASSANDRA-4277) + * Avoids double serialization of CF id in RowMutation messages + (CASSANDRA-4293) + * stream compressed sstables directly with java nio (CASSANDRA-4297) + * Support multiple ranges in SliceQueryFilter (CASSANDRA-3885) + * Add column metadata to system column families (CASSANDRA-4018) + * (cql3) Always use composite types by default (CASSANDRA-4329) + * (cql3) Add support for set, map and list (CASSANDRA-3647) + * Validate date type correctly (CASSANDRA-4441) + * (cql3) Allow definitions with only a PK (CASSANDRA-4361) + * (cql3) Add support for row key composites (CASSANDRA-4179) + * improve DynamicEndpointSnitch by using reservoir sampling (CASSANDRA-4038) + * (cql3) Add support for 2ndary indexes (CASSANDRA-3680) + * (cql3) fix defining more than one PK to be invalid (CASSANDRA-4477) + * remove schema agreement checking from all external APIs (Thrift, CQL and CQL3) (CASSANDRA-4487) + * add Murmur3Partitioner and make it default for new installations (CASSANDRA-3772) + * (cql3) update pseudo-map syntax to use map syntax (CASSANDRA-4497) + * Finer grained exceptions hierarchy and provides error code with exceptions (CASSANDRA-3979) + + 1.1.5 + * add SecondaryIndex.reload API (CASSANDRA-4581) * use millis + atomicint for commitlog segment creation instead of nanotime, which has issues under some hypervisors (CASSANDRA-4601) * fix FD leak in slice queries (CASSANDRA-4571) http://git-wip-us.apache.org/repos/asf/cassandra/blob/67cc0dc4/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java index 3a012f6,0000000..ecabf23 mode 100644,000000..100644 --- a/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/AbstractSimplePerColumnSecondaryIndex.java @@@ -1,160 -1,0 +1,166 @@@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.cassandra.db.index; + +import java.nio.ByteBuffer; +import java.util.concurrent.ExecutionException; + +import org.apache.cassandra.config.CFMetaData; +import org.apache.cassandra.config.ColumnDefinition; +import org.apache.cassandra.db.*; +import org.apache.cassandra.db.marshal.*; +import org.apache.cassandra.dht.*; +import org.apache.cassandra.utils.ByteBufferUtil; + +/** + * Implements a secondary index for a column family using a second column family + * in which the row keys are indexed values, and column names are base row keys. + */ +public abstract class AbstractSimplePerColumnSecondaryIndex extends PerColumnSecondaryIndex +{ + private ColumnFamilyStore indexCfs; + + public void init() + { + assert baseCfs != null && columnDefs != null && columnDefs.size() == 1; + + ColumnDefinition columnDef = columnDefs.iterator().next(); + init(columnDef); + + AbstractType indexComparator = SecondaryIndex.getIndexComparator(baseCfs.metadata, columnDef); + CFMetaData indexedCfMetadata = CFMetaData.newIndexMetadata(baseCfs.metadata, columnDef, indexComparator); + indexCfs = ColumnFamilyStore.createColumnFamilyStore(baseCfs.table, + indexedCfMetadata.cfName, + new LocalPartitioner(columnDef.getValidator()), + indexedCfMetadata); + + // enable and initialize row cache based on parent's setting and indexed column's cardinality + CFMetaData.Caching baseCaching = baseCfs.metadata.getCaching(); + if (baseCaching == CFMetaData.Caching.ALL || baseCaching == CFMetaData.Caching.ROWS_ONLY) + { + /* + * # of index CF's key = cardinality of indexed column. + * if # of keys stored in index CF is more than average column counts (means tall table), + * then consider it as high cardinality. + */ + double estimatedKeys = indexCfs.estimateKeys(); + double averageColumnCount = indexCfs.getMeanColumns(); + if (averageColumnCount > 0 && estimatedKeys / averageColumnCount > 1) + { + logger.debug("turning row cache on for " + indexCfs.getColumnFamilyName()); + indexCfs.metadata.caching(baseCaching); + indexCfs.initRowCache(); + } + } + } + + protected abstract void init(ColumnDefinition columnDef); + + protected abstract ByteBuffer makeIndexColumnName(ByteBuffer rowKey, IColumn column); + + public void delete(ByteBuffer rowKey, IColumn column) + { + if (column.isMarkedForDelete()) + return; + + DecoratedKey valueKey = getIndexKeyFor(column.value()); + int localDeletionTime = (int) (System.currentTimeMillis() / 1000); + ColumnFamily cfi = ColumnFamily.create(indexCfs.metadata); + cfi.addTombstone(makeIndexColumnName(rowKey, column), localDeletionTime, column.timestamp()); + indexCfs.apply(valueKey, cfi, SecondaryIndexManager.nullUpdater); + if (logger.isDebugEnabled()) + logger.debug("removed index entry for cleaned-up value {}:{}", valueKey, cfi); + } + + public void insert(ByteBuffer rowKey, IColumn column) + { + DecoratedKey valueKey = getIndexKeyFor(column.value()); + ColumnFamily cfi = ColumnFamily.create(indexCfs.metadata); + ByteBuffer name = makeIndexColumnName(rowKey, column); + if (column instanceof ExpiringColumn) + { + ExpiringColumn ec = (ExpiringColumn)column; + cfi.addColumn(new ExpiringColumn(name, ByteBufferUtil.EMPTY_BYTE_BUFFER, ec.timestamp(), ec.getTimeToLive(), ec.getLocalDeletionTime())); + } + else + { + cfi.addColumn(new Column(name, ByteBufferUtil.EMPTY_BYTE_BUFFER, column.timestamp())); + } + if (logger.isDebugEnabled()) + logger.debug("applying index row {} in {}", indexCfs.metadata.getKeyValidator().getString(valueKey.key), cfi); + + indexCfs.apply(valueKey, cfi, SecondaryIndexManager.nullUpdater); + } + + public void update(ByteBuffer rowKey, IColumn col) + { + insert(rowKey, col); + } + + public void removeIndex(ByteBuffer columnName) + { + indexCfs.invalidate(); + } + + public void forceBlockingFlush() + { + try + { + indexCfs.forceBlockingFlush(); + } + catch (ExecutionException e) + { + throw new RuntimeException(e); + } + catch (InterruptedException e) + { + throw new AssertionError(e); + } + } + + public void invalidate() + { + indexCfs.invalidate(); + } + + public void truncate(long truncatedAt) + { + indexCfs.discardSSTables(truncatedAt); + } + + public ColumnFamilyStore getIndexCfs() + { + return indexCfs; + } + + public String getIndexName() + { + return indexCfs.columnFamily; + } + + public long getLiveSize() + { + return indexCfs.getMemtableDataSize(); + } ++ ++ public void reload() ++ { ++ indexCfs.metadata.reloadSecondaryIndexMetadata(baseCfs.metadata); ++ indexCfs.reload(); ++ } +} http://git-wip-us.apache.org/repos/asf/cassandra/blob/67cc0dc4/src/java/org/apache/cassandra/db/index/SecondaryIndex.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/SecondaryIndex.java index b0ffe3a,8d734ff..fd9fabf --- a/src/java/org/apache/cassandra/db/index/SecondaryIndex.java +++ b/src/java/org/apache/cassandra/db/index/SecondaryIndex.java @@@ -15,10 -15,11 +15,10 @@@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.cassandra.db.index; + import java.io.IOException; import java.nio.ByteBuffer; - import java.nio.charset.CharacterCodingException; import java.util.*; import java.util.concurrent.*; @@@ -75,6 -66,14 +75,14 @@@ public abstract class SecondaryInde public abstract void init(); /** + * Reload an existing index following a change to its configuration, + * or that of the indexed column(s). Differs from init() in that we expect + * expect new resources (such as CFS for a KEYS index) to be created by + * init() but not here + */ - public abstract void reload() throws IOException; ++ public abstract void reload(); + + /** * Validates the index_options passed in the ColumnDef * @throws ConfigurationException */ http://git-wip-us.apache.org/repos/asf/cassandra/blob/67cc0dc4/src/java/org/apache/cassandra/db/index/SecondaryIndexManager.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/67cc0dc4/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/index/keys/KeysIndex.java index dbc32ce,bbdedce..7174ffc --- a/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java +++ b/src/java/org/apache/cassandra/db/index/keys/KeysIndex.java @@@ -15,24 -15,28 +15,16 @@@ * See the License for the specific language governing permissions and * limitations under the License. */ - package org.apache.cassandra.db.index.keys; -import java.io.IOException; import java.nio.ByteBuffer; import java.util.Set; --import java.util.concurrent.ExecutionException; --import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.ColumnDefinition; - import org.apache.cassandra.exceptions.ConfigurationException; - import org.apache.cassandra.db.*; -import org.apache.cassandra.config.ConfigurationException; -import org.apache.cassandra.db.*; -import org.apache.cassandra.db.index.PerColumnSecondaryIndex; ++import org.apache.cassandra.db.IColumn; +import org.apache.cassandra.db.index.AbstractSimplePerColumnSecondaryIndex; import org.apache.cassandra.db.index.SecondaryIndexSearcher; --import org.apache.cassandra.db.marshal.AbstractType; --import org.apache.cassandra.db.marshal.BytesType; --import org.apache.cassandra.db.marshal.LocalByPartionerType; --import org.apache.cassandra.dht.*; --import org.apache.cassandra.service.StorageService; --import org.apache.cassandra.utils.ByteBufferUtil; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; ++import org.apache.cassandra.exceptions.ConfigurationException; /** * Implements a secondary index for a column family using a second column family http://git-wip-us.apache.org/repos/asf/cassandra/blob/67cc0dc4/test/unit/org/apache/cassandra/db/SecondaryIndexColumnSizeTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/SecondaryIndexColumnSizeTest.java index e1cce4e,645131b..8872d65 --- a/test/unit/org/apache/cassandra/db/SecondaryIndexColumnSizeTest.java +++ b/test/unit/org/apache/cassandra/db/SecondaryIndexColumnSizeTest.java @@@ -127,20 -133,18 +127,25 @@@ public class SecondaryIndexColumnSizeTe { } - @Override - public void deleteFromIndex(DecoratedKey<?> key, List<IColumn> indexedColumnsInRow) - { + public void index(ByteBuffer rowKey, ColumnFamily cf) + { + } + + public void index(ByteBuffer rowKey) + { + } + + public void delete(DecoratedKey key) + { } + + @Override + public void reload() + { + } } - - + + private class MockColumnIndex extends PerColumnSecondaryIndex { @Override @@@ -208,8 -212,13 +213,13 @@@ } @Override - public void updateColumn(DecoratedKey valueKey, ByteBuffer rowKey, IColumn col) throws IOException + public void update(ByteBuffer rowKey, IColumn col) { } + + @Override + public void reload() + { + } } }
