merge from 1.2
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/c48acd9e Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/c48acd9e Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/c48acd9e Branch: refs/heads/trunk Commit: c48acd9e6e74a4fa9039df0f599fefdb948ee72b Parents: 784738e 304a25e Author: Jonathan Ellis <[email protected]> Authored: Tue Apr 23 13:11:25 2013 -0500 Committer: Jonathan Ellis <[email protected]> Committed: Tue Apr 23 13:11:25 2013 -0500 ---------------------------------------------------------------------- CHANGES.txt | 1 + src/java/org/apache/cassandra/db/ColumnFamily.java | 5 + src/java/org/apache/cassandra/db/ColumnIndex.java | 10 ++ src/java/org/apache/cassandra/db/DeletionInfo.java | 11 +-- src/java/org/apache/cassandra/db/DeletionTime.java | 4 +- .../org/apache/cassandra/db/RowIndexEntry.java | 88 +++++++++------ .../db/columniterator/IndexedSliceReader.java | 6 +- .../db/columniterator/SSTableNamesIterator.java | 16 ++-- .../db/columniterator/SimpleSliceReader.java | 3 +- .../apache/cassandra/io/sstable/Descriptor.java | 6 +- .../apache/cassandra/io/sstable/IndexHelper.java | 37 +------ .../io/sstable/SSTableIdentityIterator.java | 4 +- .../apache/cassandra/io/sstable/SSTableWriter.java | 3 +- .../corrupt-sstables/Keyspace1-Super5-f-2-Data.db | Bin 530200 -> 0 bytes .../Keyspace1-Super5-f-2-Filter.db | Bin 496 -> 0 bytes .../corrupt-sstables/Keyspace1-Super5-f-2-Index.db | Bin 1300 -> 0 bytes .../Keyspace1-Super5-f-2-Statistics.db | Bin 4264 -> 0 bytes test/data/serialization/2.0/db.RowMutation.bin | Bin 3599 -> 3599 bytes test/unit/org/apache/cassandra/Util.java | 10 +- .../org/apache/cassandra/cache/ObjectSizeTest.java | 4 +- .../apache/cassandra/db/ColumnFamilyStoreTest.java | 2 +- .../apache/cassandra/db/RangeTombstoneTest.java | 4 - test/unit/org/apache/cassandra/db/ScrubTest.java | 34 ------ 23 files changed, 101 insertions(+), 147 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index d993894,3d43fe6..4fdef2c --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,43 -1,5 +1,44 @@@ +2.0 + * removed PBSPredictor (CASSANDRA-5455) + * CAS support (CASSANDRA-5062, ) + * Leveled compaction performs size-tiered compactions in L0 + (CASSANDRA-5371, 5439) + * Add yaml network topology snitch for mixed ec2/other envs (CASSANDRA-5339) + * Log when a node is down longer than the hint window (CASSANDRA-4554) + * Optimize tombstone creation for ExpiringColumns (CASSANDRA-4917) + * Improve LeveledScanner work estimation (CASSANDRA-5250, 5407) + * Replace compaction lock with runWithCompactionsDisabled (CASSANDRA-3430) + * Change Message IDs to ints (CASSANDRA-5307) + * Move sstable level information into the Stats component, removing the + need for a separate Manifest file (CASSANDRA-4872) + * avoid serializing to byte[] on commitlog append (CASSANDRA-5199) + * make index_interval configurable per columnfamily (CASSANDRA-3961) + * add default_time_to_live (CASSANDRA-3974) + * add memtable_flush_period_in_ms (CASSANDRA-4237) + * replace supercolumns internally by composites (CASSANDRA-3237, 5123) + * upgrade thrift to 0.9.0 (CASSANDRA-3719) + * drop unnecessary keyspace parameter from user-defined compaction API + (CASSANDRA-5139) + * more robust solution to incomplete compactions + counters (CASSANDRA-5151) + * Change order of directory searching for c*.in.sh (CASSANDRA-3983) + * Add tool to reset SSTable compaction level for LCS (CASSANDRA-5271) + * Allow custom configuration loader (CASSANDRA-5045) + * Remove memory emergency pressure valve logic (CASSANDRA-3534) + * Reduce request latency with eager retry (CASSANDRA-4705) + * cqlsh: Remove ASSUME command (CASSANDRA-5331) + * Rebuild BF when loading sstables if bloom_filter_fp_chance + has changed since compaction (CASSANDRA-5015) + * remove row-level bloom filters (CASSANDRA-4885) + * Change Kernel Page Cache skipping into row preheating (disabled by default) + (CASSANDRA-4937) + * Improve repair by deciding on a gcBefore before sending + out TreeRequests (CASSANDRA-4932) + * Add an official way to disable compactions (CASSANDRA-5074) + * Reenable ALTER TABLE DROP with new semantics (CASSANDRA-3919) + + 1.2.5 + * remove per-row column name bloom filters (CASSANDRA-5492) * Include fatal errors in trace events (CASSANDRA-5447) * Ensure that PerRowSecondaryIndex is notified of row-level deletes (CASSANDRA-5445) http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/db/ColumnFamily.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/ColumnFamily.java index fffdfe4,6164900..4186460 --- a/src/java/org/apache/cassandra/db/ColumnFamily.java +++ b/src/java/org/apache/cassandra/db/ColumnFamily.java @@@ -144,110 -114,120 +144,115 @@@ public abstract class ColumnFamily impl } /** - * @return The CFMetaData for this row + * Clear this column map, removing all columns. */ - public CFMetaData metadata() - { - return cfm; - } + public abstract void clear(); - public IColumnSerializer getColumnSerializer() - { - return cfm.getColumnSerializer(); - } + /** + * Returns the factory used for this ISortedColumns implementation. + */ + public abstract Factory getFactory(); - public OnDiskAtom.Serializer getOnDiskSerializer() - { - return cfm.getOnDiskSerializer(); - } + public abstract DeletionInfo deletionInfo(); + public abstract void setDeletionInfo(DeletionInfo info); - public boolean isSuper() + public abstract void delete(DeletionInfo info); + public abstract void maybeResetDeletionTimes(int gcBefore); + ++ public void delete(DeletionTime deletionTime) + { - return getType() == ColumnFamilyType.Super; ++ delete(new DeletionInfo(deletionTime)); + } + /** - * Same as addAll() but do a cloneMe of SuperColumn if necessary to - * avoid keeping references to the structure (see #3957). + * Adds a column to this column map. + * If a column with the same name is already present in the map, it will + * be replaced by the newly added column. */ - public void addAllWithSCCopy(ColumnFamily cf, Allocator allocator) - { - if (cf.isSuper()) - { - for (IColumn c : cf) - { - columns.addColumn(((SuperColumn)c).cloneMe(), allocator); - } - delete(cf); - } - else - { - addAll(cf, allocator); - } - } + public abstract void addColumn(Column column, Allocator allocator); - public void addColumn(QueryPath path, ByteBuffer value, long timestamp) - { - addColumn(path, value, timestamp, 0); - } + /** + * Adds all the columns of a given column map to this column map. + * This is equivalent to: + * <code> + * for (Column c : cm) + * addColumn(c, ...); + * </code> + * but is potentially faster. + */ + public abstract void addAll(ColumnFamily cm, Allocator allocator, Function<Column, Column> transformation); - public void addColumn(QueryPath path, ByteBuffer value, long timestamp, int timeToLive) - { - assert path.columnName != null : path; - assert !metadata().getDefaultValidator().isCommutative(); - Column column; - if (timeToLive > 0) - column = new ExpiringColumn(path.columnName, value, timestamp, timeToLive); - else - column = new Column(path.columnName, value, timestamp); - addColumn(path.superColumnName, column); - } + /** + * Replace oldColumn if present by newColumn. + * Returns true if oldColumn was present and thus replaced. + * oldColumn and newColumn should have the same name. + */ + public abstract boolean replace(Column oldColumn, Column newColumn); - public void addCounter(QueryPath path, long value) - { - assert path.columnName != null : path; - addColumn(path.superColumnName, new CounterUpdateColumn(path.columnName, value, System.currentTimeMillis())); - } + /** + * Get a column given its name, returning null if the column is not + * present. + */ + public abstract Column getColumn(ByteBuffer name); - public void addTombstone(QueryPath path, ByteBuffer localDeletionTime, long timestamp) - { - assert path.columnName != null : path; - addColumn(path.superColumnName, new DeletedColumn(path.columnName, localDeletionTime, timestamp)); - } + /** + * Returns an iterable with the names of columns in this column map in the same order + * as the underlying columns themselves. + */ + public abstract Iterable<ByteBuffer> getColumnNames(); - public void addTombstone(QueryPath path, int localDeletionTime, long timestamp) - { - assert path.columnName != null : path; - addColumn(path.superColumnName, new DeletedColumn(path.columnName, localDeletionTime, timestamp)); - } + /** + * Returns the columns of this column map as a collection. + * The columns in the returned collection should be sorted as the columns + * in this map. + */ + public abstract Collection<Column> getSortedColumns(); - public void addTombstone(ByteBuffer name, int localDeletionTime, long timestamp) - { - addColumn(null, new DeletedColumn(name, localDeletionTime, timestamp)); - } + /** + * Returns the columns of this column map as a collection. + * The columns in the returned collection should be sorted in reverse + * order of the columns in this map. + */ + public abstract Collection<Column> getReverseSortedColumns(); + + /** + * Returns the number of columns in this map. + */ + public abstract int getColumnCount(); - public void addColumn(ByteBuffer superColumnName, Column column) + /** + * Returns true if this contains no columns or deletion info + */ + public boolean isEmpty() { - IColumn c; - if (superColumnName == null) - { - c = column; - } - else - { - assert isSuper(); - c = new SuperColumn(superColumnName, getSubComparator()); - c.addColumn(column); // checks subcolumn name - } - addColumn(c); + return deletionInfo().isLive() && getColumnCount() == 0; } - public void addAtom(OnDiskAtom atom) + /** + * Returns an iterator over the columns of this map that returns only the matching @param slices. + * The provided slices must be in order and must be non-overlapping. + */ + public abstract Iterator<Column> iterator(ColumnSlice[] slices); + + /** + * Returns a reversed iterator over the columns of this map that returns only the matching @param slices. + * The provided slices must be in reversed order and must be non-overlapping. + */ + public abstract Iterator<Column> reverseIterator(ColumnSlice[] slices); + + /** + * Returns if this map only support inserts in reverse order. + */ + public abstract boolean isInsertReversed(); + + public void delete(ColumnFamily columns) { - if (atom instanceof IColumn) - { - addColumn((IColumn)atom); - } - else - { - assert atom instanceof RangeTombstone; - delete(new DeletionInfo((RangeTombstone)atom, getComparator())); - } + delete(columns.deletionInfo()); } - public void clear() + public void addAll(ColumnFamily cf, Allocator allocator) { - columns.clear(); + addAll(cf, allocator, Functions.<Column>identity()); } /* http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/db/ColumnIndex.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/db/DeletionInfo.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/DeletionInfo.java index 59dbd91,9cd4545..70c0e76 --- a/src/java/org/apache/cassandra/db/DeletionInfo.java +++ b/src/java/org/apache/cassandra/db/DeletionInfo.java @@@ -28,14 -28,14 +28,13 @@@ import com.google.common.base.Objects import com.google.common.collect.Iterables; import org.apache.cassandra.db.marshal.AbstractType; --import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.ISSTableSerializer; ++import org.apache.cassandra.io.ISerializer; import org.apache.cassandra.io.IVersionedSerializer; import org.apache.cassandra.io.sstable.Descriptor; import org.apache.cassandra.net.MessagingService; import org.apache.cassandra.utils.ByteBufferUtil; import org.apache.cassandra.utils.IntervalTree; --import org.apache.cassandra.utils.ObjectSizes; public class DeletionInfo { http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/db/DeletionTime.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/db/RowIndexEntry.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java index f63c577,0493819..f315ee9 --- a/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java +++ b/src/java/org/apache/cassandra/db/columniterator/IndexedSliceReader.java @@@ -85,17 -88,17 +85,17 @@@ class IndexedSliceReader extends Abstra } else { - this.emptyColumnFamily = EmptyColumns.factory.create(sstable.metadata); - emptyColumnFamily.delete(indexEntry.deletionInfo()); - this.emptyColumnFamily = ColumnFamily.create(sstable.metadata); ++ emptyColumnFamily = EmptyColumns.factory.create(sstable.metadata); + emptyColumnFamily.delete(indexEntry.deletionTime()); fetcher = new IndexedBlockFetcher(indexEntry.position); } } else { setToRowStart(sstable, indexEntry, input); - IndexHelper.skipSSTableBloomFilter(file, version); + IndexHelper.skipBloomFilter(file); this.indexes = IndexHelper.deserializeIndex(file); - this.emptyColumnFamily = ColumnFamily.create(sstable.metadata); + this.emptyColumnFamily = EmptyColumns.factory.create(sstable.metadata); emptyColumnFamily.delete(DeletionInfo.serializer().deserializeFromSSTable(file, version)); fetcher = indexes.isEmpty() ? new SimpleBlockFetcher() http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java index 415a1b8,3750665..ea61781 --- a/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java +++ b/src/java/org/apache/cassandra/db/columniterator/SSTableNamesIterator.java @@@ -146,8 -146,8 +146,8 @@@ public class SSTableNamesIterator exten } else { - cf = ColumnFamily.create(sstable.metadata); + cf = ArrayBackedSortedColumns.factory.create(sstable.metadata); - cf.delete(indexEntry.deletionInfo()); + cf.delete(indexEntry.deletionTime()); } List<OnDiskAtom> result = new ArrayList<OnDiskAtom>(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/db/columniterator/SimpleSliceReader.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/io/sstable/Descriptor.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/Descriptor.java index 803969e,748bd01..3e21025 --- a/src/java/org/apache/cassandra/io/sstable/Descriptor.java +++ b/src/java/org/apache/cassandra/io/sstable/Descriptor.java @@@ -66,13 -66,7 +66,13 @@@ public class Descripto // records estimated histogram of deletion times in tombstones // bloom filter (keys and columns) upgraded to Murmur3 // ib (1.2.1): tracks min client timestamp in metadata component + // ic (1.2.5): omits per-row bloom filter of column names + // ja (2.0.0): super columns are serialized as composites (note that there is no real format change, + // this is mostly a marker to know if we should expect super columns or not. We do need + // a major version bump however, because we should not allow streaming of super columns + // into this new format) + // tracks max local deletiontime in sstable metadata + // records bloom_filter_fp_chance in metadata component - // remove row-level BF (CASSANDRA-4885) public static final Version CURRENT = new Version(current_version); @@@ -92,10 -86,6 +92,9 @@@ public final boolean hasPromotedIndexes; public final FilterFactory.Type filterType; public final boolean hasAncestors; + public final boolean hasSuperColumns; - public final boolean hasBloomFilterSizeInHeader; + public final boolean tracksMaxLocalDeletionTime; + public final boolean hasBloomFilterFPChance; public final boolean hasRowLevelBF; public Version(String version) @@@ -121,10 -110,7 +120,9 @@@ filterType = FilterFactory.Type.MURMUR2; else filterType = FilterFactory.Type.MURMUR3; - hasBloomFilterSizeInHeader = version.compareTo("ia") < 0; + hasSuperColumns = version.compareTo("ja") < 0; + hasBloomFilterFPChance = version.compareTo("ja") >= 0; - hasRowLevelBF = version.compareTo("ja") < 0; + hasRowLevelBF = version.compareTo("ic") < 0; } /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/io/sstable/IndexHelper.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java index 147472a,07d78e4..7c3f146 --- a/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java +++ b/src/java/org/apache/cassandra/io/sstable/SSTableIdentityIterator.java @@@ -151,14 -151,13 +151,14 @@@ public class SSTableIdentityIterator im if (sstable != null && !dataVersion.hasPromotedIndexes) { - IndexHelper.skipSSTableBloomFilter(inputWithTracker, dataVersion); + IndexHelper.skipBloomFilter(inputWithTracker); IndexHelper.skipIndex(inputWithTracker); } - columnFamily = ColumnFamily.create(metadata); + columnFamily = EmptyColumns.factory.create(metadata); columnFamily.delete(DeletionInfo.serializer().deserializeFromSSTable(inputWithTracker, dataVersion)); - atomSerializer = columnFamily.getOnDiskSerializer(); + columnCount = inputWithTracker.readInt(); + atomIterator = columnFamily.metadata().getOnDiskIterator(inputWithTracker, columnCount, dataVersion); columnPosition = dataStart + inputWithTracker.getBytesRead(); } catch (IOException e) http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/src/java/org/apache/cassandra/io/sstable/SSTableWriter.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/test/data/serialization/2.0/db.RowMutation.bin ---------------------------------------------------------------------- diff --cc test/data/serialization/2.0/db.RowMutation.bin index f9e634f,0000000..f779233 mode 100644,000000..100644 Binary files differ http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/test/unit/org/apache/cassandra/Util.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/Util.java index 2de8c0c,b2414ce..c1e35b4 --- a/test/unit/org/apache/cassandra/Util.java +++ b/test/unit/org/apache/cassandra/Util.java @@@ -158,18 -158,18 +158,18 @@@ public class Uti } /** -- * Writes out a bunch of rows for a single column family. ++ * Writes out a bunch of mutations for a single column family. * -- * @param rows A group of RowMutations for the same table and column family. ++ * @param mutations A group of RowMutations for the same table and column family. * @return The ColumnFamilyStore that was used. */ -- public static ColumnFamilyStore writeColumnFamily(List<IMutation> rms) throws IOException, ExecutionException, InterruptedException ++ public static ColumnFamilyStore writeColumnFamily(List<IMutation> mutations) throws IOException, ExecutionException, InterruptedException { -- IMutation first = rms.get(0); ++ IMutation first = mutations.get(0); String tablename = first.getTable(); UUID cfid = first.getColumnFamilyIds().iterator().next(); -- for (IMutation rm : rms) ++ for (IMutation rm : mutations) rm.apply(); ColumnFamilyStore store = Table.open(tablename).getColumnFamilyStore(cfid); http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java index c11fcf2,a394644..b029e29 --- a/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java +++ b/test/unit/org/apache/cassandra/db/ColumnFamilyStoreTest.java @@@ -672,11 -681,11 +672,11 @@@ public class ColumnFamilyStoreTest exte sp.getSlice_range().setStart(ArrayUtils.EMPTY_BYTE_ARRAY); sp.getSlice_range().setFinish(ArrayUtils.EMPTY_BYTE_ARRAY); - assertRowAndColCount(1, 6, scfName, false, cfs.getRangeSlice(scfName, Util.range("f", "g"), 100, ThriftValidation.asIFilter(sp, cfs.getComparator()), null)); + assertRowAndColCount(1, 6, scfName, false, cfs.getRangeSlice(Util.range("f", "g"), 100, ThriftValidation.asIFilter(sp, cfs.metadata, scfName), null)); - // deeleet. + // delete - RowMutation rm = new RowMutation(table.name, key.key); - rm.delete(new QueryPath(cfName, scfName), 2); + RowMutation rm = new RowMutation(table.getName(), key.key); + rm.deleteRange(cfName, SuperColumns.startOf(scfName), SuperColumns.endOf(scfName), 2); rm.apply(); // verify delete. http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/RangeTombstoneTest.java index ae147d4,c531461..e5f0acb --- a/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java +++ b/test/unit/org/apache/cassandra/db/RangeTombstoneTest.java @@@ -161,13 -164,9 +161,9 @@@ public class RangeTombstoneTest extend return ByteBufferUtil.bytes(i); } - private static void insertData(ColumnFamilyStore cfs, String key) throws Exception - { - } - private static void add(RowMutation rm, int value, long timestamp) { - rm.add(new QueryPath(CFNAME, null, b(value)), b(value), timestamp); + rm.add(CFNAME, b(value), b(value), timestamp); } private static void delete(ColumnFamily cf, int from, int to, long timestamp) http://git-wip-us.apache.org/repos/asf/cassandra/blob/c48acd9e/test/unit/org/apache/cassandra/db/ScrubTest.java ---------------------------------------------------------------------- diff --cc test/unit/org/apache/cassandra/db/ScrubTest.java index 93d0a0d,26f0e78..ed2bac1 --- a/test/unit/org/apache/cassandra/db/ScrubTest.java +++ b/test/unit/org/apache/cassandra/db/ScrubTest.java @@@ -50,7 -45,7 +50,6 @@@ public class ScrubTest extends SchemaLo { public String TABLE = "Keyspace1"; public String CF = "Standard1"; -- public String CF2 = "Super5"; public String CF3 = "Standard2"; public String copySSTables(String cf) throws IOException
