merge from 1.0
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/efba6788 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/efba6788 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/efba6788 Branch: refs/heads/cassandra-1.1 Commit: efba67888ef832d2c97ae04e226487b5a5fd07c2 Parents: fbb5ec0 5bcfcbc Author: Jonathan Ellis <[email protected]> Authored: Mon Jul 2 03:01:40 2012 -0500 Committer: Jonathan Ellis <[email protected]> Committed: Mon Jul 2 03:01:40 2012 -0500 ---------------------------------------------------------------------- CHANGES.txt | 3 +++ .../cassandra/db/AbstractColumnContainer.java | 4 ++-- src/java/org/apache/cassandra/db/Column.java | 2 +- src/java/org/apache/cassandra/db/IColumn.java | 2 +- .../org/apache/cassandra/db/IColumnContainer.java | 2 +- .../cassandra/db/compaction/PrecompactedRow.java | 2 +- 6 files changed, 9 insertions(+), 6 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/efba6788/CHANGES.txt ---------------------------------------------------------------------- diff --cc CHANGES.txt index 7953c4e,c752244..72991f1 --- a/CHANGES.txt +++ b/CHANGES.txt @@@ -1,128 -1,6 +1,131 @@@ -1.0.11 +1.1.3 + * restore pre-CASSANDRA-3862 approach to removing expired tombstones + from row cache during compaction (CASSANDRA-4364) + * (stress) support for CQL prepared statements (CASSANDRA-3633) ++Merged from 1.0: + * allow dropping columns shadowed by not-yet-expired supercolumn or row + tombstones in PrecompactedRow (CASSANDRA-4396) + + +1.1.2 + * Fix cleanup not deleting index entries (CASSANDRA-4379) + * Use correct partitioner when saving + loading caches (CASSANDRA-4331) + * Check schema before trying to export sstable (CASSANDRA-2760) + * Raise a meaningful exception instead of NPE when PFS encounters + an unconfigured node + no default (CASSANDRA-4349) + * fix bug in sstable blacklisting with LCS (CASSANDRA-4343) + * LCS no longer promotes tiny sstables out of L0 (CASSANDRA-4341) + * skip tombstones during hint replay (CASSANDRA-4320) + * fix NPE in compactionstats (CASSANDRA-4318) + * enforce 1m min keycache for auto (CASSANDRA-4306) + * Have DeletedColumn.isMFD always return true (CASSANDRA-4307) + * (cql3) exeption message for ORDER BY constraints said primary filter can be + an IN clause, which is misleading (CASSANDRA-4319) + * (cql3) Reject (not yet supported) creation of 2ndardy indexes on tables with + composite primary keys (CASSANDRA-4328) + * Set JVM stack size to 160k for java 7 (CASSANDRA-4275) + * cqlsh: add COPY command to load data from CSV flat files (CASSANDRA-4012) + * CFMetaData.fromThrift to throw ConfigurationException upon error (CASSANDRA-4353) + * Use CF comparator to sort indexed columns in SecondaryIndexManager + (CASSANDRA-4365) + * add strategy_options to the KSMetaData.toString() output (CASSANDRA-4248) + * (cql3) fix range queries containing unqueried results (CASSANDRA-4372) + * (cql3) allow updating column_alias types (CASSANDRA-4041) + * (cql3) Fix deletion bug (CASSANDRA-4193) + * Fix computation of overlapping sstable for leveled compaction (CASSANDRA-4321) + * Improve scrub and allow to run it offline (CASSANDRA-4321) + * Fix assertionError in StorageService.bulkLoad (CASSANDRA-4368) + * (cqlsh) add option to authenticate to a keyspace at startup (CASSANDRA-4108) + * (cqlsh) fix ASSUME functionality (CASSANDRA-4352) + * Fix ColumnFamilyRecordReader to not return progress > 100% (CASSANDRA-3942) +Merged from 1.0: + * Set gc_grace on index CF to 0 (CASSANDRA-4314) + + +1.1.1 + * allow larger cache capacities than 2GB (CASSANDRA-4150) + * add getsstables command to nodetool (CASSANDRA-4199) + * apply parent CF compaction settings to secondary index CFs (CASSANDRA-4280) + * preserve commitlog size cap when recycling segments at startup + (CASSANDRA-4201) + * (Hadoop) fix split generation regression (CASSANDRA-4259) + * ignore min/max compactions settings in LCS, while preserving + behavior that min=max=0 disables autocompaction (CASSANDRA-4233) + * log number of rows read from saved cache (CASSANDRA-4249) + * calculate exact size required for cleanup operations (CASSANDRA-1404) + * avoid blocking additional writes during flush when the commitlog + gets behind temporarily (CASSANDRA-1991) + * enable caching on index CFs based on data CF cache setting (CASSANDRA-4197) + * warn on invalid replication strategy creation options (CASSANDRA-4046) + * remove [Freeable]Memory finalizers (CASSANDRA-4222) + * include tombstone size in ColumnFamily.size, which can prevent OOM + during sudden mass delete operations by yielding a nonzero liveRatio + (CASSANDRA-3741) + * Open 1 sstableScanner per level for leveled compaction (CASSANDRA-4142) + * Optimize reads when row deletion timestamps allow us to restrict + the set of sstables we check (CASSANDRA-4116) + * add support for commitlog archiving and point-in-time recovery + (CASSANDRA-3690) + * avoid generating redundant compaction tasks during streaming + (CASSANDRA-4174) + * add -cf option to nodetool snapshot, and takeColumnFamilySnapshot to + StorageService mbean (CASSANDRA-556) + * optimize cleanup to drop entire sstables where possible (CASSANDRA-4079) + * optimize truncate when autosnapshot is disabled (CASSANDRA-4153) + * update caches to use byte[] keys to reduce memory overhead (CASSANDRA-3966) + * add column limit to cli (CASSANDRA-3012, 4098) + * clean up and optimize DataOutputBuffer, used by CQL compression and + CompositeType (CASSANDRA-4072) + * optimize commitlog checksumming (CASSANDRA-3610) + * identify and blacklist corrupted SSTables from future compactions + (CASSANDRA-2261) + * Move CfDef and KsDef validation out of thrift (CASSANDRA-4037) + * Expose API to repair a user provided range (CASSANDRA-3912) + * Add way to force the cassandra-cli to refresh its schema (CASSANDRA-4052) + * Avoid having replicate on write tasks stacking up at CL.ONE (CASSANDRA-2889) + * (cql3) Backwards compatibility for composite comparators in non-cql3-aware + clients (CASSANDRA-4093) + * (cql3) Fix order by for reversed queries (CASSANDRA-4160) + * (cql3) Add ReversedType support (CASSANDRA-4004) + * (cql3) Add timeuuid type (CASSANDRA-4194) + * (cql3) Minor fixes (CASSANDRA-4185) + * (cql3) Fix prepared statement in BATCH (CASSANDRA-4202) + * (cql3) Reduce the list of reserved keywords (CASSANDRA-4186) + * (cql3) Move max/min compaction thresholds to compaction strategy options + (CASSANDRA-4187) + * Fix exception during move when localhost is the only source (CASSANDRA-4200) + * (cql3) Allow paging through non-ordered partitioner results (CASSANDRA-3771) + * (cql3) Fix drop index (CASSANDRA-4192) + * (cql3) Don't return range ghosts anymore (CASSANDRA-3982) + * fix re-creating Keyspaces/ColumnFamilies with the same name as dropped + ones (CASSANDRA-4219) + * fix SecondaryIndex LeveledManifest save upon snapshot (CASSANDRA-4230) + * fix missing arrayOffset in FBUtilities.hash (CASSANDRA-4250) + * (cql3) Add name of parameters in CqlResultSet (CASSANDRA-4242) + * (cql3) Correctly validate order by queries (CASSANDRA-4246) + * rename stress to cassandra-stress for saner packaging (CASSANDRA-4256) + * Fix exception on colum metadata with non-string comparator (CASSANDRA-4269) + * Check for unknown/invalid compression options (CASSANDRA-4266) + * (cql3) Adds simple access to column timestamp and ttl (CASSANDRA-4217) + * (cql3) Fix range queries with secondary indexes (CASSANDRA-4257) + * Better error messages from improper input in cli (CASSANDRA-3865) + * Try to stop all compaction upon Keyspace or ColumnFamily drop (CASSANDRA-4221) + * (cql3) Allow keyspace properties to contain hyphens (CASSANDRA-4278) + * (cql3) Correctly validate keyspace access in create table (CASSANDRA-4296) + * Avoid deadlock in migration stage (CASSANDRA-3882) + * Take supercolumn names and deletion info into account in memtable throughput + (CASSANDRA-4264) + * Add back backward compatibility for old style replication factor (CASSANDRA-4294) + * Preserve compatibility with pre-1.1 index queries (CASSANDRA-4262) +Merged from 1.0: + * Fix super columns bug where cache is not updated (CASSANDRA-4190) + * fix maxTimestamp to include row tombstones (CASSANDRA-4116) + * (CLI) properly handle quotes in create/update keyspace commands (CASSANDRA-4129) + * Avoids possible deadlock during bootstrap (CASSANDRA-4159) + * fix stress tool that hangs forever on timeout or error (CASSANDRA-4128) + * stress tool to return appropriate exit code on failure (CASSANDRA-4188) + * fix compaction NPE when out of disk space and assertions disabled + (CASSANDRA-3985) * synchronize LCS getEstimatedTasks to avoid CME (CASSANDRA-4255) * ensure unique streaming session id's (CASSANDRA-4223) * kick off background compaction when min/max thresholds change http://git-wip-us.apache.org/repos/asf/cassandra/blob/efba6788/src/java/org/apache/cassandra/db/AbstractColumnContainer.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/AbstractColumnContainer.java index a87985e,af8a010..c7922b1 --- a/src/java/org/apache/cassandra/db/AbstractColumnContainer.java +++ b/src/java/org/apache/cassandra/db/AbstractColumnContainer.java @@@ -180,24 -234,26 +180,24 @@@ public abstract class AbstractColumnCon return columns.iterator(); } - protected static class DeletionInfo + public Iterator<IColumn> reverseIterator() { - public final long markedForDeleteAt; - public final int localDeletionTime; + return columns.reverseIterator(); + } - public DeletionInfo() - { - this(Long.MIN_VALUE, Integer.MIN_VALUE); - } + public Iterator<IColumn> iterator(ByteBuffer start) + { + return columns.iterator(start); + } - public DeletionInfo(long markedForDeleteAt, int localDeletionTime) - { - this.markedForDeleteAt = markedForDeleteAt; - this.localDeletionTime = localDeletionTime; - } + public Iterator<IColumn> reverseIterator(ByteBuffer start) + { + return columns.reverseIterator(start); } - public boolean hasExpiredTombstones(int gcBefore) + public boolean hasIrrelevantData(int gcBefore) { - if (isMarkedForDelete() && getLocalDeletionTime() < gcBefore) + if (getLocalDeletionTime() < gcBefore) return true; for (IColumn column : columns) http://git-wip-us.apache.org/repos/asf/cassandra/blob/efba6788/src/java/org/apache/cassandra/db/Column.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/Column.java index 9697ac6,072bfde..97aece0 --- a/src/java/org/apache/cassandra/db/Column.java +++ b/src/java/org/apache/cassandra/db/Column.java @@@ -282,52 -281,9 +282,52 @@@ public class Column implements IColum valueValidator.validate(value()); } - public boolean hasExpiredTombstones(int gcBefore) + public boolean hasIrrelevantData(int gcBefore) { - return isMarkedForDelete() && getLocalDeletionTime() < gcBefore; + return getLocalDeletionTime() < gcBefore; + } + + public static Column create(String value, long timestamp, String... names) + { + return new Column(decomposeName(names), UTF8Type.instance.decompose(value), timestamp); + } + + public static Column create(int value, long timestamp, String... names) + { + return new Column(decomposeName(names), Int32Type.instance.decompose(value), timestamp); + } + + public static Column create(boolean value, long timestamp, String... names) + { + return new Column(decomposeName(names), BooleanType.instance.decompose(value), timestamp); + } + + public static Column create(double value, long timestamp, String... names) + { + return new Column(decomposeName(names), DoubleType.instance.decompose(value), timestamp); + } + + public static Column create(ByteBuffer value, long timestamp, String... names) + { + return new Column(decomposeName(names), value, timestamp); + } + + static ByteBuffer decomposeName(String... names) + { + assert names.length > 0; + + if (names.length == 1) + return UTF8Type.instance.decompose(names[0]); + + // not super performant. at this time, only infrequently called schema code uses this. + List<AbstractType<?>> types = new ArrayList<AbstractType<?>>(names.length); + for (int i = 0; i < names.length; i++) + types.add(UTF8Type.instance); + + CompositeType.Builder builder = new CompositeType.Builder(CompositeType.getInstance(types)); + for (String name : names) + builder.add(UTF8Type.instance.decompose(name)); + return builder.build(); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/efba6788/src/java/org/apache/cassandra/db/IColumn.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/efba6788/src/java/org/apache/cassandra/db/IColumnContainer.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/IColumnContainer.java index 33753cf,3259fdd..0635387 --- a/src/java/org/apache/cassandra/db/IColumnContainer.java +++ b/src/java/org/apache/cassandra/db/IColumnContainer.java @@@ -44,9 -44,9 +44,9 @@@ public interface IColumnContaine public boolean isMarkedForDelete(); public long getMarkedForDeleteAt(); - public boolean hasExpiredTombstones(int gcBefore); + public boolean hasIrrelevantData(int gcBefore); - public AbstractType getComparator(); + public AbstractType<?> getComparator(); public Collection<IColumn> getSortedColumns(); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/efba6788/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java index 8ea4639,aced6d9..8ed21ca --- a/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java +++ b/src/java/org/apache/cassandra/db/compaction/PrecompactedRow.java @@@ -67,12 -67,11 +67,12 @@@ public class PrecompactedRow extends Ab // taking this into account. Boolean shouldPurge = null; - if (cf.hasExpiredTombstones(controller.gcBefore)) + if (cf.hasIrrelevantData(controller.gcBefore)) shouldPurge = controller.shouldPurge(key); - ColumnFamily compacted = shouldPurge != null && shouldPurge - ? ColumnFamilyStore.removeDeleted(cf, controller.gcBefore) - : cf; + // We should only gc tombstone if shouldPurge == true. But otherwise, + // it is still ok to collect column that shadowed by their (deleted) + // container, which removeDeleted(cf, Integer.MAX_VALUE) will do + ColumnFamily compacted = ColumnFamilyStore.removeDeleted(cf, shouldPurge != null && shouldPurge ? controller.gcBefore : Integer.MIN_VALUE); if (compacted != null && compacted.metadata().getDefaultValidator().isCommutative()) {
