Merge branch cassandra-3.9 into trunk
Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/d1bd71c4 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/d1bd71c4 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/d1bd71c4 Branch: refs/heads/trunk Commit: d1bd71c4ea5e02d53bfcd99d5fb390d3ee18d17c Parents: e22170e cb97649 Author: Benjamin Lerer <[email protected]> Authored: Tue Aug 16 15:50:14 2016 +0200 Committer: Benjamin Lerer <[email protected]> Committed: Tue Aug 16 15:52:20 2016 +0200 ---------------------------------------------------------------------- CHANGES.txt | 1 + NEWS.txt | 11 +- .../columniterator/AbstractSSTableIterator.java | 24 +- .../db/columniterator/SSTableIterator.java | 17 + .../columniterator/SSTableReversedIterator.java | 19 +- .../cassandra/db/compaction/Scrubber.java | 170 ++++- .../cassandra/db/marshal/ReversedType.java | 10 - .../db/partitions/AbstractBTreePartition.java | 7 +- .../db/partitions/ImmutableBTreePartition.java | 34 +- .../db/rows/UnfilteredRowIterators.java | 28 + .../validation/entities/SecondaryIndexTest.java | 131 ++++ .../cql3/validation/operations/DeleteTest.java | 113 +++ .../operations/SelectOrderByTest.java | 765 ++++++++++--------- .../cql3/validation/operations/SelectTest.java | 355 ++++++++- .../cassandra/db/marshal/ReversedTypeTest.java | 4 +- 15 files changed, 1269 insertions(+), 420 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d1bd71c4/CHANGES.txt ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d1bd71c4/NEWS.txt ---------------------------------------------------------------------- diff --cc NEWS.txt index 56d2827,2edff8e..9bdeec1 --- a/NEWS.txt +++ b/NEWS.txt @@@ -13,64 -13,16 +13,73 @@@ restore snapshots created with the prev 'sstableloader' tool. You can upgrade the file format of your snapshots using the provided 'sstableupgrade' tool. - +3.10 +==== + +New features +------------ + - Support for GROUP BY queries has been added. + - A new compaction-stress tool has been added to test the throughput of compaction + for any cassandra-stress user schema. see compaction-stress help for how to use. + - Compaction can now take into account overlapping tables that don't take part + in the compaction to look for deleted or overwritten data in the compacted tables. + Then such data is found, it can be safely discarded, which in turn should enable + the removal of tombstones over that data. + + The behavior can be engaged in two ways: + - as a "nodetool garbagecollect -g CELL/ROW" operation, which applies + single-table compaction on all sstables to discard deleted data in one step. + - as a "provide_overlapping_tombstones:CELL/ROW/NONE" compaction strategy flag, + which uses overlapping tables as a source of deletions/overwrites during all + compactions. + The argument specifies the granularity at which deleted data is to be found: + - If ROW is specified, only whole deleted rows (or sets of rows) will be + discarded. + - If CELL is specified, any columns whose value is overwritten or deleted + will also be discarded. + - NONE (default) specifies the old behavior, overlapping tables are not used to + decide when to discard data. + Which option to use depends on your workload, both ROW and CELL increase the + disk load on compaction (especially with the size-tiered compaction strategy), + with CELL being more resource-intensive. Both should lead to better read + performance if deleting rows (resp. overwriting or deleting cells) is common. + - Prepared statements are now persisted in the table prepared_statements in + the system keyspace. Upon startup, this table is used to preload all + previously prepared statements - i.e. in many cases clients do not need to + re-prepare statements against restarted nodes. + - cqlsh can now connect to older Cassandra versions by downgrading the native + protocol version. Please note that this is currently not part of our release + testing and, as a consequence, it is not guaranteed to work in all cases. + See CASSANDRA-12150 for more details. + - Snapshots that are automatically taken before a table is dropped or truncated + will have a "dropped" or "truncated" prefix on their snapshot tag name. + - Metrics are exposed for successful and failed authentication attempts. + These can be located using the object names org.apache.cassandra.metrics:type=Client,name=AuthSuccess + and org.apache.cassandra.metrics:type=Client,name=AuthFailure respectively. + - Add support to "unset" JSON fields in prepared statements by specifying DEFAULT UNSET. + See CASSANDRA-11424 for details + +Upgrading +--------- + - Logs written to stdout are now consistent with logs written to files. + Time is now local (it was UTC on the console and local in files). Date, thread, file + and line info where added to stdout. (see CASSANDRA-12004) + - The 'clientutil' jar, which has been somewhat broken on the 3.x branch, is not longer provided. + The features provided by that jar are provided by any good java driver and we advise relying on drivers rather on + that jar, but if you need that jar for backward compatiblity until you do so, you should use the version provided + on previous Cassandra branch, like the 3.0 branch (by design, the functionality provided by that jar are stable + accross versions so using the 3.0 jar for a client connecting to 3.x should work without issues). + + 3.9 + === + + Upgrading + --------- + - The ReversedType behaviour has been corrected for clustering columns of + BYTES type containing empty value. Scrub should be run on the existing + SSTables containing a descending clustering column of BYTES type to correct + their ordering. See CASSANDRA-12127 for more details. + 3.8 === http://git-wip-us.apache.org/repos/asf/cassandra/blob/d1bd71c4/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d1bd71c4/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d1bd71c4/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/d1bd71c4/src/java/org/apache/cassandra/db/compaction/Scrubber.java ---------------------------------------------------------------------- diff --cc src/java/org/apache/cassandra/db/compaction/Scrubber.java index 2cfc75d,5443d8e..0007e30 --- a/src/java/org/apache/cassandra/db/compaction/Scrubber.java +++ b/src/java/org/apache/cassandra/db/compaction/Scrubber.java @@@ -325,6 -299,38 +299,38 @@@ public class Scrubber implements Closea } } + @SuppressWarnings("resource") + private boolean tryAppend(DecoratedKey prevKey, DecoratedKey key, SSTableRewriter writer) + { + // OrderCheckerIterator will check, at iteration time, that the rows are in the proper order. If it detects + // that one row is out of order, it will stop returning them. The remaining rows will be sorted and added + // to the outOfOrder set that will be later written to a new SSTable. - OrderCheckerIterator sstableIterator = new OrderCheckerIterator(new RowMergingSSTableIterator(sstable, dataFile, key), ++ OrderCheckerIterator sstableIterator = new OrderCheckerIterator(new RowMergingSSTableIterator(SSTableIdentityIterator.create(sstable, dataFile, key)), + cfs.metadata.comparator); + + try (UnfilteredRowIterator iterator = withValidation(sstableIterator, dataFile.getPath())) + { + if (prevKey != null && prevKey.compareTo(key) > 0) + { + saveOutOfOrderRow(prevKey, key, iterator); + return false; + } + + if (writer.tryAppend(iterator) == null) + emptyRows++; + else + goodRows++; + } + + if (sstableIterator.hasRowsOutOfOrder()) + { + outputHandler.warn(String.format("Out of order rows found in partition: %s", key)); + outOfOrder.add(sstableIterator.getRowsOutOfOrder()); + } + + return true; + } + private void updateIndexKey() { currentIndexKey = nextIndexKey; http://git-wip-us.apache.org/repos/asf/cassandra/blob/d1bd71c4/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java ----------------------------------------------------------------------
