Repository: cassandra Updated Branches: refs/heads/trunk 4abd99363 -> 4fb559b58
9986: Remove SliceableUnfilteredRowIterator Unifies more read path access paths onto Slices, by constructing a Slices object for all ClusteringIndexFilter that would have previously sliced the iterator themselves. All iterators now accept a Slices object, and iterate over its contents as necessary. Project: http://git-wip-us.apache.org/repos/asf/cassandra/repo Commit: http://git-wip-us.apache.org/repos/asf/cassandra/commit/4fb559b5 Tree: http://git-wip-us.apache.org/repos/asf/cassandra/tree/4fb559b5 Diff: http://git-wip-us.apache.org/repos/asf/cassandra/diff/4fb559b5 Branch: refs/heads/trunk Commit: 4fb559b58a8a8419e91a786e44f25f6506f4d037 Parents: 4abd993 Author: Benedict Elliott Smith <bened...@apache.org> Authored: Sun Sep 13 01:35:07 2015 +0100 Committer: Benedict Elliott Smith <bened...@apache.org> Committed: Mon Jan 25 10:52:04 2016 +0000 ---------------------------------------------------------------------- .../org/apache/cassandra/db/ReadResponse.java | 3 +- .../db/SinglePartitionReadCommand.java | 8 +- src/java/org/apache/cassandra/db/Slices.java | 80 -------------------- .../columniterator/AbstractSSTableIterator.java | 33 ++++++-- .../db/columniterator/SSTableIterator.java | 8 +- .../columniterator/SSTableReversedIterator.java | 8 +- .../db/filter/ClusteringIndexFilter.java | 9 +-- .../db/filter/ClusteringIndexNamesFilter.java | 50 ++---------- .../db/filter/ClusteringIndexSliceFilter.java | 6 +- .../db/partitions/AbstractBTreePartition.java | 34 --------- .../db/partitions/PartitionUpdate.java | 11 +-- .../db/rows/SliceableUnfilteredRowIterator.java | 39 ---------- .../io/sstable/format/SSTableReader.java | 6 +- .../io/sstable/format/big/BigTableReader.java | 19 +++-- .../io/sstable/format/big/BigTableScanner.java | 2 +- .../partition/PartitionImplementationTest.java | 33 ++++---- .../cassandra/io/sstable/LegacySSTableTest.java | 7 +- .../sstable/format/ClientModeSSTableTest.java | 8 +- 18 files changed, 86 insertions(+), 278 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/ReadResponse.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/ReadResponse.java b/src/java/org/apache/cassandra/db/ReadResponse.java index 41f0d5d..c114315 100644 --- a/src/java/org/apache/cassandra/db/ReadResponse.java +++ b/src/java/org/apache/cassandra/db/ReadResponse.java @@ -283,13 +283,12 @@ public abstract class ReadResponse { ImmutableBTreePartition partition = toReturn.get(idx++); - ClusteringIndexFilter filter = command.clusteringIndexFilter(partition.partitionKey()); // Pre-3.0, we didn't have a way to express exclusivity for non-composite comparators, so all slices were // inclusive on both ends. If we have exclusive slice ends, we need to filter the results here. if (!command.metadata().isCompound()) - return filter.filter(partition.sliceableUnfilteredIterator(command.columnFilter(), filter.isReversed())); + return partition.unfilteredIterator(command.columnFilter(), filter.getSlices(command.metadata()), filter.isReversed()); return partition.unfilteredIterator(command.columnFilter(), Slices.ALL, filter.isReversed()); } http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java index a1de3d6..9ad9ba3 100644 --- a/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java +++ b/src/java/org/apache/cassandra/db/SinglePartitionReadCommand.java @@ -549,7 +549,7 @@ public class SinglePartitionReadCommand extends ReadCommand sstable.incrementReadCount(); @SuppressWarnings("resource") // 'iter' is added to iterators which is closed on exception, or through the closing of the final merged iterator - UnfilteredRowIterator iter = filter.filter(sstable.iterator(partitionKey(), columnFilter(), filter.isReversed(), isForThrift())); + UnfilteredRowIterator iter = sstable.iterator(partitionKey(), filter.getSlices(metadata()), columnFilter(), filter.isReversed(), isForThrift()); if (!sstable.isRepaired()) oldestUnrepairedTombstone = Math.min(oldestUnrepairedTombstone, sstable.getMinLocalDeletionTime()); @@ -569,7 +569,7 @@ public class SinglePartitionReadCommand extends ReadCommand sstable.incrementReadCount(); @SuppressWarnings("resource") // 'iter' is either closed right away, or added to iterators which is close on exception, or through the closing of the final merged iterator - UnfilteredRowIterator iter = filter.filter(sstable.iterator(partitionKey(), columnFilter(), filter.isReversed(), isForThrift())); + UnfilteredRowIterator iter = sstable.iterator(partitionKey(), filter.getSlices(metadata()), columnFilter(), filter.isReversed(), isForThrift()); if (iter.partitionLevelDeletion().markedForDeleteAt() > minTimestamp) { iterators.add(iter); @@ -704,7 +704,7 @@ public class SinglePartitionReadCommand extends ReadCommand // We need to get the partition deletion and include it if it's live. In any case though, we're done with that sstable. sstable.incrementReadCount(); - try (UnfilteredRowIterator iter = sstable.iterator(partitionKey(), columnFilter(), filter.isReversed(), isForThrift())) + try (UnfilteredRowIterator iter = sstable.iterator(partitionKey(), Slices.ALL, columnFilter(), filter.isReversed(), isForThrift())) { if (iter.partitionLevelDeletion().isLive()) { @@ -717,7 +717,7 @@ public class SinglePartitionReadCommand extends ReadCommand Tracing.trace("Merging data from sstable {}", sstable.descriptor.generation); sstable.incrementReadCount(); - try (UnfilteredRowIterator iter = filter.filter(sstable.iterator(partitionKey(), columnFilter(), filter.isReversed(), isForThrift()))) + try (UnfilteredRowIterator iter = sstable.iterator(partitionKey(), filter.getSlices(metadata()), columnFilter(), filter.isReversed(), isForThrift());) { if (iter.isEmpty()) continue; http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/Slices.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/Slices.java b/src/java/org/apache/cassandra/db/Slices.java index 94dea15..13533d6 100644 --- a/src/java/org/apache/cassandra/db/Slices.java +++ b/src/java/org/apache/cassandra/db/Slices.java @@ -141,16 +141,6 @@ public abstract class Slices implements Iterable<Slice> */ public abstract boolean intersects(List<ByteBuffer> minClusteringValues, List<ByteBuffer> maxClusteringValues); - /** - * Given a sliceable row iterator, returns a row iterator that only return rows selected by the slice of - * this {@code Slices} object. - * - * @param iter the sliceable iterator to filter. - * - * @return an iterator that only returns the rows (or rather Unfiltered) of {@code iter} that are selected by those slices. - */ - public abstract UnfilteredRowIterator makeSliceIterator(SliceableUnfilteredRowIterator iter); - public abstract String toCQLString(CFMetaData metadata); /** @@ -452,65 +442,6 @@ public abstract class Slices implements Iterable<Slice> return false; } - public UnfilteredRowIterator makeSliceIterator(final SliceableUnfilteredRowIterator iter) - { - return new WrappingUnfilteredRowIterator(iter) - { - private int nextSlice = iter.isReverseOrder() ? slices.length - 1 : 0; - private Iterator<Unfiltered> currentSliceIterator = Collections.emptyIterator(); - - private Unfiltered next; - - @Override - public boolean hasNext() - { - prepareNext(); - return next != null; - } - - @Override - public Unfiltered next() - { - prepareNext(); - Unfiltered toReturn = next; - next = null; - return toReturn; - } - - private boolean hasMoreSlice() - { - return isReverseOrder() - ? nextSlice >= 0 - : nextSlice < slices.length; - } - - private Slice popNextSlice() - { - return slices[isReverseOrder() ? nextSlice-- : nextSlice++]; - } - - private void prepareNext() - { - if (next != null) - return; - - while (true) - { - if (currentSliceIterator.hasNext()) - { - next = currentSliceIterator.next(); - return; - } - - if (!hasMoreSlice()) - return; - - currentSliceIterator = iter.slice(popNextSlice()); - } - } - }; - } - public Iterator<Slice> iterator() { return Iterators.forArray(slices); @@ -803,11 +734,6 @@ public abstract class Slices implements Iterable<Slice> return true; } - public UnfilteredRowIterator makeSliceIterator(SliceableUnfilteredRowIterator iter) - { - return iter; - } - public Iterator<Slice> iterator() { return Iterators.singletonIterator(Slice.ALL); @@ -883,12 +809,6 @@ public abstract class Slices implements Iterable<Slice> return false; } - public UnfilteredRowIterator makeSliceIterator(SliceableUnfilteredRowIterator iter) - { - return UnfilteredRowIterators.noRowsIterator(iter.metadata(), iter.partitionKey(), iter.staticRow(), - iter.partitionLevelDeletion(), iter.isReverseOrder()); - } - public Iterator<Slice> iterator() { return Iterators.emptyIterator(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java b/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java index 8ac3dcb..792f5ad 100644 --- a/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java +++ b/src/java/org/apache/cassandra/db/columniterator/AbstractSSTableIterator.java @@ -33,7 +33,7 @@ import org.apache.cassandra.io.util.FileDataInput; import org.apache.cassandra.io.util.FileMark; import org.apache.cassandra.utils.ByteBufferUtil; -abstract class AbstractSSTableIterator implements SliceableUnfilteredRowIterator +abstract class AbstractSSTableIterator implements UnfilteredRowIterator { protected final SSTableReader sstable; protected final DecoratedKey key; @@ -48,18 +48,23 @@ abstract class AbstractSSTableIterator implements SliceableUnfilteredRowIterator private boolean isClosed; + protected final Slices slices; + protected int slice; + @SuppressWarnings("resource") // We need this because the analysis is not able to determine that we do close // file on every path where we created it. protected AbstractSSTableIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, + Slices slices, ColumnFilter columnFilter, boolean isForThrift) { this.sstable = sstable; this.key = key; this.columns = columnFilter; + this.slices = slices; this.helper = new SerializationHelper(sstable.metadata, sstable.descriptor.version.correspondingMessagingVersion(), SerializationHelper.Flag.LOCAL, columnFilter); this.isForThrift = isForThrift; @@ -109,6 +114,9 @@ abstract class AbstractSSTableIterator implements SliceableUnfilteredRowIterator this.reader = needsReader ? createReader(indexEntry, file, shouldCloseFile) : null; } + if (reader != null && slices.size() > 0) + reader.setForSlice(slices.get(0)); + if (reader == null && file != null && shouldCloseFile) file.close(); } @@ -216,7 +224,19 @@ abstract class AbstractSSTableIterator implements SliceableUnfilteredRowIterator public boolean hasNext() { - return reader != null && reader.hasNext(); + while (true) + { + if (reader == null) + return false; + + if (reader.hasNext()) + return true; + + if (++slice >= slices.size()) + return false; + + slice(slices.get(slice)); + } } public Unfiltered next() @@ -225,15 +245,12 @@ abstract class AbstractSSTableIterator implements SliceableUnfilteredRowIterator return reader.next(); } - public Iterator<Unfiltered> slice(Slice slice) + private void slice(Slice slice) { try { - if (reader == null) - return Collections.emptyIterator(); - - reader.setForSlice(slice); - return reader; + if (reader != null) + reader.setForSlice(slice); } catch (IOException e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java index 0409310..302fc24 100644 --- a/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java +++ b/src/java/org/apache/cassandra/db/columniterator/SSTableIterator.java @@ -31,19 +31,15 @@ import org.apache.cassandra.io.util.FileDataInput; */ public class SSTableIterator extends AbstractSSTableIterator { - public SSTableIterator(SSTableReader sstable, DecoratedKey key, ColumnFilter columns, boolean isForThrift) - { - this(sstable, null, key, sstable.getPosition(key, SSTableReader.Operator.EQ), columns, isForThrift); - } - public SSTableIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, + Slices slices, ColumnFilter columns, boolean isForThrift) { - super(sstable, file, key, indexEntry, columns, isForThrift); + super(sstable, file, key, indexEntry, slices, columns, isForThrift); } protected Reader createReader(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile) http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java b/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java index 14cec36..ba5acf5 100644 --- a/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java +++ b/src/java/org/apache/cassandra/db/columniterator/SSTableReversedIterator.java @@ -34,19 +34,15 @@ import org.apache.cassandra.utils.btree.BTree; */ public class SSTableReversedIterator extends AbstractSSTableIterator { - public SSTableReversedIterator(SSTableReader sstable, DecoratedKey key, ColumnFilter columns, boolean isForThrift) - { - this(sstable, null, key, sstable.getPosition(key, SSTableReader.Operator.EQ), columns, isForThrift); - } - public SSTableReversedIterator(SSTableReader sstable, FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, + Slices slices, ColumnFilter columns, boolean isForThrift) { - super(sstable, file, key, indexEntry, columns, isForThrift); + super(sstable, file, key, indexEntry, slices, columns, isForThrift); } protected Reader createReader(RowIndexEntry indexEntry, FileDataInput file, boolean shouldCloseFile) http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java index e3f824f..d1907b1 100644 --- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexFilter.java @@ -128,14 +128,7 @@ public interface ClusteringIndexFilter */ public UnfilteredRowIterator filterNotIndexed(ColumnFilter columnFilter, UnfilteredRowIterator iterator); - /** - * Returns an iterator that only returns the rows of the provided sliceable iterator that this filter selects. - * - * @param iterator the sliceable iterator for which we should filter rows. - * - * @return an iterator that only returns the rows (or rather unfiltered) from {@code iterator} that are selected by this filter. - */ - public UnfilteredRowIterator filter(SliceableUnfilteredRowIterator iterator); + public Slices getSlices(CFMetaData metadata); /** * Given a partition, returns a row iterator for the rows of this partition that are selected by this filter. http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java index 17cd802..6a010d9 100644 --- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexNamesFilter.java @@ -126,52 +126,12 @@ public class ClusteringIndexNamesFilter extends AbstractClusteringIndexFilter return Transformation.apply(iterator, new FilterNotIndexed()); } - public UnfilteredRowIterator filter(final SliceableUnfilteredRowIterator iter) + public Slices getSlices(CFMetaData metadata) { - // Please note that this method assumes that rows from 'iter' already have their columns filtered, i.e. that - // they only include columns that we select. - return new WrappingUnfilteredRowIterator(iter) - { - private final Iterator<Clustering> clusteringIter = clusteringsInQueryOrder.iterator(); - private Iterator<Unfiltered> currentClustering; - private Unfiltered next; - - @Override - public boolean hasNext() - { - if (next != null) - return true; - - if (currentClustering != null && currentClustering.hasNext()) - { - next = currentClustering.next(); - return true; - } - - while (clusteringIter.hasNext()) - { - Clustering nextClustering = clusteringIter.next(); - currentClustering = iter.slice(Slice.make(nextClustering)); - if (currentClustering.hasNext()) - { - next = currentClustering.next(); - return true; - } - } - return false; - } - - @Override - public Unfiltered next() - { - if (next == null && !hasNext()) - throw new NoSuchElementException(); - - Unfiltered toReturn = next; - next = null; - return toReturn; - } - }; + Slices.Builder builder = new Slices.Builder(metadata.comparator, clusteringsInQueryOrder.size()); + for (Clustering clustering : clusteringsInQueryOrder) + builder.add(Slice.make(clustering)); + return builder.build(); } public UnfilteredRowIterator getUnfilteredRowIterator(final ColumnFilter columnFilter, final Partition partition) http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java b/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java index 7a174ee..ba30dcf 100644 --- a/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java +++ b/src/java/org/apache/cassandra/db/filter/ClusteringIndexSliceFilter.java @@ -114,11 +114,9 @@ public class ClusteringIndexSliceFilter extends AbstractClusteringIndexFilter return Transformation.apply(iterator, new FilterNotIndexed()); } - public UnfilteredRowIterator filter(SliceableUnfilteredRowIterator iterator) + public Slices getSlices(CFMetaData metadata) { - // Please note that this method assumes that rows from 'iter' already have their columns filtered, i.e. that - // they only include columns that we select. - return slices.makeSliceIterator(iterator); + return slices; } public UnfilteredRowIterator getUnfilteredRowIterator(ColumnFilter columnFilter, Partition partition) http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java index b7fa691..e44124f 100644 --- a/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java +++ b/src/java/org/apache/cassandra/db/partitions/AbstractBTreePartition.java @@ -282,40 +282,6 @@ public abstract class AbstractBTreePartition implements Partition, Iterable<Row> } } - public class SliceableIterator extends AbstractIterator implements SliceableUnfilteredRowIterator - { - private Iterator<Unfiltered> iterator; - - protected SliceableIterator(ColumnFilter selection, boolean isReversed) - { - super(selection, isReversed); - } - - protected Unfiltered computeNext() - { - if (iterator == null) - iterator = unfilteredIterator(selection, Slices.ALL, isReverseOrder); - if (!iterator.hasNext()) - return endOfData(); - return iterator.next(); - } - - public Iterator<Unfiltered> slice(Slice slice) - { - return sliceIterator(selection, slice, isReverseOrder, current, staticRow); - } - } - - public SliceableUnfilteredRowIterator sliceableUnfilteredIterator(ColumnFilter columns, boolean reversed) - { - return new SliceableIterator(columns, reversed); - } - - protected SliceableUnfilteredRowIterator sliceableUnfilteredIterator() - { - return sliceableUnfilteredIterator(ColumnFilter.all(metadata), false); - } - protected static Holder build(UnfilteredRowIterator iterator, int initialRowCapacity) { CFMetaData metadata = iterator.metadata(); http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java index 02369e4..9f6e900 100644 --- a/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java +++ b/src/java/org/apache/cassandra/db/partitions/PartitionUpdate.java @@ -438,13 +438,6 @@ public class PartitionUpdate extends AbstractBTreePartition return super.iterator(); } - @Override - public SliceableUnfilteredRowIterator sliceableUnfilteredIterator(ColumnFilter columns, boolean reversed) - { - maybeBuild(); - return super.sliceableUnfilteredIterator(columns, reversed); - } - /** * Validates the data contained in this update. * @@ -625,7 +618,7 @@ public class PartitionUpdate extends AbstractBTreePartition { public void serialize(PartitionUpdate update, DataOutputPlus out, int version) throws IOException { - try (UnfilteredRowIterator iter = update.sliceableUnfilteredIterator()) + try (UnfilteredRowIterator iter = update.unfilteredIterator()) { assert !iter.isReverseOrder(); @@ -714,7 +707,7 @@ public class PartitionUpdate extends AbstractBTreePartition public long serializedSize(PartitionUpdate update, int version) { - try (UnfilteredRowIterator iter = update.sliceableUnfilteredIterator()) + try (UnfilteredRowIterator iter = update.unfilteredIterator()) { if (version < MessagingService.VERSION_30) return LegacyLayout.serializedSizeAsLegacyPartition(iter, version); http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/db/rows/SliceableUnfilteredRowIterator.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/db/rows/SliceableUnfilteredRowIterator.java b/src/java/org/apache/cassandra/db/rows/SliceableUnfilteredRowIterator.java deleted file mode 100644 index 2250ee9..0000000 --- a/src/java/org/apache/cassandra/db/rows/SliceableUnfilteredRowIterator.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.cassandra.db.rows; - -import java.util.Iterator; - -import org.apache.cassandra.db.Slice; - -public interface SliceableUnfilteredRowIterator extends UnfilteredRowIterator -{ - /** - * Move forward (resp. backward if isReverseOrder() is true for the iterator) in - * the iterator and return an iterator over the Unfiltered selected by the provided - * {@code slice}. - * <p> - * Please note that successive calls to {@code slice} are allowed provided the - * slice are non overlapping and are passed in clustering (resp. reverse clustering) order. - * However, {@code slice} is allowed to leave the iterator in an unknown state and there - * is no guarantee over what a call to {@code hasNext} or {@code next} will yield after - * a call to {@code slice}. In other words, for a given iterator, you should either use - * {@code slice} or {@code hasNext/next} but not both. - */ - public Iterator<Unfiltered> slice(Slice slice); -} http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java index 9e9b98a..550b1a0 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/SSTableReader.java @@ -46,7 +46,7 @@ import org.apache.cassandra.config.Schema; import org.apache.cassandra.db.*; import org.apache.cassandra.db.commitlog.ReplayPosition; import org.apache.cassandra.db.filter.ColumnFilter; -import org.apache.cassandra.db.rows.SliceableUnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.AbstractBounds; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -1587,8 +1587,8 @@ public abstract class SSTableReader extends SSTable implements SelfRefCounted<SS */ protected abstract RowIndexEntry getPosition(PartitionPosition key, Operator op, boolean updateCacheAndStats, boolean permitMatchPastLast); - public abstract SliceableUnfilteredRowIterator iterator(DecoratedKey key, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift); - public abstract SliceableUnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift); + public abstract UnfilteredRowIterator iterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift); + public abstract UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift); /** * Finds and returns the first key beyond a given token in this SSTable or null if no such key exists. http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java index c16018a..7abf728 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableReader.java @@ -21,10 +21,12 @@ import com.google.common.util.concurrent.RateLimiter; import org.apache.cassandra.cache.KeyCacheKey; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.db.*; -import org.apache.cassandra.db.rows.SliceableUnfilteredRowIterator; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.columniterator.SSTableIterator; import org.apache.cassandra.db.columniterator.SSTableReversedIterator; +import org.apache.cassandra.db.rows.Rows; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterators; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -57,18 +59,19 @@ public class BigTableReader extends SSTableReader super(desc, components, metadata, maxDataAge, sstableMetadata, openReason, header); } - public SliceableUnfilteredRowIterator iterator(DecoratedKey key, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift) + public UnfilteredRowIterator iterator(DecoratedKey key, Slices slices, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift) { - return reversed - ? new SSTableReversedIterator(this, key, selectedColumns, isForThrift) - : new SSTableIterator(this, key, selectedColumns, isForThrift); + RowIndexEntry rie = getPosition(key, SSTableReader.Operator.EQ); + return iterator(null, key, rie, slices, selectedColumns, reversed, isForThrift); } - public SliceableUnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift) + public UnfilteredRowIterator iterator(FileDataInput file, DecoratedKey key, RowIndexEntry indexEntry, Slices slices, ColumnFilter selectedColumns, boolean reversed, boolean isForThrift) { + if (indexEntry == null) + return UnfilteredRowIterators.noRowsIterator(metadata, key, Rows.EMPTY_STATIC_ROW, DeletionTime.LIVE, reversed); return reversed - ? new SSTableReversedIterator(this, file, key, indexEntry, selectedColumns, isForThrift) - : new SSTableIterator(this, file, key, indexEntry, selectedColumns, isForThrift); + ? new SSTableReversedIterator(this, file, key, indexEntry, slices, selectedColumns, isForThrift) + : new SSTableIterator(this, file, key, indexEntry, slices, selectedColumns, isForThrift); } /** http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java index fd413fd..0ce045a 100644 --- a/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java +++ b/src/java/org/apache/cassandra/io/sstable/format/big/BigTableScanner.java @@ -330,7 +330,7 @@ public class BigTableScanner implements ISSTableScanner } ClusteringIndexFilter filter = dataRange.clusteringIndexFilter(partitionKey()); - return filter.filter(sstable.iterator(dfile, partitionKey(), currentEntry, columns, filter.isReversed(), isForThrift)); + return sstable.iterator(dfile, partitionKey(), currentEntry, filter.getSlices(BigTableScanner.this.metadata()), columns, filter.isReversed(), isForThrift); } catch (CorruptSSTableException | IOException e) { http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java b/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java index f215331..fe6ac1e 100644 --- a/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java +++ b/test/unit/org/apache/cassandra/db/partition/PartitionImplementationTest.java @@ -314,10 +314,10 @@ public class PartitionImplementationTest testSearchIterator(sortedContent, partition, cf, true); // sliceable iter - testSliceableIterator(sortedContent, partition, ColumnFilter.all(cfm), false); - testSliceableIterator(sortedContent, partition, cf, false); - testSliceableIterator(sortedContent, partition, ColumnFilter.all(cfm), true); - testSliceableIterator(sortedContent, partition, cf, true); + testSlicingOfIterators(sortedContent, partition, ColumnFilter.all(cfm), false); + testSlicingOfIterators(sortedContent, partition, cf, false); + testSlicingOfIterators(sortedContent, partition, ColumnFilter.all(cfm), true); + testSlicingOfIterators(sortedContent, partition, cf, true); } void testSearchIterator(NavigableSet<Clusterable> sortedContent, Partition partition, ColumnFilter cf, boolean reversed) @@ -361,23 +361,30 @@ public class PartitionImplementationTest return builder.build(); } - void testSliceableIterator(NavigableSet<Clusterable> sortedContent, AbstractBTreePartition partition, ColumnFilter cf, boolean reversed) + void testSlicingOfIterators(NavigableSet<Clusterable> sortedContent, AbstractBTreePartition partition, ColumnFilter cf, boolean reversed) { Function<? super Clusterable, ? extends Clusterable> colFilter = x -> x instanceof Row ? ((Row) x).filter(cf, cfm) : x; Slices slices = makeSlices(); - try (SliceableUnfilteredRowIterator sliceableIter = partition.sliceableUnfilteredIterator(cf, reversed)) + + // fetch each slice in turn + for (Slice slice : (Iterable<Slice>) () -> directed(slices, reversed)) { - for (Slice slice : (Iterable<Slice>) () -> directed(slices, reversed)) + try (UnfilteredRowIterator slicedIter = partition.unfilteredIterator(cf, Slices.with(cfm.comparator, slice), reversed)) + { assertIteratorsEqual(streamOf(directed(slice(sortedContent, slice), reversed)).map(colFilter).iterator(), - sliceableIter.slice(slice)); + slicedIter); + } } - // Try using sliceable as unfiltered iterator - try (SliceableUnfilteredRowIterator sliceableIter = partition.sliceableUnfilteredIterator(cf, reversed)) + // Fetch all slices at once + try (UnfilteredRowIterator slicedIter = partition.unfilteredIterator(cf, slices, reversed)) { - assertIteratorsEqual((reversed ? sortedContent.descendingSet() : sortedContent). - stream().map(colFilter).iterator(), - sliceableIter); + List<Iterator<? extends Clusterable>> slicelist = new ArrayList<>(); + slices.forEach(slice -> slicelist.add(directed(slice(sortedContent, slice), reversed))); + if (reversed) + Collections.reverse(slicelist); + + assertIteratorsEqual(Iterators.concat(slicelist.toArray(new Iterator[0])), slicedIter); } } http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java index 00727b8..7ea2463 100644 --- a/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/LegacySSTableTest.java @@ -44,9 +44,10 @@ import org.apache.cassandra.cql3.UntypedResultSet; import org.apache.cassandra.db.ColumnFamilyStore; import org.apache.cassandra.db.DeletionTime; import org.apache.cassandra.db.Keyspace; -import org.apache.cassandra.db.rows.SliceableUnfilteredRowIterator; +import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.BytesType; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.IPartitioner; import org.apache.cassandra.dht.Range; import org.apache.cassandra.dht.Token; @@ -174,7 +175,7 @@ public class LegacySSTableTest { ByteBuffer key = bytes(keystring); - SliceableUnfilteredRowIterator iter = sstable.iterator(Util.dk(key), ColumnFilter.selectionBuilder().add(cfs.metadata.getColumnDefinition(bytes("name"))).build(), false, false); + UnfilteredRowIterator iter = sstable.iterator(Util.dk(key), Slices.ALL, ColumnFilter.selectionBuilder().add(cfs.metadata.getColumnDefinition(bytes("name"))).build(), false, false); // check not deleted (CASSANDRA-6527) assert iter.partitionLevelDeletion().equals(DeletionTime.LIVE); @@ -215,7 +216,7 @@ public class LegacySSTableTest ByteBuffer key = bytes(keystring); - SliceableUnfilteredRowIterator iter = reader.iterator(Util.dk(key), ColumnFilter.selection(cfs.metadata.partitionColumns()), false, false); + UnfilteredRowIterator iter = reader.iterator(Util.dk(key), Slices.ALL, ColumnFilter.selection(cfs.metadata.partitionColumns()), false, false); // check not deleted (CASSANDRA-6527) assert iter.partitionLevelDeletion().equals(DeletionTime.LIVE); http://git-wip-us.apache.org/repos/asf/cassandra/blob/4fb559b5/test/unit/org/apache/cassandra/io/sstable/format/ClientModeSSTableTest.java ---------------------------------------------------------------------- diff --git a/test/unit/org/apache/cassandra/io/sstable/format/ClientModeSSTableTest.java b/test/unit/org/apache/cassandra/io/sstable/format/ClientModeSSTableTest.java index 661fcd5..6b0427b 100644 --- a/test/unit/org/apache/cassandra/io/sstable/format/ClientModeSSTableTest.java +++ b/test/unit/org/apache/cassandra/io/sstable/format/ClientModeSSTableTest.java @@ -29,15 +29,13 @@ import org.junit.Test; import org.apache.cassandra.concurrent.ScheduledExecutors; import org.apache.cassandra.config.CFMetaData; import org.apache.cassandra.config.Config; +import org.apache.cassandra.db.Slices; import org.apache.cassandra.db.filter.ColumnFilter; import org.apache.cassandra.db.marshal.BytesType; -import org.apache.cassandra.db.rows.SliceableUnfilteredRowIterator; +import org.apache.cassandra.db.rows.UnfilteredRowIterator; import org.apache.cassandra.dht.ByteOrderedPartitioner; import org.apache.cassandra.exceptions.ConfigurationException; import org.apache.cassandra.io.sstable.Descriptor; -import org.apache.cassandra.io.sstable.format.SSTableFormat; -import org.apache.cassandra.io.sstable.format.SSTableReader; -import org.apache.cassandra.io.sstable.format.Version; /** * Tests backwards compatibility for SSTables @@ -107,7 +105,7 @@ public class ClientModeSSTableTest ByteBuffer key = bytes(Integer.toString(100)); - try (SliceableUnfilteredRowIterator iter = reader.iterator(metadata.decorateKey(key), ColumnFilter.selection(metadata.partitionColumns()), false, false)) + try (UnfilteredRowIterator iter = reader.iterator(metadata.decorateKey(key), Slices.ALL, ColumnFilter.selection(metadata.partitionColumns()), false, false)) { assert iter.next().clustering().get(0).equals(key); }
