Repository: ignite Updated Branches: refs/heads/master 2a82b5104 -> ef4d576b4
IGNITE-6626: SQL: avoid row materialization when filtering out not relevant partitions. This closes #2848. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/ef4d576b Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/ef4d576b Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/ef4d576b Branch: refs/heads/master Commit: ef4d576b49815a2b1117eef1f786a5b31861e26d Parents: 2a82b51 Author: devozerov <[email protected]> Authored: Thu Nov 2 21:37:19 2017 +0300 Committer: devozerov <[email protected]> Committed: Thu Nov 2 21:37:19 2017 +0300 ---------------------------------------------------------------------- .../cache/persistence/tree/BPlusTree.java | 28 +++-- .../apache/ignite/internal/util/IgniteTree.java | 12 ++ .../query/h2/opt/GridH2SpatialIndex.java | 16 ++- .../internal/processors/query/h2/H2Cursor.java | 18 +-- .../processors/query/h2/database/H2Tree.java | 17 ++- .../query/h2/database/H2TreeIndex.java | 18 ++- .../query/h2/database/io/H2ExtrasInnerIO.java | 10 +- .../query/h2/database/io/H2ExtrasLeafIO.java | 10 +- .../query/h2/opt/GridH2IndexBase.java | 112 ++----------------- 9 files changed, 88 insertions(+), 153 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4d576b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java index c73b4c7..8e6e099 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/tree/BPlusTree.java @@ -928,23 +928,12 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements throw new IllegalStateException("Tree is being concurrently destroyed: " + getName()); } - /** - * @param lower Lower bound inclusive or {@code null} if unbounded. - * @param upper Upper bound inclusive or {@code null} if unbounded. - * @return Cursor. - * @throws IgniteCheckedException If failed. - */ + /** {@inheritDoc} */ @Override public GridCursor<T> find(L lower, L upper) throws IgniteCheckedException { return find(lower, upper, null); } - /** - * @param lower Lower bound inclusive or {@code null} if unbounded. - * @param upper Upper bound inclusive or {@code null} if unbounded. - * @param x Implementation specific argument, {@code null} always means that we need to return full detached data row. - * @return Cursor. - * @throws IgniteCheckedException If failed. - */ + /** {@inheritDoc} */ public final GridCursor<T> find(L lower, L upper, Object x) throws IgniteCheckedException { checkDestroyed(); @@ -4544,13 +4533,22 @@ public abstract class BPlusTree<L, T extends L> extends DataStructure implements if (rows == EMPTY) rows = (T[])new Object[cnt]; + int foundCnt = 0; + for (int i = 0; i < cnt; i++) { T r = getRow(io, pageAddr, startIdx + i, x); - rows = GridArrays.set(rows, i, r); + if (r != null) + rows = GridArrays.set(rows, foundCnt++, r); + } + + if (foundCnt == 0) { + rows = (T[])EMPTY; + + return false; } - GridArrays.clearTail(rows, cnt); + GridArrays.clearTail(rows, foundCnt); return true; } http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4d576b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteTree.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteTree.java b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteTree.java index 396b8a4..9e854d2 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteTree.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/util/IgniteTree.java @@ -64,6 +64,18 @@ public interface IgniteTree<L, T> { public GridCursor<T> find(L lower, L upper) throws IgniteCheckedException; /** + * Returns a cursor from lower to upper bounds inclusive. + * + * @param lower Lower bound or {@code null} if unbounded. + * @param upper Upper bound or {@code null} if unbounded. + * @param x Implementation specific argument, {@code null} always means that we need to return full detached + * data row. + * @return Cursor. + * @throws IgniteCheckedException If failed. + */ + public GridCursor<T> find(L lower, L upper, Object x) throws IgniteCheckedException; + + /** * Returns a value mapped to the lowest key, or {@code null} if tree is empty * @return Value. * @throws IgniteCheckedException If failed. http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4d576b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java ---------------------------------------------------------------------- diff --git a/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java index 2cd36b3..b6125c8 100644 --- a/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java +++ b/modules/geospatial/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2SpatialIndex.java @@ -32,6 +32,8 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.processors.query.h2.H2Cursor; import org.apache.ignite.internal.util.GridCursorIteratorWrapper; import org.apache.ignite.internal.util.lang.GridCursor; +import org.apache.ignite.spi.indexing.IndexingQueryCacheFilter; +import org.apache.ignite.spi.indexing.IndexingQueryFilter; import org.h2.engine.Session; import org.h2.index.Cursor; import org.h2.index.IndexLookupBatch; @@ -322,6 +324,12 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex if (!i.hasNext()) return EMPTY_CURSOR; + long time = System.currentTimeMillis(); + + IndexingQueryFilter qryFilter = threadLocalFilter(); + + IndexingQueryCacheFilter qryCacheFilter = qryFilter != null ? qryFilter.forCache(getTable().cacheName()) : null; + List<GridH2Row> rows = new ArrayList<>(); do { @@ -329,11 +337,15 @@ public class GridH2SpatialIndex extends GridH2IndexBase implements SpatialIndex assert row != null; - rows.add(row); + if (row.expireTime() != 0 && row.expireTime() <= time) + continue; + + if (qryCacheFilter == null || qryCacheFilter.applyPartition(row.partition())) + rows.add(row); } while (i.hasNext()); - return filter(new GridCursorIteratorWrapper(rows.iterator()), threadLocalFilter()); + return new GridCursorIteratorWrapper(rows.iterator()); } /** {@inheritDoc} */ http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4d576b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Cursor.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Cursor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Cursor.java index f4c35f2..01b3504 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Cursor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/H2Cursor.java @@ -21,7 +21,6 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; import org.apache.ignite.internal.util.lang.GridCursor; import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.spi.indexing.IndexingQueryCacheFilter; import org.h2.index.Cursor; import org.h2.message.DbException; import org.h2.result.Row; @@ -35,27 +34,15 @@ public class H2Cursor implements Cursor { private final GridCursor<GridH2Row> cursor; /** */ - private final IndexingQueryCacheFilter filter; - - /** */ private final long time = U.currentTimeMillis(); /** * @param cursor Cursor. - * @param filter Filter. */ - public H2Cursor(GridCursor<GridH2Row> cursor, IndexingQueryCacheFilter filter) { + public H2Cursor(GridCursor<GridH2Row> cursor) { assert cursor != null; this.cursor = cursor; - this.filter = filter; - } - - /** - * @param cursor Cursor. - */ - public H2Cursor(GridCursor<GridH2Row> cursor) { - this(cursor, null); } /** {@inheritDoc} */ @@ -82,8 +69,7 @@ public class H2Cursor implements Cursor { if (row.expireTime() > 0 && row.expireTime() <= time) continue; - if (filter == null || filter.applyPartition(row.partition())) - return true; + return true; } return false; http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4d576b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java index fcfeb16..4aecab7 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java @@ -21,6 +21,7 @@ import java.util.Comparator; import java.util.List; import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree; @@ -29,8 +30,10 @@ import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMeta import org.apache.ignite.internal.processors.cache.persistence.tree.reuse.ReuseList; import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasInnerIO; import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasLeafIO; +import org.apache.ignite.internal.processors.query.h2.database.io.H2RowLinkIO; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.spi.indexing.IndexingQueryCacheFilter; import org.h2.result.SearchRow; import org.h2.table.IndexColumn; import org.h2.value.Value; @@ -118,8 +121,20 @@ public abstract class H2Tree extends BPlusTree<SearchRow, GridH2Row> { } /** {@inheritDoc} */ - @Override protected GridH2Row getRow(BPlusIO<SearchRow> io, long pageAddr, int idx, Object ignore) + @Override protected GridH2Row getRow(BPlusIO<SearchRow> io, long pageAddr, int idx, Object filter) throws IgniteCheckedException { + if (filter != null) { + // Filter out not interesting partitions without deserializing the row. + IndexingQueryCacheFilter filter0 = (IndexingQueryCacheFilter)filter; + + long link = ((H2RowLinkIO)io).getLink(pageAddr, idx); + + int part = PageIdUtils.partId(PageIdUtils.pageId(link)); + + if (!filter0.applyPartition(part)) + return null; + } + return (GridH2Row)io.getLookupRow(this, pageAddr, idx); } http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4d576b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java index 3c0ab5e..b3307d0 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2TreeIndex.java @@ -178,7 +178,7 @@ public class H2TreeIndex extends GridH2IndexBase { H2Tree tree = treeForRead(seg); - return new H2Cursor(tree.find(lower, upper), p); + return new H2Cursor(tree.find(lower, upper, p)); } catch (IgniteCheckedException e) { throw DbException.convert(e); @@ -318,19 +318,27 @@ public class H2TreeIndex extends GridH2IndexBase { } /** {@inheritDoc} */ - @Override protected GridCursor<GridH2Row> doFind0( + @Override protected H2Cursor doFind0( IgniteTree t, @Nullable SearchRow first, boolean includeFirst, @Nullable SearchRow last, IndexingQueryFilter filter) { try { - GridCursor<GridH2Row> range = t.find(first, last); + IndexingQueryCacheFilter p = null; + + if (filter != null) { + String cacheName = getTable().cacheName(); + + p = filter.forCache(cacheName); + } + + GridCursor<GridH2Row> range = t.find(first, last, p); if (range == null) - return EMPTY_CURSOR; + range = EMPTY_CURSOR; - return filter(range, filter); + return new H2Cursor(range); } catch (IgniteCheckedException e) { throw DbException.convert(e); http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4d576b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java index b8877e9..a4aa600 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasInnerIO.java @@ -33,7 +33,7 @@ import org.h2.result.SearchRow; /** * Inner page for H2 row references. */ -public class H2ExtrasInnerIO extends BPlusInnerIO<SearchRow> { +public class H2ExtrasInnerIO extends BPlusInnerIO<SearchRow> implements H2RowLinkIO { /** Payload size. */ private final int payloadSize; @@ -129,12 +129,8 @@ public class H2ExtrasInnerIO extends BPlusInnerIO<SearchRow> { PageUtils.putLong(dstPageAddr, dstOff + payloadSize, link); } - /** - * @param pageAddr Page address. - * @param idx Index. - * @return Link to row. - */ - private long getLink(long pageAddr, int idx) { + /** {@inheritDoc} */ + @Override public long getLink(long pageAddr, int idx) { return PageUtils.getLong(pageAddr, offset(idx) + payloadSize); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4d576b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java index 6161f8d..8723601 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/io/H2ExtrasLeafIO.java @@ -33,7 +33,7 @@ import org.h2.result.SearchRow; /** * Leaf page for H2 row references. */ -public class H2ExtrasLeafIO extends BPlusLeafIO<SearchRow> { +public class H2ExtrasLeafIO extends BPlusLeafIO<SearchRow> implements H2RowLinkIO { /** Payload size. */ private final int payloadSize; @@ -126,12 +126,8 @@ public class H2ExtrasLeafIO extends BPlusLeafIO<SearchRow> { return ((H2Tree)tree).getRowFactory().getRow(link); } - /** - * @param pageAddr Page address. - * @param idx Index. - * @return Link to row. - */ - private long getLink(long pageAddr, int idx) { + /** {@inheritDoc} */ + @Override public long getLink(long pageAddr, int idx) { return PageUtils.getLong(pageAddr, offset(idx) + payloadSize); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/ef4d576b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java index c28d067..5d4a4e6 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/opt/GridH2IndexBase.java @@ -28,6 +28,7 @@ import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.managers.communication.GridMessageListener; import org.apache.ignite.internal.processors.cache.CacheObject; import org.apache.ignite.internal.processors.cache.GridCacheContext; +import org.apache.ignite.internal.processors.query.h2.H2Cursor; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2IndexRangeRequest; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2IndexRangeResponse; import org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2RowMessage; @@ -46,7 +47,6 @@ import org.apache.ignite.lang.IgniteBiTuple; import org.apache.ignite.logger.NullLogger; import org.apache.ignite.plugin.extensions.communication.Message; import org.apache.ignite.spi.indexing.IndexingQueryFilter; -import org.apache.ignite.spi.indexing.IndexingQueryCacheFilter; import org.h2.engine.Session; import org.h2.index.BaseIndex; import org.h2.index.Cursor; @@ -73,7 +73,6 @@ import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; -import java.util.NoSuchElementException; import java.util.UUID; import java.util.concurrent.BlockingQueue; import java.util.concurrent.Future; @@ -261,17 +260,6 @@ public abstract class GridH2IndexBase extends BaseIndex { } /** - * Filters rows from expired ones and using predicate. - * - * @param cursor GridCursor over rows. - * @param filter Optional filter. - * @return Filtered iterator. - */ - protected GridCursor<GridH2Row> filter(GridCursor<GridH2Row> cursor, IndexingQueryFilter filter) { - return new FilteringCursor(cursor, U.currentTimeMillis(), filter, getTable().cacheName()); - } - - /** * @return Filter for currently running query or {@code null} if none. */ protected static IndexingQueryFilter threadLocalFilter() { @@ -1570,7 +1558,7 @@ public abstract class GridH2IndexBase extends BaseIndex { * @param filter Filter. * @return Iterator over rows in given range. */ - protected GridCursor<GridH2Row> doFind0( + protected H2Cursor doFind0( IgniteTree t, @Nullable SearchRow first, boolean includeFirst, @@ -1580,77 +1568,11 @@ public abstract class GridH2IndexBase extends BaseIndex { } /** - * Cursor which filters by expiration time and predicate. - */ - protected static class FilteringCursor implements GridCursor<GridH2Row> { - /** */ - private final GridCursor<GridH2Row> cursor; - - /** */ - private final IndexingQueryCacheFilter fltr; - - /** */ - private final long time; - - /** */ - private GridH2Row next; - - /** - * @param cursor GridCursor. - * @param time Time for expired rows filtering. - * @param qryFilter Filter. - * @param cacheName Cache name. - */ - protected FilteringCursor(GridCursor<GridH2Row> cursor, long time, IndexingQueryFilter qryFilter, - String cacheName) { - this.cursor = cursor; - this.time = time; - this.fltr = qryFilter != null ? qryFilter.forCache(cacheName) : null; - } - - /** - * @param row Row. - * @return If this row was accepted. - */ - @SuppressWarnings({"unchecked", "SimplifiableIfStatement"}) - protected boolean accept(GridH2Row row) { - if (row.expireTime() != 0 && row.expireTime() <= time) - return false; - - return fltr == null || fltr.applyPartition(row.partition()); - } - - /** {@inheritDoc} */ - @Override public boolean next() throws IgniteCheckedException { - next = null; - - while (cursor.next()) { - GridH2Row t = cursor.get(); - - if (accept(t)) { - next = t; - return true; - } - } - - return false; - } - - /** {@inheritDoc} */ - @Override public GridH2Row get() throws IgniteCheckedException { - if (next == null) - throw new NoSuchElementException(); - - return next; - } - } - - /** * */ private static final class CursorIteratorWrapper implements Iterator<GridH2Row> { /** */ - private final GridCursor<GridH2Row> cursor; + private final H2Cursor cursor; /** Next element. */ private GridH2Row next; @@ -1658,18 +1580,13 @@ public abstract class GridH2IndexBase extends BaseIndex { /** * @param cursor Cursor. */ - private CursorIteratorWrapper(GridCursor<GridH2Row> cursor) { + private CursorIteratorWrapper(H2Cursor cursor) { assert cursor != null; this.cursor = cursor; - try { - if (cursor.next()) - next = cursor.get(); - } - catch (IgniteCheckedException e) { - throw U.convertException(e); - } + if (cursor.next()) + next = (GridH2Row)cursor.get(); } /** {@inheritDoc} */ @@ -1679,19 +1596,14 @@ public abstract class GridH2IndexBase extends BaseIndex { /** {@inheritDoc} */ @Override public GridH2Row next() { - try { - GridH2Row res = next; + GridH2Row res = next; - if (cursor.next()) - next = cursor.get(); - else - next = null; + if (cursor.next()) + next = (GridH2Row)cursor.get(); + else + next = null; - return res; - } - catch (IgniteCheckedException e) { - throw U.convertException(e); - } + return res; } /** {@inheritDoc} */
