IGNITE-4624: Scan query optimization. This closes #1509.
Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/2f57760d Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/2f57760d Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/2f57760d Branch: refs/heads/ignite-4680-sb Commit: 2f57760dbb4fba948cd035498d2c7f71869c0665 Parents: 11bbec4 Author: Andrey V. Mashenkov <[email protected]> Authored: Fri Feb 17 16:15:31 2017 +0300 Committer: Andrey V. Mashenkov <[email protected]> Committed: Fri Feb 17 18:47:21 2017 +0300 ---------------------------------------------------------------------- .../distributed/dht/GridDhtCacheAdapter.java | 19 +++- .../cache/query/GridCacheQueryManager.java | 97 ++++++++++---------- 2 files changed, 64 insertions(+), 52 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/2f57760d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java index dcd379a..be7fa55 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/distributed/dht/GridDhtCacheAdapter.java @@ -1247,14 +1247,27 @@ public abstract class GridDhtCacheAdapter<K, V> extends GridDistributedCacheAdap final boolean backup, final boolean keepBinary, final AffinityTopologyVersion topVer) { + + return iterator(localEntriesIteratorEx(primary, backup, topVer), !keepBinary); + } + + /** + * @param primary If {@code true} includes primary entries. + * @param backup If {@code true} includes backup entries. + * @param topVer Specified affinity topology version. + * @return Local entries iterator. + */ + public Iterator<? extends GridCacheEntryEx> localEntriesIteratorEx(final boolean primary, + final boolean backup, + final AffinityTopologyVersion topVer) { assert primary || backup; if (primary && backup) - return iterator(entries().iterator(), !keepBinary); + return entries().iterator(); else { final Iterator<GridDhtLocalPartition> partIt = topology().currentLocalPartitions().iterator(); - Iterator<GridCacheMapEntry> it = new Iterator<GridCacheMapEntry>() { + return new Iterator<GridCacheMapEntry>() { private GridCacheMapEntry next; private Iterator<GridCacheMapEntry> curIt; @@ -1311,8 +1324,6 @@ public abstract class GridDhtCacheAdapter<K, V> extends GridDistributedCacheAdap while (partIt.hasNext()); } }; - - return iterator(it, !keepBinary); } } http://git-wip-us.apache.org/repos/asf/ignite/blob/2f57760d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java index d64dff4..14b1106 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/query/GridCacheQueryManager.java @@ -1033,23 +1033,25 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte * @throws GridDhtUnreservedPartitionException If failed to reserve partition. */ private GridIterator<IgniteBiTuple<K, V>> onheapIterator( - GridCacheQueryAdapter<?> qry, + final GridCacheQueryAdapter<?> qry, AffinityTopologyVersion topVer, final IgniteBiPredicate<K, V> keyValFilter, - boolean backups, + final boolean backups, final ExpiryPolicy plc, final boolean locNode) throws GridDhtUnreservedPartitionException { - Iterator<K> keyIter; + Iterator<? extends GridCacheEntryEx> entryIter; GridDhtLocalPartition locPart = null; Integer part = qry.partition(); - if (part == null || cctx.isLocal()) { - // Performance optimization. - if (locNode && plc == null && !cctx.isLocal()) { - GridDhtCacheAdapter<K, V> cache = cctx.isNear() ? cctx.near().dht() : cctx.dht(); + if (cctx.isLocal()) + entryIter = cctx.local().allEntries().iterator(); + else if (part == null) { + GridDhtCacheAdapter<K, V> cache = cctx.isNear() ? cctx.near().dht() : cctx.dht(); + // Performance optimization. + if (locNode && plc == null) { final Iterator<Cache.Entry<K, V>> iter = cache.localEntriesIterator(true, backups, cache.context().keepBinary(), topVer); @@ -1099,12 +1101,10 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte }; } - IgniteInternalCache<K, V> keepBinaryCache = cctx.cache().keepBinary(); - - keyIter = backups ? keepBinaryCache.keySetx().iterator() : keepBinaryCache.primaryKeySet().iterator(); + entryIter = cache.localEntriesIteratorEx(true, backups, topVer); } else if (part < 0 || part >= cctx.affinity().partitions()) - keyIter = new GridEmptyIterator<>(); + return new GridEmptyIterator<>(); else { final GridDhtCacheAdapter dht = cctx.isNear() ? cctx.near().dht() : cctx.dht(); @@ -1115,28 +1115,12 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte throw new GridDhtUnreservedPartitionException(part, cctx.affinity().affinityTopologyVersion(), "Partition can not be reserved."); - final GridDhtLocalPartition locPart0 = locPart; - - keyIter = new Iterator<K>() { - private Iterator<KeyCacheObject> iter0 = locPart0.keySet().iterator(); - - @Override public boolean hasNext() { - return iter0.hasNext(); - } - - @Override public K next() { - return (K)iter0.next(); - } - - @Override public void remove() { - iter0.remove(); - } - }; + entryIter = locPart.allEntries().iterator(); } final GridDhtLocalPartition locPart0 = locPart; - return new PeekValueExpiryAwareIterator(keyIter, plc, topVer, keyValFilter, qry.keepBinary(), locNode, true) { + return new PeekValueExpiryAwareIterator(entryIter, plc, topVer, keyValFilter, qry.keepBinary(), locNode, true) { @Override protected void onClose() { super.onClose(); @@ -1263,18 +1247,20 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte ExpiryPolicy expPlc, final boolean keepBinary, boolean locNode) { - Iterator<K> keyIter = new Iterator<K>() { + Iterator<? extends GridCacheEntryEx> keyIter = new Iterator<GridCacheEntryEx>() { /** {@inheritDoc} */ @Override public boolean hasNext() { return it.hasNext(); } /** {@inheritDoc} */ - @Override public K next() { + @Override public GridCacheEntryEx next() { try { KeyCacheObject key = cctx.toCacheKeyObject(it.next().getKey()); - return (K)cctx.unwrapBinaryIfNeeded(key, keepBinary); + final GridCacheEntryEx entryEx = cctx.cache().entryEx(key); + + return entryEx; } catch (IgniteCheckedException e) { throw new IgniteException(e); @@ -2189,8 +2175,8 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte } /** - * Gets cache queries detailed metrics. - * Detail metrics could be enabled by setting non-zero value via {@link CacheConfiguration#setQueryDetailMetricsSize(int)} + * Gets cache queries detailed metrics. Detail metrics could be enabled by setting non-zero value via {@link + * CacheConfiguration#setQueryDetailMetricsSize(int)} * * @return Cache queries metrics aggregated by query type and query text. */ @@ -3091,8 +3077,10 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte private abstract static class CachedResult<R> extends GridFutureAdapter<IgniteSpiCloseableIterator<R>> { /** Absolute position of each recipient. */ private final Map<Object, QueueIterator> recipients = new GridLeanMap<>(1); + /** */ private CircularQueue<R> queue; + /** */ private int pruned; @@ -3529,10 +3517,10 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte private IgniteCacheExpiryPolicy expiryPlc; /** */ - private Iterator<K> keyIt; + private Iterator<? extends GridCacheEntryEx> entryIt; /** - * @param keyIt Key iterator. + * @param entryIter Key iterator. * @param plc Expiry policy. * @param topVer Topology version. * @param keyValFilter Key-value filter. @@ -3540,8 +3528,8 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte * @param locNode Local node. * @param heapOnly Heap only. */ - private PeekValueExpiryAwareIterator( - Iterator<K> keyIt, + PeekValueExpiryAwareIterator( + Iterator<? extends GridCacheEntryEx> entryIter, ExpiryPolicy plc, AffinityTopologyVersion topVer, IgniteBiPredicate<K, V> keyValFilter, @@ -3549,7 +3537,7 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte boolean locNode, boolean heapOnly ) { - this.keyIt = keyIt; + this.entryIt = entryIter; this.plc = plc; this.topVer = topVer; this.keyValFilter = keyValFilter; @@ -3593,15 +3581,27 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte private void advance() { IgniteBiTuple<K, V> next0 = null; - while (keyIt.hasNext()) { + while (entryIt.hasNext()) { next0 = null; - K key = keyIt.next(); + GridCacheEntryEx entry = entryIt.next(); + + if (entry.deleted()) + continue; + KeyCacheObject key = entry.key(); CacheObject val; try { - val = value(key); + if (heapOnly) + val = entry.peek(true, false, false, expiryPlc); + else + val = value(entry, entry.key()); + } + catch (GridCacheEntryRemovedException ignore) { + assert heapOnly; + + continue; } catch (IgniteCheckedException e) { if (log.isDebugEnabled()) @@ -3664,23 +3664,24 @@ public abstract class GridCacheQueryManager<K, V> extends GridCacheManagerAdapte } /** + * @param entry Entry. * @param key Key. * @return Value. * @throws IgniteCheckedException If failed to peek value. */ - private CacheObject value(K key) throws IgniteCheckedException { + private CacheObject value(GridCacheEntryEx entry, KeyCacheObject key) throws IgniteCheckedException { while (true) { try { - GridCacheEntryEx entry = heapOnly ? cache.peekEx(key) : cache.entryEx(key); + if (entry == null) + entry = cache.entryEx(key); - if (expiryPlc != null && !heapOnly) + if (expiryPlc != null) entry.unswap(); - return entry != null ? entry.peek(true, !heapOnly, !heapOnly, topVer, expiryPlc) : null; + return entry.peek(true, true, true, topVer, expiryPlc); } catch (GridCacheEntryRemovedException ignore) { - if (heapOnly) - return null; + entry = null; } } }
