Repository: ignite Updated Branches: refs/heads/ignite-8900-repro [created] b3707b44c
Reproducer for IGNITE-8900 issue with broken links. Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b3707b44 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b3707b44 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b3707b44 Branch: refs/heads/ignite-8900-repro Commit: b3707b44c89b3bb4cb30d07cb1c9936264b0666e Parents: 2959990 Author: devozerov <[email protected]> Authored: Fri Jun 29 23:01:26 2018 +0300 Committer: devozerov <[email protected]> Committed: Fri Jun 29 23:01:26 2018 +0300 ---------------------------------------------------------------------- .../cache/GridCacheManagerAdapter.java | 2 +- .../spi/indexing/IndexingQueryCacheFilter.java | 11 +- .../processors/query/h2/database/H2Tree.java | 23 ++- .../query/h2/twostep/GridMapQueryExecutor.java | 14 ++ .../ignite/internal/processors/cache/Main.java | 142 +++++++++++++++++++ 5 files changed, 188 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/b3707b44/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheManagerAdapter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheManagerAdapter.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheManagerAdapter.java index 0078a0d..48cf14f 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheManagerAdapter.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/GridCacheManagerAdapter.java @@ -28,7 +28,7 @@ import org.apache.ignite.lang.IgniteFuture; */ public class GridCacheManagerAdapter<K, V> implements GridCacheManager<K, V> { /** Context. */ - protected GridCacheContext cctx; + public GridCacheContext cctx; /** Logger. */ protected IgniteLogger log; http://git-wip-us.apache.org/repos/asf/ignite/blob/b3707b44/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingQueryCacheFilter.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingQueryCacheFilter.java b/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingQueryCacheFilter.java index fa3ec97..695ec44 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingQueryCacheFilter.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/indexing/IndexingQueryCacheFilter.java @@ -27,6 +27,9 @@ import java.util.Set; * Indexing query filter for specific cache. */ public class IndexingQueryCacheFilter { + + public static volatile boolean DEBUG = false; + /** Affinity manager. */ private final GridCacheAffinityManager aff; @@ -74,9 +77,13 @@ public class IndexingQueryCacheFilter { * @return {@code True} if passed. */ public boolean applyPartition(int part) { + boolean res; + if (parts == null) - return aff.primaryByPartition(locNode, part, topVer); + res = aff.primaryByPartition(locNode, part, topVer); else - return parts.contains(part); + res = parts.contains(part); + + return res; } } http://git-wip-us.apache.org/repos/asf/ignite/blob/b3707b44/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java index 8da3b05..8bf7b1b 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/database/H2Tree.java @@ -24,6 +24,8 @@ import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; +import org.apache.ignite.internal.processors.cache.KeyCacheObject; +import org.apache.ignite.internal.processors.cache.KeyCacheObjectImpl; import org.apache.ignite.internal.processors.cache.persistence.tree.BPlusTree; import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusIO; import org.apache.ignite.internal.processors.cache.persistence.tree.io.BPlusMetaIO; @@ -34,6 +36,7 @@ import org.apache.ignite.internal.processors.query.h2.database.io.H2ExtrasLeafIO import org.apache.ignite.internal.processors.query.h2.database.io.H2RowLinkIO; import org.apache.ignite.internal.processors.query.h2.opt.GridH2KeyValueRowOnheap; import org.apache.ignite.internal.processors.query.h2.opt.GridH2Row; +import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.spi.indexing.IndexingQueryCacheFilter; import org.h2.result.SearchRow; @@ -164,7 +167,25 @@ public abstract class H2Tree extends BPlusTree<SearchRow, GridH2Row> { return null; } - return (GridH2Row)io.getLookupRow(this, pageAddr, idx); + GridH2Row row = (GridH2Row)io.getLookupRow(this, pageAddr, idx); + + KeyCacheObject key = row.key(); + + if (GridMapQueryExecutor.DEBUG) { + String keyStr = key.toString(); + + if (keyStr.contains("610026643276160002")) { + IndexingQueryCacheFilter filter0 = (IndexingQueryCacheFilter)filter; + + long link = ((H2RowLinkIO)io).getLink(pageAddr, idx); + + int part = PageIdUtils.partId(PageIdUtils.pageId(link)); + + System.out.println("FAULT: " + part); + } + } + + return row; } /** http://git-wip-us.apache.org/repos/asf/ignite/blob/b3707b44/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java index 930ada2..bb49844 100644 --- a/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java +++ b/modules/indexing/src/main/java/org/apache/ignite/internal/processors/query/h2/twostep/GridMapQueryExecutor.java @@ -100,6 +100,9 @@ import static org.apache.ignite.internal.processors.query.h2.twostep.msg.GridH2V */ @SuppressWarnings("ForLoopReplaceableByForEach") public class GridMapQueryExecutor { + + public static volatile boolean DEBUG = false; + /** */ public static final boolean FORCE_LAZY = IgniteSystemProperties.getBoolean(IGNITE_SQL_FORCE_LAZY_RESULT_SET); @@ -1000,6 +1003,17 @@ public class GridMapQueryExecutor { try { boolean loc = node.isLocal(); + if (DEBUG) { + StringBuilder sb = new StringBuilder(); + + sb.append("NODE=" + ctx.grid().name() + "\n"); + + for (Value[] row : rows) + sb.append("\t" + row[0].getLong()); + + System.out.println(sb.toString()); + } + GridQueryNextPageResponse msg = new GridQueryNextPageResponse(qr.queryRequestId(), segmentId, qry, page, page == 0 ? res.rowCount() : -1, res.columnCount(), http://git-wip-us.apache.org/repos/asf/ignite/blob/b3707b44/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/Main.java ---------------------------------------------------------------------- diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/Main.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/Main.java new file mode 100644 index 0000000..c271856 --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/Main.java @@ -0,0 +1,142 @@ +package org.apache.ignite.internal.processors.cache; + +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashSet; +import java.util.List; +import java.util.Random; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import java.util.stream.LongStream; + +import org.apache.ignite.Ignite; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.Ignition; +import org.apache.ignite.cache.CacheMode; +import org.apache.ignite.cache.CacheRebalanceMode; +import org.apache.ignite.cache.CacheWriteSynchronizationMode; +import org.apache.ignite.cache.query.FieldsQueryCursor; + +import org.apache.ignite.cache.query.SqlFieldsQuery; + +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.events.EventType; +import org.apache.ignite.internal.processors.query.h2.twostep.GridMapQueryExecutor; +import org.apache.ignite.spi.discovery.tcp.TcpDiscoverySpi; +import org.apache.ignite.spi.discovery.tcp.ipfinder.vm.TcpDiscoveryVmIpFinder; +import org.apache.ignite.spi.indexing.IndexingQueryCacheFilter; + +public class Main { + + public static final String CACHE = "cache"; + + public static void main(String[] args) throws InterruptedException { + Ignition.start(getConfig().setIgniteInstanceName("node1")); + Ignition.start(getConfig().setIgniteInstanceName("node2")); + + Random random = new Random(1); + Ignite client = Ignition.start(getConfig().setClientMode(true)); + + int ctr = 0; + + while (true) { + String cacheName = CACHE + ctr++; + + System.out.println("CACHE NAME: " + cacheName); + + IgniteCache<Long, Value> cache = client.getOrCreateCache(new CacheConfiguration<Long, Value>(cacheName) + .setCacheMode(CacheMode.PARTITIONED) + .setBackups(1) + .setWriteSynchronizationMode(CacheWriteSynchronizationMode.FULL_SYNC) + .setIndexedTypes(Long.class, Value.class)); + + System.out.println("Populating cache..."); + cache.putAll(LongStream.range(610026643276160000L, 610026643276170000L).boxed() + .collect(Collectors.toMap(Function.identity(), + t -> Value.of(new byte[(random.nextInt(16)) * 1000])))); + + if (ctr == 5) { + GridMapQueryExecutor.DEBUG = true; + IndexingQueryCacheFilter.DEBUG = true; + } + + for (int i = 0; i < 10; i++) { + System.out.print(i == 9 ? "\n" : "."); + long start = 610026643276160000L; + long end = start + random.nextInt(10); + + int expectedResultCount = (int) (end - start + 1); + + String sql = String.format( + "SELECT _KEY " + + "FROM %s " + + "WHERE _KEY >= %d AND _KEY <= %d", Value.class.getSimpleName().toLowerCase(), + start, end); + + Set<Long> keySet = new HashSet<>(); + for (long l = start; l < end + 1; l++) + keySet.add(l); + int size = cache.getAll(keySet).entrySet().size(); + + List<Long> resultKeys; + try(FieldsQueryCursor<List<?>> results = cache.query(new SqlFieldsQuery(sql))) { + resultKeys = new ArrayList<>(); + results.forEach(objects -> resultKeys.add((Long)objects.get(0))); + Collections.sort(resultKeys); + } + + if (resultKeys.size() != expectedResultCount) { + System.out.printf("Expected %d results but got back %d results " + + "(query range %d to %d), cache.getAll returned %d entries.%n", expectedResultCount, resultKeys.size(), start, end, size); + System.out.printf("Query results: %s%n", resultKeys); + + System.exit(-1); + } + + Thread.sleep(100); + } + cache.destroy(); + } + } + + /** + * Ignite configuration for this example. + * + * @return Ignite configuration + */ + static IgniteConfiguration getConfig() { + IgniteConfiguration cfg = new IgniteConfiguration(); + cfg.setConsistentId(cfg.getIgniteInstanceName()); + cfg.setIncludeEventTypes(EventType.EVT_CACHE_REBALANCE_PART_DATA_LOST); + + TcpDiscoverySpi discovery = new TcpDiscoverySpi(); + TcpDiscoveryVmIpFinder finder = new TcpDiscoveryVmIpFinder(); + finder.setAddresses(Collections.singletonList("127.0.0.1:47500..47501")); + discovery.setIpFinder(finder); + cfg.setDiscoverySpi(discovery); + + DataStorageConfiguration storageConfiguration = new DataStorageConfiguration(); + storageConfiguration.setPageSize(1024 * 8); + cfg.setDataStorageConfiguration(storageConfiguration); + return cfg; + } + + /** + * Class containing value to be placed into the cache. + */ + public static class Value implements Serializable { + final byte[] data; + + public Value(final byte[] data) { + this.data = data; + } + + public static Value of(final byte[] bytes) { + return new Value(bytes); + } + } +}
