Repository: ignite Updated Branches: refs/heads/ignite-3478 54b871422 -> 3d1616141
IGNITE-6385 Offheap page eviction doesn't work if data is loaded without data streamer - Fixes #2666. Signed-off-by: Alexey Goncharuk <alexey.goncha...@gmail.com> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/64b4abe5 Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/64b4abe5 Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/64b4abe5 Branch: refs/heads/ignite-3478 Commit: 64b4abe52fcc5b506529d063cd43db43943e0f14 Parents: 40597a0 Author: Ivan Rakov <ivan.glu...@gmail.com> Authored: Mon Sep 18 11:35:17 2017 +0300 Committer: Alexey Goncharuk <alexey.goncha...@gmail.com> Committed: Mon Sep 18 11:35:17 2017 +0300 ---------------------------------------------------------------------- .../cache/persistence/freelist/PagesList.java | 35 ++++++----- .../paged/PageEvictionDataStreamerTest.java | 65 ++++++++++++++++++++ .../paged/PageEvictionMultinodeTest.java | 6 +- .../IgniteCacheEvictionSelfTestSuite.java | 2 + 4 files changed, 91 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/64b4abe5/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java index 6c355f6..8a540a0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java @@ -23,6 +23,7 @@ import java.util.Map; import java.util.concurrent.atomic.AtomicLong; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteSystemProperties; +import org.apache.ignite.internal.managers.communication.GridIoPolicy; import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.pagemem.wal.IgniteWriteAheadLogManager; @@ -154,9 +155,8 @@ public abstract class PagesList extends DataStructure { */ protected final void init(long metaPageId, boolean initNew) throws IgniteCheckedException { if (metaPageId != 0L) { - if (initNew) { + if (initNew) init(metaPageId, PagesListMetaIO.VERSIONS.latest()); - } else { Map<Integer, GridLongList> bucketsData = new HashMap<>(); @@ -509,10 +509,14 @@ public abstract class PagesList extends DataStructure { */ private Stripe getPageForPut(int bucket) throws IgniteCheckedException { // Striped pool optimization. - int stripeIdx; IgniteThread igniteThread = IgniteThread.current(); + IgniteThread igniteThread = IgniteThread.current(); - if (igniteThread != null && (stripeIdx = igniteThread.stripe()) != -1) { - Stripe[] tails = getBucket(bucket); + Stripe[] tails = getBucket(bucket); + + if (igniteThread != null && igniteThread.policy() == GridIoPolicy.DATA_STREAMER_POOL) { + int stripeIdx = igniteThread.stripe(); + + assert stripeIdx != -1 : igniteThread; while (tails == null || stripeIdx >= tails.length) { addStripe(bucket, true); @@ -523,8 +527,6 @@ public abstract class PagesList extends DataStructure { return tails[stripeIdx]; } - Stripe[] tails = getBucket(bucket); - if (tails == null) return addStripe(bucket, true); @@ -854,9 +856,8 @@ public abstract class PagesList extends DataStructure { assert nextPageAddr != 0L; - if (locked == null) { + if (locked == null) locked = new GridLongList(6); - } locked.add(nextId); locked.add(nextPage); @@ -911,9 +912,8 @@ public abstract class PagesList extends DataStructure { updateTail(bucket, pageId, prevId); // Release write. - for (int i = 0; i < locked.size(); i+=3) { + for (int i = 0; i < locked.size(); i += 3) writeUnlock(locked.get(i), locked.get(i + 1), locked.get(i + 2), FALSE, true); - } } } @@ -933,9 +933,13 @@ public abstract class PagesList extends DataStructure { int len = tails.length; // Striped pool optimization. - int stripeIdx; IgniteThread igniteThread = IgniteThread.current(); + IgniteThread igniteThread = IgniteThread.current(); + + if (igniteThread != null && igniteThread.policy() == GridIoPolicy.DATA_STREAMER_POOL) { + int stripeIdx = igniteThread.stripe(); + + assert stripeIdx != -1 : igniteThread; - if (igniteThread != null && (stripeIdx = igniteThread.stripe()) != -1) { if (stripeIdx >= len) return null; @@ -971,8 +975,11 @@ public abstract class PagesList extends DataStructure { // Striped pool optimization. IgniteThread igniteThread = IgniteThread.current(); - if (igniteThread != null && igniteThread.stripe() != -1) + if (igniteThread != null && igniteThread.policy() == GridIoPolicy.DATA_STREAMER_POOL) { + assert igniteThread.stripe() != -1 : igniteThread; + return writeLock(pageId, page); + } long pageAddr = tryWriteLock(pageId, page); http://git-wip-us.apache.org/repos/asf/ignite/blob/64b4abe5/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDataStreamerTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDataStreamerTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDataStreamerTest.java new file mode 100644 index 0000000..6b6d687 --- /dev/null +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionDataStreamerTest.java @@ -0,0 +1,65 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ +package org.apache.ignite.internal.processors.cache.eviction.paged; + +import java.util.concurrent.ThreadLocalRandom; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.IgniteDataStreamer; +import org.apache.ignite.cache.CachePeekMode; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataPageEvictionMode; +import org.apache.ignite.configuration.IgniteConfiguration; + +/** + * + */ +public class PageEvictionDataStreamerTest extends PageEvictionMultinodeTest { + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { + return setEvictionMode(DataPageEvictionMode.RANDOM_LRU, super.getConfiguration(gridName)); + } + + /** {@inheritDoc} */ + @Override protected void createCacheAndTestEvcition(CacheConfiguration<Object, Object> cfg) throws Exception { + IgniteCache<Object, Object> cache = clientGrid.getOrCreateCache(cfg); + + try (IgniteDataStreamer<Object, Object> ldr = clientGrid.dataStreamer(cfg.getName())) { + ldr.allowOverwrite(true); + + for (int i = 1; i <= ENTRIES; i++) { + ThreadLocalRandom r = ThreadLocalRandom.current(); + + if (r.nextInt() % 5 == 0) + ldr.addData(i, new TestObject(PAGE_SIZE / 4 - 50 + r.nextInt(5000))); // Fragmented object. + else + ldr.addData(i, new TestObject(r.nextInt(PAGE_SIZE / 4 - 50))); // Fits in one page. + + if (i % (ENTRIES / 10) == 0) + System.out.println(">>> Entries put: " + i); + } + } + + int resultingSize = cache.size(CachePeekMode.PRIMARY); + + System.out.println(">>> Resulting size: " + resultingSize); + + // Eviction started, no OutOfMemory occurred, success. + assertTrue(resultingSize < ENTRIES); + + clientGrid.destroyCache(cfg.getName()); + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/64b4abe5/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeTest.java index c2c0775..e4587aa 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/eviction/paged/PageEvictionMultinodeTest.java @@ -42,7 +42,7 @@ public abstract class PageEvictionMultinodeTest extends PageEvictionAbstractTest CacheWriteSynchronizationMode.FULL_SYNC, CacheWriteSynchronizationMode.FULL_ASYNC}; /** Client grid. */ - private Ignite clientGrid; + Ignite clientGrid; /** {@inheritDoc} */ @Override protected void beforeTestsStarted() throws Exception { @@ -94,7 +94,7 @@ public abstract class PageEvictionMultinodeTest extends PageEvictionAbstractTest * @param cfg Config. * @throws Exception If failed. */ - private void createCacheAndTestEvcition(CacheConfiguration<Object, Object> cfg) throws Exception { + protected void createCacheAndTestEvcition(CacheConfiguration<Object, Object> cfg) throws Exception { IgniteCache<Object, Object> cache = clientGrid.getOrCreateCache(cfg); for (int i = 1; i <= ENTRIES; i++) { @@ -121,7 +121,7 @@ public abstract class PageEvictionMultinodeTest extends PageEvictionAbstractTest System.out.println(">>> Resulting size: " + resultingSize); // Eviction started, no OutOfMemory occurred, success. - assertTrue(resultingSize < ENTRIES); + assertTrue(resultingSize < ENTRIES * 10 / 11); clientGrid.destroyCache(cfg.getName()); } http://git-wip-us.apache.org/repos/asf/ignite/blob/64b4abe5/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java index 9f03c60..7985e2e 100644 --- a/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java +++ b/modules/core/src/test/java/org/apache/ignite/testsuites/IgniteCacheEvictionSelfTestSuite.java @@ -33,6 +33,7 @@ import org.apache.ignite.internal.processors.cache.eviction.fifo.FifoEvictionPol import org.apache.ignite.internal.processors.cache.eviction.lru.LruEvictionPolicySelfTest; import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearEvictionPolicySelfTest; import org.apache.ignite.internal.processors.cache.eviction.lru.LruNearOnlyNearEvictionPolicySelfTest; +import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionDataStreamerTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionReadThroughTest; import org.apache.ignite.internal.processors.cache.eviction.paged.PageEvictionTouchOrderTest; import org.apache.ignite.internal.processors.cache.eviction.paged.Random2LruNearEnabledPageEvictionMultinodeTest; @@ -79,6 +80,7 @@ public class IgniteCacheEvictionSelfTestSuite extends TestSuite { suite.addTest(new TestSuite(Random2LruPageEvictionWithRebalanceTest.class)); suite.addTest(new TestSuite(PageEvictionTouchOrderTest.class)); suite.addTest(new TestSuite(PageEvictionReadThroughTest.class)); + suite.addTest(new TestSuite(PageEvictionDataStreamerTest.class)); return suite; }