This is an automated email from the ASF dual-hosted git repository. dpavlov pushed a commit to branch ignite-2.7.5 in repository https://gitbox.apache.org/repos/asf/ignite.git
commit df73704f409c511b6acc3dbace2acd3a2f4a683b Author: Dmitriy Pavlov <[email protected]> AuthorDate: Thu Jan 31 18:45:47 2019 +0300 IGNITE-11030 Fix Pages List to preserve partition ID consistency. (#5915) (cherry picked from commit ec488539933f1cafa0756c7d2bd22b48f0c1e16b) --- .../wal/record/delta/InitNewPageRecord.java | 10 +++ .../cache/persistence/freelist/PagesList.java | 8 ++- .../cache/persistence/pagemem/PageMemoryImpl.java | 16 ++++- .../persistence/db/IgniteTcBotInitNewPageTest.java | 81 ++++++++++++++++++++++ .../testsuites/IgnitePdsWithIndexingTestSuite.java | 2 + 5 files changed, 113 insertions(+), 4 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java index d0ba2aa..9ed7f35 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/pagemem/wal/record/delta/InitNewPageRecord.java @@ -18,6 +18,7 @@ package org.apache.ignite.internal.pagemem.wal.record.delta; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.internal.pagemem.PageIdUtils; import org.apache.ignite.internal.pagemem.PageMemory; import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.util.tostring.GridToStringExclude; @@ -51,6 +52,15 @@ public class InitNewPageRecord extends PageDeltaRecord { this.ioType = ioType; this.ioVer = ioVer; this.newPageId = newPageId; + + int newPartId = PageIdUtils.partId(newPageId); + int partId = PageIdUtils.partId(pageId); + + if (newPartId != partId) { + throw new AssertionError("Partition consistency failure: " + + "newPageId=" + Long.toHexString(newPageId) + " (newPartId: " + newPartId + ") " + + "pageId=" + Long.toHexString(pageId) + " (partId: " + partId + ")"); + } } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java index f1cc32a..9a7bee9 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/freelist/PagesList.java @@ -1145,9 +1145,11 @@ public abstract class PagesList extends DataStructure { decrementBucketSize(bucket); - if (initIoVers != null) - dataPageId = initReusedPage(tailId, tailPage, tailAddr, 0, FLAG_DATA, initIoVers.latest()); - else + if (initIoVers != null) { + int partId = PageIdUtils.partId(tailId); + + dataPageId = initReusedPage(tailId, tailPage, tailAddr, partId, FLAG_DATA, initIoVers.latest()); + } else dataPageId = recyclePage(tailId, tailPage, tailAddr, null); dirty = true; diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java index 5c91378..6e00a10 100755 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/pagemem/PageMemoryImpl.java @@ -1569,7 +1569,7 @@ public class PageMemoryImpl implements PageMemoryEx { long pageId = PageIO.getPageId(page + PAGE_OVERHEAD); assert pageId != 0 : U.hexLong(PageHeader.readPageId(page)); - assert PageIO.getVersion(page + PAGE_OVERHEAD) != 0 : U.hexLong(pageId); + assert PageIO.getVersion(page + PAGE_OVERHEAD) != 0 : dumpPage(pageId, fullId.groupId()); assert PageIO.getType(page + PAGE_OVERHEAD) != 0 : U.hexLong(pageId); try { @@ -1586,6 +1586,20 @@ public class PageMemoryImpl implements PageMemoryEx { } /** + * Prepares page details for assertion. + * @param pageId Page id. + * @param grpId Group id. + */ + @NotNull private String dumpPage(long pageId, int grpId) { + int pageIdx = PageIdUtils.pageIndex(pageId); + int partId = PageIdUtils.partId(pageId); + long off = (long)(pageIdx + 1) * pageSize(); + + return U.hexLong(pageId) + " (grpId=" + grpId + ", pageIdx=" + pageIdx + ", partId=" + partId + ", offH=" + + Long.toHexString(off) + ")"; + } + + /** * @param absPtr Absolute pointer to the page. * @return {@code True} if write lock acquired for the page. */ diff --git a/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java new file mode 100644 index 0000000..8ef517c --- /dev/null +++ b/modules/indexing/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/IgniteTcBotInitNewPageTest.java @@ -0,0 +1,81 @@ +/* +* Licensed to the Apache Software Foundation (ASF) under one or more +* contributor license agreements. See the NOTICE file distributed with +* this work for additional information regarding copyright ownership. +* The ASF licenses this file to You under the Apache License, Version 2.0 +* (the "License"); you may not use this file except in compliance with +* the License. You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +package org.apache.ignite.internal.processors.cache.persistence.db; + +import com.google.common.base.Strings; +import org.apache.ignite.IgniteCache; +import org.apache.ignite.cache.affinity.rendezvous.RendezvousAffinityFunction; +import org.apache.ignite.configuration.CacheConfiguration; +import org.apache.ignite.configuration.DataRegionConfiguration; +import org.apache.ignite.configuration.DataStorageConfiguration; +import org.apache.ignite.configuration.IgniteConfiguration; +import org.apache.ignite.configuration.WALMode; +import org.apache.ignite.internal.IgniteEx; +import org.apache.ignite.internal.pagemem.wal.record.delta.InitNewPageRecord; +import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; +import org.junit.Test; + +/** + * Test creates a lot of index pages in the cache with low number of partitions.<br> + * Then cache entries are removed to enforce all pages to come to a free list. <br> + * Then creation of data pages with long data will probably result in page rotation.<br> + * Expected behaviour: all {@link InitNewPageRecord} should have consistent partition IDs. + */ +public class IgniteTcBotInitNewPageTest extends GridCommonAbstractTest { + /** Cache name. */ + public static final String CACHE = "cache"; + + /** */ + @Test + public void testInitNewPagePageIdConsistency() throws Exception { + IgniteEx ex = startGrid(0); + + IgniteCache<Object, Object> cache = ex.cache(CACHE); + + for (int i = 0; i < 1_000_000; i++) + cache.put(i, i); + + cache.clear(); + + for (int i = 0; i < 1_000; i++) + cache.put(i, Strings.repeat("Apache Ignite", 1000)); + } + + /** {@inheritDoc} */ + @Override protected IgniteConfiguration getConfiguration(String igniteInstanceName) throws Exception { + IgniteConfiguration cfg = super.getConfiguration(igniteInstanceName); + + CacheConfiguration<Integer, Object> ccfg = new CacheConfiguration<>(CACHE); + + ccfg.setAffinity(new RendezvousAffinityFunction(false, 4)); + + cfg.setCacheConfiguration(ccfg); + + DataRegionConfiguration regCfg = new DataRegionConfiguration() + .setMaxSize(2L * 1024 * 1024 * 1024) + .setPersistenceEnabled(true); + + DataStorageConfiguration dsCfg = new DataStorageConfiguration() + .setWalMode(WALMode.LOG_ONLY) + .setDefaultDataRegionConfiguration(regCfg); + + cfg.setDataStorageConfiguration(dsCfg); + + return cfg; + } +} diff --git a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java index 67b9fad..88e3fe7 100644 --- a/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java +++ b/modules/indexing/src/test/java/org/apache/ignite/testsuites/IgnitePdsWithIndexingTestSuite.java @@ -20,6 +20,7 @@ package org.apache.ignite.testsuites; import junit.framework.TestSuite; import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingAndGroupPutGetPersistenceSelfTest; import org.apache.ignite.internal.processors.cache.IgnitePdsSingleNodeWithIndexingPutGetPersistenceTest; +import org.apache.ignite.internal.processors.cache.persistence.db.IgniteTcBotInitNewPageTest; import org.apache.ignite.internal.processors.database.IgniteDbMultiNodeWithIndexingPutGetTest; import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingPutGetTest; import org.apache.ignite.internal.processors.database.IgniteDbSingleNodeWithIndexingWalRestoreTest; @@ -46,6 +47,7 @@ public class IgnitePdsWithIndexingTestSuite extends TestSuite { suite.addTestSuite(IgnitePersistentStoreSchemaLoadTest.class); suite.addTestSuite(IgnitePersistentStoreQueryWithMultipleClassesPerCacheTest.class); suite.addTestSuite(IgniteTwoRegionsRebuildIndexTest.class); + suite.addTestSuite(IgniteTcBotInitNewPageTest.class); return suite; }
