This is an automated email from the ASF dual-hosted git repository. gvvinblade pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ignite.git
The following commit(s) were added to refs/heads/master by this push: new 55a7e3a IGNITE-10584: MVCC: Wal delta record consistency test failed. This closes #5643. 55a7e3a is described below commit 55a7e3ad3e80a7361609fb936ecbb299cca21eeb Author: Andrey V. Mashenkov <andrey.mashen...@gmail.com> AuthorDate: Mon Jan 14 14:57:32 2019 +0300 IGNITE-10584: MVCC: Wal delta record consistency test failed. This closes #5643. --- .../cache/tree/mvcc/data/MvccUpdateDataRow.java | 2 +- .../wal/CpTriggeredWalDeltaConsistencyTest.java | 4 -- .../wal/ExplicitWalDeltaConsistencyTest.java | 5 +- .../wal/SysPropWalDeltaConsistencyTest.java | 4 -- .../wal/memtracker/PageMemoryTracker.java | 84 +++++++++++++++------- 5 files changed, 62 insertions(+), 37 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java index 99d0ed0..75c510b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/tree/mvcc/data/MvccUpdateDataRow.java @@ -401,7 +401,7 @@ public class MvccUpdateDataRow extends MvccDataRow implements MvccUpdateResult, rowIo.setMvccLockCoordinatorVersion(pageAddr, idx, mvccCoordinatorVersion()); rowIo.setMvccLockCounter(pageAddr, idx, mvccCounter()); - // TODO Delta record IGNITE-7991 + // Actually, there is no need to log lock delta record into WAL. setFlags(DIRTY); } diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/CpTriggeredWalDeltaConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/CpTriggeredWalDeltaConsistencyTest.java index 325d69f..ca68ec6 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/CpTriggeredWalDeltaConsistencyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/CpTriggeredWalDeltaConsistencyTest.java @@ -19,7 +19,6 @@ package org.apache.ignite.internal.processors.cache.persistence.wal; import org.apache.ignite.IgniteCache; import org.apache.ignite.internal.IgniteEx; -import org.apache.ignite.testframework.MvccFeatureChecker; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -46,9 +45,6 @@ public class CpTriggeredWalDeltaConsistencyTest extends AbstractWalDeltaConsiste */ @Test public final void testPutRemoveCacheDestroy() throws Exception { - if (MvccFeatureChecker.forcedMvcc()) - fail("https://issues.apache.org/jira/browse/IGNITE-10584"); - IgniteEx ignite = startGrid(0); ignite.cluster().active(true); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/ExplicitWalDeltaConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/ExplicitWalDeltaConsistencyTest.java index 2005174..2cd8259 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/ExplicitWalDeltaConsistencyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/ExplicitWalDeltaConsistencyTest.java @@ -42,9 +42,6 @@ public class ExplicitWalDeltaConsistencyTest extends AbstractWalDeltaConsistency */ @Test public final void testPutRemoveAfterCheckpoint() throws Exception { - if (MvccFeatureChecker.forcedMvcc()) - fail("https://issues.apache.org/jira/browse/IGNITE-10584"); - IgniteEx ignite = startGrid(0); ignite.cluster().active(true); @@ -79,7 +76,7 @@ public class ExplicitWalDeltaConsistencyTest extends AbstractWalDeltaConsistency @Test public final void testNotEmptyPds() throws Exception { if (MvccFeatureChecker.forcedMvcc()) - fail("https://issues.apache.org/jira/browse/IGNITE-10584"); + fail("https://issues.apache.org/jira/browse/IGNITE-10822"); IgniteEx ignite = startGrid(0); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/SysPropWalDeltaConsistencyTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/SysPropWalDeltaConsistencyTest.java index 74ac777..bf32501 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/SysPropWalDeltaConsistencyTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/SysPropWalDeltaConsistencyTest.java @@ -21,7 +21,6 @@ import org.apache.ignite.IgniteCache; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.IgniteEx; import org.apache.ignite.internal.processors.cache.persistence.wal.memtracker.PageMemoryTrackerPluginProvider; -import org.apache.ignite.testframework.MvccFeatureChecker; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.JUnit4; @@ -66,9 +65,6 @@ public class SysPropWalDeltaConsistencyTest extends AbstractWalDeltaConsistencyT */ @Test public final void testPutRemoveMultinode() throws Exception { - if (MvccFeatureChecker.forcedMvcc()) - fail("https://issues.apache.org/jira/browse/IGNITE-10584"); - IgniteEx ignite0 = startGrid(0); ignite0.cluster().active(true); diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java index cee2e66..6944e21 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/wal/memtracker/PageMemoryTracker.java @@ -49,14 +49,17 @@ import org.apache.ignite.internal.pagemem.wal.record.delta.PageDeltaRecord; import org.apache.ignite.internal.processors.cache.CacheGroupContext; import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; +import org.apache.ignite.internal.processors.cache.mvcc.MvccUtils; +import org.apache.ignite.internal.processors.cache.mvcc.txlog.TxLog; import org.apache.ignite.internal.processors.cache.persistence.DataRegion; import org.apache.ignite.internal.processors.cache.persistence.DbCheckpointListener; import org.apache.ignite.internal.processors.cache.persistence.GridCacheDatabaseSharedManager; import org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager; import org.apache.ignite.internal.processors.cache.persistence.metastorage.MetaStorage; -import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryEx; import org.apache.ignite.internal.processors.cache.persistence.pagemem.PageMemoryImpl; +import org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWriteAheadLogManager; +import org.apache.ignite.internal.processors.cache.tree.AbstractDataLeafIO; import org.apache.ignite.internal.util.GridUnsafe; import org.apache.ignite.internal.util.typedef.internal.CU; import org.apache.ignite.internal.util.typedef.internal.U; @@ -66,6 +69,9 @@ import org.apache.ignite.plugin.PluginContext; import org.apache.ignite.spi.encryption.EncryptionSpi; import org.mockito.Mockito; +import static org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.T_CACHE_ID_DATA_REF_MVCC_LEAF; +import static org.apache.ignite.internal.processors.cache.persistence.tree.io.PageIO.T_DATA_REF_MVCC_LEAF; + /** * Page memory tracker. * @@ -457,6 +463,9 @@ public class PageMemoryTracker implements IgnitePlugin { long totalAllocated = pageStoreMgr.pagesAllocated(MetaStorage.METASTORAGE_CACHE_ID); + if (MvccUtils.mvccEnabled(gridCtx)) + totalAllocated += pageStoreMgr.pagesAllocated(TxLog.TX_LOG_CACHE_ID); + for (CacheGroupContext ctx : gridCtx.cache().cacheGroups()) totalAllocated += pageStoreMgr.pagesAllocated(ctx.groupId()); @@ -480,15 +489,6 @@ public class PageMemoryTracker implements IgnitePlugin { synchronized (pageAllocatorMux) { long totalAllocated = pageStoreAllocatedPages(); - long metaId = ((PageMemoryEx)cacheProc.context().database().metaStorage().pageMemory()).metaPageId( - MetaStorage.METASTORAGE_CACHE_ID); - - // Meta storage meta page is counted as allocated, but never used in current implementation. - // This behavior will be fixed by https://issues.apache.org/jira/browse/IGNITE-8735 - if (!pages.containsKey(new FullPageId(metaId, MetaStorage.METASTORAGE_CACHE_ID)) - && pages.containsKey(new FullPageId(metaId + 1, MetaStorage.METASTORAGE_CACHE_ID))) - totalAllocated--; - log.info(">>> Total tracked pages: " + pages.size()); log.info(">>> Total allocated pages: " + totalAllocated); @@ -513,6 +513,8 @@ public class PageMemoryTracker implements IgnitePlugin { if (fullPageId.groupId() == MetaStorage.METASTORAGE_CACHE_ID) pageMem = cacheProc.context().database().metaStorage().pageMemory(); + else if (fullPageId.groupId() == TxLog.TX_LOG_CACHE_ID) + pageMem = cacheProc.context().database().dataRegion(TxLog.TX_LOG_CACHE_NAME).pageMemory(); else { CacheGroupContext ctx = cacheProc.cacheGroup(fullPageId.groupId()); @@ -547,20 +549,8 @@ public class PageMemoryTracker implements IgnitePlugin { dumpHistory(page); } - else { - ByteBuffer locBuf = GridUnsafe.wrapPointer(page.address(), pageSize); - ByteBuffer rmtBuf = GridUnsafe.wrapPointer(rmtPageAddr, pageSize); - - if (!locBuf.equals(rmtBuf)) { - res = false; - - log.error("Page buffers are not equals: " + fullPageId); - - dumpDiff(locBuf, rmtBuf); - - dumpHistory(page); - } - } + else if (!comparePages(fullPageId, page, rmtPageAddr)) + res = false; if (!res && !checkAll) return false; @@ -583,6 +573,52 @@ public class PageMemoryTracker implements IgnitePlugin { } /** + * Compare pages content. + * + * @param fullPageId Full page ID. + * @param expectedPage Expected page. + * @param actualPageAddr Actual page address. + * @return {@code True} if pages are equals, {@code False} otherwise. + * @throws IgniteCheckedException If fails. + */ + private boolean comparePages(FullPageId fullPageId, DirectMemoryPage expectedPage, long actualPageAddr) throws IgniteCheckedException { + long expPageArrd = expectedPage.address(); + + GridCacheProcessor cacheProc = gridCtx.cache(); + + ByteBuffer locBuf = GridUnsafe.wrapPointer(expPageArrd, pageSize); + ByteBuffer rmtBuf = GridUnsafe.wrapPointer(actualPageAddr, pageSize); + + PageIO pageIo = PageIO.getPageIO(actualPageAddr); + + if (pageIo.getType() == T_DATA_REF_MVCC_LEAF || pageIo.getType() == T_CACHE_ID_DATA_REF_MVCC_LEAF) { + assert cacheProc.cacheGroup(fullPageId.groupId()).mvccEnabled(); + + AbstractDataLeafIO io = (AbstractDataLeafIO)pageIo; + + int cnt = io.getCount(actualPageAddr); + + // Reset lock info as there is no sense to log it into WAL. + for (int i = 0; i < cnt; i++) { + io.setMvccLockCoordinatorVersion(expPageArrd, i, io.getMvccLockCoordinatorVersion(actualPageAddr, i)); + io.setMvccLockCounter(expPageArrd, i, io.getMvccLockCounter(actualPageAddr, i)); + } + } + + if (!locBuf.equals(rmtBuf)) { + log.error("Page buffers are not equals: " + fullPageId); + + dumpDiff(locBuf, rmtBuf); + + dumpHistory(expectedPage); + + return false; + } + + return true; + } + + /** * Dump statistics to log. */ private void dumpStats() {