ignite-6788 Ignite WAL reader fails on Tx marker record for persistent store with new style folder naming
Signed-off-by: agura <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/5542c70d Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/5542c70d Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/5542c70d Branch: refs/heads/ignite-6778 Commit: 5542c70da913ff1006973e25f5112152674880db Parents: 67ca745 Author: dpavlov <[email protected]> Authored: Wed Nov 1 03:17:00 2017 +0300 Committer: agura <[email protected]> Committed: Wed Nov 1 03:17:00 2017 +0300 ---------------------------------------------------------------------- .../wal/reader/StandaloneGridKernalContext.java | 20 +- .../db/wal/reader/IgniteWalReaderTest.java | 283 ++++++++++++------- 2 files changed, 195 insertions(+), 108 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/5542c70d/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java index c0c3650..485458b 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/reader/StandaloneGridKernalContext.java @@ -18,7 +18,6 @@ package org.apache.ignite.internal.processors.cache.persistence.wal.reader; import java.io.File; -import java.io.Serializable; import java.lang.reflect.Field; import java.util.Iterator; import java.util.List; @@ -49,8 +48,8 @@ import org.apache.ignite.internal.managers.loadbalancer.GridLoadBalancerManager; import org.apache.ignite.internal.processors.affinity.GridAffinityProcessor; import org.apache.ignite.internal.processors.cache.GridCacheProcessor; import org.apache.ignite.internal.processors.cache.binary.CacheObjectBinaryProcessorImpl; -import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver; import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFolderSettings; +import org.apache.ignite.internal.processors.cache.persistence.filename.PdsFoldersResolver; import org.apache.ignite.internal.processors.cacheobject.IgniteCacheObjectProcessor; import org.apache.ignite.internal.processors.closure.GridClosureProcessor; import org.apache.ignite.internal.processors.cluster.ClusterProcessor; @@ -95,6 +94,9 @@ import org.jetbrains.annotations.Nullable; * Dummy grid kernal context */ public class StandaloneGridKernalContext implements GridKernalContext { + /** Binary metadata file store folderÑ */ + public static final String BINARY_META_FOLDER = "binary_meta"; + /** Config for fake Ignite instance. */ private final IgniteConfiguration cfg; @@ -123,13 +125,12 @@ public class StandaloneGridKernalContext implements GridKernalContext { * Providing {@code null} will disable unmarshall for non primitive objects, BinaryObjects will be provided <br> */ StandaloneGridKernalContext(IgniteLogger log, - @Nullable final File binaryMetadataFileStoreDir, - @Nullable final File marshallerMappingFileStoreDir) throws IgniteCheckedException { + @Nullable File binaryMetadataFileStoreDir, + @Nullable File marshallerMappingFileStoreDir) throws IgniteCheckedException { this.log = log; try { - pluginProc = new StandaloneIgnitePluginProcessor( - this, config()); + pluginProc = new StandaloneIgnitePluginProcessor(this, config()); } catch (IgniteCheckedException e) { throw new IllegalStateException("Must not fail on empty providers list.", e); @@ -137,7 +138,12 @@ public class StandaloneGridKernalContext implements GridKernalContext { this.marshallerCtx = new MarshallerContextImpl(null); this.cfg = prepareIgniteConfiguration(); - this.cacheObjProcessor = binaryMetadataFileStoreDir != null ? binaryProcessor(this, binaryMetadataFileStoreDir) : null; + + // Fake folder provided to perform processor startup on empty folder. + if (binaryMetadataFileStoreDir == null) + binaryMetadataFileStoreDir = new File(BINARY_META_FOLDER).getAbsoluteFile(); + + this.cacheObjProcessor = binaryProcessor(this, binaryMetadataFileStoreDir); if (marshallerMappingFileStoreDir != null) { marshallerCtx.setMarshallerMappingFileStoreDir(marshallerMappingFileStoreDir); http://git-wip-us.apache.org/repos/asf/ignite/blob/5542c70d/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java ---------------------------------------------------------------------- diff --git a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java index 88eacef..9348a68 100644 --- a/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java +++ b/modules/core/src/test/java/org/apache/ignite/internal/processors/cache/persistence/db/wal/reader/IgniteWalReaderTest.java @@ -31,6 +31,7 @@ import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import java.util.UUID; import java.util.concurrent.CountDownLatch; import java.util.concurrent.TimeUnit; @@ -40,6 +41,7 @@ import org.apache.ignite.Ignite; import org.apache.ignite.IgniteCache; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteEvents; +import org.apache.ignite.IgniteSystemProperties; import org.apache.ignite.binary.BinaryObject; import org.apache.ignite.cache.CacheAtomicityMode; import org.apache.ignite.cache.CacheRebalanceMode; @@ -68,8 +70,12 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.reader.Ignite import org.apache.ignite.internal.processors.cache.version.GridCacheVersion; import org.apache.ignite.internal.util.typedef.internal.A; import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteBiInClosure; import org.apache.ignite.lang.IgniteBiTuple; +import org.apache.ignite.lang.IgniteInClosure; import org.apache.ignite.lang.IgnitePredicate; +import org.apache.ignite.logger.NullLogger; +import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.ignite.transactions.Transaction; import org.jetbrains.annotations.NotNull; @@ -77,6 +83,7 @@ import org.jetbrains.annotations.Nullable; import org.junit.Assert; import static org.apache.ignite.events.EventType.EVT_WAL_SEGMENT_ARCHIVED; +import static org.apache.ignite.internal.processors.cache.GridCacheOperation.CREATE; import static org.apache.ignite.internal.processors.cache.GridCacheOperation.DELETE; import static org.apache.ignite.internal.processors.cache.persistence.file.FilePageStoreManager.DFLT_STORE_DIR; import static org.apache.ignite.internal.processors.cache.persistence.filename.PdsConsistentIdProcessor.genNewStyleSubfolderName; @@ -94,12 +101,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { /** additional cache for testing different combinations of types in WAL */ private static final String CACHE_ADDL_NAME = "cache1"; - /** Delete DB dir before test. */ - private static final boolean deleteBefore = true; - - /** Delete DB dir after test. */ - private static final boolean deleteAfter = true; - /** Dump records to logger. Should be false for non local run */ private static final boolean dumpRecords = false; @@ -115,6 +116,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { /** Custom wal mode. */ private WALMode customWalMode; + /** Clear properties in afterTest method() */ + private boolean clearProperties; + /** {@inheritDoc} */ @Override protected IgniteConfiguration getConfiguration(String gridName) throws Exception { final IgniteConfiguration cfg = super.getConfiguration(gridName); @@ -151,23 +155,22 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { @Override protected void beforeTest() throws Exception { stopAllGrids(); - if (deleteBefore) - deleteWorkFiles(); + deleteWorkFiles(); } /** {@inheritDoc} */ @Override protected void afterTest() throws Exception { stopAllGrids(); - if (deleteAfter) - deleteWorkFiles(); + if (clearProperties) + System.clearProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS); } /** * @throws IgniteCheckedException If failed. */ - private void deleteWorkFiles() throws IgniteCheckedException { - deleteRecursively(U.resolveWorkDirectory(U.defaultWorkDirectory(), DFLT_STORE_DIR, false)); + private void deleteWorkFiles() throws Exception { + GridTestUtils.deleteDbFiles(); } /** @@ -197,17 +200,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { final int cntUsingMockIter = iterateAndCount(it, false); log.info("Total records loaded " + cntUsingMockIter); - assert cntUsingMockIter > 0; - assert cntUsingMockIter > cacheObjectsToWrite; + assertTrue(cntUsingMockIter > 0); + assertTrue(cntUsingMockIter > cacheObjectsToWrite); final File walArchiveDirWithConsistentId = new File(walArchive, subfolderName); final File walWorkDirWithConsistentId = new File(wal, subfolderName); - - final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false); - final File binaryMetaWithConsId = new File(binaryMeta, subfolderName); - final File marshaller = U.resolveWorkDirectory(workDir, "marshaller", false); - - final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE, binaryMetaWithConsId, marshaller); + final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName); final int cntArchiveDir = iterateAndCount(factory.iteratorArchiveDirectory(walArchiveDirWithConsistentId)); log.info("Total records loaded using directory : " + cntArchiveDir); @@ -218,12 +216,13 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { log.info("Total records loaded using archive directory (file-by-file): " + cntArchiveFileByFile); - assert cntArchiveFileByFile > cacheObjectsToWrite; - assert cntArchiveDir > cacheObjectsToWrite; - assert cntArchiveDir == cntArchiveFileByFile; + assertTrue(cntArchiveFileByFile > cacheObjectsToWrite); + assertTrue(cntArchiveDir > cacheObjectsToWrite); + assertTrue(cntArchiveDir == cntArchiveFileByFile); //really count2 may be less because work dir correct loading is not supported yet - assert cntUsingMockIter >= cntArchiveDir - : "Mock based reader loaded " + cntUsingMockIter + " records but standalone has loaded only " + cntArchiveDir; + assertTrue("Mock based reader loaded " + cntUsingMockIter + " records " + + "but standalone has loaded only " + cntArchiveDir, + cntUsingMockIter >= cntArchiveDir); final File[] workFiles = walWorkDirWithConsistentId.listFiles(FileWriteAheadLogManager.WAL_SEGMENT_FILE_FILTER); @@ -231,10 +230,10 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { log.info("Total records loaded from work: " + cntWork); - assert cntWork + cntArchiveFileByFile == cntUsingMockIter - : "Work iterator loaded [" + cntWork + "] " + - "Archive iterator loaded [" + cntArchiveFileByFile + "]; " + - "mock iterator [" + cntUsingMockIter + "]"; + assertTrue("Work iterator loaded [" + cntWork + "] " + + "Archive iterator loaded [" + cntArchiveFileByFile + "]; " + + "mock iterator [" + cntUsingMockIter + "]", + cntWork + cntArchiveFileByFile == cntUsingMockIter); } /** @@ -312,7 +311,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { putDummyRecords(ignite, 500); stopGrid("node0"); - assert evtRecorded.get(); + assertTrue(evtRecorded.get()); } /** @@ -407,7 +406,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { archiveSegmentForInactivity.await(archiveIncompleteSegmentAfterInactivityMs + 1001, TimeUnit.MILLISECONDS); stopGrid("node0"); - assert recordedAfterSleep; + assertTrue(recordedAfterSleep); } /** @@ -460,8 +459,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { binaryMetaWithConsId, marshallerMapping); - final BiConsumer<Object, Object> objConsumer = new BiConsumer<Object, Object>() { - @Override public void accept(Object key, Object val) { + final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object, Object>() { + @Override public void apply(Object key, Object val) { boolean rmv = remove(ctrlMap, key, val); if (!rmv) log.error("Unable to remove Key and value from control Map K:[" + key + "] V: [" + val + "]"); @@ -471,14 +470,14 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { assertEquals(indexedObj.iVal, indexedObj.jVal); assertEquals(indexedObj.iVal, key); for (byte datum : indexedObj.getData()) { - assert datum >= 'A' && datum <= 'A' + 10; + assertTrue(datum >= 'A' && datum <= 'A' + 10); } } } }; scanIterateAndCount(factory, workDir, subfolderName, cntEntries, txCnt, objConsumer, null); - assert ctrlMap.isEmpty() : " Control Map is not empty after reading entries " + ctrlMap; + assertTrue(" Control Map is not empty after reading entries " + ctrlMap, ctrlMap.isEmpty()); } /** @@ -510,8 +509,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { final String subfolderName, final int minCntEntries, final int minTxCnt, - @Nullable final BiConsumer<Object, Object> objConsumer, - @Nullable final Consumer<DataRecord> dataRecordHnd) throws IgniteCheckedException { + @Nullable final IgniteBiInClosure<Object, Object> objConsumer, + @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException { final File db = U.resolveWorkDirectory(workDir, DFLT_STORE_DIR, false); final File wal = new File(db, "wal"); @@ -544,8 +543,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { final int entriesWork = valuesSum(cntWork.values()); log.info("Archive directory: Tx found " + txCntObservedWork + " entries " + entriesWork); - assert entriesArch + entriesWork >= minCntEntries; - assert txCntObservedWork + txCntObservedArch >= minTxCnt; + assertTrue("entriesArch=" + entriesArch + " + entriesWork=" + entriesWork + + " >= minCntEntries=" + minCntEntries, + entriesArch + entriesWork >= minCntEntries); + assertTrue("txCntObservedWork=" + txCntObservedWork + " + txCntObservedArch=" + txCntObservedArch + + " >= minTxCnt=" + minTxCnt, + txCntObservedWork + txCntObservedArch >= minTxCnt); } /** @@ -611,11 +614,9 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { final File binaryMetaWithNodeSubfolder = new File(binaryMeta, subfolderName); final File marshallerMapping = U.resolveWorkDirectory(workDir, "marshaller", false); - final IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(log, PAGE_SIZE, - binaryMetaWithNodeSubfolder, - marshallerMapping); - final BiConsumer<Object, Object> objConsumer = new BiConsumer<Object, Object>() { - @Override public void accept(Object key, Object val) { + final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName); + final IgniteBiInClosure<Object, Object> objConsumer = new IgniteBiInClosure<Object, Object>() { + @Override public void apply(Object key, Object val) { log.info("K: [" + key + ", " + (key != null ? key.getClass().getName() : "?") + "]" + " V: [" + val + ", " + @@ -625,12 +626,12 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { String msg = "Unable to remove pair from control map " + "K: [" + key + "] V: [" + val + "]"; log.error(msg); } - assert !(val instanceof BinaryObject); + assertFalse(val instanceof BinaryObject); } }; - final Consumer<DataRecord> toStrChecker = new Consumer<DataRecord>() { - @Override public void accept(DataRecord record) { + final IgniteInClosure<DataRecord> toStrChecker = new IgniteInClosure<DataRecord>() { + @Override public void apply(DataRecord record) { String strRepresentation = record.toString(); for (Iterator<String> iter = ctrlStringsToSearch.iterator(); iter.hasNext(); ) { final String next = iter.next(); @@ -643,17 +644,17 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { }; scanIterateAndCount(factory, workDir, subfolderName, cntEntries, 0, objConsumer, toStrChecker); - assert ctrlMap.isEmpty() : " Control Map is not empty after reading entries: " + ctrlMap; - assert ctrlStringsToSearch.isEmpty() : " Control Map for strings in entries is not empty after" + - " reading records: " + ctrlStringsToSearch; + assertTrue(" Control Map is not empty after reading entries: " + ctrlMap, ctrlMap.isEmpty()); + assertTrue(" Control Map for strings in entries is not empty after" + + " reading records: " + ctrlStringsToSearch, ctrlStringsToSearch.isEmpty()); //Validate same WAL log with flag binary objects only final IgniteWalIteratorFactory keepBinFactory = new IgniteWalIteratorFactory(log, PAGE_SIZE, binaryMetaWithNodeSubfolder, marshallerMapping, true); - final BiConsumer<Object, Object> binObjConsumer = new BiConsumer<Object, Object>() { - @Override public void accept(Object key, Object val) { + final IgniteBiInClosure<Object, Object> binObjConsumer = new IgniteBiInClosure<Object, Object>() { + @Override public void apply(Object key, Object val) { log.info("K(KeepBinary): [" + key + ", " + (key != null ? key.getClass().getName() : "?") + "]" + " V(KeepBinary): [" + val + ", " + @@ -693,15 +694,15 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { byte data[] = binaryObj.field("data"); for (byte datum : data) { - assert datum >= 'A' && datum <= 'A' + 10; + assertTrue(datum >= 'A' && datum <= 'A' + 10); } } } } }; - final Consumer<DataRecord> binObjToStrChecker = new Consumer<DataRecord>() { - @Override public void accept(DataRecord record) { + final IgniteInClosure<DataRecord> binObjToStrChecker = new IgniteInClosure<DataRecord>() { + @Override public void apply(DataRecord record) { String strRepresentation = record.toString(); for (Iterator<String> iter = ctrlStringsForBinaryObjSearch.iterator(); iter.hasNext(); ) { final String next = iter.next(); @@ -714,9 +715,11 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { }; scanIterateAndCount(keepBinFactory, workDir, subfolderName, cntEntries, 0, binObjConsumer, binObjToStrChecker); - assert ctrlMapForBinaryObjects.isEmpty() : " Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects; - assert ctrlStringsForBinaryObjSearch.isEmpty() : " Control Map for strings in entries is not empty after" + - " reading records: " + ctrlStringsForBinaryObjSearch; + assertTrue(" Control Map is not empty after reading entries: " + ctrlMapForBinaryObjects, + ctrlMapForBinaryObjects.isEmpty()); + assertTrue(" Control Map for strings in entries is not empty after" + + " reading records: " + ctrlStringsForBinaryObjSearch, + ctrlStringsForBinaryObjSearch.isEmpty()); } @@ -764,7 +767,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { stopGrid("node0"); final String workDir = U.defaultWorkDirectory(); - final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName, workDir); + final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName); scanIterateAndCount(factory, workDir, subfolderName, totalEntries, 0, null, null); } @@ -787,7 +790,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { stopGrid("node0"); final String workDir = U.defaultWorkDirectory(); - final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName, workDir); + final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName); scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, null); } @@ -859,13 +862,13 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { stopGrid("node0"); final String workDir = U.defaultWorkDirectory(); - final IgniteWalIteratorFactory factory = createWalIteratorFactory(subfolderName, workDir); + final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName); final StringBuilder builder = new StringBuilder(); final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class); - scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new Consumer<DataRecord>() { - @Override public void accept(DataRecord dataRecord) { + scanIterateAndCount(factory, workDir, subfolderName, 0, 0, null, new IgniteInClosure<DataRecord>() { + @Override public void apply(DataRecord dataRecord) { final List<DataEntry> entries = dataRecord.writeEntries(); builder.append("{"); @@ -902,14 +905,121 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { } /** - * @param subfolderName Subfolder name. + * Tests transaction generation and WAL for putAll cache operation. + * @throws Exception if failed. + */ + public void testPutAllTxIntoTwoNodes() throws Exception { + final Ignite ignite = startGrid("node0"); + final Ignite ignite1 = startGrid(1); + + ignite.active(true); + + final Map<Object, IndexedObject> map = new TreeMap<>(); + + final int cntEntries = 1000; + for (int i = 0; i < cntEntries; i++) + map.put(i, new IndexedObject(i)); + + ignite.cache(CACHE_NAME).putAll(map); + + ignite.active(false); + + final String subfolderName = genDbSubfolderName(ignite, 0); + final String subfolderName1 = genDbSubfolderName(ignite1, 1); + + stopAllGrids(); + + final String workDir = U.defaultWorkDirectory(); + final IgniteWalIteratorFactory factory = createWalIteratorFactory(workDir, subfolderName); + + final StringBuilder builder = new StringBuilder(); + final Map<GridCacheOperation, Integer> operationsFound = new EnumMap<>(GridCacheOperation.class); + + final IgniteInClosure<DataRecord> drHnd = new IgniteInClosure<DataRecord>() { + @Override public void apply(DataRecord dataRecord) { + final List<DataEntry> entries = dataRecord.writeEntries(); + + builder.append("{"); + for (DataEntry entry : entries) { + final GridCacheOperation op = entry.op(); + final Integer cnt = operationsFound.get(op); + + operationsFound.put(op, cnt == null ? 1 : (cnt + 1)); + + if (entry instanceof UnwrapDataEntry) { + final UnwrapDataEntry entry1 = (UnwrapDataEntry)entry; + + builder.append(entry1.op()).append(" for ").append(entry1.unwrappedKey()); + final GridCacheVersion ver = entry.nearXidVersion(); + + builder.append(", "); + + if (ver != null) + builder.append("tx=").append(ver).append(", "); + } + } + + builder.append("}\n"); + } + }; + scanIterateAndCount(factory, workDir, subfolderName, 1, 1, null, drHnd); + scanIterateAndCount(factory, workDir, subfolderName1, 1, 1, null, drHnd); + + final Integer createsFound = operationsFound.get(CREATE); + + if (log.isInfoEnabled()) + log.info(builder.toString()); + + assertTrue("Create operations should be found in log: " + operationsFound, + createsFound != null && createsFound > 0); + + assertTrue("Create operations count should be at least " + cntEntries + " in log: " + operationsFound, + createsFound != null && createsFound >= cntEntries); + + } + + /** + * Tests transaction generation and WAL for putAll cache operation. + * @throws Exception if failed. + */ + public void testTxRecordsReadWoBinaryMeta() throws Exception { + clearProperties = true; + System.setProperty(IgniteSystemProperties.IGNITE_WAL_LOG_TX_RECORDS, "true"); + + final Ignite ignite = startGrid("node0"); + ignite.active(true); + + final Map<Object, IndexedObject> map = new TreeMap<>(); + + for (int i = 0; i < 1000; i++) + map.put(i, new IndexedObject(i)); + + ignite.cache(CACHE_NAME).putAll(map); + + ignite.active(false); + + final String workDir = U.defaultWorkDirectory(); + final String subfolderName = genDbSubfolderName(ignite, 0); + stopAllGrids(); + + IgniteWalIteratorFactory factory = new IgniteWalIteratorFactory(new NullLogger(), + PAGE_SIZE, + null, + null, + false); + + scanIterateAndCount(factory, workDir, subfolderName, 1000, 1, null, null); + } + + /** * @param workDir Work directory. + * @param subfolderName Subfolder name. * @return WAL iterator factory. * @throws IgniteCheckedException If failed. */ @NotNull private IgniteWalIteratorFactory createWalIteratorFactory( - String subfolderName, - String workDir + final String workDir, + final String subfolderName ) throws IgniteCheckedException { final File binaryMeta = U.resolveWorkDirectory(workDir, "binary_meta", false); final File binaryMetaWithConsId = new File(binaryMeta, subfolderName); @@ -918,7 +1028,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { return new IgniteWalIteratorFactory(log, PAGE_SIZE, binaryMetaWithConsId, - marshallerMapping); + marshallerMapping, + false); } /** @@ -943,8 +1054,8 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { */ private Map<GridCacheVersion, Integer> iterateAndCountDataRecord( final WALIterator walIter, - @Nullable final BiConsumer<Object, Object> cacheObjHnd, - @Nullable final Consumer<DataRecord> dataRecordHnd) throws IgniteCheckedException { + @Nullable final IgniteBiInClosure<Object, Object> cacheObjHnd, + @Nullable final IgniteInClosure<DataRecord> dataRecordHnd) throws IgniteCheckedException { final Map<GridCacheVersion, Integer> entriesUnderTxFound = new HashMap<>(); @@ -957,7 +1068,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { final DataRecord dataRecord = (DataRecord)walRecord; if (dataRecordHnd != null) - dataRecordHnd.accept(dataRecord); + dataRecordHnd.apply(dataRecord); final List<DataEntry> entries = dataRecord.writeEntries(); for (DataEntry entry : entries) { @@ -990,7 +1101,7 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { "; Value: " + unwrappedValObj); if (cacheObjHnd != null && (unwrappedKeyObj != null || unwrappedValObj != null)) - cacheObjHnd.accept(unwrappedKeyObj, unwrappedValObj); + cacheObjHnd.apply(unwrappedKeyObj, unwrappedValObj); final Integer entriesUnderTx = entriesUnderTxFound.get(globalTxId); entriesUnderTxFound.put(globalTxId, entriesUnderTx == null ? 1 : entriesUnderTx + 1); @@ -1008,36 +1119,6 @@ public class IgniteWalReaderTest extends GridCommonAbstractTest { return entriesUnderTxFound; } - /** - * Represents an operation that accepts a single input argument and returns no - * result. - * - * @param <T> - */ - private interface Consumer<T> { - /** - * Performs this operation on the given argument. - * - * @param t the input argument - */ - public void accept(T t); - } - - /** - * Represents an operation that accepts two input arguments and returns no - * result. - * - * @param <T> - */ - private interface BiConsumer<T, U> { - /** - * Performs this operation on the given argument. - * - * @param t the input argument - */ - public void accept(T t, U u); - } - /** Enum for cover binaryObject enum save/load */ enum TestEnum { /** */A, /** */B, /** */C
