This is an automated email from the ASF dual-hosted git repository. amashenkov pushed a commit to branch gg-19225 in repository https://gitbox.apache.org/repos/asf/ignite.git
commit 7de25e5256e9026c12d051c5f06cff7d91e56c35 Author: Anton Kalashnikov <[email protected]> AuthorDate: Mon Jun 3 17:24:48 2019 +0300 GG-18670 [IGNITE-11749] Implement automatic pages history dump on CorruptedTreeException. (cherry picked from commit 1f746e8) --- .../wal/scanner/PrintToFileHandler.java | 5 +- .../persistence/wal/scanner/PrintToLogHandler.java | 6 ++- .../persistence/wal/scanner/ScannerHandler.java | 18 +++++++ .../cache/persistence/wal/scanner/WalScanner.java | 63 +++++++++++++--------- .../diagnostic/PageHistoryDiagnoster.java | 47 ++-------------- 5 files changed, 70 insertions(+), 69 deletions(-) diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToFileHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToFileHandler.java index 4693700..5c72bf0 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToFileHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToFileHandler.java @@ -18,7 +18,6 @@ package org.apache.ignite.internal.processors.cache.persistence.wal.scanner; import java.io.File; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.StandardOpenOption; import org.apache.ignite.IgniteException; import org.apache.ignite.configuration.DataStorageConfiguration; @@ -29,6 +28,8 @@ import org.apache.ignite.internal.processors.cache.persistence.file.FileIOFactor import org.apache.ignite.lang.IgniteBiTuple; import org.jetbrains.annotations.Nullable; +import static java.nio.charset.StandardCharsets.UTF_8; +import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandler.toStringRecord; import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.DEFAULT_WAL_RECORD_PREFIX; /** @@ -67,7 +68,7 @@ class PrintToFileHandler implements ScannerHandler { * @return Bytes repersentation of data to be written in dump file. */ protected byte[] getBytes(IgniteBiTuple<WALPointer, WALRecord> record) { - return (DEFAULT_WAL_RECORD_PREFIX + record.get2() + "\n").getBytes(StandardCharsets.UTF_8); + return (DEFAULT_WAL_RECORD_PREFIX + toStringRecord(record.get2()) + System.lineSeparator()).getBytes(UTF_8); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToLogHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToLogHandler.java index 2d1af63..2061a44 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToLogHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/PrintToLogHandler.java @@ -22,6 +22,7 @@ import org.apache.ignite.internal.pagemem.wal.WALPointer; import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.lang.IgniteBiTuple; +import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandler.toStringRecord; import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.DEFAULT_WAL_RECORD_PREFIX; /** @@ -47,7 +48,10 @@ class PrintToLogHandler implements ScannerHandler { @Override public void handle(IgniteBiTuple<WALPointer, WALRecord> record) { ensureNotFinished(); - resultString.append(DEFAULT_WAL_RECORD_PREFIX).append(record.get2()).append("\n"); + resultString + .append(DEFAULT_WAL_RECORD_PREFIX) + .append(toStringRecord(record.get2())) + .append(System.lineSeparator()); } /** {@inheritDoc} */ diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/ScannerHandler.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/ScannerHandler.java index f88815c..adbfae8 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/ScannerHandler.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/ScannerHandler.java @@ -66,4 +66,22 @@ public interface ScannerHandler { } }; } + + /** + * Make string from given wal record. + * + * @param walRecord Source WAL record. + * @return Representation of WAL record. + */ + public static String toStringRecord(WALRecord walRecord) { + String walRecordStr; + + try { + walRecordStr = walRecord != null ? walRecord.toString() : "Record is null"; + } + catch (RuntimeException e) { + walRecordStr = "Record : " + walRecord.type() + " - Unable to convert to string representation."; + } + return walRecordStr; + } } diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/WalScanner.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/WalScanner.java index fe133a0..c475b91 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/WalScanner.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/cache/persistence/wal/scanner/WalScanner.java @@ -45,21 +45,45 @@ import static org.apache.ignite.internal.processors.cache.persistence.wal.reader * Scanning WAL by specific condition. */ public class WalScanner { - /** Parameters for iterator. */ - private final IteratorParametersBuilder parametersBuilder; - /** Wal iterator factory. */ - private final IgniteWalIteratorFactory iteratorFactory; + /** Low level WAL iterator. */ + private final IgniteThrowableSupplier<WALIterator> walIteratorSupplier; /** + * @param preconfiguredIter Preconfgured iterator. * @param parametersBuilder Parameters for iterator. * @param factory Factory of iterator. */ WalScanner( + WALIterator preconfiguredIter, IteratorParametersBuilder parametersBuilder, IgniteWalIteratorFactory factory ) { - this.parametersBuilder = parametersBuilder; - iteratorFactory = factory == null ? new IgniteWalIteratorFactory() : factory; + if (preconfiguredIter != null) + walIteratorSupplier = () -> preconfiguredIter; + else + walIteratorSupplier = () -> standaloneWalIterator( + factory == null ? new IgniteWalIteratorFactory() : factory, + parametersBuilder + ); + } + + /** + * @param iteratorFactory Factory of iterator. + * @param parametersBuilder Parameters for iterator. + * @return Standalone WAL iterator created by given parameters. + * @throws IgniteCheckedException If failed. + */ + private static WALIterator standaloneWalIterator( + IgniteWalIteratorFactory iteratorFactory, + IteratorParametersBuilder parametersBuilder + ) throws IgniteCheckedException { + return iteratorFactory.iterator( + parametersBuilder.copy().addFilter((type, pointer) -> + // PHYSICAL need fo page shanpshot or delta record. + // MIXED need for partiton meta state update. + type.purpose() == PHYSICAL || type.purpose() == MIXED + ) + ); } /** @@ -85,26 +109,17 @@ public class WalScanner { .or(pageOwner(groupAndPageIds0)) .or(partitionMetaStateUpdate(groupAndParts)); - return new ScanTerminateStep(() -> iterator(filter, - parametersBuilder.copy().addFilter((type, pointer) -> - // PHYSICAL need fo page shanpshot or delta record. - // MIXED need for partiton meta state update. - type.purpose() == PHYSICAL || type.purpose() == MIXED - ) - )); + return new ScanTerminateStep(() -> new FilteredWalIterator(walIteratorSupplier.get(), filter)); } /** - * @param filter Record filter. - * @param parametersBuilder Iterator parameters for customization. - * @return Instance of {@link FilteredWalIterator}. - * @throws IgniteCheckedException If initialization of iterator will be failed. + * Factory method of {@link WalScanner}. + * + * @param walIterator Preconfigured WAL iterator. + * @return Instance of {@link WalScanner}. */ - @NotNull private FilteredWalIterator iterator( - Predicate<IgniteBiTuple<WALPointer, WALRecord>> filter, - IteratorParametersBuilder parametersBuilder - ) throws IgniteCheckedException { - return new FilteredWalIterator(iteratorFactory.iterator(parametersBuilder), filter); + public static WalScanner buildWalScanner(WALIterator walIterator) { + return new WalScanner(walIterator, null, null); } /** @@ -114,7 +129,7 @@ public class WalScanner { * @return Instance of {@link WalScanner}. */ public static WalScanner buildWalScanner(IteratorParametersBuilder parametersBuilder) { - return new WalScanner(parametersBuilder, null); + return buildWalScanner(parametersBuilder, null); } /** @@ -128,7 +143,7 @@ public class WalScanner { IteratorParametersBuilder parametersBuilder, IgniteWalIteratorFactory factory ) { - return new WalScanner(parametersBuilder, factory); + return new WalScanner(null, parametersBuilder, factory); } /** diff --git a/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/PageHistoryDiagnoster.java b/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/PageHistoryDiagnoster.java index 3572fc8..9d1b2fc 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/PageHistoryDiagnoster.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/processors/diagnostic/PageHistoryDiagnoster.java @@ -21,21 +21,14 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; import java.util.EnumSet; -import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.Set; import java.util.function.BiFunction; -import java.util.function.Predicate; -import java.util.stream.Collectors; import org.apache.ignite.IgniteCheckedException; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.internal.GridKernalContext; -import org.apache.ignite.internal.pagemem.PageIdUtils; -import org.apache.ignite.internal.pagemem.wal.WALIterator; -import org.apache.ignite.internal.pagemem.wal.WALPointer; -import org.apache.ignite.internal.pagemem.wal.record.WALRecord; import org.apache.ignite.internal.processors.cache.GridCacheSharedContext; import org.apache.ignite.internal.processors.cache.persistence.wal.FileDescriptor; import org.apache.ignite.internal.processors.cache.persistence.wal.FileWALPointer; @@ -50,17 +43,13 @@ import org.apache.ignite.internal.processors.cache.persistence.wal.serializer.Re import org.apache.ignite.internal.processors.diagnostic.DiagnosticProcessor.DiagnosticFileWriteMode; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.typedef.T2; -import org.apache.ignite.lang.IgniteBiTuple; import org.jetbrains.annotations.NotNull; import static java.util.Objects.requireNonNull; import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.IgniteWalIteratorFactory.IteratorParametersBuilder.withIteratorParameters; -import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.checkpoint; -import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.pageOwner; -import static org.apache.ignite.internal.processors.cache.persistence.wal.reader.WalFilters.partitionMetaStateUpdate; +import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printRawToFile; import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printToFile; import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printToLog; -import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.ScannerHandlers.printRawToFile; import static org.apache.ignite.internal.processors.cache.persistence.wal.scanner.WalScanner.buildWalScanner; /** @@ -199,43 +188,17 @@ public class PageHistoryDiagnoster { ScannerHandler action, FileWALPointer from ) throws IgniteCheckedException { - IgniteBiTuple<WALPointer, WALRecord> lastReadRec = null; // Try scan via WAL manager. More safety way on working node. try { - Set<T2<Integer, Long>> groupAndPageIds0 = new HashSet<>(builder.pageIds); - - // Collect all (group, partition) partition pairs. - Set<T2<Integer, Integer>> groupAndParts = groupAndPageIds0.stream() - .map((tup) -> new T2<>(tup.get1(), PageIdUtils.partId(tup.get2()))) - .collect(Collectors.toSet()); - - // Build WAL filter. (Checkoint, Page, Partition meta) - Predicate<IgniteBiTuple<WALPointer, WALRecord>> filter = checkpoint() - .or(pageOwner(groupAndPageIds0)) - .or(partitionMetaStateUpdate(groupAndParts)); - - try (WALIterator it = wal.replay(from)) { - while (it.hasNext()) { - IgniteBiTuple<WALPointer, WALRecord> recTup = lastReadRec = it.next(); - - if (filter.test(recTup)) - action.handle(recTup); - } - } - finally { - action.finish(); - } + buildWalScanner(wal.replay(from)) + .findAllRecordsFor(builder.pageIds) + .forEach(action); return; } catch (IgniteCheckedException e) { - if (lastReadRec != null) { - log.warning("Failed to diagnosric scan via WAL manager, lastReadRec:[" - + lastReadRec.get1() + ", " + lastReadRec.get2() + "]",e); - } - else - log.warning("Failed to diagnosric scan via WAL manager", e); + log.warning("Failed to diagnosric scan via WAL manager", e); } // Try scan via stand alone iterator is not safety if wal still generated and moving to archive.
