This is an automated email from the ASF dual-hosted git repository. thomasm pushed a commit to branch OAK-11509 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit 6a5e4610fe00045462ae62880f65ab4ea2f7dc4b Author: Thomas Mueller <[email protected]> AuthorDate: Thu Feb 20 17:03:17 2025 +0100 OAK-11509 Incremental flat file store: reduce logging --- .../oak/index/indexer/document/DocumentStoreIndexerBase.java | 1 - .../flatfile/pipelined/PipelinedTreeStoreStrategy.java | 7 ------- .../document/flatfile/pipelined/PipelinedTreeStoreTask.java | 6 ++---- .../document/incrementalstore/IncrementalStoreBuilder.java | 9 ++++----- .../document/incrementalstore/MergeIncrementalTreeStore.java | 10 +++++----- .../jackrabbit/oak/index/indexer/document/tree/Prefetcher.java | 2 +- .../jackrabbit/oak/index/indexer/document/tree/TreeStore.java | 6 ++---- .../oak/index/indexer/document/tree/TreeStoreUtils.java | 2 +- .../oak/index/indexer/document/tree/store/FileStore.java | 3 --- .../oak/index/indexer/document/tree/store/TreeSession.java | 5 ----- .../jackrabbit/oak/plugins/document/DocumentNodeStore.java | 2 +- 11 files changed, 16 insertions(+), 37 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java index f2b22e34fa..071ed395b6 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java @@ -219,7 +219,6 @@ public abstract class DocumentStoreIndexerBase implements Closeable { if (splitFlatFile) { storeList = builder.buildList(indexHelper, indexerSupport, indexDefinitions); } else { - log.info("Building index store"); IndexStore store = builder.build(indexHelper, indexDefinitions); int threads = IndexerConfiguration.indexThreadPoolSize(); if (store instanceof ParallelIndexStore && threads > 1) { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java index 1f6764ff3c..7d066a39fb 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java @@ -420,7 +420,6 @@ public class PipelinedTreeStoreStrategy extends IndexStoreSortStrategyBase { )); try { - LOG.info("Waiting for tasks to complete"); int tasksFinished = 0; int transformTasksFinished = 0; boolean monitorQueues = true; @@ -441,17 +440,13 @@ public class PipelinedTreeStoreStrategy extends IndexStoreSortStrategyBase { Object result = completedTask.get(); if (result instanceof PipelinedMongoDownloadTask.Result) { PipelinedMongoDownloadTask.Result downloadResult = (PipelinedMongoDownloadTask.Result) result; - LOG.info("Download finished. Documents downloaded: {}", downloadResult.getDocumentsDownloaded()); downloadFuture = null; } else if (result instanceof PipelinedTransformTask.Result) { PipelinedTransformTask.Result transformResult = (PipelinedTransformTask.Result) result; transformTasksFinished++; nodeStateEntriesExtracted += transformResult.getEntryCount(); - LOG.info("Transform task {} finished. Entries processed: {}", - transformResult.getThreadId(), transformResult.getEntryCount()); if (transformTasksFinished == numberOfTransformThreads) { - LOG.info("All transform tasks finished. Total entries processed: {}", nodeStateEntriesExtracted); // No need to keep monitoring the queues, the download and transform threads are done. monitorQueues = false; // Terminate the sort thread. @@ -462,7 +457,6 @@ public class PipelinedTreeStoreStrategy extends IndexStoreSortStrategyBase { } else if (result instanceof PipelinedSortBatchTask.Result) { PipelinedSortBatchTask.Result sortTaskResult = (PipelinedSortBatchTask.Result) result; - LOG.info("Sort batch task finished. Entries processed: {}", sortTaskResult.getTotalEntries()); // The buffers between transform and merge sort tasks are no longer needed, so remove them // from the queues so they can be garbage collected. // These buffers can be very large, so this is important to avoid running out of memory in @@ -510,7 +504,6 @@ public class PipelinedTreeStoreStrategy extends IndexStoreSortStrategyBase { treeStore.close(); return resultDir; } finally { - LOG.info("Shutting down build thread pool"); new ExecutorCloser(threadPool).close(); } } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java index 5dab7c3048..dbf548128e 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java @@ -117,7 +117,6 @@ public class PipelinedTreeStoreTask implements Callable<PipelinedSortBatchTask.R MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL, 0); MetricsUtils.addMetric(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL, 0); MetricsUtils.addMetricByteSize(statisticsProvider, reporter, PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES, 0); - LOG.info("Final merge done, {} roots", session.getRootCount()); } long totalTimeMillis = taskStartTime.elapsed().toMillis(); String timeCreatingSortArrayPercentage = formatAsPercentage(timeCreatingSortArrayMillis, totalTimeMillis); @@ -189,8 +188,6 @@ public class PipelinedTreeStoreTask implements Callable<PipelinedSortBatchTask.R private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception { ByteBuffer buffer = nseb.getBuffer(); - LOG.info("Going to sort batch in memory. Entries: {}, Size: {}", - nseb.numberOfEntries(), humanReadableByteCountBin(nseb.sizeOfEntriesBytes())); ArrayList<SortKeyPath> sortBuffer = buildSortArray(nseb); if (sortBuffer.isEmpty()) { return; @@ -198,7 +195,8 @@ public class PipelinedTreeStoreTask implements Callable<PipelinedSortBatchTask.R Stopwatch sortClock = Stopwatch.createStarted(); Collections.sort(sortBuffer); timeSortingMillis += sortClock.elapsed().toMillis(); - LOG.info("Sorted batch in {}. Saving.", sortClock); + LOG.info("Sorted batch with {} entries, size {}, in {}", + nseb.numberOfEntries(), humanReadableByteCountBin(nseb.sizeOfEntriesBytes()), sortClock); Stopwatch saveClock = Stopwatch.createStarted(); long textSize = 0; batchesProcessed++; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java index aa00438459..99e310c63e 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java @@ -38,8 +38,6 @@ import java.util.Set; import java.util.function.Predicate; import static java.util.Collections.unmodifiableSet; -import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_LZ4; -import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_ZIP; public class IncrementalStoreBuilder { private final Logger log = LoggerFactory.getLogger(getClass()); @@ -149,9 +147,10 @@ public class IncrementalStoreBuilder { } private void logFlags() { - log.info("Preferred path elements are {}", Iterables.toString(preferredPathElements)); - log.info("Compression enabled while sorting : {} ({})", IndexStoreUtils.compressionEnabled(), OAK_INDEXER_USE_ZIP); - log.info("LZ4 enabled for compression algorithm : {} ({})", IndexStoreUtils.useLZ4(), OAK_INDEXER_USE_LZ4); + log.info("Preferred path elements {}, compression enabled {}, algorithm {}", + Iterables.toString(preferredPathElements), + IndexStoreUtils.compressionEnabled(), + IndexStoreUtils.useLZ4()); } } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java index d3c138b6de..f5466dac44 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java @@ -72,11 +72,13 @@ public class MergeIncrementalTreeStore implements MergeIncrementalStore { FilePacker.unpack(baseFile, baseDir, true); File topup = new File(incrementalFile.getParent(), TOPUP_FILE); if (topup.exists()) { - LOG.info("Merging diff {}", incrementalFile.getAbsolutePath()); + LOG.info("Merging diff {}, topup {}, to {}", + incrementalFile.getAbsolutePath(), + topup.getAbsolutePath(), + mergedFile.getAbsolutePath() + ); updateIndexStore(baseDir, incrementalFile, algorithm); - LOG.info("Merging topup {}", topup.getAbsolutePath()); updateIndexStore(baseDir, topup, algorithm); - LOG.info("Packing to {}", mergedFile.getAbsolutePath()); FilePacker.pack(baseDir, TreeSession.getFileNameRegex(), mergedFile, true); } else { File mergedDir = new File(mergedFile.getAbsolutePath() + ".files"); @@ -88,9 +90,7 @@ public class MergeIncrementalTreeStore implements MergeIncrementalStore { LOG.info("Packing to {}", mergedFile.getAbsolutePath()); FilePacker.pack(mergedDir, TreeSession.getFileNameRegex(), mergedFile, true); } - LOG.info("Merging metadata"); mergeMetadataFiles(); - LOG.info("Completed"); } @Override diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java index ede8449584..56f5fce4b4 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java @@ -79,7 +79,7 @@ public class Prefetcher { public Thread newThread(Runnable r) { Thread t = new Thread(r); t.setDaemon(true); - t.setName("BlobPrefetcher-" + threadNumber.getAndIncrement()); + t.setName("TreeStorePrefetcher-" + threadNumber.getAndIncrement()); return t; } }); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java index 92367e2163..387c836711 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java @@ -84,7 +84,7 @@ public class TreeStore implements ParallelIndexStore { private final AtomicLong nodeCacheHits = new AtomicLong(); private final AtomicLong nodeCacheMisses = new AtomicLong(); private final AtomicLong nodeCacheFills = new AtomicLong(); - private int iterationCount; + private long iterationCount; private PathIteratorFilter filter = new PathIteratorFilter(); private ArrayList<String> splitPoints; @@ -120,7 +120,6 @@ public class TreeStore implements ParallelIndexStore { this.session = new TreeSession(store); // we don not want to merge too early during the download session.setMaxRoots(1000); - LOG.info("Open " + toString()); } public void init() { @@ -137,7 +136,6 @@ public class TreeStore implements ParallelIndexStore { } public void setIncludedPaths(SortedSet<String> includedPaths) { - LOG.info("Included paths {}", includedPaths.toString()); filter = new PathIteratorFilter(includedPaths); } @@ -197,7 +195,7 @@ public class TreeStore implements ParallelIndexStore { continue; } } - if (++iterationCount % 1_000_000 == 0) { + if (++iterationCount % 5_000_000 == 0) { LOG.info("Fetching {} in {}", iterationCount, TreeStore.this.toString()); } current = key; diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java index f2eb586b62..7ff5e8e1e1 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java @@ -67,7 +67,7 @@ public class TreeStoreUtils { String fileName = args[1]; try (BufferedReader lineReader = new BufferedReader( new FileReader(fileName, StandardCharsets.UTF_8))) { - int count = 0; + long count = 0; long start = System.nanoTime(); while (true) { String line = lineReader.readLine(); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java index 5894f58926..dad247681e 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java @@ -41,8 +41,6 @@ import org.slf4j.LoggerFactory; */ public class FileStore implements Store { - private static final Logger LOG = LoggerFactory.getLogger(FileStore.class); - private final Properties config; private final String directory; private Compression compression = Compression.NO; @@ -58,7 +56,6 @@ public class FileStore implements Store { this.directory = config.getProperty("dir"); this.maxFileSizeBytes = Long.parseLong(config.getProperty( Store.MAX_FILE_SIZE_BYTES, "" + Store.DEFAULT_MAX_FILE_SIZE_BYTES)); - LOG.info("Max file size {} bytes", maxFileSizeBytes); new File(directory).mkdirs(); } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java index 55d3dbfd5c..eb724c631a 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java @@ -30,16 +30,12 @@ import java.util.Random; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.ConcurrentLRUCache; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Position; import org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SortedStream; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * A session that allows reading and writing keys and values in a tree store. */ public class TreeSession { - private static final Logger LOG = LoggerFactory.getLogger(TreeSession.class); - public static final String CACHE_SIZE_MB = "cacheSizeMB"; private static final int DEFAULT_CACHE_SIZE_MB = 256; private static final int DEFAULT_MAX_ROOTS = 10; @@ -72,7 +68,6 @@ public class TreeSession { long cacheSizeMB = Long.parseLong(store.getConfig().getProperty( CACHE_SIZE_MB, "" + DEFAULT_CACHE_SIZE_MB)); long cacheSizeBytes = cacheSizeMB * 1024 * 1024; - LOG.info("Cache size {} bytes", cacheSizeBytes); this.cache = new ConcurrentLRUCache<>(cacheSizeBytes) { private static final long serialVersionUID = 1L; diff --git a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java index 95bd69017a..adbc55cc4b 100644 --- a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java +++ b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java @@ -3429,7 +3429,7 @@ public final class DocumentNodeStore // see OAK-6016 and OAK-6011 if (e instanceof IllegalStateException && "Root document does not have a lastRev entry for local clusterId 0".equals(e.getMessage())) { - LOG.warn("diffJournalChildren failed with " + + LOG.debug("diffJournalChildren failed with " + e.getClass().getSimpleName() + ", falling back to classic diff : " + e.getMessage()); } else {
