This is an automated email from the ASF dual-hosted git repository.
thomasm pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4984c929b0 OAK-11509 Incremental flat file store: reduce logging
(#2102)
4984c929b0 is described below
commit 4984c929b01444fa982fef8b8e6fbea6d140d2c8
Author: Thomas Mueller <[email protected]>
AuthorDate: Fri Feb 21 08:52:24 2025 +0100
OAK-11509 Incremental flat file store: reduce logging (#2102)
* OAK-11509 Incremental flat file store: reduce logging
* OAK-11509 Incremental flat file store: reduce logging
* OAK-11509 Incremental flat file store: reduce logging
---
.../oak/index/indexer/document/DocumentStoreIndexerBase.java | 1 -
.../flatfile/pipelined/PipelinedTreeStoreStrategy.java | 7 -------
.../document/flatfile/pipelined/PipelinedTreeStoreTask.java | 6 ++----
.../document/incrementalstore/IncrementalStoreBuilder.java | 9 ++++-----
.../document/incrementalstore/MergeIncrementalTreeStore.java | 10 +++++-----
.../jackrabbit/oak/index/indexer/document/tree/Prefetcher.java | 2 +-
.../jackrabbit/oak/index/indexer/document/tree/TreeStore.java | 6 ++----
.../oak/index/indexer/document/tree/TreeStoreUtils.java | 2 +-
.../oak/index/indexer/document/tree/store/FileStore.java | 3 ---
.../oak/index/indexer/document/tree/store/TreeSession.java | 5 -----
.../jackrabbit/oak/plugins/document/DocumentNodeStore.java | 5 ++---
11 files changed, 17 insertions(+), 39 deletions(-)
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
index f2b22e34fa..071ed395b6 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/DocumentStoreIndexerBase.java
@@ -219,7 +219,6 @@ public abstract class DocumentStoreIndexerBase implements
Closeable {
if (splitFlatFile) {
storeList = builder.buildList(indexHelper, indexerSupport,
indexDefinitions);
} else {
- log.info("Building index store");
IndexStore store = builder.build(indexHelper,
indexDefinitions);
int threads = IndexerConfiguration.indexThreadPoolSize();
if (store instanceof ParallelIndexStore && threads > 1) {
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java
index 1f6764ff3c..7d066a39fb 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreStrategy.java
@@ -420,7 +420,6 @@ public class PipelinedTreeStoreStrategy extends
IndexStoreSortStrategyBase {
));
try {
- LOG.info("Waiting for tasks to complete");
int tasksFinished = 0;
int transformTasksFinished = 0;
boolean monitorQueues = true;
@@ -441,17 +440,13 @@ public class PipelinedTreeStoreStrategy extends
IndexStoreSortStrategyBase {
Object result = completedTask.get();
if (result instanceof
PipelinedMongoDownloadTask.Result) {
PipelinedMongoDownloadTask.Result
downloadResult = (PipelinedMongoDownloadTask.Result) result;
- LOG.info("Download finished. Documents
downloaded: {}", downloadResult.getDocumentsDownloaded());
downloadFuture = null;
} else if (result instanceof
PipelinedTransformTask.Result) {
PipelinedTransformTask.Result transformResult
= (PipelinedTransformTask.Result) result;
transformTasksFinished++;
nodeStateEntriesExtracted +=
transformResult.getEntryCount();
- LOG.info("Transform task {} finished. Entries
processed: {}",
- transformResult.getThreadId(),
transformResult.getEntryCount());
if (transformTasksFinished ==
numberOfTransformThreads) {
- LOG.info("All transform tasks finished.
Total entries processed: {}", nodeStateEntriesExtracted);
// No need to keep monitoring the queues,
the download and transform threads are done.
monitorQueues = false;
// Terminate the sort thread.
@@ -462,7 +457,6 @@ public class PipelinedTreeStoreStrategy extends
IndexStoreSortStrategyBase {
} else if (result instanceof
PipelinedSortBatchTask.Result) {
PipelinedSortBatchTask.Result sortTaskResult =
(PipelinedSortBatchTask.Result) result;
- LOG.info("Sort batch task finished. Entries
processed: {}", sortTaskResult.getTotalEntries());
// The buffers between transform and merge
sort tasks are no longer needed, so remove them
// from the queues so they can be garbage
collected.
// These buffers can be very large, so this is
important to avoid running out of memory in
@@ -510,7 +504,6 @@ public class PipelinedTreeStoreStrategy extends
IndexStoreSortStrategyBase {
treeStore.close();
return resultDir;
} finally {
- LOG.info("Shutting down build thread pool");
new ExecutorCloser(threadPool).close();
}
}
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java
index 5dab7c3048..dbf548128e 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedTreeStoreTask.java
@@ -117,7 +117,6 @@ public class PipelinedTreeStoreTask implements
Callable<PipelinedSortBatchTask.R
MetricsUtils.addMetric(statisticsProvider, reporter,
PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL, 0);
MetricsUtils.addMetric(statisticsProvider, reporter,
PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL,
0);
MetricsUtils.addMetricByteSize(statisticsProvider,
reporter,
PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES,
0);
- LOG.info("Final merge done, {} roots",
session.getRootCount());
}
long totalTimeMillis = taskStartTime.elapsed().toMillis();
String timeCreatingSortArrayPercentage =
formatAsPercentage(timeCreatingSortArrayMillis, totalTimeMillis);
@@ -189,8 +188,6 @@ public class PipelinedTreeStoreTask implements
Callable<PipelinedSortBatchTask.R
private void sortAndSaveBatch(NodeStateEntryBatch nseb) throws Exception {
ByteBuffer buffer = nseb.getBuffer();
- LOG.info("Going to sort batch in memory. Entries: {}, Size: {}",
- nseb.numberOfEntries(),
humanReadableByteCountBin(nseb.sizeOfEntriesBytes()));
ArrayList<SortKeyPath> sortBuffer = buildSortArray(nseb);
if (sortBuffer.isEmpty()) {
return;
@@ -198,7 +195,8 @@ public class PipelinedTreeStoreTask implements
Callable<PipelinedSortBatchTask.R
Stopwatch sortClock = Stopwatch.createStarted();
Collections.sort(sortBuffer);
timeSortingMillis += sortClock.elapsed().toMillis();
- LOG.info("Sorted batch in {}. Saving.", sortClock);
+ LOG.info("Sorted batch with {} entries, size {}, in {}",
+ nseb.numberOfEntries(),
humanReadableByteCountBin(nseb.sizeOfEntriesBytes()), sortClock);
Stopwatch saveClock = Stopwatch.createStarted();
long textSize = 0;
batchesProcessed++;
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java
index aa00438459..99e310c63e 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/IncrementalStoreBuilder.java
@@ -38,8 +38,6 @@ import java.util.Set;
import java.util.function.Predicate;
import static java.util.Collections.unmodifiableSet;
-import static
org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_LZ4;
-import static
org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.OAK_INDEXER_USE_ZIP;
public class IncrementalStoreBuilder {
private final Logger log = LoggerFactory.getLogger(getClass());
@@ -149,9 +147,10 @@ public class IncrementalStoreBuilder {
}
private void logFlags() {
- log.info("Preferred path elements are {}",
Iterables.toString(preferredPathElements));
- log.info("Compression enabled while sorting : {} ({})",
IndexStoreUtils.compressionEnabled(), OAK_INDEXER_USE_ZIP);
- log.info("LZ4 enabled for compression algorithm : {} ({})",
IndexStoreUtils.useLZ4(), OAK_INDEXER_USE_LZ4);
+ log.info("Preferred path elements {}, compression enabled {},
algorithm {}",
+ Iterables.toString(preferredPathElements),
+ IndexStoreUtils.compressionEnabled(),
+ IndexStoreUtils.useLZ4());
}
}
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java
index d3c138b6de..f5466dac44 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/incrementalstore/MergeIncrementalTreeStore.java
@@ -72,11 +72,13 @@ public class MergeIncrementalTreeStore implements
MergeIncrementalStore {
FilePacker.unpack(baseFile, baseDir, true);
File topup = new File(incrementalFile.getParent(), TOPUP_FILE);
if (topup.exists()) {
- LOG.info("Merging diff {}", incrementalFile.getAbsolutePath());
+ LOG.info("Merging diff {}, topup {}, to {}",
+ incrementalFile.getAbsolutePath(),
+ topup.getAbsolutePath(),
+ mergedFile.getAbsolutePath()
+ );
updateIndexStore(baseDir, incrementalFile, algorithm);
- LOG.info("Merging topup {}", topup.getAbsolutePath());
updateIndexStore(baseDir, topup, algorithm);
- LOG.info("Packing to {}", mergedFile.getAbsolutePath());
FilePacker.pack(baseDir, TreeSession.getFileNameRegex(),
mergedFile, true);
} else {
File mergedDir = new File(mergedFile.getAbsolutePath() + ".files");
@@ -88,9 +90,7 @@ public class MergeIncrementalTreeStore implements
MergeIncrementalStore {
LOG.info("Packing to {}", mergedFile.getAbsolutePath());
FilePacker.pack(mergedDir, TreeSession.getFileNameRegex(),
mergedFile, true);
}
- LOG.info("Merging metadata");
mergeMetadataFiles();
- LOG.info("Completed");
}
@Override
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java
index ede8449584..56f5fce4b4 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/Prefetcher.java
@@ -79,7 +79,7 @@ public class Prefetcher {
public Thread newThread(Runnable r) {
Thread t = new Thread(r);
t.setDaemon(true);
- t.setName("BlobPrefetcher-" + threadNumber.getAndIncrement());
+ t.setName("TreeStorePrefetcher-" +
threadNumber.getAndIncrement());
return t;
}
});
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java
index 92367e2163..387c836711 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStore.java
@@ -84,7 +84,7 @@ public class TreeStore implements ParallelIndexStore {
private final AtomicLong nodeCacheHits = new AtomicLong();
private final AtomicLong nodeCacheMisses = new AtomicLong();
private final AtomicLong nodeCacheFills = new AtomicLong();
- private int iterationCount;
+ private long iterationCount;
private PathIteratorFilter filter = new PathIteratorFilter();
private ArrayList<String> splitPoints;
@@ -120,7 +120,6 @@ public class TreeStore implements ParallelIndexStore {
this.session = new TreeSession(store);
// we don not want to merge too early during the download
session.setMaxRoots(1000);
- LOG.info("Open " + toString());
}
public void init() {
@@ -137,7 +136,6 @@ public class TreeStore implements ParallelIndexStore {
}
public void setIncludedPaths(SortedSet<String> includedPaths) {
- LOG.info("Included paths {}", includedPaths.toString());
filter = new PathIteratorFilter(includedPaths);
}
@@ -197,7 +195,7 @@ public class TreeStore implements ParallelIndexStore {
continue;
}
}
- if (++iterationCount % 1_000_000 == 0) {
+ if (++iterationCount % 5_000_000 == 0) {
LOG.info("Fetching {} in {}", iterationCount,
TreeStore.this.toString());
}
current = key;
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java
index f2eb586b62..7ff5e8e1e1 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/TreeStoreUtils.java
@@ -67,7 +67,7 @@ public class TreeStoreUtils {
String fileName = args[1];
try (BufferedReader lineReader = new BufferedReader(
new FileReader(fileName, StandardCharsets.UTF_8))) {
- int count = 0;
+ long count = 0;
long start = System.nanoTime();
while (true) {
String line = lineReader.readLine();
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java
index 5894f58926..dad247681e 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/FileStore.java
@@ -41,8 +41,6 @@ import org.slf4j.LoggerFactory;
*/
public class FileStore implements Store {
- private static final Logger LOG = LoggerFactory.getLogger(FileStore.class);
-
private final Properties config;
private final String directory;
private Compression compression = Compression.NO;
@@ -58,7 +56,6 @@ public class FileStore implements Store {
this.directory = config.getProperty("dir");
this.maxFileSizeBytes = Long.parseLong(config.getProperty(
Store.MAX_FILE_SIZE_BYTES, "" +
Store.DEFAULT_MAX_FILE_SIZE_BYTES));
- LOG.info("Max file size {} bytes", maxFileSizeBytes);
new File(directory).mkdirs();
}
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java
index 55d3dbfd5c..eb724c631a 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/tree/store/TreeSession.java
@@ -30,16 +30,12 @@ import java.util.Random;
import
org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.ConcurrentLRUCache;
import
org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.Position;
import
org.apache.jackrabbit.oak.index.indexer.document.tree.store.utils.SortedStream;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
/**
* A session that allows reading and writing keys and values in a tree store.
*/
public class TreeSession {
- private static final Logger LOG =
LoggerFactory.getLogger(TreeSession.class);
-
public static final String CACHE_SIZE_MB = "cacheSizeMB";
private static final int DEFAULT_CACHE_SIZE_MB = 256;
private static final int DEFAULT_MAX_ROOTS = 10;
@@ -72,7 +68,6 @@ public class TreeSession {
long cacheSizeMB = Long.parseLong(store.getConfig().getProperty(
CACHE_SIZE_MB, "" + DEFAULT_CACHE_SIZE_MB));
long cacheSizeBytes = cacheSizeMB * 1024 * 1024;
- LOG.info("Cache size {} bytes", cacheSizeBytes);
this.cache = new ConcurrentLRUCache<>(cacheSizeBytes) {
private static final long serialVersionUID = 1L;
diff --git
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
index 95bd69017a..2964c840f0 100644
---
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
+++
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/DocumentNodeStore.java
@@ -3429,9 +3429,8 @@ public final class DocumentNodeStore
// see OAK-6016 and OAK-6011
if (e instanceof IllegalStateException &&
"Root document does not have a lastRev entry for local
clusterId 0".equals(e.getMessage())) {
- LOG.warn("diffJournalChildren failed with " +
- e.getClass().getSimpleName() +
- ", falling back to classic diff : " +
e.getMessage());
+ LOG.debug("diffJournalChildren failed with {}" +
+ ", falling back to classic diff: {}",
e.getClass().getSimpleName(), e.getMessage());
} else {
LOG.warn("diffJournalChildren failed with " +
e.getClass().getSimpleName() +
