This is an automated email from the ASF dual-hosted git repository. daim pushed a commit to branch DetailedGC/OAK-10199 in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
commit a5ff165b2845806ab1d35d41e0991970badf9ef3 Author: Nuno Santos <[email protected]> AuthorDate: Mon Dec 18 15:57:21 2023 +0000 OAK-10571 - Names of metrics exported by indexing logic are inconsistent (#1241) * Rename metrics to follow a more consistent naming convention. * Add missing metrics. --- .../flatfile/pipelined/PipelinedMergeSortTask.java | 20 ++++++----- .../flatfile/pipelined/PipelinedMetrics.java | 42 +++++++++++++--------- .../pipelined/PipelinedMongoDownloadTask.java | 9 ++++- .../flatfile/pipelined/PipelinedSortBatchTask.java | 16 ++++++++- .../flatfile/pipelined/PipelinedStrategy.java | 7 ++-- .../flatfile/pipelined/PipelinedUtils.java | 7 ++++ .../pipelined/TransformStageStatistics.java | 28 ++++++++------- .../document/flatfile/pipelined/PipelinedIT.java | 39 +++++++++++--------- .../pipelined/PipelinedMergeSortTaskTest.java | 13 +++---- .../pipelined/PipelinedMongoDownloadTaskTest.java | 6 +++- .../pipelined/PipelinedSortBatchTaskTest.java | 5 +-- 11 files changed, 121 insertions(+), 71 deletions(-) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTask.java index bf4083e796..c8b1a7fa5c 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTask.java @@ -49,10 +49,6 @@ import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Function; import java.util.stream.Collectors; -import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS; -import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT; -import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME; -import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT; import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy.SENTINEL_SORTED_FILES_QUEUE; import static org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils.getSortedStoreFileName; @@ -223,16 +219,24 @@ public class PipelinedMergeSortTask implements Callable<PipelinedMergeSortTask.R String metrics = MetricsFormatter.newBuilder() .add("duration", FormattingUtils.formatToSeconds(w)) .add("durationSeconds", durationSeconds) + .add("intermediateFilesCount", intermediateFilesCount) + .add("eagerMergesRuns", eagerMergeRuns) .add("filesMerged", sortedFiles.size()) .add("ffsSizeBytes", ffsSizeBytes) .add("ffsSize", IOUtils.humanReadableByteCountBin(ffsSizeBytes)) .build(); LOG.info("[TASK:{}:END] Metrics: {}", THREAD_NAME.toUpperCase(Locale.ROOT), metrics); - MetricsUtils.setCounterOnce(statisticsProvider, OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT, intermediateFilesCount); - MetricsUtils.setCounterOnce(statisticsProvider, OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS, eagerMergeRuns); - MetricsUtils.setCounterOnce(statisticsProvider, OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT, sortedFiles.size()); - MetricsUtils.setCounterOnce(statisticsProvider, OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME, durationSeconds); + MetricsUtils.setCounterOnce(statisticsProvider, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS, durationSeconds); + MetricsUtils.setCounterOnce(statisticsProvider, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL, intermediateFilesCount); + MetricsUtils.setCounterOnce(statisticsProvider, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL, eagerMergeRuns); + MetricsUtils.setCounterOnce(statisticsProvider, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL, sortedFiles.size()); + MetricsUtils.setCounterOnce(statisticsProvider, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES, ffsSizeBytes); return new Result(flatFileStore, intermediateFilesCount, sortedFiles.size(), eagerMergeRuns); } else { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMetrics.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMetrics.java index f44f79af40..f03cd75082 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMetrics.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMetrics.java @@ -19,24 +19,32 @@ package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined; public final class PipelinedMetrics { + // Conventions for naming metrics: https://prometheus.io/docs/practices/naming/ public static final String METRIC_NAME_PREFIX = "oak_indexer_pipelined_"; - public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED = METRIC_NAME_PREFIX + "documentsDownloaded"; - public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED = METRIC_NAME_PREFIX + "documentsTraversed"; - public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT = METRIC_NAME_PREFIX + "documentsRejectedSplit"; - public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED = METRIC_NAME_PREFIX + "documentsAccepted"; - public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED = METRIC_NAME_PREFIX + "documentsRejected"; - public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE = METRIC_NAME_PREFIX + "documentsRejectedEmptyNodeState"; - public static final String OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED = METRIC_NAME_PREFIX + "entriesTraversed"; - public static final String OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED = METRIC_NAME_PREFIX + "entriesAccepted"; - public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED = METRIC_NAME_PREFIX + "entriesRejected"; - public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS = METRIC_NAME_PREFIX + "entriesRejectedHiddenPaths"; - public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED = METRIC_NAME_PREFIX + "entriesRejectedPathFiltered"; - public static final String OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_SIZE = METRIC_NAME_PREFIX + "extractedEntriesTotalSize"; - public static final String OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE = METRIC_NAME_PREFIX + "mongoDownloadEnqueueDelayPercentage"; - public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT = METRIC_NAME_PREFIX + "mergeSortIntermediateFilesCount"; - public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS = METRIC_NAME_PREFIX + "mergeSortEagerMergesRuns"; - public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT = METRIC_NAME_PREFIX + "mergeSortFinalMergeFilesCount"; - public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME = METRIC_NAME_PREFIX + "mergeSortFinalMergeTime"; + public static final String OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_DURATION_SECONDS = METRIC_NAME_PREFIX + "mongo_download_duration_seconds"; + public static final String OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE = METRIC_NAME_PREFIX + "mongo_download_enqueue_delay_percentage"; + public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED_TOTAL = METRIC_NAME_PREFIX + "documents_downloaded_total"; + public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED_TOTAL = METRIC_NAME_PREFIX + "documents_traversed_total"; + public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT_TOTAL = METRIC_NAME_PREFIX + "documents_rejected_split_total"; + public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_TOTAL = METRIC_NAME_PREFIX + "documents_accepted_total"; + public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_TOTAL = METRIC_NAME_PREFIX + "documents_rejected_total"; + public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_PERCENTAGE = METRIC_NAME_PREFIX + "documents_accepted_percentage"; + public static final String OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE_TOTAL = METRIC_NAME_PREFIX + "documents_rejected_empty_node_state_total"; + public static final String OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED_TOTAL = METRIC_NAME_PREFIX + "entries_traversed_total"; + public static final String OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_TOTAL = METRIC_NAME_PREFIX + "entries_accepted_total"; + public static final String OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_PERCENTAGE = METRIC_NAME_PREFIX + "entries_accepted_percentage"; + public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_TOTAL = METRIC_NAME_PREFIX + "entries_rejected_total"; + public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS_TOTAL = METRIC_NAME_PREFIX + "entries_rejected_hidden_paths_total"; + public static final String OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED_TOTAL = METRIC_NAME_PREFIX + "entries_rejected_path_filtered_total"; + public static final String OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_BYTES = METRIC_NAME_PREFIX + "extracted_entries_total_bytes"; + public static final String OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_CREATE_SORT_ARRAY_PERCENTAGE = METRIC_NAME_PREFIX + "sort_batch_phase_create_sort_array_percentage"; + public static final String OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_SORT_ARRAY_PERCENTAGE = METRIC_NAME_PREFIX + "sort_batch_phase_sort_array_percentage"; + public static final String OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_WRITE_TO_DISK_PERCENTAGE = METRIC_NAME_PREFIX + "sort_batch_phase_write_to_disk_percentage"; + public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL = METRIC_NAME_PREFIX + "merge_sort_intermediate_files_total"; + public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL = METRIC_NAME_PREFIX + "merge_sort_eager_merges_runs_total"; + public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL = METRIC_NAME_PREFIX + "merge_sort_final_merge_files_total"; + public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES = METRIC_NAME_PREFIX + "merge_sort_flat_file_store_size_bytes"; + public static final String OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS = METRIC_NAME_PREFIX + "merge_sort_final_merge_duration_seconds"; private PipelinedMetrics() { } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java index caa8a1ee75..00c5c8f792 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTask.java @@ -206,7 +206,14 @@ public class PipelinedMongoDownloadTask implements Callable<PipelinedMongoDownlo .add("enqueueingDelayMillis", totalEnqueueWaitTimeMillis) .add("enqueueingDelayPercentage", enqueueingDelayPercentage) .build(); - + MetricsUtils.setCounterOnce(statisticsProvider, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_DURATION_SECONDS, + durationMillis / 1000 + ); + MetricsUtils.setCounterOnce(statisticsProvider, + PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED_TOTAL, + documentsRead + ); MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE, PipelinedUtils.toPercentage(totalEnqueueWaitTimeMillis, durationMillis) diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTask.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTask.java index c53c72a955..26f85e73a1 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTask.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTask.java @@ -23,6 +23,8 @@ import org.apache.jackrabbit.guava.common.base.Stopwatch; import org.apache.jackrabbit.oak.commons.Compression; import org.apache.jackrabbit.oak.index.indexer.document.indexstore.IndexStoreUtils; import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter; +import org.apache.jackrabbit.oak.plugins.index.MetricsUtils; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -68,6 +70,7 @@ class PipelinedSortBatchTask implements Callable<PipelinedSortBatchTask.Result> private final BlockingQueue<NodeStateEntryBatch> nonEmptyBuffersQueue; private final BlockingQueue<Path> sortedFilesQueue; private final Path sortWorkDir; + private final StatisticsProvider statisticsProvider; private final ArrayList<SortKey> sortBuffer = new ArrayList<>(32 * 1024); private long entriesProcessed = 0; private long batchesProcessed = 0; @@ -80,13 +83,15 @@ class PipelinedSortBatchTask implements Callable<PipelinedSortBatchTask.Result> Compression algorithm, BlockingQueue<NodeStateEntryBatch> emptyBuffersQueue, BlockingQueue<NodeStateEntryBatch> nonEmptyBuffersQueue, - BlockingQueue<Path> sortedFilesQueue) throws IOException { + BlockingQueue<Path> sortedFilesQueue, + StatisticsProvider statisticsProvider) throws IOException { this.pathComparator = (e1, e2) -> pathComparator.compare(e1.getPathElements(), e2.getPathElements()); this.algorithm = algorithm; this.emptyBuffersQueue = emptyBuffersQueue; this.nonEmptyBuffersQueue = nonEmptyBuffersQueue; this.sortedFilesQueue = sortedFilesQueue; this.sortWorkDir = createdSortWorkDir(storeDir); + this.statisticsProvider = statisticsProvider; } @Override @@ -118,6 +123,15 @@ class PipelinedSortBatchTask implements Callable<PipelinedSortBatchTask.Result> .add("totalTimeSeconds", totalTimeMillis / 1000) .build(); LOG.info("[TASK:{}:END] Metrics: {}", THREAD_NAME.toUpperCase(Locale.ROOT), metrics); + MetricsUtils.setCounterOnce(statisticsProvider, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_CREATE_SORT_ARRAY_PERCENTAGE, + PipelinedUtils.toPercentage(timeCreatingSortArrayMillis, totalTimeMillis)); + MetricsUtils.setCounterOnce(statisticsProvider, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_SORT_ARRAY_PERCENTAGE, + PipelinedUtils.toPercentage(timeSortingMillis, totalTimeMillis)); + MetricsUtils.setCounterOnce(statisticsProvider, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_WRITE_TO_DISK_PERCENTAGE, + PipelinedUtils.toPercentage(timeWritingMillis, totalTimeMillis)); return new Result(entriesProcessed); } sortAndSaveBatch(nseBuffer); diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedStrategy.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedStrategy.java index 14bf4e8bdb..74fcce1f6a 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedStrategy.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedStrategy.java @@ -33,7 +33,6 @@ import org.apache.jackrabbit.oak.plugins.document.RevisionVector; import org.apache.jackrabbit.oak.plugins.document.mongo.MongoDocumentStore; import org.apache.jackrabbit.oak.plugins.index.FormattingUtils; import org.apache.jackrabbit.oak.plugins.index.MetricsFormatter; -import org.apache.jackrabbit.oak.plugins.index.MetricsUtils; import org.apache.jackrabbit.oak.spi.blob.BlobStore; import org.apache.jackrabbit.oak.spi.filter.PathFilter; import org.apache.jackrabbit.oak.stats.StatisticsProvider; @@ -62,7 +61,6 @@ import java.util.concurrent.TimeUnit; import java.util.function.Predicate; import static org.apache.jackrabbit.oak.commons.IOUtils.humanReadableByteCountBin; -import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED; /** * Downloads the contents of the MongoDB repository dividing the tasks in a pipeline with the following stages: @@ -435,7 +433,8 @@ public class PipelinedStrategy extends IndexStoreSortStrategyBase { this.getAlgorithm(), emptyBatchesQueue, nonEmptyBatchesQueue, - sortedFilesQueue + sortedFilesQueue, + statisticsProvider )); PipelinedMergeSortTask mergeSortTask = new PipelinedMergeSortTask( @@ -463,7 +462,6 @@ public class PipelinedStrategy extends IndexStoreSortStrategyBase { mongoDocQueue.put(SENTINEL_MONGO_DOCUMENT); } mergeSortTask.stopEagerMerging(); - MetricsUtils.setCounterOnce(statisticsProvider, OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED, downloadResult.getDocumentsDownloaded()); } else if (result instanceof PipelinedTransformTask.Result) { PipelinedTransformTask.Result transformResult = (PipelinedTransformTask.Result) result; @@ -527,7 +525,6 @@ public class PipelinedStrategy extends IndexStoreSortStrategyBase { .add("durationSeconds", start.elapsed(TimeUnit.SECONDS)) .add("nodeStateEntriesExtracted", nodeStateEntriesExtracted) .build()); - printStatistics(mongoDocQueue, emptyBatchesQueue, nonEmptyBatchesQueue, sortedFilesQueue, transformStageStatistics, true); } catch (InterruptedException e) { throw new RuntimeException(e); } finally { diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedUtils.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedUtils.java index 85fad87ae0..50062d25b4 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedUtils.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedUtils.java @@ -55,4 +55,11 @@ public class PipelinedUtils { } } + public static int toPercentageAsInt(long numerator, long denominator) { + if (denominator == 0) { + return -1; + } else { + return (int) Math.round((100.0 * numerator) / denominator); + } + } } diff --git a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/TransformStageStatistics.java b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/TransformStageStatistics.java index 61cf67e02a..2b70a8cc4a 100644 --- a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/TransformStageStatistics.java +++ b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/TransformStageStatistics.java @@ -145,18 +145,22 @@ public class TransformStageStatistics { long entriesAcceptedTotalSizeSum = entriesAcceptedTotalSize.sum(); long entriesRejectedSum = entriesRejected.sum(); long entriesTraversed = entriesAcceptedSum + entriesRejectedSum; - - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED, mongoDocumentsTraversedSum); - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT, documentsRejectedSplitSum); - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE, documentsRejectedEmptyNodeStateSum); - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED, documentsAcceptedTotal); - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED, documentsRejectedTotal); - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED, entriesTraversed); - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED, entriesAcceptedSum); - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED, entriesRejectedSum); - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS, entriesRejectedHiddenPaths.sum()); - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED, entriesRejectedPathFiltered.sum()); - MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_SIZE, entriesAcceptedTotalSizeSum); + int documentsAcceptedPercentage = PipelinedUtils.toPercentageAsInt(documentsAcceptedTotal, mongoDocumentsTraversedSum); + int entriesAcceptedPercentage = PipelinedUtils.toPercentageAsInt(entriesAcceptedSum, entriesAcceptedSum + entriesRejectedSum); + + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED_TOTAL, mongoDocumentsTraversedSum); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT_TOTAL, documentsRejectedSplitSum); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE_TOTAL, documentsRejectedEmptyNodeStateSum); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_TOTAL, documentsAcceptedTotal); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_TOTAL, documentsRejectedTotal); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_PERCENTAGE, documentsAcceptedPercentage); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED_TOTAL, entriesTraversed); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_TOTAL, entriesAcceptedSum); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_PERCENTAGE, entriesAcceptedPercentage); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_TOTAL, entriesRejectedSum); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS_TOTAL, entriesRejectedHiddenPaths.sum()); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED_TOTAL, entriesRejectedPathFiltered.sum()); + MetricsUtils.setCounterOnce(statisticsProvider, PipelinedMetrics.OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_BYTES, entriesAcceptedTotalSizeSum); } public String formatStats() { diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java index 1da9b7f86e..122342f5d2 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedIT.java @@ -249,23 +249,30 @@ public class PipelinedIT { Set<String> metricsNames = statsProvider.getRegistry().getCounters().keySet(); assertEquals(Set.of( - PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED, - PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED, - PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT, - PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED, - PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED, - PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE, - PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED, - PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED, - PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED, - PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS, - PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED, - PipelinedMetrics.OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_SIZE, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_DURATION_SECONDS, PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE, - PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT, - PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS, - PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT, - PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME + PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_TRAVERSED_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_SPLIT_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_ACCEPTED_PERCENTAGE, + PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_REJECTED_EMPTY_NODE_STATE_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_TRAVERSED_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_ACCEPTED_PERCENTAGE, + PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_HIDDEN_PATHS_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_ENTRIES_REJECTED_PATH_FILTERED_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_EXTRACTED_ENTRIES_TOTAL_BYTES, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_CREATE_SORT_ARRAY_PERCENTAGE, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_SORT_ARRAY_PERCENTAGE, + PipelinedMetrics.OAK_INDEXER_PIPELINED_SORT_BATCH_PHASE_WRITE_TO_DISK_PERCENTAGE, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS ), metricsNames); String pipelinedMetrics = statsProvider.getRegistry() diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTaskTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTaskTest.java index 1dbc10632f..fc25bd7a8e 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTaskTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMergeSortTaskTest.java @@ -39,10 +39,6 @@ import java.util.concurrent.ScheduledExecutorService; import java.util.stream.Collectors; import java.util.stream.Stream; -import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS; -import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT; -import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME; -import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT; import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy.FLATFILESTORE_CHARSET; import static org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined.PipelinedStrategy.SENTINEL_SORTED_FILES_QUEUE; import static org.junit.Assert.assertArrayEquals; @@ -137,10 +133,11 @@ public class PipelinedMergeSortTaskTest extends PipelinedMergeSortTaskTestBase { assertTrue(Files.exists(result.getFlatFileStoreFile())); Set<String> metricNames = metricStatisticsProvider.getRegistry().getCounters().keySet(); assertEquals(metricNames, Set.of( - OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_TIME, - OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_COUNT, - OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS, - OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_DURATION_SECONDS, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_INTERMEDIATE_FILES_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_EAGER_MERGES_RUNS_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FINAL_MERGE_FILES_COUNT_TOTAL, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MERGE_SORT_FLAT_FILE_STORE_SIZE_BYTES )); return result; } diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java index fe1ca75945..ae8f2ec5db 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedMongoDownloadTaskTest.java @@ -121,7 +121,11 @@ public class PipelinedMongoDownloadTaskTest { assertEquals(documents, actualDocuments); Set<String> metricNames = metricStatisticsProvider.getRegistry().getCounters().keySet(); - assertEquals(metricNames, Set.of(PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE)); + assertEquals(metricNames, Set.of( + PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_ENQUEUE_DELAY_PERCENTAGE, + PipelinedMetrics.OAK_INDEXER_PIPELINED_MONGO_DOWNLOAD_DURATION_SECONDS, + PipelinedMetrics.OAK_INDEXER_PIPELINED_DOCUMENTS_DOWNLOADED_TOTAL + )); } } finally { executor.shutdown(); diff --git a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTaskTest.java b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTaskTest.java index 2e88536fd0..735649b8b3 100644 --- a/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTaskTest.java +++ b/oak-run-commons/src/test/java/org/apache/jackrabbit/oak/index/indexer/document/flatfile/pipelined/PipelinedSortBatchTaskTest.java @@ -19,6 +19,7 @@ package org.apache.jackrabbit.oak.index.indexer.document.flatfile.pipelined; import org.apache.jackrabbit.oak.commons.Compression; +import org.apache.jackrabbit.oak.stats.StatisticsProvider; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; @@ -169,8 +170,8 @@ public class PipelinedSortBatchTaskTest { nonEmptyBatchesQueue.put(PipelinedStrategy.SENTINEL_NSE_BUFFER); PipelinedSortBatchTask sortTask = new PipelinedSortBatchTask( - sortRoot, pathComparator, algorithm, emptyBatchesQueue, nonEmptyBatchesQueue, sortedFilesQueue - ); + sortRoot, pathComparator, algorithm, emptyBatchesQueue, nonEmptyBatchesQueue, sortedFilesQueue, + StatisticsProvider.NOOP); PipelinedSortBatchTask.Result taskResult = sortTask.call(); LOG.info("Result: {}", taskResult.getTotalEntries()); LOG.info("Empty batches: {}", emptyBatchesQueue.size());
