This is an automated email from the ASF dual-hosted git repository.
ngupta pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new b213b4696c OAK-9755 | Lucene metrics using labels (#565)
b213b4696c is described below
commit b213b4696cb4deb241915f8c7aeb08061bc5acc4
Author: nit0906 <[email protected]>
AuthorDate: Thu May 12 19:59:11 2022 +0530
OAK-9755 | Lucene metrics using labels (#565)
* OAK-9755 | Introducing labels for Lucene jmx metrics
* Exporting org.apache.jackrabbit.oak.plugins.metric.util package from
oak-core
---
oak-core/pom.xml | 1 +
.../oak/plugins/metric/util/StatsProviderUtil.java | 67 ++++++++++++++++++++++
.../oak/plugins/metric/util/package-info.java | 24 ++++++++
.../jackrabbit/oak/query/ast/SelectorImpl.java | 8 ++-
.../lucene/LuceneIndexStatsUpdateCallback.java | 19 ++++--
.../oak/plugins/index/lucene/hybrid/NRTIndex.java | 17 ++++--
.../index/elastic/ElasticMetricHandler.java | 14 ++---
7 files changed, 129 insertions(+), 21 deletions(-)
diff --git a/oak-core/pom.xml b/oak-core/pom.xml
index a75bd7e9f1..706a27fbf1 100644
--- a/oak-core/pom.xml
+++ b/oak-core/pom.xml
@@ -71,6 +71,7 @@
org.apache.jackrabbit.oak.plugins.observation.filter,
org.apache.jackrabbit.oak.plugins.tree.factories,
org.apache.jackrabbit.oak.plugins.version,
+ org.apache.jackrabbit.oak.plugins.metric.util
</Export-Package>
<Jaas-ModuleClass>
org.apache.jackrabbit.oak.spi.security.authentication.GuestLoginModule,
diff --git
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/metric/util/StatsProviderUtil.java
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/metric/util/StatsProviderUtil.java
new file mode 100644
index 0000000000..5e6c243354
--- /dev/null
+++
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/metric/util/StatsProviderUtil.java
@@ -0,0 +1,67 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.metric.util;
+
+import org.apache.jackrabbit.oak.stats.CounterStats;
+import org.apache.jackrabbit.oak.stats.HistogramStats;
+import org.apache.jackrabbit.oak.stats.MeterStats;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
+import org.apache.jackrabbit.oak.stats.StatsOptions;
+import org.apache.jackrabbit.oak.stats.TimerStats;
+import org.jetbrains.annotations.NotNull;
+
+import java.util.Map;
+import java.util.function.BiFunction;
+
+/**
+ * Util class to generate a name for Stats implementations that can be used
for creating labels in prometheus.
+ * Usage - StatsProviderUtil(<StatisticsProvider
Object>).getHistoStats().apply(metricName, labels)
+ * where metricName is a String to denote the metric name and labels is map of
label values.
+ * Resultant metric will be created with a name as follows -
+ * metricName;labelName1=labelValue1;labelName2=labelValue2
+ * This can then be translated by a consuming alerting system like prometheus
into metric name and labels separately.
+ */
+public class StatsProviderUtil {
+
+ private final StatisticsProvider statisticsProvider;
+ private final BiFunction<String, Map<String, String>, String> METRIC =
(name, labels) -> labels.entrySet().stream().reduce(name,
+ (n, e) -> n + ";" + e.getKey() + "=" + e.getValue(),
+ (n1, n2) -> n1 + n2);
+
+ public StatsProviderUtil(@NotNull StatisticsProvider statisticsProvider) {
+ this.statisticsProvider = statisticsProvider;
+ }
+
+ public BiFunction<String, Map<String, String>, HistogramStats>
getHistoStats() {
+ return (name, labels) ->
statisticsProvider.getHistogram(METRIC.apply(name, labels),
StatsOptions.METRICS_ONLY);
+ }
+
+ public BiFunction<String, Map<String, String>, CounterStats>
getCounterStats() {
+ return (name, labels) ->
statisticsProvider.getCounterStats(METRIC.apply(name, labels),
StatsOptions.METRICS_ONLY);
+ }
+
+ public BiFunction<String, Map<String, String>, TimerStats> getTimerStats()
{
+ return (name, labels) ->
statisticsProvider.getTimer(METRIC.apply(name, labels),
StatsOptions.METRICS_ONLY);
+ }
+
+ public BiFunction<String, Map<String, String>, MeterStats> getMeterStats()
{
+ return (name, labels) ->
statisticsProvider.getMeter(METRIC.apply(name, labels),
StatsOptions.METRICS_ONLY);
+ }
+
+}
diff --git
a/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/metric/util/package-info.java
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/metric/util/package-info.java
new file mode 100644
index 0000000000..f06587bfd4
--- /dev/null
+++
b/oak-core/src/main/java/org/apache/jackrabbit/oak/plugins/metric/util/package-info.java
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Oak plugins metric util. This package contains StatsProvideUtil.
+ */
+@Version("0.0.1")
+package org.apache.jackrabbit.oak.plugins.metric.util;
+
+import org.osgi.annotation.versioning.Version;
\ No newline at end of file
diff --git
a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java
b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java
index 5607fc0d05..bf59d9db56 100644
---
a/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java
+++
b/oak-core/src/main/java/org/apache/jackrabbit/oak/query/ast/SelectorImpl.java
@@ -23,6 +23,7 @@ import static com.google.common.collect.Lists.newArrayList;
import static org.apache.jackrabbit.JcrConstants.NT_BASE;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
@@ -36,6 +37,7 @@ import org.apache.jackrabbit.oak.commons.LazyValue;
import org.apache.jackrabbit.oak.commons.PathUtils;
import org.apache.jackrabbit.oak.core.ImmutableRoot;
import org.apache.jackrabbit.oak.plugins.memory.PropertyBuilder;
+import org.apache.jackrabbit.oak.plugins.metric.util.StatsProviderUtil;
import org.apache.jackrabbit.oak.plugins.tree.TreeUtil;
import org.apache.jackrabbit.oak.query.ExecutionContext;
import org.apache.jackrabbit.oak.query.QueryEngineSettings;
@@ -392,8 +394,10 @@ public class SelectorImpl extends SourceImpl {
TimerStats t = timerDuration;
if (t == null) {
// reuse the timer (in the normal case)
- t = timerDuration = query.getSettings().getStatisticsProvider().
- getTimer("QUERY_DURATION_" + planIndexName,
StatsOptions.METRICS_ONLY);
+ // QUERY_DURATION;index=<planIndexName> will be translated as
metric name = QUERY_DURATION
+ // and index=<planIndexName> as a label by a downstream consumer
like prometheus.
+ StatsProviderUtil statsProviderUtil = new
StatsProviderUtil(query.getSettings().getStatisticsProvider());
+ t = timerDuration =
statsProviderUtil.getTimerStats().apply("QUERY_DURATION",
Collections.singletonMap("index", planIndexName));
}
t.update(timeNanos, TimeUnit.NANOSECONDS);
}
diff --git
a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexStatsUpdateCallback.java
b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexStatsUpdateCallback.java
index e9e3eb3597..e41edec3b5 100644
---
a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexStatsUpdateCallback.java
+++
b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/LuceneIndexStatsUpdateCallback.java
@@ -22,23 +22,26 @@ import org.apache.jackrabbit.oak.api.PropertyState;
import org.apache.jackrabbit.oak.plugins.index.IndexingContext;
import org.apache.jackrabbit.oak.plugins.index.search.PropertyDefinition;
import org.apache.jackrabbit.oak.plugins.index.search.PropertyUpdateCallback;
+import org.apache.jackrabbit.oak.plugins.metric.util.StatsProviderUtil;
import org.apache.jackrabbit.oak.stats.HistogramStats;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
-import org.apache.jackrabbit.oak.stats.StatsOptions;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
+import java.util.Collections;
+import java.util.Map;
+import java.util.function.BiFunction;
/**
* {@link PropertyUpdateCallback} that records statistics about a certain
index size (on disk) and number of documents.
*/
public class LuceneIndexStatsUpdateCallback implements PropertyUpdateCallback {
- private static final String NO_DOCS = "_NO_DOCS";
- private static final String INDEX_SIZE = "_INDEX_SIZE";
+ private static final String NO_DOCS = "NO_DOCS";
+ private static final String INDEX_SIZE = "INDEX_SIZE";
private final Logger log = LoggerFactory.getLogger(getClass());
@@ -48,6 +51,9 @@ public class LuceneIndexStatsUpdateCallback implements
PropertyUpdateCallback {
private final AsyncIndexesSizeStatsUpdate asyncIndexesSizeStatsUpdate;
private final IndexingContext indexingContext;
+ private final BiFunction<String, Map<String, String>, HistogramStats>
histogram;
+ private final StatsProviderUtil statsProviderUtil;
+
LuceneIndexStatsUpdateCallback(String indexPath, @NotNull LuceneIndexMBean
luceneIndexMBean,
@NotNull StatisticsProvider
statisticsProvider,
AsyncIndexesSizeStatsUpdate
asyncIndexesSizeStatsUpdate,
@@ -57,6 +63,8 @@ public class LuceneIndexStatsUpdateCallback implements
PropertyUpdateCallback {
this.statisticsProvider = statisticsProvider;
this.asyncIndexesSizeStatsUpdate = asyncIndexesSizeStatsUpdate;
this.indexingContext = indexingContext;
+ statsProviderUtil = new StatsProviderUtil(this.statisticsProvider);
+ this.histogram = statsProviderUtil.getHistoStats();
}
@Override
@@ -68,13 +76,14 @@ public class LuceneIndexStatsUpdateCallback implements
PropertyUpdateCallback {
public void done() {
if (shouldUpdateStats()) {
try {
+ Map<String, String> labels = Collections.singletonMap("index",
indexPath);
long startTime = System.currentTimeMillis();
int docCount =
Integer.parseInt(luceneIndexMBean.getDocCount(indexPath));
- HistogramStats docCountHistogram =
statisticsProvider.getHistogram(indexPath + NO_DOCS, StatsOptions.METRICS_ONLY);
+ HistogramStats docCountHistogram = histogram.apply(NO_DOCS,
labels);
docCountHistogram.update(docCount);
log.trace("{} stats updated, docCount {}, timeToUpdate {}",
indexPath, docCount, System.currentTimeMillis() - startTime);
long indexSize =
Long.parseLong(luceneIndexMBean.getSize(indexPath));
- HistogramStats indexSizeHistogram =
statisticsProvider.getHistogram(indexPath + INDEX_SIZE,
StatsOptions.METRICS_ONLY);
+ HistogramStats indexSizeHistogram =
histogram.apply(INDEX_SIZE, labels);
indexSizeHistogram.update(indexSize);
long endTime = System.currentTimeMillis();
asyncIndexesSizeStatsUpdate.setLastStatsUpdateTime(indexPath,
endTime);
diff --git
a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
index 4e818de641..c7e7547fe3 100644
---
a/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
+++
b/oak-lucene/src/main/java/org/apache/jackrabbit/oak/plugins/index/lucene/hybrid/NRTIndex.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.Collections;
import java.util.LinkedList;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import com.google.common.collect.ImmutableList;
@@ -37,10 +38,10 @@ import
org.apache.jackrabbit.oak.plugins.index.lucene.writer.IndexWriterUtils;
import org.apache.jackrabbit.oak.plugins.index.lucene.writer.LuceneIndexWriter;
import
org.apache.jackrabbit.oak.plugins.index.search.update.IndexUpdateListener;
import
org.apache.jackrabbit.oak.plugins.index.search.update.ReaderRefreshPolicy;
+import org.apache.jackrabbit.oak.plugins.metric.util.StatsProviderUtil;
import org.apache.jackrabbit.oak.stats.HistogramStats;
import org.apache.jackrabbit.oak.stats.MeterStats;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
-import org.apache.jackrabbit.oak.stats.StatsOptions;
import org.apache.jackrabbit.oak.stats.TimerStats;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexReader;
@@ -94,6 +95,8 @@ public class NRTIndex implements Closeable {
private List<LuceneIndexReader> readers;
private final List<IndexReader> openedReaders;
private final boolean assertAllReadersClosed;
+ private final StatsProviderUtil statsProviderUtil;
+ private final Map<String, String> labels;
public NRTIndex(LuceneIndexDefinition definition, IndexCopier indexCopier,
@@ -108,10 +111,12 @@ public class NRTIndex implements Closeable {
this.directoryFactory = directoryFactory;
this.assertAllReadersClosed = assertAllReadersClosed;
this.openedReaders = assertAllReadersClosed ? new LinkedList<>() :
Collections.emptyList();
+ statsProviderUtil = new StatsProviderUtil(statisticsProvider);
+ labels = Collections.singletonMap("index", definition.getIndexPath());
- this.refreshTimer =
statisticsProvider.getTimer(metricName("REFRESH_TIME"),
StatsOptions.METRICS_ONLY);
- this.sizeHisto = statisticsProvider.getHistogram(metricName("SIZE"),
StatsOptions.METRICS_ONLY);
- this.openTime = statisticsProvider.getTimer(metricName("OPEN_TIME"),
StatsOptions.METRICS_ONLY).time();
+ this.refreshTimer =
statsProviderUtil.getTimerStats().apply(metricName("REFRESH_TIME"), labels);
+ this.sizeHisto =
statsProviderUtil.getHistoStats().apply(metricName("SIZE"), labels);
+ this.openTime =
statsProviderUtil.getTimerStats().apply(metricName("OPEN_TIME"), labels).time();
}
/**
@@ -394,7 +399,7 @@ public class NRTIndex implements Closeable {
public NRTIndexWriter(IndexWriter indexWriter) {
this.indexWriter = indexWriter;
- this.updateMeter =
statisticsProvider.getMeter(metricName("UPDATES"), StatsOptions.METRICS_ONLY);
+ this.updateMeter =
statsProviderUtil.getMeterStats().apply(metricName("UPDATES"), labels);
}
@Override
@@ -419,6 +424,6 @@ public class NRTIndex implements Closeable {
}
private String metricName(String suffix){
- return String.format("%s_NRT_%s", definition.getIndexPath(), suffix);
+ return String.format("NRT_%s", suffix);
}
}
diff --git
a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticMetricHandler.java
b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticMetricHandler.java
index 2ec5c78308..d7f03fe53e 100644
---
a/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticMetricHandler.java
+++
b/oak-search-elastic/src/main/java/org/apache/jackrabbit/oak/plugins/index/elastic/ElasticMetricHandler.java
@@ -18,10 +18,10 @@
*/
package org.apache.jackrabbit.oak.plugins.index.elastic;
+import org.apache.jackrabbit.oak.plugins.metric.util.StatsProviderUtil;
import org.apache.jackrabbit.oak.stats.HistogramStats;
import org.apache.jackrabbit.oak.stats.MeterStats;
import org.apache.jackrabbit.oak.stats.StatisticsProvider;
-import org.apache.jackrabbit.oak.stats.StatsOptions;
import org.apache.jackrabbit.oak.stats.TimerStats;
import java.util.Collections;
@@ -50,15 +50,13 @@ public class ElasticMetricHandler {
private final BiFunction<String, Map<String, String>, MeterStats> meter;
private final BiFunction<String, Map<String, String>, HistogramStats>
histogram;
private final BiFunction<String, Map<String, String>, TimerStats> timer;
+ private final StatsProviderUtil statsProviderUtil;
public ElasticMetricHandler(StatisticsProvider sp) {
- BiFunction<String, Map<String, String>, String> metricName = (name,
labels) -> labels.entrySet().stream().reduce(name,
- (n, e) -> n + ";" + e.getKey() + "=" + e.getValue(),
- (n1, n2) -> n1 + n2);
-
- meter = (name, labels) -> sp.getMeter(metricName.apply(name, labels),
StatsOptions.METRICS_ONLY);
- histogram = (name, labels) -> sp.getHistogram(metricName.apply(name,
labels), StatsOptions.METRICS_ONLY);
- timer = (name, labels) -> sp.getTimer(metricName.apply(name, labels),
StatsOptions.METRICS_ONLY);
+ statsProviderUtil = new StatsProviderUtil(sp);
+ meter = statsProviderUtil.getMeterStats();
+ histogram = statsProviderUtil.getHistoStats();
+ timer = statsProviderUtil.getTimerStats();
}
/**
