This is an automated email from the ASF dual-hosted git repository.
joscorbe pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/jackrabbit-oak.git
The following commit(s) were added to refs/heads/trunk by this push:
new 4b7c14a6dd OAK-11609 : add support for metrics export to Prometheus
via pushgateway for VersionGarbageCollector (#2216)
4b7c14a6dd is described below
commit 4b7c14a6dd4d36d546306503c397d0a1ee627f9e
Author: horia_poradici <[email protected]>
AuthorDate: Fri Apr 25 18:42:54 2025 +0300
OAK-11609 : add support for metrics export to Prometheus via pushgateway
for VersionGarbageCollector (#2216)
---
.../oak/run/cli/NodeStoreFixtureProvider.java | 20 +++--
.../oak/run/FullGCMetricsExporterFixture.java | 28 +++++++
.../run/FullGCMetricsExporterFixtureProvider.java | 94 ++++++++++++++++++++++
.../jackrabbit/oak/run/RevisionsCommand.java | 38 +++++++--
.../plugins/document/FullGCMetricsExporter.java | 35 ++++++++
.../plugins/document/FullGCStatsCollectorImpl.java | 10 ++-
.../plugins/document/VersionGarbageCollector.java | 17 +++-
7 files changed, 226 insertions(+), 16 deletions(-)
diff --git
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/NodeStoreFixtureProvider.java
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/NodeStoreFixtureProvider.java
index e96a75c5ab..710ccc9a59 100644
---
a/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/NodeStoreFixtureProvider.java
+++
b/oak-run-commons/src/main/java/org/apache/jackrabbit/oak/run/cli/NodeStoreFixtureProvider.java
@@ -100,15 +100,19 @@ public class NodeStoreFixtureProvider {
return !manifest.exists();
}
+ public static StatisticsProvider createStatsProvider(Whiteboard wb, Closer
closer) {
+ ScheduledExecutorService executorService =
+ MoreExecutors.getExitingScheduledExecutorService(new
ScheduledThreadPoolExecutor(1));
+ MetricStatisticsProvider statsProvider = new
MetricStatisticsProvider(getPlatformMBeanServer(), executorService);
+ closer.register(statsProvider);
+ closer.register(() -> reportMetrics(statsProvider));
+ wb.register(MetricRegistry.class, statsProvider.getRegistry(),
emptyMap());
+ return statsProvider;
+ }
+
private static StatisticsProvider createStatsProvider(Options options,
Whiteboard wb, Closer closer) {
if (options.getCommonOpts().isMetricsEnabled()) {
- ScheduledExecutorService executorService =
- MoreExecutors.getExitingScheduledExecutorService(new
ScheduledThreadPoolExecutor(1));
- MetricStatisticsProvider statsProvider = new
MetricStatisticsProvider(getPlatformMBeanServer(), executorService);
- closer.register(statsProvider);
- closer.register(() -> reportMetrics(statsProvider));
- wb.register(MetricRegistry.class, statsProvider.getRegistry(),
emptyMap());
- return statsProvider;
+ return createStatsProvider(wb, closer);
}
return StatisticsProvider.NOOP;
}
@@ -163,7 +167,7 @@ public class NodeStoreFixtureProvider {
}
}
- private static class ClosingWhiteboard implements Whiteboard {
+ public static class ClosingWhiteboard implements Whiteboard {
private final Whiteboard delegate;
private final Closer closer;
diff --git
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FullGCMetricsExporterFixture.java
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FullGCMetricsExporterFixture.java
new file mode 100644
index 0000000000..ac0396e178
--- /dev/null
+++
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FullGCMetricsExporterFixture.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import org.apache.jackrabbit.oak.plugins.document.FullGCMetricsExporter;
+
+/**
+ * Fixture encapsulating FullGC metrics exporter instance of T
+ * @param <T>
+ */
+public interface FullGCMetricsExporterFixture<T> extends
FullGCMetricsExporter<T>, MetricsExporterFixture<T> {
+}
diff --git
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FullGCMetricsExporterFixtureProvider.java
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FullGCMetricsExporterFixtureProvider.java
new file mode 100644
index 0000000000..c42f3ed9f7
--- /dev/null
+++
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/FullGCMetricsExporterFixtureProvider.java
@@ -0,0 +1,94 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.run;
+
+import com.codahale.metrics.MetricRegistry;
+import io.prometheus.client.CollectorRegistry;
+import io.prometheus.client.dropwizard.DropwizardExports;
+import io.prometheus.client.exporter.PushGateway;
+import
org.apache.jackrabbit.oak.run.MetricsExporterFixtureProvider.ExportMetricsArgs;
+import
org.apache.jackrabbit.oak.run.MetricsExporterFixtureProvider.ExporterType;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
+import org.jetbrains.annotations.Nullable;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+import static java.util.Collections.emptyMap;
+import static
org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.getService;
+
+/**
+ * Initializes metrics exported fixture for fullGC. For use in oak-run
RevisionsCommand.
+ */
+class FullGCMetricsExporterFixtureProvider {
+
+ private static final Logger log =
LoggerFactory.getLogger(NodeStoreFixtureProvider.class);
+
+ @Nullable
+ static FullGCMetricsExporterFixture<PushGateway>
create(RevisionsCommand.RevisionsOptions options, Whiteboard wb) {
+ if (options.exportMetrics()) {
+ CollectorRegistry collectorRegistry = new CollectorRegistry();
+ wb.register(CollectorRegistry.class, collectorRegistry,
emptyMap());
+
+ MetricRegistry metricRegistry = getService(wb,
MetricRegistry.class);
+
+ ExportMetricsArgs metricsArgs = new
ExportMetricsArgs(options.exportMetricsArgs());
+ if (metricsArgs.getExporterType() == ExporterType.pushgateway) {
+ PushGateway pg = new PushGateway(metricsArgs.getPushUri());
+ new
DropwizardExports(metricRegistry).register(collectorRegistry);
+
+ wb.register(PushGateway.class, pg, emptyMap());
+ return new FullGCMetricsExporterFixture<>() {
+ public ExporterType getExporterType() {
+ return ExporterType.pushgateway;
+ }
+
+ public PushGateway getMetricsExporter() {
+ return pg;
+ }
+
+ @Override
+ public void close() {
+ pushMetrics(collectorRegistry, pg, metricsArgs);
+ }
+
+ /**
+ * Push the metricsMap that is passed from
VersionGarbageCollector to pushgateway.
+ */
+ @Override
+ public void onIterationComplete() {
+ pushMetrics(collectorRegistry, pg, metricsArgs);
+ }
+ };
+ }
+ }
+ return null;
+ }
+
+ private static void pushMetrics(CollectorRegistry collectorRegistry,
PushGateway pg, ExportMetricsArgs metricsArgs) {
+ try {
+ log.info("Pushing metrics to pushgateway: ",
metricsArgs.getPushMap());
+ pg.pushAdd(collectorRegistry, PushGateway.class.getName(),
metricsArgs.getPushMap());
+ } catch (IOException e) {
+ log.error("Error pushing metrics to pushgateway", e);
+ }
+ }
+}
diff --git
a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
index 0d9c611d1e..d992546861 100644
--- a/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
+++ b/oak-run/src/main/java/org/apache/jackrabbit/oak/run/RevisionsCommand.java
@@ -18,6 +18,7 @@ package org.apache.jackrabbit.oak.run;
import java.io.IOException;
import java.util.Arrays;
+import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Optional;
@@ -42,6 +43,7 @@ import
org.apache.jackrabbit.oak.plugins.document.DocumentNodeStore;
import org.apache.jackrabbit.oak.plugins.document.DocumentNodeStoreBuilder;
import org.apache.jackrabbit.oak.plugins.document.DocumentStore;
import org.apache.jackrabbit.oak.plugins.document.FormatVersion;
+import org.apache.jackrabbit.oak.plugins.document.FullGCMetricsExporter;
import org.apache.jackrabbit.oak.plugins.document.MissingLastRevSeeker;
import org.apache.jackrabbit.oak.plugins.document.NodeDocument;
import org.apache.jackrabbit.oak.plugins.document.RevisionContextWrapper;
@@ -50,12 +52,16 @@ import
org.apache.jackrabbit.oak.plugins.document.VersionGCSupport;
import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCInfo;
import
org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector.VersionGCStats;
import org.apache.jackrabbit.oak.plugins.document.util.MongoConnection;
+import org.apache.jackrabbit.oak.run.cli.NodeStoreFixtureProvider;
import org.apache.jackrabbit.oak.run.commons.Command;
import org.apache.jackrabbit.oak.plugins.document.VersionGCOptions;
import org.apache.jackrabbit.oak.plugins.document.VersionGarbageCollector;
import org.apache.jackrabbit.oak.spi.blob.MemoryBlobStore;
import org.apache.jackrabbit.oak.spi.gc.LoggingGCMonitor;
+import org.apache.jackrabbit.oak.spi.whiteboard.DefaultWhiteboard;
+import org.apache.jackrabbit.oak.spi.whiteboard.Whiteboard;
import org.apache.jackrabbit.oak.stats.DefaultStatisticsProvider;
+import org.apache.jackrabbit.oak.stats.StatisticsProvider;
import org.bson.Document;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -75,6 +81,7 @@ import static
org.apache.jackrabbit.oak.plugins.document.util.Utils.timestampToS
import static org.apache.jackrabbit.oak.run.Utils.asCloseable;
import static org.apache.jackrabbit.oak.run.Utils.createDocumentMKBuilder;
import static org.apache.jackrabbit.oak.run.Utils.getMongoConnection;
+import static
org.apache.jackrabbit.oak.spi.whiteboard.WhiteboardUtils.getService;
/**
* Gives information about current node revisions state.
@@ -118,7 +125,7 @@ public class RevisionsCommand implements Command {
this.exitWhenDone = exitWhenDone;
}
- private static class RevisionsOptions extends Utils.NodeStoreOptions {
+ static class RevisionsOptions extends Utils.NodeStoreOptions {
static final String CMD_INFO = "info";
static final String CMD_COLLECT = "collect";
@@ -148,6 +155,7 @@ public class RevisionsCommand implements Command {
final OptionSpec<Boolean> embeddedVerification;
final OptionSpec<Integer> fullGcMode;
final OptionSpec<Boolean> fullGCAuditLoggingEnabled;
+ final OptionSpec<String> exportMetrics;
RevisionsOptions(String usage) {
super(usage);
@@ -211,6 +219,8 @@ public class RevisionsCommand implements Command {
.withOptionalArg().ofType(Long.class).defaultsTo(TimeUnit.DAYS.toSeconds(1));
fullGCAuditLoggingEnabled =
parser.accepts("fullGCAuditLoggingEnabled", "Enable audit logging for Full GC")
.withOptionalArg().ofType(Boolean.class).defaultsTo(FALSE);
+ exportMetrics = parser.accepts("exportMetrics",
+ "type, URI to export the metrics and optional metadata all
delimeted by semi-colon(;)").withRequiredArg();
}
public RevisionsOptions parse(String[] args) {
@@ -313,6 +323,14 @@ public class RevisionsCommand implements Command {
Boolean isFullGCAuditLoggingEnabled() {
return options.has(fullGCAuditLoggingEnabled);
}
+
+ boolean exportMetrics() {
+ return options.has(exportMetrics);
+ }
+
+ String exportMetricsArgs() {
+ return exportMetrics.value(options);
+ }
}
@Override
@@ -382,7 +400,6 @@ public class RevisionsCommand implements Command {
builder.setFullGCBatchSize(options.getFullGcBatchSize());
builder.setFullGCProgressSize(options.getFullGcProgressSize());
builder.setFullGcMaxAgeMillis(SECONDS.toMillis(options.getFullGcMaxAge()));
-
builder.setFullGCAuditLoggingEnabled(options.isFullGCAuditLoggingEnabled());
// create a VersionGCSupport while builder is read-write
VersionGCSupport gcSupport = builder.createVersionGCSupport();
@@ -416,7 +433,6 @@ public class RevisionsCommand implements Command {
System.out.println("FullGcProgressSize is : " +
options.getFullGcProgressSize());
System.out.println("FullGcMaxAgeInSecs is : " +
options.getFullGcMaxAge());
System.out.println("FullGcMaxAgeMillis is : " +
builder.getFullGcMaxAgeMillis());
- System.out.println("FullGCAuditLoggingEnabled is : " +
options.isFullGCAuditLoggingEnabled());
VersionGarbageCollector gc = createVersionGC(builder.build(),
gcSupport, options.isDryRun(), builder);
VersionGCOptions gcOptions = gc.getOptions();
@@ -460,11 +476,22 @@ public class RevisionsCommand implements Command {
private void collect(final RevisionsOptions options, Closer closer,
boolean fullGCEnabled) throws IOException {
VersionGarbageCollector gc = bootstrapVGC(options, closer,
fullGCEnabled);
- // Set a default statistics provider
- gc.setStatisticsProvider(new
DefaultStatisticsProvider(Executors.newSingleThreadScheduledExecutor()));
+
+ // setup metrics exporter
+ Whiteboard whiteboard = new
NodeStoreFixtureProvider.ClosingWhiteboard(new DefaultWhiteboard(), closer);
+ StatisticsProvider statsProvider =
NodeStoreFixtureProvider.createStatsProvider(whiteboard, closer);
+ whiteboard.register(StatisticsProvider.class, statsProvider,
Collections.emptyMap());
+ gc.setStatisticsProvider(statsProvider, true);
+
+ FullGCMetricsExporter metricsExporter =
FullGCMetricsExporterFixtureProvider.create(options, whiteboard);
+ gc.setFullGCMetricsExporter(metricsExporter);
+
ExecutorService executor = Executors.newSingleThreadExecutor();
final Semaphore finished = new Semaphore(0);
try {
+ // register metrics exporter to closer
+ closer.register(metricsExporter);
+
// collect until shutdown hook is called
final AtomicBoolean running = new AtomicBoolean(true);
Runtime.getRuntime().addShutdownHook(new Thread(() -> {
@@ -490,6 +517,7 @@ public class RevisionsCommand implements Command {
}
System.out.println("retrieving gc info");
printInfo(gc, options);
+ } catch (Exception e) {
} finally {
finished.release();
if (options.isDryRun()) {
diff --git
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCMetricsExporter.java
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCMetricsExporter.java
new file mode 100644
index 0000000000..9a2cbde092
--- /dev/null
+++
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCMetricsExporter.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.jackrabbit.oak.plugins.document;
+
+import java.io.Closeable;
+import java.util.Map;
+
+/**
+ * Exporter interface for setting dependency for VersionGarbageCollector that
allows
+ * for export of fullGC metrics to Prometheus via pushgateway.
+ * @param <T>
+ */
+public interface FullGCMetricsExporter<T> extends Closeable {
+
+ /**
+ * Called from VersionGarbageCollector when a fullGC iteration completes.
+ */
+ void onIterationComplete();
+}
diff --git
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImpl.java
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImpl.java
index 8ff70733fa..7e8377362b 100644
---
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImpl.java
+++
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/FullGCStatsCollectorImpl.java
@@ -37,6 +37,7 @@ import static
org.apache.jackrabbit.oak.stats.StatsOptions.METRICS_ONLY;
*/
class FullGCStatsCollectorImpl implements FullGCStatsCollector {
+ static final String OAK_RUN_METRICS_PREFIX = "oak_FullGC";
static final String FULL_GC = "FullGC";
static final String READ_DOC = "READ_DOC";
static final String DELETED_ORPHAN_NODE = "DELETED_ORPHAN_NODE";
@@ -83,9 +84,15 @@ class FullGCStatsCollectorImpl implements
FullGCStatsCollector {
private final CounterStats counter;
private final CounterStats failureCounter;
+ private static String METRICS_QUALIFIED_NAME_PREFIX;
FullGCStatsCollectorImpl(StatisticsProvider provider) {
+ this(provider, false);
+ }
+
+ FullGCStatsCollectorImpl(StatisticsProvider provider, boolean isOakRunJob)
{
this.provider = provider;
+ this.METRICS_QUALIFIED_NAME_PREFIX = isOakRunJob ?
OAK_RUN_METRICS_PREFIX : FULL_GC;
readDoc = meter(provider, READ_DOC);
deletedOrphanNode = meter(provider, DELETED_ORPHAN_NODE);
@@ -232,11 +239,10 @@ class FullGCStatsCollectorImpl implements
FullGCStatsCollector {
}
private static String qualifiedName(String metricName) {
- return FULL_GC + "." + metricName;
+ return METRICS_QUALIFIED_NAME_PREFIX + "." + metricName;
}
private MeterStats getMeter(Map<GCPhase, MeterStats> map, GCPhase phase,
String name) {
return map.computeIfAbsent(phase, p -> meter(provider, name + "." +
p.name()));
}
-
}
diff --git
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
index 9a187adaff..d41ce304cd 100644
---
a/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
+++
b/oak-store-document/src/main/java/org/apache/jackrabbit/oak/plugins/document/VersionGarbageCollector.java
@@ -271,6 +271,7 @@ public class VersionGarbageCollector {
private GCMonitor gcMonitor = GCMonitor.EMPTY;
private RevisionGCStats gcStats = new RevisionGCStats(NOOP);
private FullGCStatsCollector fullGCStats = new
FullGCStatsCollectorImpl(NOOP);
+ private FullGCMetricsExporter fullGCMetricsExporter;
VersionGarbageCollector(DocumentNodeStore nodeStore,
VersionGCSupport gcSupport,
@@ -330,8 +331,16 @@ public class VersionGarbageCollector {
}
public void setStatisticsProvider(StatisticsProvider provider) {
+ setStatisticsProvider(provider, false);
+ }
+
+ public void setStatisticsProvider(StatisticsProvider provider, boolean
pushMetrics) {
this.gcStats = new RevisionGCStats(provider);
- this.fullGCStats = new FullGCStatsCollectorImpl(provider);
+ this.fullGCStats = new FullGCStatsCollectorImpl(provider, pushMetrics);
+ }
+
+ public void setFullGCMetricsExporter(FullGCMetricsExporter exporter) {
+ this.fullGCMetricsExporter = exporter;
}
@NotNull
@@ -380,6 +389,9 @@ public class VersionGarbageCollector {
gcStats.finished(overall);
if (fullGCEnabled) {
fullGCStats.finished(overall);
+ if (fullGCMetricsExporter != null) {
+ fullGCMetricsExporter.onIterationComplete();
+ }
}
if (overall.iterationCount > 1) {
gcMonitor.info("Revision garbage collection finished after
{} iterations - aggregate statistics: {}",
@@ -972,6 +984,9 @@ public class VersionGarbageCollector {
// now remove the garbage in one go, if any
if (gc.hasGarbage() &&
phases.start(GCPhase.FULL_GC_CLEANUP)) {
gc.removeGarbage(phases.stats);
+ if (fullGCMetricsExporter != null) {
+
fullGCMetricsExporter.onIterationComplete();
+ }
phases.stop(GCPhase.FULL_GC_CLEANUP);
} else {
if (log.isDebugEnabled()) {
