EBernhardson has uploaded a new change for review.
https://gerrit.wikimedia.org/r/258720
Change subject: [WIP] Collect per-node fetch/query latency percentiles
......................................................................
[WIP] Collect per-node fetch/query latency percentiles
This records latency percentiles on a per node basis and reports
them via JMX. If we think this is a good idea (i'm not sure) i
can add some settings for turning it on/off along with changing
the default reporting to Graphite.
I tried doing it without rewriting bytecode, but the stats collection
is injected multiple levels down and unaccessible from a plugin. This
is a fairly non-invasive rewrite though so maybe it's ok.
Change-Id: I91c6c989578aee0b320eeac916a0aa6a912f900c
---
M pom.xml
M src/main/java/org/wikimedia/search/extra/ExtraPlugin.java
A
src/main/java/org/wikimedia/search/extra/stats/NodeQueryStatsShardSearchModule.java
A
src/main/java/org/wikimedia/search/extra/stats/NodeQueryStatsShardSearchService.java
4 files changed, 123 insertions(+), 1 deletion(-)
git pull ssh://gerrit.wikimedia.org:29418/search/extra
refs/changes/20/258720/1
diff --git a/pom.xml b/pom.xml
index 357a3a8..665087e 100644
--- a/pom.xml
+++ b/pom.xml
@@ -45,6 +45,8 @@
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<elasticsearch.version>1.7.0</elasticsearch.version>
<lucene.version>4.10.4</lucene.version>
+ <metrics.version>3.1.0</metrics.version>
+ <javassist.version>3.20.0-GA</javassist.version>
</properties>
<build>
@@ -338,5 +340,15 @@
<version>18.0</version>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>io.dropwizard.metrics</groupId>
+ <artifactId>metrics-core</artifactId>
+ <version>${metrics.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.javassist</groupId>
+ <artifactId>javassist</artifactId>
+ <version>${javassist.version}</version>
+ </dependency>
</dependencies>
</project>
diff --git a/src/main/java/org/wikimedia/search/extra/ExtraPlugin.java
b/src/main/java/org/wikimedia/search/extra/ExtraPlugin.java
index 57b8b71..4f99bb6 100644
--- a/src/main/java/org/wikimedia/search/extra/ExtraPlugin.java
+++ b/src/main/java/org/wikimedia/search/extra/ExtraPlugin.java
@@ -5,7 +5,11 @@
import org.elasticsearch.common.collect.ImmutableList;
import org.elasticsearch.common.inject.AbstractModule;
import org.elasticsearch.common.inject.Module;
+import org.elasticsearch.common.inject.Provides;
+import org.elasticsearch.common.inject.Singleton;
import org.elasticsearch.common.inject.multibindings.Multibinder;
+import org.elasticsearch.common.logging.ESLogger;
+import org.elasticsearch.common.logging.Loggers;
import org.elasticsearch.common.settings.Settings;
import org.elasticsearch.index.query.QueryParser;
import org.elasticsearch.index.query.functionscore.FunctionScoreModule;
@@ -25,6 +29,15 @@
import org.wikimedia.search.extra.superdetectnoop.SuperDetectNoopScript;
import org.wikimedia.search.extra.superdetectnoop.WithinAbsoluteHandler;
import org.wikimedia.search.extra.superdetectnoop.WithinPercentageHandler;
+import org.wikimedia.search.extra.stats.NodeQueryStatsShardSearchService;
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.JmxReporter;
+import javassist.ClassPool;
+import javassist.CtClass;
+import javassist.CtMethod;
+import javassist.expr.ExprEditor;
+import javassist.expr.NewExpr;
+import javassist.CannotCompileException;
/**
* Setup the Elasticsearch plugin.
@@ -67,7 +80,7 @@
@Override
public Collection<Class<? extends Module>> modules() {
- return ImmutableList.<Class<? extends Module>>
of(SafeifierActionsModule.class, CloseEnoughDetectorsModule.class);
+ return ImmutableList.<Class<? extends Module>>
of(SafeifierActionsModule.class, CloseEnoughDetectorsModule.class,
StatsModule.class);
}
public static class SafeifierActionsModule extends AbstractModule {
@@ -97,4 +110,49 @@
handlers.addBinding().toInstance(new SetHandler.Recognizer());
}
}
+
+ public static class StatsModule extends AbstractModule {
+
+ public StatsModule(Settings settings) {
+ }
+
+ @Override
+ protected void configure() {
+ final ESLogger logger = Loggers.getLogger(getClass());
+ // here be dragons. Open up IndexService class and replace new
ShardSearchModule();
+ // with our own module.
+ try {
+ ClassPool pool = ClassPool.getDefault();
+ pool.importPackage("org.wikimedia.search.extra.stats");
+ CtClass ct = pool.get("org.elasticsearch.index.IndexService");
+ final String lookFor =
"org.elasticsearch.index.search.stats.ShardSearchModule";
+ for(CtMethod method : ct.getMethods()) {
+ if (!method.getName().equals("createShard")) {
+ continue;
+ }
+ method.instrument(new ExprEditor() {
+ public void edit(NewExpr expr) throws
CannotCompileException {
+ if (!expr.getClassName().equals(lookFor)) {
+ return;
+ }
+ expr.replace("{ $_ = new
NodeQueryStatsShardSearchModule(); }");
+ }
+ });
+ }
+ // Force the JVM to load our new class
+ ct.toClass();
+ logger.info("Replaced IndexService implementation for stats
collection");
+ } catch (Throwable e) {
+ logger.warn("Failed replacing ShardSearchModule", e);
+ }
+ }
+
+ @Provides @Singleton
+ public MetricRegistry provideMetricRegistry() {
+ MetricRegistry registry = new MetricRegistry();
+ JmxReporter reporter = JmxReporter.forRegistry(registry).build();
+ reporter.start();
+ return registry;
+ }
+ }
}
diff --git
a/src/main/java/org/wikimedia/search/extra/stats/NodeQueryStatsShardSearchModule.java
b/src/main/java/org/wikimedia/search/extra/stats/NodeQueryStatsShardSearchModule.java
new file mode 100644
index 0000000..9308b57
--- /dev/null
+++
b/src/main/java/org/wikimedia/search/extra/stats/NodeQueryStatsShardSearchModule.java
@@ -0,0 +1,17 @@
+package org.wikimedia.search.extra.stats;
+
+import org.elasticsearch.common.inject.AbstractModule;
+import org.elasticsearch.common.logging.Loggers;
+import org.elasticsearch.index.search.stats.ShardSearchService;
+import org.elasticsearch.index.search.slowlog.ShardSlowLogSearchService;
+/**
+ */
+public class NodeQueryStatsShardSearchModule extends AbstractModule {
+
+ @Override
+ protected void configure() {
+ Loggers.getLogger(getClass()).warn("Running in custom
ShardSearchModule");
+
bind(ShardSearchService.class).to(NodeQueryStatsShardSearchService.class).asEagerSingleton();
+ bind(ShardSlowLogSearchService.class).asEagerSingleton();
+ }
+}
diff --git
a/src/main/java/org/wikimedia/search/extra/stats/NodeQueryStatsShardSearchService.java
b/src/main/java/org/wikimedia/search/extra/stats/NodeQueryStatsShardSearchService.java
new file mode 100644
index 0000000..f45cf2f
--- /dev/null
+++
b/src/main/java/org/wikimedia/search/extra/stats/NodeQueryStatsShardSearchService.java
@@ -0,0 +1,35 @@
+package org.wikimedia.search.extra.stats;
+
+import org.elasticsearch.common.inject.Inject;
+import org.elasticsearch.common.settings.Settings;
+import org.elasticsearch.index.search.stats.ShardSearchService;
+import org.elasticsearch.index.search.slowlog.ShardSlowLogSearchService;
+import org.elasticsearch.index.settings.IndexSettings;
+import org.elasticsearch.index.shard.ShardId;
+import org.elasticsearch.search.internal.SearchContext;
+
+import com.codahale.metrics.MetricRegistry;
+import com.codahale.metrics.Histogram;
+
+public class NodeQueryStatsShardSearchService extends ShardSearchService {
+ final Histogram queryTiming;
+ final Histogram fetchTiming;
+
+ @Inject
+ public NodeQueryStatsShardSearchService(ShardId shardId, @IndexSettings
Settings indexSettings,
+ ShardSlowLogSearchService
slowLogSearchService, MetricRegistry registry) {
+ super(shardId, indexSettings, slowLogSearchService);
+ queryTiming = registry.histogram(MetricRegistry.name(getClass(),
"queryNanos"));
+ fetchTiming = registry.histogram(MetricRegistry.name(getClass(),
"fetchNanos"));
+ }
+
+ public void onQueryPhase(SearchContext searchContext, long tookInNanos) {
+ super.onQueryPhase(searchContext, tookInNanos);
+ queryTiming.update(tookInNanos);
+ }
+
+ public void onFetchPhase(SearchContext searchContext, long tookInNanos) {
+ super.onFetchPhase(searchContext, tookInNanos);
+ fetchTiming.update(tookInNanos);
+ }
+}
--
To view, visit https://gerrit.wikimedia.org/r/258720
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: I91c6c989578aee0b320eeac916a0aa6a912f900c
Gerrit-PatchSet: 1
Gerrit-Project: search/extra
Gerrit-Branch: master
Gerrit-Owner: EBernhardson <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
