KYLIN-2776 Using dropwizard as default metric framework
Project: http://git-wip-us.apache.org/repos/asf/kylin/repo Commit: http://git-wip-us.apache.org/repos/asf/kylin/commit/8e078847 Tree: http://git-wip-us.apache.org/repos/asf/kylin/tree/8e078847 Diff: http://git-wip-us.apache.org/repos/asf/kylin/diff/8e078847 Branch: refs/heads/master Commit: 8e078847a42b4f7853a2c1a75afa405c09c4ea7b Parents: d85f6ab Author: yiming.xu <100650...@qq.com> Authored: Wed Aug 23 13:12:58 2017 +0800 Committer: Hongbin Ma <m...@kyligence.io> Committed: Wed Aug 23 13:49:23 2017 +0800 ---------------------------------------------------------------------- core-common/pom.xml | 17 + .../apache/kylin/common/KylinConfigBase.java | 20 + .../kylin/common/metrics/common/Metrics.java | 162 ++++++ .../common/metrics/common/MetricsConstant.java | 38 ++ .../common/metrics/common/MetricsFactory.java | 62 +++ .../common/metrics/common/MetricsScope.java | 33 ++ .../common/metrics/common/MetricsVariable.java | 27 + .../kylin/common/metrics/common/Metricss.java | 40 ++ .../metrics/metrics2/CodahaleMetrics.java | 508 +++++++++++++++++++ .../metrics/metrics2/CodahaleReporter.java | 31 ++ .../metrics2/ConsoleMetricsReporter.java | 51 ++ .../metrics/metrics2/JmxMetricsReporter.java | 55 ++ .../metrics2/JsonFileMetricsReporter.java | 136 +++++ .../metrics2/KylinObjectNameFactory.java | 68 +++ .../metrics2/MetricVariableRatioGauge.java | 47 ++ .../metrics/metrics2/Metrics2Reporter.java | 60 +++ .../metrics/metrics2/MetricsReporting.java | 26 + .../common/metrics/perflog/IPerfLogger.java | 48 ++ .../common/metrics/perflog/PerfLogger.java | 158 ++++++ .../metrics/perflog/PerfLoggerFactory.java | 56 ++ .../metrics/perflog/SimplePerfLogger.java | 73 +++ pom.xml | 3 +- .../kylin/rest/init/InitialTaskManager.java | 4 +- .../kylin/rest/metrics/QueryMetrics2Facade.java | 106 ++++ .../apache/kylin/rest/service/QueryService.java | 2 + .../kylin/rest/metrics/QueryMetrics2Test.java | 128 +++++ 26 files changed, 1957 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/pom.xml ---------------------------------------------------------------------- diff --git a/core-common/pom.xml b/core-common/pom.xml index 8852743..d58de36 100644 --- a/core-common/pom.xml +++ b/core-common/pom.xml @@ -68,6 +68,23 @@ <artifactId>junit</artifactId> <scope>test</scope> </dependency> + + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-jvm</artifactId> + <version>${dropwizard.version}</version> + </dependency> + <dependency> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-json</artifactId> + <version>${dropwizard.version}</version> + </dependency> + + <dependency> + <groupId>com.github.joshelser</groupId> + <artifactId>dropwizard-metrics-hadoop-metrics2-reporter</artifactId> + <version>0.1.2</version> + </dependency> </dependencies> </project> http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java index b622825..99b9bb2 100644 --- a/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java +++ b/core-common/src/main/java/org/apache/kylin/common/KylinConfigBase.java @@ -1173,4 +1173,24 @@ abstract public class KylinConfigBase implements Serializable { public boolean isWebCrossDomainEnabled() { return Boolean.parseBoolean(getOptional("kylin.web.cross-domain-enabled", "true")); } + + /** + * metric + */ + public String getCoadhaleMetricReportClassesName() { + return getOptional("kylin.metric.codahale-metric-report-classes", + "org.apache.kylin.common.metrics.metrics2.JsonFileMetricsReporter,org.apache.kylin.common.metrics.metrics2.JmxMetricsReporter"); + } + + public String getMetricFileLocation() { + return getOptional("kylin.metric.file.location", "/tmp/report.json"); + } + + public Long getJsonFileMetricsReporterInterval() { + return Long.parseLong(getOptional("kylin.metric.json-file-metric-reporter.interval", "5000")); + } + + public String getPerfLoggerClassName() { + return getOptional("kylin.metric.perf-logger.class", "org.apache.kylin.common.metrics.perflog.PerfLogger"); + } } http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java new file mode 100644 index 0000000..dcda3cd --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metrics.java @@ -0,0 +1,162 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.common.metrics.common; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.codahale.metrics.Timer; + +import java.util.concurrent.TimeUnit; + +/** + * Generic Metics interface. + */ +public interface Metrics { + + /** + * Deinitializes the Metrics system. + */ + public void close() throws Exception; + + /** + * + * @param name starts a scope of a given name. Scopes is stored as thread-local variable. + */ + public void startStoredScope(String name); + + /** + * Closes the stored scope of a given name. + * Note that this must be called on the same thread as where the scope was started. + * @param name + */ + public void endStoredScope(String name); + + /** + * Create scope with given name and returns it. + * @param name + * @return + */ + public MetricsScope createScope(String name); + + /** + * Close the given scope. + * @param scope + */ + public void endScope(MetricsScope scope); + + //Counter-related methods + + /** + * Increments a counter of the given name by 1. + * @param name + * @return + */ + public Long incrementCounter(String name); + + /** + * Increments a counter of the given name by "increment" + * @param name + * @param increment + * @return + */ + public Long incrementCounter(String name, long increment); + + /** + * Decrements a counter of the given name by 1. + * @param name + * @return + */ + public Long decrementCounter(String name); + + /** + * Decrements a counter of the given name by "decrement" + * @param name + * @param decrement + * @return + */ + public Long decrementCounter(String name, long decrement); + + /** + * Adds a metrics-gauge to track variable. For example, number of open database connections. + * @param name name of gauge + * @param variable variable to track. + */ + public void addGauge(String name, final MetricsVariable variable); + + /** + * Add a ratio metric to track the correlation between two variables + * @param name name of the ratio gauge + * @param numerator numerator of the ratio + * @param denominator denominator of the ratio + */ + public void addRatio(String name, MetricsVariable<Integer> numerator, MetricsVariable<Integer> denominator); + + /** + * Mark an event occurance for a meter. Meters measure the rate of an event and track + * 1/5/15 minute moving averages + * @param name name of the meter + */ + public void markMeter(String name); + + /** + * + * @param name name of the Timer + * @param duration + * @param unit + */ + + public void updateTimer(String name, long duration, TimeUnit unit); + + + /** + * + * @param name name of the histogram + * @param count + */ + public void updateHistogram(String name, long count); + + /** + * + * @param name name of the time + * @return + */ + public Timer getTimer(String name); + + /** + * + * @param name name of the counter + * @return + */ + public Counter getCounter(String name); + + /** + * + * @param name name of the time + * @return + */ + public Histogram getHistogram(String name); + + /** + * + * @param name name of the meter + * @return + */ + public Meter getMeter(String name); + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsConstant.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsConstant.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsConstant.java new file mode 100644 index 0000000..49c8d8e --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsConstant.java @@ -0,0 +1,38 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.common.metrics.common; + +/** + * This class defines some metrics generated by Hive processes. + */ +public class MetricsConstant { + + public static final String API_PREFIX = "api_"; + public static final String ACTIVE_CALLS = "metrics:name=active_calls,method="; + public static final String CALLS = "metrics:name=calls,method="; + + public static final String QUERY_SUCCESS_COUNT = "QuerySuccessCount"; + public static final String QUERY_FAIL_COUNT = "QueryFailCount"; + public static final String QUERY_CACHE_COUNT = "QueryCacheCount"; + public static final String QUERY_COUNT = "QueryCount"; + public static final String QUERY_DURATION = "QueryDuration"; + public static final String QUERY_RESULT_ROWCOUNT = "QueryResultRowcount"; + public static final String QUERY_SCAN_ROWCOUNT = "QueryScanRowcount"; + public static final String TOTAL = "total"; + +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsFactory.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsFactory.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsFactory.java new file mode 100644 index 0000000..ff0ab7d --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsFactory.java @@ -0,0 +1,62 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.common; + + +import org.apache.kylin.common.metrics.metrics2.CodahaleMetrics; + +/** + * Class that manages a static Metric instance for this process. + */ +public class MetricsFactory { + + //Volatile ensures that static access returns Metrics instance in fully-initialized state. + //Alternative is to synchronize static access, which has performance penalties. + private volatile static Metrics metrics; + static { + MetricsFactory.init(); + } + + /** + * Initializes static Metrics instance. + */ + public synchronized static void init() { + if (metrics == null) { + Class metricsClass = MetricsFactory.class; + metrics = new CodahaleMetrics(); + } + } + + /** + * Returns static Metrics instance, null if not initialized or closed. + */ + public static Metrics getInstance() { + return metrics; + } + + /** + * Closes and removes static Metrics instance. + */ + public synchronized static void close() throws Exception { + if (metrics != null) { + metrics.close(); + metrics = null; + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsScope.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsScope.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsScope.java new file mode 100644 index 0000000..8a72073 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsScope.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ +package org.apache.kylin.common.metrics.common; + +/** + * Metrics Scope to represent duration of an event. + * + * Implementation can capture information like the average duration of open scopes, + * number of open scopes, number of completed scopes. + * + * Scopes are created via the Metrics framework (see Metrics#createScope or Metrics$createStoredScope) + * + * Scope may be stored by the Metrics framework via 'storedScope' concept for further reference. + * + * In either case, it is the caller's responsibility to end the scope via the Metrics framework (see Metrics#endScope) + */ +public interface MetricsScope { +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsVariable.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsVariable.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsVariable.java new file mode 100644 index 0000000..3273c7f --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/MetricsVariable.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.common; + +/** + * Interface for metrics variables. For example a the database service could expose the number of + * currently active connections. + */ +public interface MetricsVariable<T> { + public T getValue(); +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metricss.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metricss.java b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metricss.java new file mode 100644 index 0000000..105ab5d --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/common/Metricss.java @@ -0,0 +1,40 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.common; + +public final class Metricss { + public final static String METRICS = "metrics:"; + public final static String PROJECT_TEMPLATE = METRICS + "project=%s"; + public final static String CUBE_TEMPLATE = METRICS + "project=%s,cube=%s"; + + + public static String buildMetricName(String prefix, String name) { + return String.format(prefix+",name=%s", name); + } + + public static String buildCubeMetricPrefix(String project) { + return String.format(PROJECT_TEMPLATE, project); + } + + public static String buildCubeMetricPrefix(String project, String cube) { + return String.format(CUBE_TEMPLATE, project, cube); + } + + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java new file mode 100644 index 0000000..d7e2f8c --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleMetrics.java @@ -0,0 +1,508 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.metrics2; + +import java.io.Closeable; +import java.lang.reflect.Constructor; +import java.lang.reflect.InvocationTargetException; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; + +import org.apache.commons.lang3.ClassUtils; +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.metrics.common.Metrics; +import org.apache.kylin.common.metrics.common.MetricsConstant; +import org.apache.kylin.common.metrics.common.MetricsScope; +import org.apache.kylin.common.metrics.common.MetricsVariable; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codahale.metrics.Counter; +import com.codahale.metrics.ExponentiallyDecayingReservoir; +import com.codahale.metrics.Gauge; +import com.codahale.metrics.Histogram; +import com.codahale.metrics.Meter; +import com.codahale.metrics.Metric; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.MetricSet; +import com.codahale.metrics.Timer; +import com.codahale.metrics.json.MetricsModule; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.base.Preconditions; +import com.google.common.base.Splitter; +import com.google.common.cache.CacheBuilder; +import com.google.common.cache.CacheLoader; +import com.google.common.cache.LoadingCache; +import com.google.common.collect.Lists; + +/** + * Codahale-backed Metrics implementation. + */ +public class CodahaleMetrics implements Metrics { + + public static final Logger LOGGER = LoggerFactory.getLogger(CodahaleMetrics.class); + + public final MetricRegistry metricRegistry = new MetricRegistry(); + private final Lock timersLock = new ReentrantLock(); + private final Lock countersLock = new ReentrantLock(); + private final Lock gaugesLock = new ReentrantLock(); + private final Lock metersLock = new ReentrantLock(); + private final Lock histogramLock = new ReentrantLock(); + private final Set<Closeable> reporters = new HashSet<Closeable>(); + private final ThreadLocal<HashMap<String, CodahaleMetricsScope>> threadLocalScopes = new ThreadLocal<HashMap<String, CodahaleMetricsScope>>() { + @Override + protected HashMap<String, CodahaleMetricsScope> initialValue() { + return new HashMap<String, CodahaleMetricsScope>(); + } + }; + private LoadingCache<String, Timer> timers; + private LoadingCache<String, Counter> counters; + private LoadingCache<String, Meter> meters; + private LoadingCache<String, Histogram> histograms; + private ConcurrentHashMap<String, Gauge> gauges; + private KylinConfig conf; + + public CodahaleMetrics() { + this.conf = conf; + //Codahale artifacts are lazily-created. + timers = CacheBuilder.newBuilder().build(new CacheLoader<String, Timer>() { + @Override + public Timer load(String key) { + Timer timer = new Timer(new ExponentiallyDecayingReservoir()); + metricRegistry.register(key, timer); + return timer; + } + }); + counters = CacheBuilder.newBuilder().build(new CacheLoader<String, Counter>() { + @Override + public Counter load(String key) { + Counter counter = new Counter(); + metricRegistry.register(key, counter); + return counter; + } + }); + meters = CacheBuilder.newBuilder().build(new CacheLoader<String, Meter>() { + @Override + public Meter load(String key) { + Meter meter = new Meter(); + metricRegistry.register(key, meter); + return meter; + } + }); + histograms = CacheBuilder.newBuilder().build(new CacheLoader<String, Histogram>() { + @Override + public Histogram load(String key) { + Histogram histogram = new Histogram(new ExponentiallyDecayingReservoir()); + metricRegistry.register(key, histogram); + return histogram; + } + }); + gauges = new ConcurrentHashMap<String, Gauge>(); + //register JVM metrics + // registerAll("gc", new GarbageCollectorMetricSet()); + // registerAll("buffers", new BufferPoolMetricSet(ManagementFactory.getPlatformMBeanServer())); + // registerAll("memory", new MemoryUsageGaugeSet()); + // registerAll("threads", new ThreadStatesGaugeSet()); + // registerAll("classLoadingz", new ClassLoadingGaugeSet()); + + //initialize reporters + initReporting(); + } + + @Override + public void close() throws Exception { + if (reporters != null) { + for (Closeable reporter : reporters) { + reporter.close(); + } + } + for (Map.Entry<String, Metric> metric : metricRegistry.getMetrics().entrySet()) { + metricRegistry.remove(metric.getKey()); + } + timers.invalidateAll(); + counters.invalidateAll(); + meters.invalidateAll(); + } + + @Override + public void startStoredScope(String name) { + if (threadLocalScopes.get().containsKey(name)) { + threadLocalScopes.get().get(name).open(); + } else { + threadLocalScopes.get().put(name, new CodahaleMetricsScope(name)); + } + } + + public MetricsScope getStoredScope(String name) throws IllegalArgumentException { + if (threadLocalScopes.get().containsKey(name)) { + return threadLocalScopes.get().get(name); + } else { + throw new IllegalArgumentException("No metrics scope named " + name); + } + } + + @Override + public void endStoredScope(String name) { + if (threadLocalScopes.get().containsKey(name)) { + threadLocalScopes.get().get(name).close(); + threadLocalScopes.get().remove(name); + } + } + + public MetricsScope createScope(String name) { + return new CodahaleMetricsScope(name); + } + + public void endScope(MetricsScope scope) { + ((CodahaleMetricsScope) scope).close(); + } + + @Override + public Long incrementCounter(String name) { + return incrementCounter(name, 1L); + } + + @Override + public Long incrementCounter(String name, long increment) { + String key = name; + try { + countersLock.lock(); + counters.get(key).inc(increment); + return counters.get(key).getCount(); + } catch (ExecutionException ee) { + throw new IllegalStateException("Error retrieving counter from the metric registry ", ee); + } finally { + countersLock.unlock(); + } + } + + @Override + public Long decrementCounter(String name) { + return decrementCounter(name, 1L); + } + + @Override + public Long decrementCounter(String name, long decrement) { + String key = name; + try { + countersLock.lock(); + counters.get(key).dec(decrement); + return counters.get(key).getCount(); + } catch (ExecutionException ee) { + throw new IllegalStateException("Error retrieving counter from the metric registry ", ee); + } finally { + countersLock.unlock(); + } + } + + @Override + public void addGauge(String name, final MetricsVariable variable) { + Gauge gauge = new Gauge() { + @Override + public Object getValue() { + return variable.getValue(); + } + }; + addGaugeInternal(name, gauge); + } + + @Override + public void addRatio(String name, MetricsVariable<Integer> numerator, MetricsVariable<Integer> denominator) { + Preconditions.checkArgument(numerator != null, "Numerator must not be null"); + Preconditions.checkArgument(denominator != null, "Denominator must not be null"); + + MetricVariableRatioGauge gauge = new MetricVariableRatioGauge(numerator, denominator); + addGaugeInternal(name, gauge); + } + + private void addGaugeInternal(String name, Gauge gauge) { + try { + gaugesLock.lock(); + gauges.put(name, gauge); + // Metrics throws an Exception if we don't do this when the key already exists + if (metricRegistry.getGauges().containsKey(name)) { + LOGGER.warn("A Gauge with name [" + name + "] already exists. " + + " The old gauge will be overwritten, but this is not recommended"); + metricRegistry.remove(name); + } + metricRegistry.register(name, gauge); + } finally { + gaugesLock.unlock(); + } + } + + @Override + public void markMeter(String name) { + String key = name; + try { + metersLock.lock(); + Meter meter = meters.get(name); + meter.mark(); + } catch (ExecutionException e) { + throw new IllegalStateException("Error retrieving meter " + name + " from the metric registry ", e); + } finally { + metersLock.unlock(); + } + } + + @Override + public void updateHistogram(String name, long count) { + try { + histogramLock.lock(); + Histogram histogram = histograms.get(name); + histogram.update(count); + } catch (ExecutionException e) { + throw new IllegalStateException("Error retrieving meter " + name + " from the metric registry ", e); + } finally { + histogramLock.unlock(); + } + } + + @Override + public void updateTimer(String name, long duration, TimeUnit unit) { + String key = name; + try { + timersLock.lock(); + Timer timer = timers.get(key); + timer.update(duration, unit); + } catch (ExecutionException e) { + throw new IllegalStateException("Error retrieving timer " + name + " from the metric registry ", e); + } finally { + timersLock.unlock(); + } + } + + // This method is necessary to synchronize lazy-creation to the timers. + public Timer getTimer(String name) { + try { + timersLock.lock(); + return timers.get(name); + } catch (ExecutionException e) { + throw new IllegalStateException("Error retrieving timer " + name + " from the metric registry ", e); + } finally { + timersLock.unlock(); + } + } + + public Counter getCounter(String name) { + try { + countersLock.lock(); + return counters.get(name); + } catch (ExecutionException e) { + throw new IllegalStateException("Error retrieving counter " + name + " from the metric registry ", e); + } finally { + countersLock.unlock(); + } + } + + public Histogram getHistogram(String name) { + try { + histogramLock.lock(); + return histograms.get(name); + } catch (ExecutionException e) { + throw new IllegalStateException("Error retrieving histogram " + name + " from the metric registry ", e); + } finally { + histogramLock.unlock(); + } + } + + public Meter getMeter(String name) { + try { + metersLock.lock(); + return meters.get(name); + } catch (ExecutionException e) { + throw new IllegalStateException("Error retrieving meter " + name + " from the metric registry ", e); + } finally { + metersLock.unlock(); + } + } + + private void registerAll(String prefix, MetricSet metricSet) { + for (Map.Entry<String, Metric> entry : metricSet.getMetrics().entrySet()) { + if (entry.getValue() instanceof MetricSet) { + registerAll(prefix + "." + entry.getKey(), (MetricSet) entry.getValue()); + } else { + metricRegistry.register(prefix + "." + entry.getKey(), entry.getValue()); + } + } + } + + @VisibleForTesting + public MetricRegistry getMetricRegistry() { + return metricRegistry; + } + + @VisibleForTesting + public String dumpJson() throws Exception { + ObjectMapper jsonMapper = new ObjectMapper() + .registerModule(new MetricsModule(TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS, false)); + return jsonMapper.writerWithDefaultPrettyPrinter().writeValueAsString(metricRegistry); + } + + /** + * Initializes reporters from HIVE_CODAHALE_METRICS_REPORTER_CLASSES or HIVE_METRICS_REPORTER if the former is not defined. + * Note: if both confs are defined, only HIVE_CODAHALE_METRICS_REPORTER_CLASSES will be used. + */ + private void initReporting() { + + if (!(initCodahaleMetricsReporterClasses() || initMetricsReporter())) { + LOGGER.warn("Unable to initialize metrics reporting"); + } + if (reporters.isEmpty()) { + // log a warning incase no reporters were successfully added + LOGGER.warn("No reporters configured for codahale metrics!"); + } + } + + /** + * Initializes reporting using HIVE_CODAHALE_METRICS_REPORTER_CLASSES. + * @return whether initialization was successful or not + */ + private boolean initCodahaleMetricsReporterClasses() { + + List<String> reporterClasses = Lists.newArrayList(Splitter.on(",").trimResults().omitEmptyStrings() + .split(KylinConfig.getInstanceFromEnv().getCoadhaleMetricReportClassesName())); + if (reporterClasses.isEmpty()) { + return false; + } + + for (String reporterClass : reporterClasses) { + Class name = null; + try { + name = ClassUtils.getClass(reporterClass); + } catch (ClassNotFoundException e) { + LOGGER.error("Unable to instantiate metrics reporter class " + reporterClass + + " from conf HIVE_CODAHALE_METRICS_REPORTER_CLASSES", e); + throw new IllegalArgumentException(e); + } + try { + Constructor constructor = name.getConstructor(MetricRegistry.class, KylinConfig.class); + CodahaleReporter reporter = (CodahaleReporter) constructor.newInstance(metricRegistry, conf); + reporter.start(); + reporters.add(reporter); + } catch (NoSuchMethodException | InstantiationException | IllegalAccessException + | InvocationTargetException e) { + LOGGER.error("Unable to instantiate using constructor(MetricRegistry, HiveConf) for" + " reporter " + + reporterClass + " from conf HIVE_CODAHALE_METRICS_REPORTER_CLASSES", e); + throw new IllegalArgumentException(e); + } + } + return true; + } + + /** + * Initializes reporting using HIVE_METRICS+REPORTER. + * @return whether initialization was successful or not + */ + private boolean initMetricsReporter() { + + List<String> metricsReporterNames = Lists.newArrayList(Splitter.on(",").trimResults().omitEmptyStrings() + .split(KylinConfig.getInstanceFromEnv().getCoadhaleMetricReportClassesName())); + if (metricsReporterNames.isEmpty()) { + return false; + } + + MetricsReporting reporter = null; + for (String metricsReportingName : metricsReporterNames) { + try { + reporter = MetricsReporting.valueOf(metricsReportingName.trim().toUpperCase()); + } catch (IllegalArgumentException e) { + LOGGER.error("Invalid reporter name " + metricsReportingName, e); + throw e; + } + CodahaleReporter codahaleReporter = null; + switch (reporter) { + case CONSOLE: + codahaleReporter = new ConsoleMetricsReporter(metricRegistry, conf); + break; + case JMX: + codahaleReporter = new JmxMetricsReporter(metricRegistry, conf); + break; + case JSON_FILE: + codahaleReporter = new JsonFileMetricsReporter(metricRegistry, conf); + break; + case HADOOP2: + codahaleReporter = new Metrics2Reporter(metricRegistry, conf); + break; + default: + LOGGER.warn("Unhandled reporter " + reporter + " provided."); + } + if (codahaleReporter != null) { + codahaleReporter.start(); + reporters.add(codahaleReporter); + } + } + return true; + } + + public class CodahaleMetricsScope implements MetricsScope { + + private final String name; + private final Timer timer; + private Timer.Context timerContext; + + private boolean isOpen = false; + + /** + * Instantiates a named scope - intended to only be called by Metrics, so locally scoped. + * @param name - name of the variable + */ + private CodahaleMetricsScope(String name) { + this.name = name; + this.timer = CodahaleMetrics.this.getTimer(MetricsConstant.CALLS + name); + open(); + } + + /** + * Opens scope, and makes note of the time started, increments run counter + * + */ + public void open() { + if (!isOpen) { + isOpen = true; + this.timerContext = timer.time(); + CodahaleMetrics.this.incrementCounter(MetricsConstant.ACTIVE_CALLS + name); + } else { + LOGGER.warn("Scope named " + name + " is not closed, cannot be opened."); + } + } + + /** + * Closes scope, and records the time taken + */ + public void close() { + if (isOpen) { + timerContext.close(); + CodahaleMetrics.this.decrementCounter(MetricsConstant.ACTIVE_CALLS + name); + } else { + LOGGER.warn("Scope named " + name + " is not open, cannot be closed."); + } + isOpen = false; + } + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleReporter.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleReporter.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleReporter.java new file mode 100644 index 0000000..354f427 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/CodahaleReporter.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.metrics2; + +import java.io.Closeable; + +import com.codahale.metrics.Reporter; + +public interface CodahaleReporter extends Closeable, Reporter { + + /** + * Start the reporter. + */ + public void start(); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/ConsoleMetricsReporter.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/ConsoleMetricsReporter.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/ConsoleMetricsReporter.java new file mode 100644 index 0000000..b292443 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/ConsoleMetricsReporter.java @@ -0,0 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.metrics2; + +import java.util.concurrent.TimeUnit; + +import org.apache.kylin.common.KylinConfig; + +import com.codahale.metrics.ConsoleReporter; +import com.codahale.metrics.MetricRegistry; + +/** + * A wrapper around Codahale ConsoleReporter to make it a pluggable/configurable Hive Metrics reporter. + */ +public class ConsoleMetricsReporter implements CodahaleReporter { + + private final ConsoleReporter reporter; + + public ConsoleMetricsReporter(MetricRegistry registry, KylinConfig conf) { + + reporter = ConsoleReporter.forRegistry(registry).convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS).build(); + + } + + @Override + public void start() { + reporter.start(10, TimeUnit.SECONDS); + } + + @Override + public void close() { + reporter.close(); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JmxMetricsReporter.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JmxMetricsReporter.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JmxMetricsReporter.java new file mode 100644 index 0000000..2b18f2e --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JmxMetricsReporter.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.metrics2; + +import java.util.concurrent.TimeUnit; + +import org.apache.kylin.common.KylinConfig; + +import com.codahale.metrics.JmxReporter; +import com.codahale.metrics.MetricRegistry; + +/** + * A wrapper around Codahale JmxReporter to make it a pluggable/configurable Hive Metrics reporter. + */ +public class JmxMetricsReporter implements CodahaleReporter { + + private final MetricRegistry registry; + private final KylinConfig conf; + private final JmxReporter jmxReporter; + + public JmxMetricsReporter(MetricRegistry registry, KylinConfig conf) { + this.registry = registry; + this.conf = conf; + + jmxReporter = JmxReporter.forRegistry(registry).convertRatesTo(TimeUnit.SECONDS) + .createsObjectNamesWith(new KylinObjectNameFactory()).convertDurationsTo(TimeUnit.MILLISECONDS).build(); + } + + @Override + public void start() { + jmxReporter.start(); + } + + @Override + public void close() { + jmxReporter.close(); + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JsonFileMetricsReporter.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JsonFileMetricsReporter.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JsonFileMetricsReporter.java new file mode 100644 index 0000000..6e8ad76 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/JsonFileMetricsReporter.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.metrics2; + +import java.io.BufferedWriter; +import java.io.IOException; +import java.io.OutputStreamWriter; +import java.net.URI; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.kylin.common.KylinConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.json.MetricsModule; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; + +/** + * A metrics reporter for CodahaleMetrics that dumps metrics periodically into a file in JSON format. + */ + +public class JsonFileMetricsReporter implements CodahaleReporter { + + private static final Logger LOGGER = LoggerFactory.getLogger(JsonFileMetricsReporter.class); + private final MetricRegistry metricRegistry; + private final ObjectWriter jsonWriter; + private final ScheduledExecutorService executorService; + private final KylinConfig conf; + private final long interval; + private final String pathString; + private final Path path; + + public JsonFileMetricsReporter(MetricRegistry registry, KylinConfig conf) { + this.metricRegistry = registry; + this.jsonWriter = new ObjectMapper() + .registerModule(new MetricsModule(TimeUnit.MILLISECONDS, TimeUnit.MILLISECONDS, false)) + .writerWithDefaultPrettyPrinter(); + executorService = Executors.newSingleThreadScheduledExecutor(); + this.conf = conf; + + interval = KylinConfig.getInstanceFromEnv().getJsonFileMetricsReporterInterval(); + pathString = KylinConfig.getInstanceFromEnv().getMetricFileLocation(); + path = new Path(pathString); + } + + @Override + public void start() { + + final Path tmpPath = new Path(pathString + ".tmp"); + URI tmpPathURI = tmpPath.toUri(); + final FileSystem fs; + try { + if (tmpPathURI.getScheme() == null && tmpPathURI.getAuthority() == null) { + //default local + fs = FileSystem.getLocal(new Configuration()); + } else { + fs = FileSystem.get(tmpPathURI, new Configuration()); + } + } catch (IOException e) { + LOGGER.error("Unable to access filesystem for path " + tmpPath + ". Aborting reporting", e); + return; + } + + Runnable task = new Runnable() { + public void run() { + try { + String json = null; + try { + json = jsonWriter.writeValueAsString(metricRegistry); + } catch (JsonProcessingException e) { + LOGGER.error("Unable to convert json to string ", e); + return; + } + + BufferedWriter bw = null; + try { + fs.delete(tmpPath, true); + bw = new BufferedWriter(new OutputStreamWriter(fs.create(tmpPath, true))); + bw.write(json); + fs.setPermission(tmpPath, FsPermission.createImmutable((short) 0644)); + } catch (IOException e) { + LOGGER.error("Unable to write to temp file " + tmpPath, e); + return; + } finally { + if (bw != null) { + bw.close(); + } + } + + try { + fs.rename(tmpPath, path); + fs.setPermission(path, FsPermission.createImmutable((short) 0644)); + } catch (IOException e) { + LOGGER.error("Unable to rename temp file " + tmpPath + " to " + pathString, e); + return; + } + } catch (Throwable t) { + // catch all errors (throwable and execptions to prevent subsequent tasks from being suppressed) + LOGGER.error("Error executing scheduled task ", t); + } + } + }; + + executorService.scheduleWithFixedDelay(task, 0, interval, TimeUnit.MILLISECONDS); + } + + @Override + public void close() { + executorService.shutdown(); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/KylinObjectNameFactory.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/KylinObjectNameFactory.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/KylinObjectNameFactory.java new file mode 100644 index 0000000..3348999 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/KylinObjectNameFactory.java @@ -0,0 +1,68 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.metrics2; + +import java.util.Hashtable; + +import javax.management.MalformedObjectNameException; +import javax.management.ObjectName; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.codahale.metrics.ObjectNameFactory; + +public class KylinObjectNameFactory implements ObjectNameFactory { + + private static final Logger LOGGER = LoggerFactory.getLogger(KylinObjectNameFactory.class); + + @Override + public ObjectName createName(String type, String domain, String name) { + try { + if (name.startsWith(domain)) { + ObjectName objectName = new ObjectName(name); + return objectName; + } + + ObjectName objectName = new ObjectName(domain, "name", name); + if (objectName.isPattern()) { + objectName = new ObjectName(domain, "name", ObjectName.quote(name)); + } + return objectName; + } catch (MalformedObjectNameException e) { + try { + return new ObjectName(domain, "name", ObjectName.quote(name)); + } catch (MalformedObjectNameException e1) { + LOGGER.warn("Unable to register {} {}", type, name, e1); + throw new RuntimeException(e1); + } + } + } + + public ObjectName process(String domain, String name) throws MalformedObjectNameException { + String[] kvArry = name.split(","); + Hashtable<String, String> hashTable = new Hashtable<>(); + for (int i = 0; i < kvArry.length; i++) { + String[] split = kvArry[i].split("="); + hashTable.put(split[0], split[1]); + } + ObjectName objectName = new ObjectName(domain, hashTable); + return objectName; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricVariableRatioGauge.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricVariableRatioGauge.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricVariableRatioGauge.java new file mode 100644 index 0000000..b49093d --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricVariableRatioGauge.java @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.metrics2; + +import com.codahale.metrics.RatioGauge; +import org.apache.kylin.common.metrics.common.MetricsVariable; + + +/** + * Combines two numeric metric variables into one gauge type metric displaying their ratio + */ +public class MetricVariableRatioGauge extends RatioGauge { + + private final MetricsVariable<Integer> numerator; + private final MetricsVariable<Integer> denominator; + + public MetricVariableRatioGauge(MetricsVariable<Integer> numerator, MetricsVariable<Integer> denominator) { + this.numerator = numerator; + this.denominator = denominator; + } + + @Override + protected Ratio getRatio() { + Integer numValue = numerator.getValue(); + Integer denomValue = denominator.getValue(); + if (numValue != null && denomValue != null) { + return Ratio.of(numValue.doubleValue(), denomValue.doubleValue()); + } + return Ratio.of(0d, 0d); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/Metrics2Reporter.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/Metrics2Reporter.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/Metrics2Reporter.java new file mode 100644 index 0000000..d1c3c7f --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/Metrics2Reporter.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.metrics2; + +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem; +import org.apache.kylin.common.KylinConfig; + +import com.codahale.metrics.MetricRegistry; +import com.github.joshelser.dropwizard.metrics.hadoop.HadoopMetrics2Reporter; + +/** + * A wrapper around Codahale HadoopMetrics2Reporter to make it a pluggable/configurable Hive Metrics reporter. + */ +public class Metrics2Reporter implements CodahaleReporter { + + private final MetricRegistry metricRegistry; + private final KylinConfig conf; + private final HadoopMetrics2Reporter reporter; + + public Metrics2Reporter(MetricRegistry registry, KylinConfig conf) { + this.metricRegistry = registry; + this.conf = conf; + String applicationName = "kylin"; + + reporter = HadoopMetrics2Reporter.forRegistry(metricRegistry).convertRatesTo(TimeUnit.SECONDS) + .convertDurationsTo(TimeUnit.MILLISECONDS).build(DefaultMetricsSystem.initialize(applicationName), // The application-level name + applicationName, // Component name + applicationName, // Component description + "General"); // Name for each metric record + } + + @Override + public void start() { + long reportingInterval = 30; + reporter.start(reportingInterval, TimeUnit.SECONDS); + } + + @Override + public void close() { + reporter.close(); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricsReporting.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricsReporting.java b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricsReporting.java new file mode 100644 index 0000000..fc1b663 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/metrics2/MetricsReporting.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.metrics2; + +/** + * Reporting options for org.apache.hadoop.hive.common.metrics.metrics2.Metrics. + */ +public enum MetricsReporting { + JMX, CONSOLE, JSON_FILE, HADOOP2 +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/IPerfLogger.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/IPerfLogger.java b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/IPerfLogger.java new file mode 100644 index 0000000..83a4210 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/IPerfLogger.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.perflog; + +public interface IPerfLogger { + + /** + * Call this function when you start to measure time spent by a piece of code. + * + * @param callerName the logging object to be used. + * @param method method or ID that identifies this perf log element. + */ + public void perfLogBegin(String callerName, String method); + + /** + * Call this function in correspondence of perfLogBegin to mark the end of the measurement. + * + * @param callerName + * @param method + * @return long duration the difference between now and startTime, or -1 if startTime is null + */ + public long perfLogEnd(String callerName, String method); + + /** + * Call this function in correspondence of perfLogBegin to mark the end of the measurement. + * + * @param callerName + * @param method + * @return long duration the difference between now and startTime, or -1 if startTime is null + */ + public long perfLogEnd(String callerName, String method, String additionalInfo); +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLogger.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLogger.java b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLogger.java new file mode 100644 index 0000000..cf235bb --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLogger.java @@ -0,0 +1,158 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.perflog; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.kylin.common.metrics.common.Metrics; +import org.apache.kylin.common.metrics.common.MetricsFactory; +import org.apache.kylin.common.metrics.common.MetricsScope; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.google.common.collect.ImmutableMap; + + +/** + * PerfLogger. + * <p> + * Can be used to measure and log the time spent by a piece of code. + */ +public class PerfLogger implements IPerfLogger { + + static final private Logger LOG = LoggerFactory.getLogger(PerfLogger.class.getName()); + protected final Map<String, Long> startTimes = new HashMap<String, Long>(); + protected final Map<String, Long> endTimes = new HashMap<String, Long>(); + //Methods for metrics integration. Each thread-local PerfLogger will open/close scope during each perf-log method. + transient Map<String, MetricsScope> openScopes = new HashMap<String, MetricsScope>(); + + public void perfLogBegin(String callerName, String method) { + long startTime = System.currentTimeMillis(); + startTimes.put(method, new Long(startTime)); + if (LOG.isDebugEnabled()) { + LOG.debug("<PERFLOG method=" + method + " from=" + callerName + ">"); + } + beginMetrics(callerName + "." + method); + } + + public long perfLogEnd(String callerName, String method) { + return perfLogEnd(callerName, method, null); + } + + public long perfLogEnd(String callerName, String method, String additionalInfo) { + Long startTime = startTimes.get(method); + long endTime = System.currentTimeMillis(); + endTimes.put(method, new Long(endTime)); + long duration = startTime == null ? -1 : endTime - startTime.longValue(); + + if (LOG.isDebugEnabled()) { + StringBuilder sb = new StringBuilder("</PERFLOG method=").append(method); + if (startTime != null) { + sb.append(" start=").append(startTime); + } + sb.append(" end=").append(endTime); + if (startTime != null) { + sb.append(" duration=").append(duration); + } + sb.append(" from=").append(callerName); + if (additionalInfo != null) { + sb.append(" ").append(additionalInfo); + } + sb.append(">"); + LOG.debug(sb.toString()); + } + endMetrics(callerName + "." + method); + return duration; + } + + public Long getStartTime(String method) { + long startTime = 0L; + + if (startTimes.containsKey(method)) { + startTime = startTimes.get(method); + } + return startTime; + } + + public Long getEndTime(String method) { + long endTime = 0L; + + if (endTimes.containsKey(method)) { + endTime = endTimes.get(method); + } + return endTime; + } + + public boolean startTimeHasMethod(String method) { + return startTimes.containsKey(method); + } + + public boolean endTimeHasMethod(String method) { + return endTimes.containsKey(method); + } + + public Long getDuration(String method) { + long duration = 0; + if (startTimes.containsKey(method) && endTimes.containsKey(method)) { + duration = endTimes.get(method) - startTimes.get(method); + } + return duration; + } + + public ImmutableMap<String, Long> getStartTimes() { + return ImmutableMap.copyOf(startTimes); + } + + public ImmutableMap<String, Long> getEndTimes() { + return ImmutableMap.copyOf(endTimes); + } + + private void beginMetrics(String method) { + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + MetricsScope scope = metrics.createScope(method); + openScopes.put(method, scope); + } + + } + + private void endMetrics(String method) { + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + MetricsScope scope = openScopes.remove(method); + if (scope != null) { + metrics.endScope(scope); + } + } + } + + /** + * Cleans up any dangling perfLog metric call scopes. + */ + public void cleanupPerfLogMetrics() { + Metrics metrics = MetricsFactory.getInstance(); + if (metrics != null) { + for (MetricsScope openScope : openScopes.values()) { + metrics.endScope(openScope); + } + } + openScopes.clear(); + } +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLoggerFactory.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLoggerFactory.java b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLoggerFactory.java new file mode 100644 index 0000000..4fc43a8 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/PerfLoggerFactory.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.perflog; + +import org.apache.commons.lang3.ClassUtils; +import org.apache.kylin.common.KylinConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class PerfLoggerFactory { + + protected static final ThreadLocal<IPerfLogger> perfLogger = new ThreadLocal<IPerfLogger>(); + static final private Logger LOG = LoggerFactory.getLogger(PerfLoggerFactory.class.getName()); + + public static IPerfLogger getPerfLogger() { + return getPerfLogger(false); + } + + public static void setPerfLogger(IPerfLogger iPerfLogger) { + perfLogger.set(iPerfLogger); + } + + public static IPerfLogger getPerfLogger(boolean resetPerfLogger) { + IPerfLogger result = perfLogger.get(); + if (resetPerfLogger || result == null) { + try { + result = (IPerfLogger) ClassUtils.getClass(KylinConfig.getInstanceFromEnv().getPerfLoggerClassName()) + .newInstance(); + } catch (ClassNotFoundException e) { + LOG.error("Performance Logger Class not found:" + e.getMessage()); + result = new SimplePerfLogger(); + } catch (IllegalAccessException | InstantiationException e) { + e.printStackTrace(); + } + perfLogger.set(result); + } + return result; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/SimplePerfLogger.java ---------------------------------------------------------------------- diff --git a/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/SimplePerfLogger.java b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/SimplePerfLogger.java new file mode 100644 index 0000000..cd88c65 --- /dev/null +++ b/core-common/src/main/java/org/apache/kylin/common/metrics/perflog/SimplePerfLogger.java @@ -0,0 +1,73 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.common.metrics.perflog; + +import java.util.HashMap; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +public class SimplePerfLogger implements IPerfLogger { + + static final private Logger LOG = LoggerFactory.getLogger(SimplePerfLogger.class.getName()); + protected final Map<String, Long> startTimes = new HashMap<String, Long>(); + protected final Map<String, Long> endTimes = new HashMap<String, Long>(); + + protected SimplePerfLogger() { + } + + public void perfLogBegin(String callerName, String method) { + long startTime = System.currentTimeMillis(); + startTimes.put(method, new Long(startTime)); + if (LOG.isDebugEnabled()) { + LOG.debug("<PERFLOG method=" + method + " from=" + callerName + ">"); + } + } + + public long perfLogEnd(String callerName, String method) { + return perfLogEnd(callerName, method, null); + } + + public long perfLogEnd(String callerName, String method, String additionalInfo) { + Long startTime = startTimes.get(method); + long endTime = System.currentTimeMillis(); + endTimes.put(method, new Long(endTime)); + long duration = startTime == null ? -1 : endTime - startTime.longValue(); + + if (LOG.isDebugEnabled()) { + StringBuilder sb = new StringBuilder("</PERFLOG method=").append(method); + if (startTime != null) { + sb.append(" start=").append(startTime); + } + sb.append(" end=").append(endTime); + if (startTime != null) { + sb.append(" duration=").append(duration); + } + sb.append(" from=").append(callerName); + if (additionalInfo != null) { + sb.append(" ").append(additionalInfo); + } + sb.append(">"); + LOG.debug(sb.toString()); + } + return duration; + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index f7444cd..5b00798 100644 --- a/pom.xml +++ b/pom.xml @@ -114,7 +114,8 @@ <cors.version>2.5</cors.version> <tomcat.version>7.0.69</tomcat.version> <t-digest.version>3.1</t-digest.version> - + <!--metric--> + <dropwizard.version>3.1.2</dropwizard.version> <!-- REST Service, ref https://github.com/spring-projects/spring-boot/blob/v1.3.8.RELEASE/spring-boot-dependencies/pom.xml --> <spring.boot.version>1.3.8.RELEASE</spring.boot.version> <spring.framework.version>4.2.8.RELEASE</spring.framework.version> http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java index 8bba674..0194334 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java +++ b/server-base/src/main/java/org/apache/kylin/rest/init/InitialTaskManager.java @@ -20,6 +20,7 @@ package org.apache.kylin.rest.init; import org.apache.commons.lang.StringUtils; import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.rest.metrics.QueryMetrics2Facade; import org.apache.kylin.rest.metrics.QueryMetricsFacade; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -42,7 +43,8 @@ public class InitialTaskManager implements InitializingBean { private void runInitialTasks() { // init metrics system for kylin QueryMetricsFacade.init(); - + QueryMetrics2Facade.init(); + KylinConfig kylinConfig = KylinConfig.getInstanceFromEnv(); String initTasks = kylinConfig.getInitTasks(); if (!StringUtils.isEmpty(initTasks)) { http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetrics2Facade.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetrics2Facade.java b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetrics2Facade.java new file mode 100644 index 0000000..b42e65c --- /dev/null +++ b/server-base/src/main/java/org/apache/kylin/rest/metrics/QueryMetrics2Facade.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + +package org.apache.kylin.rest.metrics; + +import static org.apache.kylin.common.metrics.common.MetricsConstant.TOTAL; +import static org.apache.kylin.common.metrics.common.Metricss.buildCubeMetricPrefix; + +import java.util.concurrent.TimeUnit; + +import javax.annotation.concurrent.ThreadSafe; + +import org.apache.kylin.common.KylinConfig; +import org.apache.kylin.common.metrics.common.Metrics; +import org.apache.kylin.common.metrics.common.MetricsConstant; +import org.apache.kylin.common.metrics.common.MetricsFactory; +import org.apache.kylin.common.metrics.common.Metricss; +import org.apache.kylin.rest.request.SQLRequest; +import org.apache.kylin.rest.response.SQLResponse; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * The entrance of metrics features. + */ +@ThreadSafe +public class QueryMetrics2Facade { + + private static final Logger logger = LoggerFactory.getLogger(QueryMetrics2Facade.class); + private static Metrics metrics; + private static boolean enabled = false; + + public static void init() { + enabled = KylinConfig.getInstanceFromEnv().getQueryMetricsEnabled(); + } + + public static void updateMetrics(SQLRequest sqlRequest, SQLResponse sqlResponse) { + if (!enabled) { + return; + } + if (metrics == null) { + metrics = MetricsFactory.getInstance(); + } + String projectName = sqlRequest.getProject(); + String cube = sqlResponse.getCube(); + if (cube == null) { + return; + } + String cubeName = cube.replace("=", "->"); + + // update(getQueryMetrics("Server_Total"), sqlResponse); + update(buildCubeMetricPrefix(TOTAL), sqlResponse); + update(buildCubeMetricPrefix(projectName), sqlResponse); + String cubeMetricName = buildCubeMetricPrefix(projectName, cubeName); + update(cubeMetricName, sqlResponse); + } + + private static void update(String name, SQLResponse sqlResponse) { + try { + incrQueryCount(name, sqlResponse); + incrCacheHitCount(name, sqlResponse); + if (!sqlResponse.getIsException()) { + metrics.updateTimer(Metricss.buildMetricName(name, MetricsConstant.QUERY_DURATION), + sqlResponse.getDuration(), TimeUnit.MILLISECONDS); + metrics.updateHistogram(Metricss.buildMetricName(name, MetricsConstant.QUERY_RESULT_ROWCOUNT), + sqlResponse.getResults().size()); + metrics.updateHistogram(Metricss.buildMetricName(name, MetricsConstant.QUERY_SCAN_ROWCOUNT), + sqlResponse.getTotalScanCount()); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + + } + + private static void incrQueryCount(String name, SQLResponse sqlResponse) { + if (!sqlResponse.isHitExceptionCache() && !sqlResponse.getIsException()) { + metrics.incrementCounter(Metricss.buildMetricName(name, MetricsConstant.QUERY_SUCCESS_COUNT)); + } else { + metrics.incrementCounter(Metricss.buildMetricName(name, MetricsConstant.QUERY_FAIL_COUNT)); + } + metrics.incrementCounter(Metricss.buildMetricName(name, MetricsConstant.QUERY_COUNT)); + } + + private static void incrCacheHitCount(String name, SQLResponse sqlResponse) { + if (sqlResponse.isStorageCacheUsed()) { + metrics.incrementCounter(Metricss.buildMetricName(name, MetricsConstant.QUERY_CACHE_COUNT)); + } + } + +} http://git-wip-us.apache.org/repos/asf/kylin/blob/8e078847/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java ---------------------------------------------------------------------- diff --git a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java index 24ac5a0..533b93d 100644 --- a/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java +++ b/server-base/src/main/java/org/apache/kylin/rest/service/QueryService.java @@ -87,6 +87,7 @@ import org.apache.kylin.query.util.QueryUtil; import org.apache.kylin.rest.constant.Constant; import org.apache.kylin.rest.exception.BadRequestException; import org.apache.kylin.rest.exception.InternalErrorException; +import org.apache.kylin.rest.metrics.QueryMetrics2Facade; import org.apache.kylin.rest.metrics.QueryMetricsFacade; import org.apache.kylin.rest.model.Query; import org.apache.kylin.rest.msg.Message; @@ -436,6 +437,7 @@ public class QueryService extends BasicService { logQuery(sqlRequest, sqlResponse); QueryMetricsFacade.updateMetrics(sqlRequest, sqlResponse); + QueryMetrics2Facade.updateMetrics(sqlRequest, sqlResponse); if (sqlResponse.getIsException()) throw new InternalErrorException(sqlResponse.getExceptionMessage());
