http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/FastLongHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/FastLongHistogram.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/FastLongHistogram.java new file mode 100644 index 0000000..4e83e1b --- /dev/null +++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/FastLongHistogram.java @@ -0,0 +1,406 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.metrics.Snapshot; +import org.apache.hadoop.hbase.util.AtomicUtils; +import org.apache.hadoop.hbase.util.LongAdder; + +/** + * FastLongHistogram is a thread-safe class that estimate distribution of data and computes the + * quantiles. + */ [email protected] [email protected] +public class FastLongHistogram { + + /** + * Default number of bins. + */ + public static final int DEFAULT_NBINS = 255; + + public static final double[] DEFAULT_QUANTILES = + new double[]{0.25, 0.5, 0.75, 0.90, 0.95, 0.98, 0.99, 0.999}; + + /** + * Bins is a class containing a list of buckets(or bins) for estimation histogram of some data. + */ + private static class Bins { + private final LongAdder[] counts; + // inclusive + private final long binsMin; + // exclusive + private final long binsMax; + private final long bins10XMax; + private final AtomicLong min = new AtomicLong(Long.MAX_VALUE); + private final AtomicLong max = new AtomicLong(0L); + + private final LongAdder count = new LongAdder(); + private final LongAdder total = new LongAdder(); + + // set to true when any of data has been inserted to the Bins. It is set after the counts are + // updated. + private final AtomicBoolean hasData = new AtomicBoolean(false); + + /** + * The constructor for creating a Bins without any prior data. + */ + public Bins(int numBins) { + counts = createCounters(numBins + 3); + this.binsMin = 1L; + + // These two numbers are total guesses + // and should be treated as highly suspect. + this.binsMax = 1000; + this.bins10XMax = binsMax * 10; + } + + /** + * The constructor for creating a Bins with last Bins. + */ + public Bins(Bins last, int numOfBins, double minQ, double maxQ) { + long[] values = last.getQuantiles(new double[] { minQ, maxQ }); + long wd = values[1] - values[0] + 1; + // expand minQ and maxQ in two ends back assuming uniform distribution + this.binsMin = Math.max(0L, (long) (values[0] - wd * minQ)); + long binsMax = (long) (values[1] + wd * (1 - maxQ)) + 1; + // make sure each of bins is at least of width 1 + this.binsMax = Math.max(binsMax, this.binsMin + numOfBins); + this.bins10XMax = Math.max((long) (values[1] + (binsMax - 1) * 9), this.binsMax + 1); + + this.counts = createCounters(numOfBins + 3); + } + + private LongAdder[] createCounters(int num) { + LongAdder[] counters = new LongAdder[num]; + for (int i = 0; i < num; i++) { + counters[i] = new LongAdder(); + } + return counters; + } + + private int getIndex(long value) { + if (value < this.binsMin) { + return 0; + } else if (value > this.bins10XMax) { + return this.counts.length - 1; + } else if (value >= this.binsMax) { + return this.counts.length - 2; + } + // compute the position + return 1 + (int) ((value - this.binsMin) * (this.counts.length - 3) / + (this.binsMax - this.binsMin)); + + } + + /** + * Adds a value to the histogram. + */ + public void add(long value, long count) { + if (value < 0) { + // The whole computation is completely thrown off if there are negative numbers + // + // Normally we would throw an IllegalArgumentException however this is the metrics + // system and it should be completely safe at all times. + // So silently throw it away. + return; + } + AtomicUtils.updateMin(min, value); + AtomicUtils.updateMax(max, value); + + this.count.add(count); + this.total.add(value * count); + + int pos = getIndex(value); + this.counts[pos].add(count); + + // hasData needs to be updated as last + this.hasData.set(true); + } + + /** + * Computes the quantiles give the ratios. + */ + public long[] getQuantiles(double[] quantiles) { + if (!this.hasData.get()) { + // No data yet. + return new long[quantiles.length]; + } + + // Make a snapshot of lowerCounter, higherCounter and bins.counts to counts. + // This is not synchronized, but since the counter are accumulating, the result is a good + // estimation of a snapshot. + long[] counts = new long[this.counts.length]; + long total = 0L; + for (int i = 0; i < this.counts.length; i++) { + counts[i] = this.counts[i].sum(); + total += counts[i]; + } + + int rIndex = 0; + double qCount = total * quantiles[0]; + long cum = 0L; + + long[] res = new long[quantiles.length]; + countsLoop: for (int i = 0; i < counts.length; i++) { + // mn and mx define a value range + long mn, mx; + if (i == 0) { + mn = this.min.get(); + mx = this.binsMin; + } else if (i == counts.length - 1) { + mn = this.bins10XMax; + mx = this.max.get(); + } else if (i == counts.length - 2) { + mn = this.binsMax; + mx = this.bins10XMax; + } else { + mn = this.binsMin + (i - 1) * (this.binsMax - this.binsMin) / (this.counts.length - 3); + mx = this.binsMin + i * (this.binsMax - this.binsMin) / (this.counts.length - 3); + } + + if (mx < this.min.get()) { + continue; + } + if (mn > this.max.get()) { + break; + } + mn = Math.max(mn, this.min.get()); + mx = Math.min(mx, this.max.get()); + + // lastCum/cum are the corresponding counts to mn/mx + double lastCum = cum; + cum += counts[i]; + + // fill the results for qCount is within current range. + while (qCount <= cum) { + if (cum == lastCum) { + res[rIndex] = mn; + } else { + res[rIndex] = (long) ((qCount - lastCum) * (mx - mn) / (cum - lastCum) + mn); + } + + // move to next quantile + rIndex++; + if (rIndex >= quantiles.length) { + break countsLoop; + } + qCount = total * quantiles[rIndex]; + } + } + // In case quantiles contains values >= 100% + for (; rIndex < quantiles.length; rIndex++) { + res[rIndex] = this.max.get(); + } + + return res; + } + + + long getNumAtOrBelow(long val) { + final int targetIndex = getIndex(val); + long totalToCurrentIndex = 0; + for (int i = 0; i <= targetIndex; i++) { + totalToCurrentIndex += this.counts[i].sum(); + } + return totalToCurrentIndex; + } + + public long getMin() { + long min = this.min.get(); + return min == Long.MAX_VALUE ? 0 : min; // in case it is not initialized + } + + public long getMean() { + long count = this.count.sum(); + long total = this.total.sum(); + if (count == 0) { + return 0; + } + return total / count; + } + } + + // The bins counting values. It is replaced with a new one in calling of reset(). + private volatile Bins bins; + + /** + * Constructor. + */ + public FastLongHistogram() { + this(DEFAULT_NBINS); + } + + /** + * Constructor. + * @param numOfBins the number of bins for the histogram. A larger value results in more precise + * results but with lower efficiency, and vice versus. + */ + public FastLongHistogram(int numOfBins) { + this.bins = new Bins(numOfBins); + } + + /** + * Constructor setting the bins assuming a uniform distribution within a range. + * @param numOfBins the number of bins for the histogram. A larger value results in more precise + * results but with lower efficiency, and vice versus. + * @param min lower bound of the region, inclusive. + * @param max higher bound of the region, inclusive. + */ + public FastLongHistogram(int numOfBins, long min, long max) { + this(numOfBins); + Bins bins = new Bins(numOfBins); + bins.add(min, 1); + bins.add(max, 1); + this.bins = new Bins(bins, numOfBins, 0.01, 0.999); + } + + private FastLongHistogram(Bins bins) { + this.bins = bins; + } + + /** + * Adds a value to the histogram. + */ + public void add(long value, long count) { + this.bins.add(value, count); + } + + /** + * Computes the quantiles give the ratios. + */ + public long[] getQuantiles(double[] quantiles) { + return this.bins.getQuantiles(quantiles); + } + + public long[] getQuantiles() { + return this.bins.getQuantiles(DEFAULT_QUANTILES); + } + + public long getMin() { + return this.bins.getMin(); + } + + public long getMax() { + return this.bins.max.get(); + } + + public long getCount() { + return this.bins.count.sum(); + } + + public long getMean() { + return this.bins.getMean(); + } + + public long getNumAtOrBelow(long value) { + return this.bins.getNumAtOrBelow(value); + } + + /** + * Resets the histogram for new counting. + */ + public Snapshot snapshotAndReset() { + final Bins oldBins = this.bins; + this.bins = new Bins(this.bins, this.bins.counts.length - 3, 0.01, 0.99); + final long[] percentiles = oldBins.getQuantiles(DEFAULT_QUANTILES); + final long count = oldBins.count.sum(); + + return new Snapshot() { + @Override + public long[] getQuantiles(double[] quantiles) { + return oldBins.getQuantiles(quantiles); + } + + @Override + public long[] getQuantiles() { + return percentiles; + } + + @Override + public long getCount() { + return count; + } + + @Override + public long getCountAtOrBelow(long val) { + return oldBins.getNumAtOrBelow(val); + } + + @Override + public long get25thPercentile() { + return percentiles[0]; + } + + @Override + public long get75thPercentile() { + return percentiles[2]; + } + + @Override + public long get90thPercentile() { + return percentiles[3]; + } + + @Override + public long get95thPercentile() { + return percentiles[4]; + } + + @Override + public long get98thPercentile() { + return percentiles[5]; + } + + @Override + public long get99thPercentile() { + return percentiles[6]; + } + + @Override + public long get999thPercentile() { + return percentiles[7]; + } + + @Override + public long getMedian() { + return percentiles[1]; + } + + @Override + public long getMax() { + return oldBins.max.get(); + } + + @Override + public long getMean() { + return oldBins.getMean(); + } + + @Override + public long getMin() { + return oldBins.getMin(); + } + }; + } +}
http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java new file mode 100644 index 0000000..b52caf8 --- /dev/null +++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/HistogramImpl.java @@ -0,0 +1,81 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.Histogram; +import org.apache.hadoop.hbase.metrics.Snapshot; + +/** + * Custom histogram implementation based on FastLongHistogram. Dropwizard-based histograms are + * slow compared to this implementation, so we are using our implementation here. + * See HBASE-15222. + */ [email protected] +public class HistogramImpl implements Histogram { + // Double buffer the two FastLongHistograms. + // As they are reset they learn how the buckets should be spaced + // So keep two around and use them + protected final FastLongHistogram histogram; + private final CounterImpl counter; + + public HistogramImpl() { + this(Integer.MAX_VALUE << 2); + } + + public HistogramImpl(long maxExpected) { + this(FastLongHistogram.DEFAULT_NBINS, 1, maxExpected); + } + + public HistogramImpl(int numBins, long min, long maxExpected) { + this.counter = new CounterImpl(); + this.histogram = new FastLongHistogram(numBins, min, maxExpected); + } + + protected HistogramImpl(CounterImpl counter, FastLongHistogram histogram) { + this.counter = counter; + this.histogram = histogram; + } + + @Override + public void update(int value) { + counter.increment(); + histogram.add(value, 1); + } + + @Override + public void update(long value) { + counter.increment(); + histogram.add(value, 1); + } + + public long getCount() { + return counter.getCount(); + } + + public long getMax() { + return this.histogram.getMax(); + } + + public Snapshot snapshot() { + return histogram.snapshotAndReset(); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java new file mode 100644 index 0000000..3788bd1 --- /dev/null +++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistriesImpl.java @@ -0,0 +1,82 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.Collection; +import java.util.Collections; +import java.util.Set; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryFactory; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; + +import com.google.common.base.Optional; +import com.google.common.base.Supplier; + +/** + * Implementation of MetricRegistries that does ref-counting. + */ [email protected] +public class MetricRegistriesImpl extends MetricRegistries { + private final MetricRegistryFactory factory; + private final RefCountingMap<MetricRegistryInfo, MetricRegistry> registries; + + public MetricRegistriesImpl() { + this(new MetricRegistryFactoryImpl()); + } + + public MetricRegistriesImpl(MetricRegistryFactory factory) { + this.factory = factory; + this.registries = new RefCountingMap<>(); + } + + @Override + public MetricRegistry create(final MetricRegistryInfo info) { + return registries.put(info, new Supplier<MetricRegistry>() { + @Override + public MetricRegistry get() { + return factory.create(info); + } + }); + } + + public boolean remove(MetricRegistryInfo key) { + return registries.remove(key) == null; + } + + public Optional<MetricRegistry> get(MetricRegistryInfo info) { + return Optional.fromNullable(registries.get(info)); + } + + public Collection<MetricRegistry> getMetricRegistries() { + return Collections.unmodifiableCollection(registries.values()); + } + + public void clear() { + registries.clear(); + } + + public Set<MetricRegistryInfo> getMetricRegistryInfos() { + return Collections.unmodifiableSet(registries.keySet()); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryFactoryImpl.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryFactoryImpl.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryFactoryImpl.java new file mode 100644 index 0000000..b47153e --- /dev/null +++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryFactoryImpl.java @@ -0,0 +1,34 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryFactory; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; + [email protected] +public class MetricRegistryFactoryImpl implements MetricRegistryFactory { + @Override + public MetricRegistry create(MetricRegistryInfo info) { + return new MetricRegistryImpl(info); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java new file mode 100644 index 0000000..c29e2b5 --- /dev/null +++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/MetricRegistryImpl.java @@ -0,0 +1,183 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.Map; +import java.util.Map.Entry; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Gauge; +import org.apache.hadoop.hbase.metrics.Histogram; +import org.apache.hadoop.hbase.metrics.Meter; +import org.apache.hadoop.hbase.metrics.Metric; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.hbase.metrics.MetricSet; +import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.hbase.util.CollectionUtils; + +import com.google.common.base.Optional; + +/** + * Custom implementation of {@link MetricRegistry}. + */ [email protected] +public class MetricRegistryImpl implements MetricRegistry { + + private final MetricRegistryInfo info; + + private final ConcurrentMap<String, Metric> metrics; + + public MetricRegistryImpl(MetricRegistryInfo info) { + this.info = info; + this.metrics = new ConcurrentHashMap<>(); + } + + @Override + public Timer timer(String name) { + Timer metric = (Timer) metrics.get(name); + if (metric != null) { + return metric; + } + + Timer newTimer = createTimer(); + metric = (Timer) metrics.putIfAbsent(name, newTimer); + if (metric != null) { + return metric; + } + + return newTimer; + } + + protected Timer createTimer() { + return new TimerImpl(); + } + + @Override + public Histogram histogram(String name) { + Histogram metric = (Histogram) metrics.get(name); + if (metric != null) { + return metric; + } + + Histogram newHistogram = createHistogram(); + metric = (Histogram) metrics.putIfAbsent(name, newHistogram); + if (metric != null) { + return metric; + } + + return newHistogram; + } + + protected Histogram createHistogram() { + return new HistogramImpl(); + } + + @Override + public Meter meter(String name) { + Meter metric = (Meter) metrics.get(name); + if (metric != null) { + return metric; + } + + Meter newmeter = createMeter(); + metric = (Meter) metrics.putIfAbsent(name, newmeter); + if (metric != null) { + return metric; + } + + return newmeter; + } + + protected Meter createMeter() { + return new DropwizardMeter(); + } + + @Override + public Counter counter(String name) { + Counter metric = (Counter) metrics.get(name); + if (metric != null) { + return metric; + } + + Counter newCounter = createCounter(); + metric = (Counter) metrics.putIfAbsent(name, newCounter); + if (metric != null) { + return metric; + } + + return newCounter; + } + + protected Counter createCounter() { + return new CounterImpl(); + } + + @Override + public Optional<Metric> get(String name) { + + return Optional.fromNullable(metrics.get(name)); + } + + @Override + public Metric register(String name, Metric metric) { + Metric m = metrics.get(name); + if (m != null) { + return m; + } + + Metric oldMetric = metrics.putIfAbsent(name, metric); + if (oldMetric != null) { + return oldMetric; + } + + return metric; + } + + @Override + public <T> Gauge<T> register(String name, Gauge<T> gauge) { + return (Gauge) register(name, (Metric)gauge); + } + + @Override + public void registerAll(MetricSet metricSet) { + Set<Entry<String,Metric>> entrySet = metricSet.getMetrics().entrySet(); + for (Entry<String, Metric> entry : entrySet) { + register(entry.getKey(), entry.getValue()); + } + } + + @Override + public Map<String, Metric> getMetrics() { + return metrics; + } + + @Override + public boolean remove(String name) { + return metrics.remove(name) != null; + } + + @Override + public MetricRegistryInfo getMetricRegistryInfo() { + return info; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/RefCountingMap.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/RefCountingMap.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/RefCountingMap.java new file mode 100644 index 0000000..889b026 --- /dev/null +++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/RefCountingMap.java @@ -0,0 +1,108 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.ArrayList; +import java.util.Collection; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; + +import com.google.common.base.Supplier; + +/** + * A map of K to V, but does ref counting for added and removed values. The values are + * not added directly, but instead requested from the given Supplier if ref count == 0. Each put() + * call will increment the ref count, and each remove() will decrement it. The values are removed + * from the map iff ref count == 0. + */ [email protected] +class RefCountingMap<K, V> { + + private ConcurrentHashMap<K, Payload<V>> map = new ConcurrentHashMap<>(); + private static class Payload<V> { + V v; + volatile int refCount; + Payload(V v) { + this.v = v; + this.refCount = 1; // create with ref count = 1 + } + } + + V put(K key, Supplier<V> supplier) { + synchronized (map) { + Payload<V> oldValue = map.get(key); + if (oldValue == null) { + oldValue = new Payload<V>(supplier.get()); + map.put(key, oldValue); + return oldValue.v; + } + oldValue.refCount++; + return oldValue.v; + } + } + + V get(K k) { + Payload<V> p = map.get(k); + return p == null ? null : p.v; + } + + /** + * Decrements the ref count of k, and removes from map if ref count == 0. + * @param k the key to remove + * @return the value associated with the specified key or null if key is removed from map. + */ + V remove(K key) { + synchronized (map) { + Payload<V> oldValue = map.get(key); + if (oldValue != null) { + if (--oldValue.refCount == 0) { + map.remove(key); + return null; + } + return oldValue.v; + } + } + return null; + } + + void clear() { + map.clear(); + } + + Set<K> keySet() { + return map.keySet(); + } + + Collection<V> values() { + ArrayList<V> values = new ArrayList<V>(map.size()); + for (Payload<V> v : map.values()) { + values.add(v.v); + } + return values; + } + + int size() { + return map.size(); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/TimerImpl.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/TimerImpl.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/TimerImpl.java new file mode 100644 index 0000000..1cd8b17 --- /dev/null +++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/TimerImpl.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.Timer; + +/** + * Custom implementation of {@link Timer}. + */ [email protected] +public class TimerImpl implements Timer { + private final HistogramImpl histogram; + private final DropwizardMeter meter; + + // track time events in micros + private static final TimeUnit DEFAULT_UNIT = TimeUnit.MICROSECONDS; + + public TimerImpl() { + this.histogram = new HistogramImpl(); + this.meter = new DropwizardMeter(); + } + + @Override + public void update(long duration, TimeUnit unit) { + if (duration >= 0) { + histogram.update(DEFAULT_UNIT.convert(duration, unit)); + meter.mark(); + } + } + + @Override + public HistogramImpl getHistogram() { + return histogram; + } + + @Override + public DropwizardMeter getMeter() { + return meter; + } + + @Override + public void updateMillis(long durationMillis) { + update(durationMillis, TimeUnit.NANOSECONDS); + } + + @Override + public void updateMicros(long durationMicros) { + update(durationMicros, TimeUnit.MICROSECONDS); + } + + @Override + public void updateNanos(long durationNanos) { + update(durationNanos, TimeUnit.NANOSECONDS); + + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/package-info.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/package-info.java b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/package-info.java new file mode 100644 index 0000000..9c2e952 --- /dev/null +++ b/hbase-metrics/src/main/java/org/apache/hadoop/hbase/metrics/impl/package-info.java @@ -0,0 +1,25 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/** + * Implementation of the HBase Metrics framework. + */ +@PackageMarker +package org.apache.hadoop.hbase.metrics.impl; + +import org.apache.hadoop.hbase.metrics.PackageMarker; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/main/resources/META-INF/services/org.apache.hadoop.hbase.metrics.MetricRegistries ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/main/resources/META-INF/services/org.apache.hadoop.hbase.metrics.MetricRegistries b/hbase-metrics/src/main/resources/META-INF/services/org.apache.hadoop.hbase.metrics.MetricRegistries new file mode 100644 index 0000000..02edf2e --- /dev/null +++ b/hbase-metrics/src/main/resources/META-INF/services/org.apache.hadoop.hbase.metrics.MetricRegistries @@ -0,0 +1,18 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java new file mode 100644 index 0000000..f0b4f8c --- /dev/null +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestCounterImpl.java @@ -0,0 +1,59 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; + +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test class for {@link CounterImpl}. + */ +@Category(SmallTests.class) +public class TestCounterImpl { + + private Counter counter; + + @Before public void setup() { + this.counter = new CounterImpl(); + } + + @Test public void testCounting() { + counter.increment(); + assertEquals(1L, counter.getCount()); + counter.increment(); + assertEquals(2L, counter.getCount()); + counter.increment(2L); + assertEquals(4L, counter.getCount()); + counter.increment(-1L); + assertEquals(3L, counter.getCount()); + + counter.decrement(); + assertEquals(2L, counter.getCount()); + counter.decrement(); + assertEquals(1L, counter.getCount()); + counter.decrement(4L); + assertEquals(-3L, counter.getCount()); + counter.decrement(-3L); + assertEquals(0L, counter.getCount()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java new file mode 100644 index 0000000..3ba9821 --- /dev/null +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestDropwizardMeter.java @@ -0,0 +1,51 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.mockito.Mockito; + +import com.codahale.metrics.Meter; + +/** + * Test class for {@link DropwizardMeter}. + */ +@Category(SmallTests.class) +public class TestDropwizardMeter { + + private Meter meter; + + @Before public void setup() { + this.meter = Mockito.mock(Meter.class); + } + + @Test public void test() { + DropwizardMeter dwMeter = new DropwizardMeter(this.meter); + + dwMeter.mark(); + dwMeter.mark(10L); + dwMeter.mark(); + dwMeter.mark(); + + Mockito.verify(meter, Mockito.times(3)).mark(); + Mockito.verify(meter).mark(10L); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java new file mode 100644 index 0000000..3dcd4fe --- /dev/null +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestFastLongHistogram.java @@ -0,0 +1,132 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; + +import java.util.Arrays; +import java.util.Random; + + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Testcases for FastLongHistogram. + */ +@Category(SmallTests.class) +public class TestFastLongHistogram { + + private static void doTestUniform(FastLongHistogram hist) { + long[] VALUES = { 0, 10, 20, 30, 40, 50 }; + double[] qs = new double[VALUES.length]; + for (int i = 0; i < qs.length; i++) { + qs[i] = (double) VALUES[i] / VALUES[VALUES.length - 1]; + } + + for (int i = 0; i < 10; i++) { + for (long v : VALUES) { + hist.add(v, 1); + } + long[] vals = hist.getQuantiles(qs); + System.out.println(Arrays.toString(vals)); + for (int j = 0; j < qs.length; j++) { + Assert.assertTrue(j + "-th element org: " + VALUES[j] + ", act: " + vals[j], + Math.abs(vals[j] - VALUES[j]) <= 10); + } + hist.snapshotAndReset(); + } + } + + @Test + public void testUniform() { + FastLongHistogram hist = new FastLongHistogram(100, 0, 50); + doTestUniform(hist); + } + + @Test + public void testAdaptionOfChange() { + // assumes the uniform distribution + FastLongHistogram hist = new FastLongHistogram(100, 0, 100); + + Random rand = new Random(); + + for (int n = 0; n < 10; n++) { + for (int i = 0; i < 900; i++) { + hist.add(rand.nextInt(100), 1); + } + + // add 10% outliers, this breaks the assumption, hope bin10xMax works + for (int i = 0; i < 100; i++) { + hist.add(1000 + rand.nextInt(100), 1); + } + + long[] vals = hist.getQuantiles(new double[] { 0.25, 0.75, 0.95 }); + System.out.println(Arrays.toString(vals)); + if (n == 0) { + Assert.assertTrue("Out of possible value", vals[0] >= 0 && vals[0] <= 50); + Assert.assertTrue("Out of possible value", vals[1] >= 50 && vals[1] <= 100); + Assert.assertTrue("Out of possible value", vals[2] >= 900 && vals[2] <= 1100); + } + + hist.snapshotAndReset(); + } + } + + + @Test + public void testGetNumAtOrBelow() { + long[] VALUES = { 1, 10, 20, 30, 40, 50 }; + + FastLongHistogram h = new FastLongHistogram(); + for (long v : VALUES) { + for (int i = 0; i < 100; i++) { + h.add(v, 1); + } + } + + h.add(Integer.MAX_VALUE, 1); + + h.snapshotAndReset(); + + for (long v : VALUES) { + for (int i = 0; i < 100; i++) { + h.add(v, 1); + } + } + // Add something way out there to make sure it doesn't throw off the counts. + h.add(Integer.MAX_VALUE, 1); + + assertEquals(100, h.getNumAtOrBelow(1)); + assertEquals(200, h.getNumAtOrBelow(11)); + assertEquals(601, h.getNumAtOrBelow(Long.MAX_VALUE)); + } + + + @Test + public void testSameValues() { + FastLongHistogram hist = new FastLongHistogram(100); + + hist.add(50, 100); + + hist.snapshotAndReset(); + doTestUniform(hist); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java new file mode 100644 index 0000000..7927110 --- /dev/null +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestGauge.java @@ -0,0 +1,61 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; + +import java.util.concurrent.atomic.AtomicLong; + +import org.apache.hadoop.hbase.metrics.Gauge; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test class for {@link Gauge}. + */ +@Category(SmallTests.class) +public class TestGauge { + + @Test + public void testGetValue() { + SimpleGauge gauge = new SimpleGauge(); + + assertEquals(0, (long)gauge.getValue()); + + gauge.setValue(1000L); + + assertEquals(1000L, (long)gauge.getValue()); + } + + /** + * Gauge implementation with a setter. + */ + private static class SimpleGauge implements Gauge<Long> { + + private final AtomicLong value = new AtomicLong(0L); + + @Override public Long getValue() { + return this.value.get(); + } + + public void setValue(long value) { + this.value.set(value); + } + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java new file mode 100644 index 0000000..5d3b1fa --- /dev/null +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestHistogramImpl.java @@ -0,0 +1,103 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; + + +import org.apache.hadoop.hbase.metrics.Snapshot; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test case for {@link HistogramImpl} + */ +@Category(SmallTests.class) +public class TestHistogramImpl { + + @Test + public void testUpdate() { + HistogramImpl histogram = new HistogramImpl(); + + assertEquals(0, histogram.getCount()); + + histogram.update(0); + assertEquals(1, histogram.getCount()); + + histogram.update(10); + assertEquals(2, histogram.getCount()); + + histogram.update(20); + histogram.update(30); + + assertEquals(4, histogram.getCount()); + } + + @Test + public void testSnapshot() { + HistogramImpl histogram = new HistogramImpl(); + for (int i = 0; i < 100; i++) { + histogram.update(i); + } + + Snapshot snapshot = histogram.snapshot(); + + assertEquals(100, snapshot.getCount()); + assertEquals(50, snapshot.getMedian()); + assertEquals(49, snapshot.getMean()); + assertEquals(0, snapshot.getMin()); + assertEquals(99, snapshot.getMax()); + assertEquals(25, snapshot.get25thPercentile()); + assertEquals(75, snapshot.get75thPercentile()); + assertEquals(90, snapshot.get90thPercentile()); + assertEquals(95, snapshot.get95thPercentile()); + assertEquals(98, snapshot.get98thPercentile()); + assertEquals(99, snapshot.get99thPercentile()); + assertEquals(99, snapshot.get999thPercentile()); + + assertEquals(51, snapshot.getCountAtOrBelow(50)); + + // check that histogram is reset. + assertEquals(100, histogram.getCount()); // count does not reset + + // put more data after reset + for (int i = 100; i < 200; i++) { + histogram.update(i); + } + + assertEquals(200, histogram.getCount()); + + snapshot = histogram.snapshot(); + assertEquals(100, snapshot.getCount()); // only 100 more events + assertEquals(150, snapshot.getMedian()); + assertEquals(149, snapshot.getMean()); + assertEquals(100, snapshot.getMin()); + assertEquals(199, snapshot.getMax()); + assertEquals(125, snapshot.get25thPercentile()); + assertEquals(175, snapshot.get75thPercentile()); + assertEquals(190, snapshot.get90thPercentile()); + assertEquals(195, snapshot.get95thPercentile()); + assertEquals(198, snapshot.get98thPercentile()); + assertEquals(199, snapshot.get99thPercentile()); + assertEquals(199, snapshot.get999thPercentile()); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java new file mode 100644 index 0000000..1917ed4 --- /dev/null +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestMetricRegistryImpl.java @@ -0,0 +1,164 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +import java.util.Map; + +import org.apache.hadoop.hbase.metrics.Counter; +import org.apache.hadoop.hbase.metrics.Gauge; +import org.apache.hadoop.hbase.metrics.Meter; +import org.apache.hadoop.hbase.metrics.Metric; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; +import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.base.Optional; + +@Category(SmallTests.class) +public class TestMetricRegistryImpl { + private MetricRegistryInfo info; + private MetricRegistryImpl registry; + + @Before + public void setUp() { + info = new MetricRegistryInfo("foo", "bar", "baz", "foobar", false); + registry = new MetricRegistryImpl(info); + } + + @Test + public void testCounter() { + Counter counter = registry.counter("mycounter"); + assertNotNull(counter); + counter.increment(42L); + Optional<Metric> metric = registry.get("mycounter"); + assertTrue(metric.isPresent()); + assertEquals(42L, (long)((Counter)metric.get()).getCount()); + } + + @Test + public void testRegisterGauge() { + registry.register("mygauge", new Gauge<Long>() { + @Override + public Long getValue() { + return 42L; + } + }); + Optional<Metric> metric = registry.get("mygauge"); + assertTrue(metric.isPresent()); + assertEquals(42L, (long)((Gauge<Long>)metric.get()).getValue()); + } + + @Test + public void testRegisterGaugeLambda() { + // register a Gauge using lambda expression + registry.register("gaugeLambda", new Gauge<Long>(){ + + @Override + public Long getValue() { + return 42L; + } + }); + Optional<Metric> metric = registry.get("gaugeLambda"); + assertTrue(metric.isPresent()); + assertEquals(42L, (long)((Gauge<Long>)metric.get()).getValue()); + } + + @Test + public void testTimer() { + Timer timer = registry.timer("mytimer"); + assertNotNull(timer); + timer.updateNanos(100); + } + + @Test + public void testMeter() { + Meter meter = registry.meter("mymeter"); + assertNotNull(meter); + meter.mark(); + } + + @Test + public void testRegister() { + CounterImpl counter = new CounterImpl(); + registry.register("mycounter", counter); + counter.increment(42L); + + Optional<Metric> metric = registry.get("mycounter"); + assertTrue(metric.isPresent()); + assertEquals(42L, (long)((Counter)metric.get()).getCount()); + } + + @Test + public void testDoubleRegister() { + Gauge g1 = registry.register("mygauge", new Gauge<Long>(){ + + @Override + public Long getValue() { + return 42L; + }}); + Gauge g2 = registry.register("mygauge", new Gauge<Long>(){ + + @Override + public Long getValue() { + return 52L; + }}); + + // second gauge is ignored if it exists + assertEquals(g1, g2); + + Optional<Metric> metric = registry.get("mygauge"); + assertTrue(metric.isPresent()); + assertEquals(42L, (long)((Gauge<Long>)metric.get()).getValue()); + + + Counter c1 = registry.counter("mycounter"); + Counter c2 = registry.counter("mycounter"); + + assertEquals(c1, c2); + } + + @Test + public void testGetMetrics() { + CounterImpl counter = new CounterImpl(); + registry.register("mycounter", counter); + Gauge gauge = registry.register("mygauge", new Gauge<Long>(){ + + @Override + public Long getValue() { + return 42L; + }}); + Timer timer = registry.timer("mytimer"); + + Map<String, Metric> metrics = registry.getMetrics(); + assertEquals(3, metrics.size()); + + assertEquals(counter, metrics.get("mycounter")); + assertEquals(gauge, metrics.get("mygauge")); + assertEquals(timer, metrics.get("mytimer")); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java new file mode 100644 index 0000000..128f615 --- /dev/null +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestRefCountingMap.java @@ -0,0 +1,284 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +import java.util.Collection; +import java.util.Set; + +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +import com.google.common.base.Supplier; +import com.google.common.collect.Lists; + +@Category(SmallTests.class) +public class TestRefCountingMap { + + private RefCountingMap<String, String> map; + + @Before + public void setUp() { + map = new RefCountingMap<>(); + } + + @Test + public void testPutGet() { + map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue"; + } + }); + + String v = map.get("foo"); + assertNotNull(v); + assertEquals("foovalue", v); + } + + @Test + public void testPutMulti() { + String v1 = map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue"; + } + }); + String v2 = map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue2"; + } + }); + String v3 = map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue3"; + } + }); + + String v = map.get("foo"); + assertEquals("foovalue", v); + assertEquals(v, v1); + assertEquals(v, v2); + assertEquals(v, v3); + } + + @Test + public void testPutRemove() { + map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue"; + } + }); + String v = map.remove("foo"); + assertNull(v); + v = map.get("foo"); + assertNull(v); + } + + @Test + public void testPutRemoveMulti() { + map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue"; + } + }); + map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue2"; + } + }); + map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue3"; + } + }); + + // remove 1 + String v = map.remove("foo"); + assertEquals("foovalue", v); + + // remove 2 + v = map.remove("foo"); + assertEquals("foovalue", v); + + // remove 3 + v = map.remove("foo"); + assertNull(v); + v = map.get("foo"); + assertNull(v); + } + + @Test + public void testSize() { + assertEquals(0, map.size()); + + // put a key + map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue"; + } + }); + assertEquals(1, map.size()); + + // put a different key + map.put("bar", new Supplier<String>() { + + @Override + public String get() { + return "foovalue2"; + } + }); + assertEquals(2, map.size()); + + // put the same key again + map.put("bar", new Supplier<String>() { + + @Override + public String get() { + return "foovalue3"; + } + }); + assertEquals(2, map.size()); // map should be same size + } + + @Test + public void testClear() { + map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue"; + } + }); + map.put("bar", new Supplier<String>() { + + @Override + public String get() { + return "foovalue2"; + } + }); + map.put("baz", new Supplier<String>() { + + @Override + public String get() { + return "foovalue3"; + } + }); + + map.clear(); + + assertEquals(0, map.size()); + } + + + @Test + public void testKeySet() { + map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue"; + } + }); + map.put("bar", new Supplier<String>() { + + @Override + public String get() { + return "foovalue2"; + } + }); + map.put("baz", new Supplier<String>() { + + @Override + public String get() { + return "foovalue3"; + } + }); + + Set<String> keys = map.keySet(); + assertEquals(3, keys.size()); + + for (String v : Lists.newArrayList("foo", "bar", "baz")) { + assertTrue(keys.contains(v)); + } + } + + @Test + public void testValues() { + map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue"; + } + }); + map.put("foo", new Supplier<String>() { + + @Override + public String get() { + return "foovalue2"; + } + }); + map.put("bar", new Supplier<String>() { + + @Override + public String get() { + return "foovalue3"; + } + }); + map.put("baz", new Supplier<String>() { + + @Override + public String get() { + return "foovalue4"; + } + }); + + Collection<String> values = map.values(); + assertEquals(3, values.size()); + + for (String v : Lists.newArrayList("foovalue", "foovalue3", "foovalue4")) { + assertTrue(values.contains(v)); + } + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java ---------------------------------------------------------------------- diff --git a/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java new file mode 100644 index 0000000..2b3dc8f --- /dev/null +++ b/hbase-metrics/src/test/java/org/apache/hadoop/hbase/metrics/impl/TestTimerImpl.java @@ -0,0 +1,53 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.hbase.metrics.impl; + +import static org.junit.Assert.assertEquals; + +import java.util.concurrent.TimeUnit; + +import org.apache.hadoop.hbase.metrics.Timer; +import org.apache.hadoop.hbase.testclassification.SmallTests; +import org.junit.Before; +import org.junit.Test; +import org.junit.experimental.categories.Category; + +/** + * Test class for {@link TimerImpl} + */ +@Category(SmallTests.class) +public class TestTimerImpl { + + private Timer timer; + + @Before + public void setup() { + this.timer = new TimerImpl(); + } + + @Test + public void testUpdate() { + timer.update(40, TimeUnit.MILLISECONDS); + timer.update(41, TimeUnit.MILLISECONDS); + timer.update(42, TimeUnit.MILLISECONDS); + assertEquals(3, timer.getHistogram().getCount()); + assertEquals(3, timer.getMeter().getCount()); + + assertEquals(TimeUnit.MILLISECONDS.toMicros(41), timer.getHistogram().snapshot().getMedian()); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-resource-bundle/src/main/resources/supplemental-models.xml ---------------------------------------------------------------------- diff --git a/hbase-resource-bundle/src/main/resources/supplemental-models.xml b/hbase-resource-bundle/src/main/resources/supplemental-models.xml index eac4cad..3f01f09 100644 --- a/hbase-resource-bundle/src/main/resources/supplemental-models.xml +++ b/hbase-resource-bundle/src/main/resources/supplemental-models.xml @@ -1065,6 +1065,20 @@ under the License. </licenses> </project> </supplement> + <supplement> + <project> + <groupId>io.dropwizard.metrics</groupId> + <artifactId>metrics-core</artifactId> + + <licenses> + <license> + <name>Apache License, Version 2.0</name> + <url>http://www.apache.org/licenses/LICENSE-2.0.txt</url> + <distribution>repo</distribution> + </license> + </licenses> + </project> + </supplement> <supplement> <project> <groupId>org.codehaus.jettison</groupId> http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-server/pom.xml ---------------------------------------------------------------------- diff --git a/hbase-server/pom.xml b/hbase-server/pom.xml index c1a5329..e765b1d 100644 --- a/hbase-server/pom.xml +++ b/hbase-server/pom.xml @@ -397,6 +397,14 @@ <type>test-jar</type> <scope>test</scope> </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-metrics-api</artifactId> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-metrics</artifactId> + </dependency> <!-- resource bundle only needed at build time --> <dependency> <groupId>org.apache.hbase</groupId> http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java index abe934a..a6e54e0 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterCoprocessorEnvironment.java @@ -24,10 +24,20 @@ import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.master.MasterServices; +import org.apache.hadoop.hbase.metrics.MetricRegistry; @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) @InterfaceStability.Evolving public interface MasterCoprocessorEnvironment extends CoprocessorEnvironment { /** @return reference to the HMaster services */ MasterServices getMasterServices(); + + /** + * Returns a MetricRegistry that can be used to track metrics at the master level. + * + * <p>See ExampleMasterObserverWithMetrics class in the hbase-examples modules for examples + * of how metrics can be instantiated and used.</p> + * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + MetricRegistry getMetricRegistryForMaster(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetricsCoprocessor.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetricsCoprocessor.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetricsCoprocessor.java new file mode 100644 index 0000000..d564002 --- /dev/null +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MetricsCoprocessor.java @@ -0,0 +1,136 @@ +/** + * + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.apache.hadoop.hbase.coprocessor; + +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.MetricRegistries; +import org.apache.hadoop.hbase.metrics.MetricRegistry; +import org.apache.hadoop.hbase.metrics.MetricRegistryInfo; + +import com.google.common.annotations.VisibleForTesting; + +/** + * Utility class for tracking metrics for various types of coprocessors. Each coprocessor instance + * creates its own MetricRegistry which is exported as an individual MetricSource. MetricRegistries + * are ref counted using the hbase-metric module interfaces. + */ [email protected] +public class MetricsCoprocessor { + + // Master coprocessor metrics + private static final String MASTER_COPROC_METRICS_NAME = "Coprocessor.Master"; + private static final String MASTER_COPROC_METRICS_CONTEXT = "master"; + private static final String MASTER_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase MasterObservers"; + private static final String MASTER_COPROC_METRICS_JMX_CONTEXT + = "Master,sub=" + MASTER_COPROC_METRICS_NAME; + + // RegionServer coprocessor metrics + private static final String RS_COPROC_METRICS_NAME = "Coprocessor.RegionServer"; + private static final String RS_COPROC_METRICS_CONTEXT = "regionserver"; + private static final String RS_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase RegionServerObservers"; + private static final String RS_COPROC_METRICS_JMX_CONTEXT = "RegionServer,sub=" + + RS_COPROC_METRICS_NAME; + + // Region coprocessor metrics + private static final String REGION_COPROC_METRICS_NAME = "Coprocessor.Region"; + private static final String REGION_COPROC_METRICS_CONTEXT = "regionserver"; + private static final String REGION_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase RegionObservers"; + private static final String REGION_COPROC_METRICS_JMX_CONTEXT + = "RegionServer,sub=" + REGION_COPROC_METRICS_NAME; + + // WAL coprocessor metrics + private static final String WAL_COPROC_METRICS_NAME = "Coprocessor.WAL"; + private static final String WAL_COPROC_METRICS_CONTEXT = "regionserver"; + private static final String WAL_COPROC_METRICS_DESCRIPTION + = "Metrics about HBase WALObservers"; + private static final String WAL_COPROC_METRICS_JMX_CONTEXT + = "RegionServer,sub=" + WAL_COPROC_METRICS_NAME; + + private static String suffix(String metricName, String cpName) { + return new StringBuilder(metricName) + .append(".") + .append("CP_") + .append(cpName) + .toString(); + } + + @VisibleForTesting + static MetricRegistryInfo createRegistryInfoForMasterCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(MASTER_COPROC_METRICS_NAME, clazz), + MASTER_COPROC_METRICS_DESCRIPTION, + suffix(MASTER_COPROC_METRICS_JMX_CONTEXT, clazz), + MASTER_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForMasterCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForMasterCoprocessor(clazz)); + } + + @VisibleForTesting + static MetricRegistryInfo createRegistryInfoForRSCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(RS_COPROC_METRICS_NAME, clazz), + RS_COPROC_METRICS_DESCRIPTION, + suffix(RS_COPROC_METRICS_JMX_CONTEXT, clazz), + RS_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForRSCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForRSCoprocessor(clazz)); + } + + @VisibleForTesting + public static MetricRegistryInfo createRegistryInfoForRegionCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(REGION_COPROC_METRICS_NAME, clazz), + REGION_COPROC_METRICS_DESCRIPTION, + suffix(REGION_COPROC_METRICS_JMX_CONTEXT, clazz), + REGION_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForRegionCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForRegionCoprocessor(clazz)); + } + + @VisibleForTesting + public static MetricRegistryInfo createRegistryInfoForWALCoprocessor(String clazz) { + return new MetricRegistryInfo( + suffix(WAL_COPROC_METRICS_NAME, clazz), + WAL_COPROC_METRICS_DESCRIPTION, + suffix(WAL_COPROC_METRICS_JMX_CONTEXT, clazz), + WAL_COPROC_METRICS_CONTEXT, false); + } + + public static MetricRegistry createRegistryForWALCoprocessor(String clazz) { + return MetricRegistries.global().create(createRegistryInfoForWALCoprocessor(clazz)); + } + + public static void removeRegistry(MetricRegistry registry) { + if (registry == null) { + return; + } + MetricRegistries.global().remove(registry.getMetricRegistryInfo()); + } +} http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java index bdf88af..3566f06 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionCoprocessorEnvironment.java @@ -21,11 +21,12 @@ package org.apache.hadoop.hbase.coprocessor; import java.util.concurrent.ConcurrentMap; -import org.apache.hadoop.hbase.classification.InterfaceAudience; -import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; import org.apache.hadoop.hbase.HRegionInfo; +import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.classification.InterfaceStability; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.regionserver.Region; import org.apache.hadoop.hbase.regionserver.RegionServerServices; @@ -43,4 +44,23 @@ public interface RegionCoprocessorEnvironment extends CoprocessorEnvironment { /** @return shared data between all instances of this coprocessor */ ConcurrentMap<String, Object> getSharedData(); + + /** + * Returns a MetricRegistry that can be used to track metrics at the region server level. All + * metrics tracked at this level will be shared by all the coprocessor instances + * of the same class in the same region server process. Note that there will be one + * region coprocessor environment per region in the server, but all of these instances will share + * the same MetricRegistry. The metric instances (like Counter, Timer, etc) will also be shared + * among all of the region coprocessor instances. + * + * <p>See ExampleRegionObserverWithMetrics class in the hbase-examples modules to see examples of how + * metrics can be instantiated and used.</p> + * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + // Note: we are not exposing getMetricRegistryForRegion(). per-region metrics are already costly + // so we do not want to allow coprocessors to export metrics at the region level. We can allow + // getMetricRegistryForTable() to allow coprocessors to track metrics per-table, per-regionserver. + MetricRegistry getMetricRegistryForRegionServer(); + + } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java index 527df45..f42556a 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/RegionServerCoprocessorEnvironment.java @@ -19,6 +19,7 @@ package org.apache.hadoop.hbase.coprocessor; import org.apache.hadoop.hbase.CoprocessorEnvironment; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.regionserver.RegionServerServices; public interface RegionServerCoprocessorEnvironment extends CoprocessorEnvironment { @@ -28,4 +29,13 @@ public interface RegionServerCoprocessorEnvironment extends CoprocessorEnvironme * @return the region server services */ RegionServerServices getRegionServerServices(); + + /** + * Returns a MetricRegistry that can be used to track metrics at the region server level. + * + * <p>See ExampleMasterObserverWithMetrics class in the hbase-examples modules for examples + * of how metrics can be instantiated and used.</p> + * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + MetricRegistry getMetricRegistryForRegionServer(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java index a4ce5f1..0865d96 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/WALCoprocessorEnvironment.java @@ -23,6 +23,7 @@ import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.classification.InterfaceStability; import org.apache.hadoop.hbase.CoprocessorEnvironment; import org.apache.hadoop.hbase.HBaseInterfaceAudience; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.wal.WAL; @InterfaceAudience.LimitedPrivate(HBaseInterfaceAudience.COPROC) @@ -30,4 +31,13 @@ import org.apache.hadoop.hbase.wal.WAL; public interface WALCoprocessorEnvironment extends CoprocessorEnvironment { /** @return reference to the region server's WAL */ WAL getWAL(); + + /** + * Returns a MetricRegistry that can be used to track metrics at the region server level. + * + * <p>See ExampleRegionServerObserverWithMetrics class in the hbase-examples modules for examples + * of how metrics can be instantiated and used.</p> + * @return A MetricRegistry for the coprocessor class to track and export metrics. + */ + MetricRegistry getMetricRegistryForRegionServer(); } http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java index 4c1ad23..dd3bf25 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/AgeSnapshot.java @@ -17,7 +17,7 @@ */ package org.apache.hadoop.hbase.io.hfile; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; import org.codehaus.jackson.annotate.JsonIgnoreProperties; /** http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java index d868a1a..e5bb83b 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/BlockCacheUtil.java @@ -25,7 +25,7 @@ import java.util.concurrent.ConcurrentSkipListSet; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hbase.util.FastLongHistogram; +import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; import org.codehaus.jackson.JsonGenerationException; import org.codehaus.jackson.annotate.JsonIgnoreProperties; import org.codehaus.jackson.map.JsonMappingException; http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java index f38ec70..9249271 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/io/hfile/CacheStats.java @@ -21,9 +21,9 @@ package org.apache.hadoop.hbase.io.hfile; import java.util.concurrent.atomic.AtomicLong; import org.apache.hadoop.hbase.classification.InterfaceAudience; +import org.apache.hadoop.hbase.metrics.impl.FastLongHistogram; import org.apache.hadoop.hbase.util.Counter; -import org.apache.hadoop.hbase.util.FastLongHistogram; /** * Class that implements cache metrics. http://git-wip-us.apache.org/repos/asf/hbase/blob/a3c3f101/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java index 9fb8d81..c7dd282 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java @@ -36,8 +36,14 @@ import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.classification.InterfaceAudience; import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.coprocessor.*; +import org.apache.hadoop.hbase.coprocessor.CoprocessorHost; +import org.apache.hadoop.hbase.coprocessor.CoprocessorService; +import org.apache.hadoop.hbase.coprocessor.MasterCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.MasterObserver; +import org.apache.hadoop.hbase.coprocessor.MetricsCoprocessor; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv; +import org.apache.hadoop.hbase.metrics.MetricRegistry; import org.apache.hadoop.hbase.procedure2.ProcedureExecutor; import org.apache.hadoop.hbase.protobuf.generated.QuotaProtos.Quotas; import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotDescription; @@ -59,18 +65,33 @@ public class MasterCoprocessorHost */ static class MasterEnvironment extends CoprocessorHost.Environment implements MasterCoprocessorEnvironment { - private MasterServices masterServices; + private final MasterServices masterServices; + private final MetricRegistry metricRegistry; public MasterEnvironment(final Class<?> implClass, final Coprocessor impl, final int priority, final int seq, final Configuration conf, final MasterServices services) { super(impl, priority, seq, conf); this.masterServices = services; + this.metricRegistry = + MetricsCoprocessor.createRegistryForMasterCoprocessor(implClass.getName()); } + @Override public MasterServices getMasterServices() { return masterServices; } + + @Override + public MetricRegistry getMetricRegistryForMaster() { + return metricRegistry; + } + + @Override + protected void shutdown() { + super.shutdown(); + MetricsCoprocessor.removeRegistry(this.metricRegistry); + } } private MasterServices masterServices;
