This is an automated email from the ASF dual-hosted git repository.
adar pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 9339f6d KUDU-2934: add MeanGauge type to metrics
9339f6d is described below
commit 9339f6dda1c30e007bab3cfb0f37a9dcb327e683
Author: zhangyifan27 <[email protected]>
AuthorDate: Wed Sep 18 15:30:28 2019 +0800
KUDU-2934: add MeanGauge type to metrics
The current merge method is not suitable for the metric
'average_diskrowset_height', it is a mean value and we
could not just sum up the metric values when merging
metrics.
There is another issue with the metric 'average_diskrowset_height',
the value of the metric is double but it was used as an AtomicGauge
metric, which is only for types that are convertible to/from int64_t.
This patch added a new 'MeanGauge' type for metrics which show
the mean value of something.
Change-Id: I827d3acb5d4f47a7e28ac40cb7df392c29c82a40
Reviewed-on: http://gerrit.cloudera.org:8080/14252
Reviewed-by: Adar Dembo <[email protected]>
Tested-by: Adar Dembo <[email protected]>
---
src/kudu/tablet/compaction_policy-test.cc | 10 +++++--
src/kudu/tablet/compaction_policy.cc | 3 +-
src/kudu/tablet/rowset_info.cc | 9 +++---
src/kudu/tablet/rowset_info.h | 8 ++++--
src/kudu/tablet/tablet.cc | 20 +++++++++----
src/kudu/tablet/tablet_metrics.cc | 4 ++-
src/kudu/tablet/tablet_metrics.h | 4 +--
src/kudu/util/metrics-test.cc | 33 +++++++++++++++++++++
src/kudu/util/metrics.cc | 48 +++++++++++++++++++++++++++++++
src/kudu/util/metrics.h | 48 +++++++++++++++++++++++++++++++
10 files changed, 167 insertions(+), 20 deletions(-)
diff --git a/src/kudu/tablet/compaction_policy-test.cc
b/src/kudu/tablet/compaction_policy-test.cc
index 34824b0..69c1bdf 100644
--- a/src/kudu/tablet/compaction_policy-test.cc
+++ b/src/kudu/tablet/compaction_policy-test.cc
@@ -445,9 +445,13 @@ double ComputeAverageRowsetHeight(
RowSetTree tree;
CHECK_OK(tree.Reset(rowsets));
- double avg_height;
- RowSetInfo::ComputeCdfAndCollectOrdered(tree, &avg_height, nullptr, nullptr);
- return avg_height;
+ double rowset_total_height, rowset_total_width;
+ RowSetInfo::ComputeCdfAndCollectOrdered(tree,
+ &rowset_total_height,
+ &rowset_total_width,
+ nullptr,
+ nullptr);
+ return rowset_total_width > 0 ? rowset_total_height / rowset_total_width :
0.0;
}
} // anonymous namespace
diff --git a/src/kudu/tablet/compaction_policy.cc
b/src/kudu/tablet/compaction_policy.cc
index e740ac4..0d96cfc 100644
--- a/src/kudu/tablet/compaction_policy.cc
+++ b/src/kudu/tablet/compaction_policy.cc
@@ -107,7 +107,8 @@ void BudgetedCompactionPolicy::SetupKnapsackInput(
vector<RowSetInfo>* asc_min_key,
vector<RowSetInfo>* asc_max_key) const {
RowSetInfo::ComputeCdfAndCollectOrdered(tree,
- /*average_height=*/nullptr,
+ /*rowset_total_height=*/nullptr,
+ /*rowset_total_width=*/nullptr,
asc_min_key,
asc_max_key);
diff --git a/src/kudu/tablet/rowset_info.cc b/src/kudu/tablet/rowset_info.cc
index 187a96f..f276125 100644
--- a/src/kudu/tablet/rowset_info.cc
+++ b/src/kudu/tablet/rowset_info.cc
@@ -253,7 +253,8 @@ void RowSetInfo::Collect(const RowSetTree& tree,
vector<RowSetInfo>* rsvec) {
}
void RowSetInfo::ComputeCdfAndCollectOrdered(const RowSetTree& tree,
- double* average_height,
+ double* rowset_total_height,
+ double* rowset_total_width,
vector<RowSetInfo>*
info_by_min_key,
vector<RowSetInfo>*
info_by_max_key) {
DCHECK((info_by_min_key && info_by_max_key) ||
@@ -352,9 +353,9 @@ void RowSetInfo::ComputeCdfAndCollectOrdered(const
RowSetTree& tree,
FinalizeCDFVector(total_width, &info_by_min_key_tmp);
FinalizeCDFVector(total_width, &info_by_max_key_tmp);
- if (average_height) {
- *average_height = total_width > 0 ? weighted_height_sum / total_width
- : 0.0;
+ if (rowset_total_height && rowset_total_width) {
+ *rowset_total_height = weighted_height_sum;
+ *rowset_total_width = total_width;
}
if (info_by_min_key && info_by_max_key) {
diff --git a/src/kudu/tablet/rowset_info.h b/src/kudu/tablet/rowset_info.h
index f133e3e..585c9cb 100644
--- a/src/kudu/tablet/rowset_info.h
+++ b/src/kudu/tablet/rowset_info.h
@@ -47,14 +47,16 @@ class RowSetInfo {
// From the rowset tree 'tree', computes the keyspace cdf and collects rowset
// information in min-key- and max-key-sorted order into 'info_by_min_key'
- // and 'info_by_max_key', respectively. The average value of the height of
the
- // rowset tree is set into 'average_height', if it is not nullptr.
+ // and 'info_by_max_key', respectively.
+ // The total weighted height and the total width of the rowset tree is set
into
+ // 'rowset_total_height' and 'rowset_total_width', if they are not nullptr.
// If one of 'info_by_min_key' and 'info_by_max_key' is nullptr, the other
// must be.
// Requires holding the compact_select_lock_ for the tablet that the
// rowsets in 'tree' references.
static void ComputeCdfAndCollectOrdered(const RowSetTree& tree,
- double* average_height,
+ double* rowset_total_height,
+ double* rowset_total_width,
std::vector<RowSetInfo>*
info_by_min_key,
std::vector<RowSetInfo>*
info_by_max_key);
diff --git a/src/kudu/tablet/tablet.cc b/src/kudu/tablet/tablet.cc
index 78eff66..5c3db23 100644
--- a/src/kudu/tablet/tablet.cc
+++ b/src/kudu/tablet/tablet.cc
@@ -1725,12 +1725,13 @@ void Tablet::UpdateAverageRowsetHeight() {
scoped_refptr<TabletComponents> comps;
GetComponents(&comps);
std::lock_guard<std::mutex> l(compact_select_lock_);
- double avg_height;
+ double rowset_total_height, rowset_total_width;
RowSetInfo::ComputeCdfAndCollectOrdered(*comps->rowsets,
- &avg_height,
+ &rowset_total_height,
+ &rowset_total_width,
nullptr,
nullptr);
- metrics_->average_diskrowset_height->set_value(avg_height);
+ metrics_->average_diskrowset_height->set_value(rowset_total_height,
rowset_total_width);
}
Status Tablet::Compact(CompactFlags flags) {
@@ -2358,9 +2359,16 @@ void Tablet::PrintRSLayout(ostream* o) {
out << "</p>";
}
- double avg_height;
+ double rowset_total_height, rowset_total_width;
vector<RowSetInfo> min, max;
- RowSetInfo::ComputeCdfAndCollectOrdered(*rowsets_copy, &avg_height, &min,
&max);
+ RowSetInfo::ComputeCdfAndCollectOrdered(*rowsets_copy,
+ &rowset_total_height,
+ &rowset_total_width,
+ &min,
+ &max);
+ double average_rowset_height = rowset_total_width > 0
+ ? rowset_total_height / rowset_total_width
+ : 0.0;
DumpCompactionSVG(min, picked, o, /*print_xml_header=*/false);
// Compaction policy ignores rowsets unavailable for compaction. This is
good,
@@ -2422,7 +2430,7 @@ void Tablet::PrintRSLayout(ostream* o) {
HumanReadableNumBytes::ToString(size_bytes_median),
HumanReadableNumBytes::ToString(size_bytes_third_quartile),
HumanReadableNumBytes::ToString(size_bytes_max),
- avg_height);
+ average_rowset_height);
out << "</table>" << endl;
}
diff --git a/src/kudu/tablet/tablet_metrics.cc
b/src/kudu/tablet/tablet_metrics.cc
index 5aa4fa2..e266af1 100644
--- a/src/kudu/tablet/tablet_metrics.cc
+++ b/src/kudu/tablet/tablet_metrics.cc
@@ -269,6 +269,7 @@ namespace tablet {
#define MINIT(x) x(METRIC_##x.Instantiate(entity))
#define GINIT(x) x(METRIC_##x.Instantiate(entity, 0))
+#define MEANINIT(x) x(METRIC_##x.InstantiateMeanGauge(entity))
TabletMetrics::TabletMetrics(const scoped_refptr<MetricEntity>& entity)
: MINIT(rows_inserted),
MINIT(rows_upserted),
@@ -313,10 +314,11 @@ TabletMetrics::TabletMetrics(const
scoped_refptr<MetricEntity>& entity)
MINIT(undo_delta_block_gc_delete_duration),
MINIT(undo_delta_block_gc_perform_duration),
MINIT(leader_memory_pressure_rejections),
- GINIT(average_diskrowset_height) {
+ MEANINIT(average_diskrowset_height) {
}
#undef MINIT
#undef GINIT
+#undef MEANINIT
void TabletMetrics::AddProbeStats(const ProbeStats* stats_array, int len,
Arena* work_arena) {
diff --git a/src/kudu/tablet/tablet_metrics.h b/src/kudu/tablet/tablet_metrics.h
index c60b9aa..b17098f 100644
--- a/src/kudu/tablet/tablet_metrics.h
+++ b/src/kudu/tablet/tablet_metrics.h
@@ -33,7 +33,7 @@ struct ProbeStats;
// Container for all metrics specific to a single tablet.
struct TabletMetrics {
- explicit TabletMetrics(const scoped_refptr<MetricEntity>& metric_entity);
+ explicit TabletMetrics(const scoped_refptr<MetricEntity>& entity);
// Add a batch of probe stats to the metrics.
//
@@ -100,7 +100,7 @@ struct TabletMetrics {
scoped_refptr<Counter> leader_memory_pressure_rejections;
// Compaction metrics.
- scoped_refptr<AtomicGauge<double>> average_diskrowset_height;
+ scoped_refptr<MeanGauge> average_diskrowset_height;
};
} // namespace tablet
diff --git a/src/kudu/util/metrics-test.cc b/src/kudu/util/metrics-test.cc
index 7e5bbbc..55ddecf 100644
--- a/src/kudu/util/metrics-test.cc
+++ b/src/kudu/util/metrics-test.cc
@@ -168,6 +168,39 @@ TEST_F(MetricsTest, SimpleStringGaugeForMergeTest) {
state_for_merge->unique_values());
}
+METRIC_DEFINE_gauge_double(test_entity, test_mean_gauge, "Test mean Gauge",
+ MetricUnit::kUnits, "Description of mean Gauge");
+
+TEST_F(MetricsTest, SimpleMeanGaugeTest) {
+ scoped_refptr<MeanGauge> average_usage =
+ METRIC_test_mean_gauge.InstantiateMeanGauge(entity_);
+ ASSERT_EQ(METRIC_test_mean_gauge.description(),
average_usage->prototype()->description());
+ ASSERT_EQ(0, average_usage->value());
+ average_usage->set_value(10.0, 2.0);
+ ASSERT_EQ(5, average_usage->value());
+ average_usage->set_value(5.0, 2.0);
+ ASSERT_EQ(2.5, average_usage->value());
+}
+
+TEST_F(MetricsTest, SimpleMeanGaugeMergeTest) {
+ scoped_refptr<MeanGauge> average_usage =
+ METRIC_test_mean_gauge.InstantiateMeanGauge(entity_);
+ scoped_refptr<MeanGauge> average_usage_for_merge =
+ METRIC_test_mean_gauge.InstantiateMeanGauge(entity_same_attr_);
+ average_usage_for_merge->MergeFrom(average_usage);
+ ASSERT_EQ(0, average_usage->value());
+ ASSERT_EQ(0, average_usage_for_merge->value());
+ average_usage->set_value(10.0, 1.0);
+ average_usage_for_merge->set_value(2.0, 2.0);
+ ASSERT_EQ(10, average_usage->value());
+ ASSERT_EQ(1, average_usage_for_merge->value());
+ average_usage_for_merge->MergeFrom(average_usage);
+ ASSERT_EQ(10, average_usage->value());
+ ASSERT_EQ(4, average_usage_for_merge->value());
+ average_usage_for_merge->MergeFrom(average_usage_for_merge);
+ ASSERT_EQ(4, average_usage_for_merge->value());
+}
+
METRIC_DEFINE_gauge_uint64(test_entity, test_gauge, "Test uint64 Gauge",
MetricUnit::kBytes, "Description of Test Gauge");
diff --git a/src/kudu/util/metrics.cc b/src/kudu/util/metrics.cc
index 6102ef7..fc6b538 100644
--- a/src/kudu/util/metrics.cc
+++ b/src/kudu/util/metrics.cc
@@ -727,6 +727,54 @@ void StringGauge::WriteValue(JsonWriter* writer) const {
}
//
+// MeanGauge
+//
+
+scoped_refptr<Metric> MeanGauge::snapshot() const {
+ std::lock_guard<simple_spinlock> l(lock_);
+ scoped_refptr<Metric> m
+ = new MeanGauge(down_cast<const GaugePrototype<double>*>(prototype_));
+ return m;
+}
+
+double MeanGauge::value() const {
+ std::lock_guard<simple_spinlock> l(lock_);
+ return total_count_ > 0 ? total_sum_ / total_count_
+ : 0.0;
+}
+
+double MeanGauge::total_sum() const {
+ std::lock_guard<simple_spinlock> l(lock_);
+ return total_sum_;
+}
+
+double MeanGauge::total_count() const {
+ std::lock_guard<simple_spinlock> l(lock_);
+ return total_count_;
+}
+
+void MeanGauge::set_value(double total_sum, double total_count) {
+ std::lock_guard<simple_spinlock> l(lock_);
+ total_sum_ = total_sum;
+ total_count_ = total_count;
+}
+
+void MeanGauge::MergeFrom(const scoped_refptr<Metric>& other) {
+ if (PREDICT_FALSE(this == other.get())) {
+ return;
+ }
+
+ scoped_refptr<MeanGauge> other_ptr = down_cast<MeanGauge*>(other.get());
+ std::lock_guard<simple_spinlock> l(lock_);
+ total_sum_ += other_ptr->total_sum();
+ total_count_ += other_ptr->total_count();
+}
+
+void MeanGauge::WriteValue(JsonWriter* writer) const {
+ writer->Double(value());
+}
+
+//
// Counter
//
// This implementation is optimized by using a striped counter. See LongAdder
for details.
diff --git a/src/kudu/util/metrics.h b/src/kudu/util/metrics.h
index 072c556..eccca0a 100644
--- a/src/kudu/util/metrics.h
+++ b/src/kudu/util/metrics.h
@@ -340,6 +340,7 @@ class CounterPrototype;
class Histogram;
class HistogramPrototype;
class HistogramSnapshotPB;
+class MeanGauge;
class Metric;
class MetricEntity;
class MetricEntityPrototype;
@@ -617,6 +618,8 @@ class MetricEntity : public
RefCountedThreadSafe<MetricEntity> {
scoped_refptr<AtomicGauge<T> > FindOrCreateGauge(const GaugePrototype<T>*
proto,
const T& initial_value);
+ scoped_refptr<MeanGauge> FindOrCreateMeanGauge(const GaugePrototype<double>*
proto);
+
template<typename T>
scoped_refptr<FunctionGauge<T> > FindOrCreateFunctionGauge(const
GaugePrototype<T>* proto,
const
Callback<T()>& function);
@@ -895,6 +898,11 @@ class GaugePrototype : public MetricPrototype {
return entity->FindOrCreateGauge(this, initial_value);
}
+ scoped_refptr<MeanGauge> InstantiateMeanGauge(
+ const scoped_refptr<MetricEntity>& entity) const {
+ return entity->FindOrCreateMeanGauge(this);
+ }
+
// Instantiate a gauge that is backed by the given callback.
scoped_refptr<FunctionGauge<T> > InstantiateFunctionGauge(
const scoped_refptr<MetricEntity>& entity,
@@ -967,6 +975,33 @@ class StringGauge : public Gauge {
DISALLOW_COPY_AND_ASSIGN(StringGauge);
};
+// Gauge implementation for mean that uses locks to ensure thread safety.
+class MeanGauge : public Gauge {
+ public:
+ explicit MeanGauge(const GaugePrototype<double>* proto)
+ : Gauge(proto),
+ total_sum_(0.0),
+ total_count_(0.0) {
+ }
+ scoped_refptr<Metric> snapshot() const override;
+ double value() const;
+ double total_count() const;
+ double total_sum() const;
+ void set_value(double total_sum, double total_count);
+ virtual bool IsUntouched() const override {
+ return false;
+ }
+ void MergeFrom(const scoped_refptr<Metric>& other) override;
+
+ protected:
+ virtual void WriteValue(JsonWriter* writer) const override;
+ private:
+ double total_sum_;
+ double total_count_;
+ mutable simple_spinlock lock_; // Guards total_sum_ and total_count_
+ DISALLOW_COPY_AND_ASSIGN(MeanGauge);
+};
+
// Lock-free implementation for types that are convertible to/from int64_t.
template <typename T>
class AtomicGauge : public Gauge {
@@ -1349,6 +1384,19 @@ inline scoped_refptr<AtomicGauge<T> >
MetricEntity::FindOrCreateGauge(
return m;
}
+inline scoped_refptr<MeanGauge> MetricEntity::FindOrCreateMeanGauge(
+ const GaugePrototype<double>* proto) {
+ CheckInstantiation(proto);
+ std::lock_guard<simple_spinlock> l(lock_);
+ scoped_refptr<MeanGauge> m = down_cast<MeanGauge*>(
+ FindPtrOrNull(metric_map_, proto).get());
+ if (!m) {
+ m = new MeanGauge(proto);
+ InsertOrDie(&metric_map_, proto, m);
+ }
+ return m;
+}
+
template<typename T>
inline scoped_refptr<FunctionGauge<T> >
MetricEntity::FindOrCreateFunctionGauge(
const GaugePrototype<T>* proto,
