This is an automated email from the ASF dual-hosted git repository.
alexey pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kudu.git
The following commit(s) were added to refs/heads/master by this push:
new 00efc6826 KUDU-3375 Expose Prometheus Metrics in Kudu
00efc6826 is described below
commit 00efc6826ac9a1f5d10750296c7357790a041fec
Author: mammadli.khazar <[email protected]>
AuthorDate: Tue Oct 11 15:41:43 2022 +0200
KUDU-3375 Expose Prometheus Metrics in Kudu
This patch introduces an endpoint which emits server level metrics in
Prometheus Text Explosion format for Kudu. The endpoint can be found at
/metrics_prometheus path of the webserver.
More information on the Prometheus format can be found at:
https://prometheus.io/docs/instrumenting/exposition_formats
Prefix kudu_ has been added to all of the metric names according to
Prometheus naming convention to specify the domain and differentiate
from any other metrics belonging to different services. Following the
kudu_ prefix either master_ or tserver_ prefix is attached based on the
server type that is exposing the metrics. In order to differentiate
between different master and tablet servers running on the same host
Prometheus automatically attaches an instance label which holds the
"hostname:portname".
eg: kudu_master_metric{instance="localhost:8765", job="kudu-master",
unit_type="unit"}
Not all of the Kudu metric names follow the Prometheus metric naming
recommendations which state metric units should be attached at the end
of the metric name. In Kudu some metrics have these units at the end of
the metric name, some in the middle of the metric and for some metrics
it is unclear what the metric unit is just by looking at the metric
name. Due to this and also because Kudu supports more Metric Units
than base Prometheus Metric units, metric units are represented as
Prometheus labels for each metric. This also makes it possible to keep
time based units more accurate as Milliseconds, Nanoseconds and etc.
do not need to be converted to seconds which is the base time unit in
Prometheus.
More information on Prometheus Metric units and naming conventions
can be found at:
https://prometheus.io/docs/practices/naming/#metric-names
Tested by feeding all of the generated metrics to a prometheus server
running on localhost. Added unit tests to each metric type.
Change-Id: I4b2d01bd56f6f0f1b6f31cbce15e671d16521739
Reviewed-on: http://gerrit.cloudera.org:8080/19133
Tested-by: Kudu Jenkins
Reviewed-by: Alexey Serbin <[email protected]>
---
src/kudu/server/default_path_handlers.cc | 22 ++-
src/kudu/server/default_path_handlers.h | 4 +-
src/kudu/server/server_base.cc | 1 +
src/kudu/util/CMakeLists.txt | 1 +
src/kudu/util/metrics-test.cc | 124 +++++++++++++++
src/kudu/util/metrics.cc | 172 +++++++++++++++++++++
src/kudu/util/metrics.h | 80 ++++++++--
.../prometheus_writer.cc} | 25 +--
.../prometheus_writer.h} | 28 ++--
9 files changed, 405 insertions(+), 52 deletions(-)
diff --git a/src/kudu/server/default_path_handlers.cc
b/src/kudu/server/default_path_handlers.cc
index 682db402d..7b2ecc2f7 100644
--- a/src/kudu/server/default_path_handlers.cc
+++ b/src/kudu/server/default_path_handlers.cc
@@ -25,11 +25,13 @@
#include <functional>
#include <memory>
#include <string>
+#include <type_traits>
#include <unordered_map>
#include <vector>
#include <gflags/gflags.h>
#include <glog/logging.h>
+#include "kudu/util/prometheus_writer.h"
#include "kudu/util/version_info.h"
#include "kudu/util/version_info.pb.h"
@@ -510,6 +512,13 @@ static void WriteMetricsAsJson(const MetricRegistry* const
metrics,
}
}
+static void WriteMetricsAsPrometheus(const MetricRegistry* const metrics,
+ const Webserver::WebRequest& /*req*/,
+ Webserver::PrerenderedWebResponse* resp) {
+ PrometheusWriter writer(&resp->output);
+ WARN_NOT_OK(metrics->WriteAsPrometheus(&writer), "Couldn't write Prometheus
metrics over HTTP");
+}
+
void RegisterMetricsJsonHandler(Webserver* webserver, const MetricRegistry*
const metrics) {
auto callback = [metrics](const Webserver::WebRequest& req,
Webserver::PrerenderedWebResponse* resp) {
@@ -518,7 +527,7 @@ void RegisterMetricsJsonHandler(Webserver* webserver, const
MetricRegistry* cons
bool not_styled = false;
bool not_on_nav_bar = false;
bool is_on_nav_bar = true;
- webserver->RegisterPrerenderedPathHandler("/metrics", "Metrics", callback,
+ webserver->RegisterPrerenderedPathHandler("/metrics", "JSON Metrics",
callback,
not_styled, is_on_nav_bar);
// The old name -- this is preserved for compatibility with older releases of
@@ -527,4 +536,15 @@ void RegisterMetricsJsonHandler(Webserver* webserver,
const MetricRegistry* cons
not_styled, not_on_nav_bar);
}
+void RegisterMetricsPrometheusHandler(Webserver* webserver, const
MetricRegistry* const metrics) {
+ auto callback = [metrics](const Webserver::WebRequest& req,
+ Webserver::PrerenderedWebResponse* resp) {
+ WriteMetricsAsPrometheus(metrics, req, resp);
+ };
+ constexpr bool not_styled = false;
+ constexpr bool is_on_nav_bar = true;
+ webserver->RegisterPrerenderedPathHandler("/metrics_prometheus", "Prometheus
Metrics", callback,
+ not_styled, is_on_nav_bar);
+}
+
} // namespace kudu
diff --git a/src/kudu/server/default_path_handlers.h
b/src/kudu/server/default_path_handlers.h
index 1a5891546..aadf00fd9 100644
--- a/src/kudu/server/default_path_handlers.h
+++ b/src/kudu/server/default_path_handlers.h
@@ -32,8 +32,10 @@ void AddPreInitializedDefaultPathHandlers(Webserver*
webserver);
void AddPostInitializedDefaultPathHandlers(Webserver* webserver);
// Adds an endpoint to get metrics in JSON format.
-void RegisterMetricsJsonHandler(Webserver* webserver, const MetricRegistry*
const metrics);
+void RegisterMetricsJsonHandler(Webserver* webserver, const MetricRegistry*
metrics);
+// Adds an endpoint to get metrics in Prometheus format.
+void RegisterMetricsPrometheusHandler(Webserver* webserver, const
MetricRegistry* metrics);
} // namespace kudu
#endif // KUDU_SERVER_DEFAULT_PATH_HANDLERS_H
diff --git a/src/kudu/server/server_base.cc b/src/kudu/server/server_base.cc
index b3aee0960..5c9029534 100644
--- a/src/kudu/server/server_base.cc
+++ b/src/kudu/server/server_base.cc
@@ -969,6 +969,7 @@ Status ServerBase::Start() {
AddPostInitializedDefaultPathHandlers(web_server_.get());
AddRpczPathHandlers(messenger_, web_server_.get());
RegisterMetricsJsonHandler(web_server_.get(), metric_registry_.get());
+ RegisterMetricsPrometheusHandler(web_server_.get(),
metric_registry_.get());
TracingPathHandlers::RegisterHandlers(web_server_.get());
web_server_->set_footer_html(FooterHtml());
web_server_->SetStartupComplete(true);
diff --git a/src/kudu/util/CMakeLists.txt b/src/kudu/util/CMakeLists.txt
index e1713e16d..3a1fab523 100644
--- a/src/kudu/util/CMakeLists.txt
+++ b/src/kudu/util/CMakeLists.txt
@@ -221,6 +221,7 @@ set(UTIL_SRCS
pb_util.cc
pb_util-internal.cc
process_memory.cc
+ prometheus_writer.cc
random_util.cc
rolling_log.cc
rw_mutex.cc
diff --git a/src/kudu/util/metrics-test.cc b/src/kudu/util/metrics-test.cc
index e80a2d59a..2ab2d15d3 100644
--- a/src/kudu/util/metrics-test.cc
+++ b/src/kudu/util/metrics-test.cc
@@ -41,6 +41,7 @@
#include "kudu/util/jsonwriter.h"
#include "kudu/util/monotime.h"
#include "kudu/util/oid_generator.h"
+#include "kudu/util/prometheus_writer.h"
#include "kudu/util/random.h"
#include "kudu/util/status.h"
#include "kudu/util/test_macros.h"
@@ -131,6 +132,22 @@ TEST_F(MetricsTest, ResetCounter) {
ASSERT_EQ(0, c->value());
}
+TEST_F(MetricsTest, CounterPrometheusTest) {
+ scoped_refptr<Counter> requests =
+ new Counter(&METRIC_test_counter);
+
+ std::ostringstream output;
+ string prefix;
+ PrometheusWriter writer(&output);
+ ASSERT_OK(requests->WriteAsPrometheus(&writer, prefix));
+
+ const string expected_output = "# HELP test_counter Description of test
counter\n"
+ "# TYPE test_counter counter\n"
+ "test_counter{unit_type=\"requests\"} 0\n";
+
+ ASSERT_EQ(expected_output, output.str());
+}
+
METRIC_DEFINE_gauge_string(test_entity, test_string_gauge, "Test string Gauge",
MetricUnit::kState, "Description of string Gauge",
kudu::MetricLevel::kInfo);
@@ -186,6 +203,18 @@ TEST_F(MetricsTest, SimpleStringGaugeForMergeTest) {
state_for_merge->unique_values());
}
+TEST_F(MetricsTest, StringGaugePrometheusTest) {
+ scoped_refptr<StringGauge> state =
+ new StringGauge(&METRIC_test_string_gauge, "Healthy");
+
+ std::ostringstream output;
+ string prefix;
+ PrometheusWriter writer(&output);
+ ASSERT_OK(state->WriteAsPrometheus(&writer, prefix));
+ // String Gauges are not representable in Prometheus, empty output is
expected
+ ASSERT_EQ("", output.str());
+}
+
METRIC_DEFINE_gauge_double(test_entity, test_mean_gauge, "Test mean Gauge",
MetricUnit::kUnits, "Description of mean Gauge",
kudu::MetricLevel::kInfo);
@@ -260,10 +289,33 @@ TEST_F(MetricsTest, TestMeanGaugeJsonPrint) {
ASSERT_EQ(value, kTotalSum/kTotalCount);
}
+
+TEST_F(MetricsTest, MeanGaugePrometheusTest) {
+ scoped_refptr<MeanGauge> average_usage =
+ METRIC_test_mean_gauge.InstantiateMeanGauge(entity_);
+
+ std::ostringstream output;
+ string prefix;
+ PrometheusWriter writer(&output);
+ ASSERT_OK(average_usage->WriteAsPrometheus(&writer, prefix));
+
+ const string expected_output = "# HELP test_mean_gauge Description of mean
Gauge\n"
+ "# TYPE test_mean_gauge gauge\n"
+ "test_mean_gauge{unit_type=\"units\"} 0\n"
+ "test_mean_gauge_count{unit_type=\"units\"}
0\n"
+ "test_mean_gauge_sum{unit_type=\"units\"}
0\n";
+
+ ASSERT_EQ(expected_output, output.str());
+}
+
METRIC_DEFINE_gauge_uint64(test_entity, test_gauge, "Test uint64 Gauge",
MetricUnit::kBytes, "Description of Test Gauge",
kudu::MetricLevel::kInfo);
+METRIC_DEFINE_gauge_bool(test_entity, test_gauge_bool, "Test boolean Gauge",
+ MetricUnit::kState, "Description of Test boolean
Gauge",
+ kudu::MetricLevel::kInfo);
+
TEST_F(MetricsTest, SimpleAtomicGaugeTest) {
scoped_refptr<AtomicGauge<uint64_t> > mem_usage =
METRIC_test_gauge.Instantiate(entity_, 0);
@@ -323,6 +375,38 @@ TEST_F(MetricsTest, SimpleAtomicGaugeMinTypeMergeTest) {
ASSERT_EQ(1, start_time_for_merge->value());
}
+TEST_F(MetricsTest, AtomicGaugePrometheusTest) {
+ scoped_refptr<AtomicGauge<uint64_t> > mem_usage =
+ METRIC_test_gauge.Instantiate(entity_, 0);
+
+ std::ostringstream output;
+ string prefix;
+ PrometheusWriter writer(&output);
+ ASSERT_OK(mem_usage->WriteAsPrometheus(&writer, prefix));
+
+ const string expected_output = "# HELP test_gauge Description of Test
Gauge\n"
+ "# TYPE test_gauge gauge\n"
+ "test_gauge{unit_type=\"bytes\"} 0\n";
+
+ ASSERT_EQ(expected_output, output.str());
+}
+
+TEST_F(MetricsTest, AtomicGaugeBooleanPrometheusTest) {
+ scoped_refptr<AtomicGauge<bool> > clock_extrapolating =
+ METRIC_test_gauge_bool.Instantiate(entity_, false);
+
+ std::ostringstream output;
+ string prefix;
+ PrometheusWriter writer(&output);
+ ASSERT_OK(clock_extrapolating->WriteAsPrometheus(&writer, prefix));
+
+ const string expected_output = "# HELP test_gauge_bool Description of Test
boolean Gauge\n"
+ "# TYPE test_gauge_bool gauge\n"
+ "test_gauge_bool{unit_type=\"state\"} 0\n";
+
+ ASSERT_EQ(expected_output, output.str());
+}
+
METRIC_DEFINE_gauge_int64(test_entity, test_func_gauge, "Test Function Gauge",
MetricUnit::kBytes, "Test Gauge 2",
kudu::MetricLevel::kInfo);
@@ -462,6 +546,24 @@ TEST_F(MetricsTest, AutoDetachToConstant) {
ASSERT_EQ(12345, gauge->value());
}
+TEST_F(MetricsTest, FunctionGaugePrometheusTest) {
+ int metric_val = 1000;
+ scoped_refptr<FunctionGauge<int64_t> > gauge =
+ METRIC_test_func_gauge.InstantiateFunctionGauge(
+ entity_, [&metric_val]() { return MyFunction(&metric_val); });
+
+ std::ostringstream output;
+ string prefix;
+ PrometheusWriter writer(&output);
+ ASSERT_OK(gauge->WriteAsPrometheus(&writer, prefix));
+
+ const string expected_output = "# HELP test_func_gauge Test Gauge 2\n"
+ "# TYPE test_func_gauge gauge\n"
+ "test_func_gauge{unit_type=\"bytes\"} 1000\n";
+
+ ASSERT_EQ(expected_output, output.str());
+}
+
METRIC_DEFINE_gauge_uint64(test_entity, counter_as_gauge, "Gauge exposed as
Counter",
MetricUnit::kBytes, "Gauge exposed as Counter",
kudu::MetricLevel::kInfo,
@@ -515,6 +617,28 @@ TEST_F(MetricsTest, SimpleHistogramMergeTest) {
ASSERT_EQ(18, hist_for_merge->histogram()->TotalSum());
}
+TEST_F(MetricsTest, HistogramPrometheusTest) {
+ scoped_refptr<Histogram> hist = METRIC_test_hist.Instantiate(entity_);
+
+ std::ostringstream output;
+ PrometheusWriter writer(&output);
+ string prefix;
+ ASSERT_OK(hist->WriteAsPrometheus(&writer, prefix));
+
+ const string expected_output = "# HELP test_hist foo\n"
+ "# TYPE test_hist histogram\n"
+ "test_hist_bucket{unit_type=\"milliseconds\",
le=\"0.75\"} 0\n"
+ "test_hist_bucket{unit_type=\"milliseconds\",
le=\"0.95\"} 0\n"
+ "test_hist_bucket{unit_type=\"milliseconds\",
le=\"0.99\"} 0\n"
+ "test_hist_bucket{unit_type=\"milliseconds\",
le=\"0.999\"} 0\n"
+ "test_hist_bucket{unit_type=\"milliseconds\",
le=\"0.9999\"} 0\n"
+ "test_hist_bucket{unit_type=\"milliseconds\",
le=\"+Inf\"} 0\n"
+ "test_hist_sum{unit_type=\"milliseconds\"}
0\n"
+ "test_hist_count{unit_type=\"milliseconds\"}
0\n";
+
+ ASSERT_EQ(expected_output, output.str());
+}
+
TEST_F(MetricsTest, JsonPrintTest) {
scoped_refptr<Counter> test_counter =
METRIC_test_counter.Instantiate(entity_);
test_counter->Increment();
diff --git a/src/kudu/util/metrics.cc b/src/kudu/util/metrics.cc
index 663ffe604..d60cde079 100644
--- a/src/kudu/util/metrics.cc
+++ b/src/kudu/util/metrics.cc
@@ -48,6 +48,7 @@ using std::string;
using std::unordered_set;
using std::vector;
using strings::Substitute;
+using strings::SubstituteAndAppend;
template<typename Collection>
void WriteMetricsToJson(JsonWriter* writer,
@@ -68,6 +69,16 @@ void WriteMetricsToJson(JsonWriter* writer,
writer->EndArray();
}
+template<typename Collection>
+void WriteMetricsToPrometheus(PrometheusWriter* writer,
+ const Collection& metrics,
+ const std::string& prefix) {
+ for (const auto& [name, val] : metrics) {
+ WARN_NOT_OK(val->WriteAsPrometheus(writer, prefix),
+ Substitute("Failed to write $0 as Prometheus", name));
+ }
+}
+
void WriteToJson(JsonWriter* writer,
const MergedEntityMetrics &merged_entity_metrics,
const MetricJsonOptions &opts) {
@@ -392,6 +403,40 @@ Status MetricEntity::WriteAsJson(JsonWriter* writer, const
MetricJsonOptions& op
return Status::OK();
}
+Status MetricEntity::WriteAsPrometheus(PrometheusWriter* writer) const {
+ MetricMap metrics;
+ AttributeMap attrs;
+ MetricFilters filters;
+ filters.entity_level = "debug";
+ const string master_prefix = "kudu_master_";
+ const string tserver_prefix = "kudu_tserver_";
+ const string master_server = "kudu.master";
+ const string tablet_server = "kudu.tabletserver";
+ // Empty filters results in getting all the metrics for this MetricEntity.
+ const auto s = GetMetricsAndAttrs(filters, &metrics, &attrs);
+ if (s.IsNotFound()) {
+ // Status::NotFound is returned when this entity has been filtered, treat
it
+ // as OK, and skip printing it.
+ return Status::OK();
+ }
+ RETURN_NOT_OK(s);
+ // Only emit server level metrics
+ if (strcmp(prototype_->name(), "server") == 0) {
+ if (id_ == master_server) {
+ // attach kudu_master_ as prefix to metrics
+ WriteMetricsToPrometheus(writer, metrics, master_prefix);
+ return Status::OK();
+ }
+ if (id_ == tablet_server) {
+ // attach kudu_tserver_ as prefix to metrics
+ WriteMetricsToPrometheus(writer, metrics, tserver_prefix);
+ return Status::OK();
+ }
+ }
+
+ return Status::NotFound("Entity is not relevant to Prometheus");
+}
+
Status MetricEntity::CollectTo(MergedEntityMetrics* collections,
const MetricFilters& filters,
const MetricMergeRules& merge_rules) const {
@@ -540,6 +585,22 @@ Status MetricRegistry::WriteAsJson(JsonWriter* writer,
const MetricJsonOptions&
return Status::OK();
}
+Status MetricRegistry::WriteAsPrometheus(PrometheusWriter* writer) const {
+ EntityMap entities;
+ {
+ std::lock_guard<simple_spinlock> l(lock_);
+ entities = entities_;
+ }
+ for (const auto& e : entities) {
+ WARN_NOT_OK(e.second->WriteAsPrometheus(writer),
+ Substitute("Failed to write entity $0 as Prometheus",
e.second->id()));
+ }
+
+ entities.clear(); // necessary to deref metrics we just dumped before doing
retirement scan.
+ const_cast<MetricRegistry*>(this)->RetireOldMetrics();
+ return Status::OK();
+}
+
void MetricRegistry::RetireOldMetrics() {
std::lock_guard<simple_spinlock> l(lock_);
for (auto it = entities_.begin(); it != entities_.end();) {
@@ -674,6 +735,12 @@ void MetricPrototype::WriteFields(JsonWriter* writer,
}
}
+void MetricPrototype::WriteFields(PrometheusWriter* writer, const string
&prefix) const {
+ writer->WriteEntry(Substitute("# HELP $0$1 $2\n# TYPE $3$4 $5\n",
+ prefix, name(), description(),
+ prefix, name(), MetricType::Name(type())));
+}
+
//
// FunctionGaugeDetacher
//
@@ -751,6 +818,14 @@ Status Gauge::WriteAsJson(JsonWriter* writer,
return Status::OK();
}
+Status Gauge::WriteAsPrometheus(PrometheusWriter* writer, const std::string&
prefix) const {
+ prototype_->WriteFields(writer, prefix);
+
+ WriteValue(writer, prefix);
+
+ return Status::OK();
+}
+
//
// StringGauge
//
@@ -822,6 +897,24 @@ void StringGauge::WriteValue(JsonWriter* writer) const {
writer->String(value());
}
+// A string gauge's value can be anything, but Prometheus does not support
+// non-numeric values for gauges with exception of {+,-}Inf and NaN
+// (see https://prometheus.io/docs/instrumenting/exposition_formats/).
+// DCHECK() is added to make sure this method is not called from anywhere,
+// but overriding it is necessary since Gauge::WriteValue() is a pure virtual
one.
+// An alternative could be defining a empty implementation for
Gauge::WriteValue()
+// virtual method and not adding this empty override here.
+void StringGauge::WriteValue(PrometheusWriter* writer, const std::string&
prefix) const {
+ DCHECK(false);
+}
+
+Status StringGauge::WriteAsPrometheus(PrometheusWriter* /*writer*/,
+ const std::string& /*prefix*/) const {
+ // Prometheus doesn't support string gauges.
+ // This function ensures that output written to Prometheus is empty.
+ return Status::OK();
+}
+
//
// MeanGauge
//
@@ -883,6 +976,21 @@ void MeanGauge::WriteValue(JsonWriter* writer) const {
writer->Double(total_count());
}
+void MeanGauge::WriteValue(PrometheusWriter* writer, const std::string&
prefix) const {
+ auto output = Substitute("$0$1{unit_type=\"$2\"} $3\n", prefix,
prototype_->name(),
+ MetricUnit::Name(prototype_->unit()), value());
+
+ SubstituteAndAppend(&output, "$0$1$2{unit_type=\"$3\"} $4\n",
+ prefix, prototype_->name(), "_count",
+ MetricUnit::Name(prototype_->unit()), total_count());
+
+ SubstituteAndAppend(&output, "$0$1$2{unit_type=\"$3\"} $4\n",
+ prefix, prototype_->name(), "_sum",
+ MetricUnit::Name(prototype_->unit()), total_sum());
+
+ writer->WriteEntry(output);
+}
+
//
// Counter
//
@@ -921,6 +1029,15 @@ Status Counter::WriteAsJson(JsonWriter* writer,
return Status::OK();
}
+Status Counter::WriteAsPrometheus(PrometheusWriter* writer, const std::string&
prefix) const {
+ prototype_->WriteFields(writer, prefix);
+
+ writer->WriteEntry(Substitute("$0$1{unit_type=\"$2\"} $3\n", prefix,
prototype_->name(),
+ MetricUnit::Name(prototype_->unit()),
value()));
+
+ return Status::OK();
+}
+
/////////////////////////////////////////////////
// HistogramPrototype
/////////////////////////////////////////////////
@@ -977,6 +1094,61 @@ Status Histogram::WriteAsJson(JsonWriter* writer,
return Status::OK();
}
+Status Histogram::WriteAsPrometheus(PrometheusWriter* writer, const
std::string& prefix) const {
+ prototype_->WriteFields(writer, prefix);
+ std::string output = "";
+ MetricJsonOptions opts;
+ // Snapshot is taken to preserve the consistency across metrics and
+ // requirements given by Prometheus. The value for the _bucket in +Inf
+ // quantile needs to be equal to the total _count
+ HistogramSnapshotPB snapshot;
+ RETURN_NOT_OK(GetHistogramSnapshotPB(&snapshot, opts));
+
+ output = Substitute("$0$1$2{unit_type=\"$3\", le=\"0.75\"} $4\n",
+ prefix, prototype_->name(), "_bucket",
+ MetricUnit::Name(prototype_->unit()),
+ snapshot.percentile_75());
+
+ SubstituteAndAppend(&output, "$0$1$2{unit_type=\"$3\", le=\"0.95\"} $4\n",
+ prefix, prototype_->name(), "_bucket",
+ MetricUnit::Name(prototype_->unit()),
+ snapshot.percentile_95());
+
+ SubstituteAndAppend(&output, "$0$1$2{unit_type=\"$3\", le=\"0.99\"} $4\n",
+ prefix, prototype_->name(), "_bucket",
+ MetricUnit::Name(prototype_->unit()),
+ snapshot.percentile_99());
+
+ SubstituteAndAppend(&output, "$0$1$2{unit_type=\"$3\", le=\"0.999\"} $4\n",
+ prefix, prototype_->name(), "_bucket",
+ MetricUnit::Name(prototype_->unit()),
+ snapshot.percentile_99_9());
+
+ SubstituteAndAppend(&output, "$0$1$2{unit_type=\"$3\", le=\"0.9999\"} $4\n",
+ prefix, prototype_->name(), "_bucket",
+ MetricUnit::Name(prototype_->unit()),
+ snapshot.percentile_99_99());
+
+ SubstituteAndAppend(&output, "$0$1$2{unit_type=\"$3\", le=\"+Inf\"} $4\n",
+ prefix, prototype_->name(), "_bucket",
+ MetricUnit::Name(prototype_->unit()),
+ snapshot.total_count());
+
+ SubstituteAndAppend(&output, "$0$1$2{unit_type=\"$3\"} $4\n",
+ prefix, prototype_->name(), "_sum",
+ MetricUnit::Name(prototype_->unit()),
+ snapshot.total_sum());
+
+ SubstituteAndAppend(&output, "$0$1$2{unit_type=\"$3\"} $4\n",
+ prefix, prototype_->name(), "_count",
+ MetricUnit::Name(prototype_->unit()),
+ snapshot.total_count());
+
+ writer->WriteEntry(output);
+
+ return Status::OK();
+}
+
Status Histogram::GetHistogramSnapshotPB(HistogramSnapshotPB* snapshot_pb,
const MetricJsonOptions& opts) const {
snapshot_pb->set_name(prototype_->name());
diff --git a/src/kudu/util/metrics.h b/src/kudu/util/metrics.h
index db8bccbfe..28e49b827 100644
--- a/src/kudu/util/metrics.h
+++ b/src/kudu/util/metrics.h
@@ -249,6 +249,7 @@
#include "kudu/gutil/map-util.h"
#include "kudu/gutil/port.h"
#include "kudu/gutil/ref_counted.h"
+#include "kudu/gutil/strings/substitute.h"
#include "kudu/util/atomic.h"
#include "kudu/util/hdr_histogram.h"
#include "kudu/util/jsonwriter.h" // IWYU pragma: keep
@@ -256,6 +257,7 @@
#include "kudu/util/monotime.h"
#include "kudu/util/status.h"
#include "kudu/util/striped64.h"
+#include "kudu/util/prometheus_writer.h"
// Define a new entity type.
//
@@ -581,6 +583,8 @@ class MetricPrototype {
void WriteFields(JsonWriter* writer,
const MetricJsonOptions& opts) const;
+ void WriteFields(PrometheusWriter* writer, const std::string& prefix) const;
+
protected:
explicit MetricPrototype(CtorArgs args);
virtual ~MetricPrototype() {
@@ -665,6 +669,8 @@ class MetricEntity : public
RefCountedThreadSafe<MetricEntity> {
// See MetricRegistry::WriteAsJson()
Status WriteAsJson(JsonWriter* writer, const MetricJsonOptions& opts) const;
+ Status WriteAsPrometheus(PrometheusWriter* writer) const;
+
// Collect metrics of this entity to 'collections'. Metrics will be filtered
by 'filters',
// and will be merged under the rule of 'merge_rules'.
Status CollectTo(MergedEntityMetrics* collections,
@@ -756,6 +762,8 @@ class Metric : public RefCountedThreadSafe<Metric> {
// All metrics must be able to render themselves as JSON.
virtual Status WriteAsJson(JsonWriter* writer,
const MetricJsonOptions& opts) const = 0;
+ // All metrics must be able to render themselves as Prometheus.
+ virtual Status WriteAsPrometheus(PrometheusWriter* writer, const
std::string& prefix) const = 0;
const MetricPrototype* prototype() const { return prototype_; }
@@ -877,6 +885,8 @@ class MetricRegistry {
// output of this function.
Status WriteAsJson(JsonWriter* writer, const MetricJsonOptions& opts) const;
+ // Writes metrics in this registry to given Prometheus 'writer'.
+ Status WriteAsPrometheus(PrometheusWriter* writer) const;
// For each registered entity, retires orphaned metrics. If an entity has no
more
// metrics and there are no external references, entities are removed as
well.
//
@@ -1015,12 +1025,13 @@ class Gauge : public Metric {
explicit Gauge(const MetricPrototype* prototype)
: Metric(prototype) {
}
- virtual ~Gauge() {}
- virtual Status WriteAsJson(JsonWriter* w,
- const MetricJsonOptions& opts) const OVERRIDE;
+ ~Gauge() override {}
+ Status WriteAsJson(JsonWriter* w, const MetricJsonOptions& opts) const
OVERRIDE;
+ Status WriteAsPrometheus(PrometheusWriter* w, const std::string& prefix)
const OVERRIDE;
protected:
virtual void WriteValue(JsonWriter* writer) const = 0;
+ virtual void WriteValue(PrometheusWriter* writer, const std::string& prefix)
const = 0;
private:
DISALLOW_COPY_AND_ASSIGN(Gauge);
};
@@ -1042,7 +1053,10 @@ class StringGauge : public Gauge {
protected:
FRIEND_TEST(MetricsTest, SimpleStringGaugeForMergeTest);
- virtual void WriteValue(JsonWriter* writer) const OVERRIDE;
+ FRIEND_TEST(MetricsTest, StringGaugePrometheusTest);
+ Status WriteAsPrometheus(PrometheusWriter* w, const std::string& prefix)
const OVERRIDE;
+ void WriteValue(JsonWriter* writer) const OVERRIDE;
+ void WriteValue(PrometheusWriter* writer, const std::string& prefix) const
OVERRIDE;
void FillUniqueValuesUnlocked();
std::unordered_set<std::string> unique_values();
private:
@@ -1071,7 +1085,8 @@ class MeanGauge : public Gauge {
void MergeFrom(const scoped_refptr<Metric>& other) override;
protected:
- virtual void WriteValue(JsonWriter* writer) const override;
+ void WriteValue(JsonWriter* writer) const override;
+ void WriteValue(PrometheusWriter* writer, const std::string& prefix) const
override;
private:
double total_sum_;
double total_count_;
@@ -1144,9 +1159,29 @@ class AtomicGauge : public Gauge {
}
}
protected:
- virtual void WriteValue(JsonWriter* writer) const OVERRIDE {
+ void WriteValue(JsonWriter* writer) const OVERRIDE {
writer->Value(value());
}
+
+ void WriteValue(PrometheusWriter* writer,const std::string& prefix) const
OVERRIDE {
+ std::string output = "";
+
+ // If Boolean Gauge, convert false/true to 0/1 for Prometheus
+ if constexpr (std::is_same_v<T, bool> ) {
+ int check = 0;
+ if (value() == true) {
+ check = 1;
+ } else {
+ check = 0;
+ }
+ output = strings::Substitute("$0$1{unit_type=\"$2\"} $3\n", prefix,
prototype_->name(),
+ MetricUnit::Name(prototype_->unit()),
check);
+ } else {
+ output = strings::Substitute("$0$1{unit_type=\"$2\"} $3\n", prefix,
prototype_->name(),
+ MetricUnit::Name(prototype_->unit()),
value());
+ }
+ writer->WriteEntry(output);
+ }
private:
AtomicInt<int64_t> value_;
MergeType type_;
@@ -1235,10 +1270,28 @@ class FunctionGauge : public Gauge {
return function_();
}
- virtual void WriteValue(JsonWriter* writer) const OVERRIDE {
+ void WriteValue(JsonWriter* writer) const OVERRIDE {
writer->Value(value());
}
+ void WriteValue(PrometheusWriter* writer, const std::string& prefix) const
OVERRIDE {
+ std::string output = "";
+ // If Boolean Gauge, convert false/true to 0/1 for Prometheus
+ if constexpr (std::is_same_v<T, bool> ) {
+ int check = 0;
+ if (value() == true) {
+ check = 1;
+ } else {
+ check = 0;
+ }
+ output = strings::Substitute("$0$1{unit_type=\"$2\"} $3\n", prefix,
prototype_->name(),
+ MetricUnit::Name(prototype_->unit()),
check);
+ } else {
+ output = strings::Substitute("$0$1{unit_type=\"$2\"} $3\n", prefix,
prototype_->name(),
+ MetricUnit::Name(prototype_->unit()),
value());
+ }
+ writer->WriteEntry(output);
+ }
// Reset this FunctionGauge to return a specific value.
// This should be used during destruction. If you want a settable
// Gauge, use a normal Gauge instead of a FunctionGauge.
@@ -1357,10 +1410,11 @@ class Counter : public Metric {
int64_t value() const;
void Increment();
void IncrementBy(int64_t amount);
- virtual Status WriteAsJson(JsonWriter* w,
- const MetricJsonOptions& opts) const OVERRIDE;
+ Status WriteAsJson(JsonWriter* w, const MetricJsonOptions& opts) const
OVERRIDE;
- virtual bool IsUntouched() const override {
+ Status WriteAsPrometheus(PrometheusWriter* w, const std::string& prefix)
const OVERRIDE;
+
+ bool IsUntouched() const override {
return value() == 0;
}
@@ -1385,6 +1439,7 @@ class Counter : public Metric {
FRIEND_TEST(MetricsTest, SimpleCounterTest);
FRIEND_TEST(MetricsTest, SimpleCounterMergeTest);
FRIEND_TEST(MultiThreadedMetricsTest, CounterIncrementTest);
+ FRIEND_TEST(MetricsTest, CounterPrometheusTest);
friend class MetricEntity;
explicit Counter(const CounterPrototype* proto);
@@ -1431,8 +1486,9 @@ class Histogram : public Metric {
// or IncrementBy()).
uint64_t TotalCount() const;
- virtual Status WriteAsJson(JsonWriter* w,
- const MetricJsonOptions& opts) const OVERRIDE;
+ Status WriteAsJson(JsonWriter* w, const MetricJsonOptions& opts) const
OVERRIDE;
+
+ Status WriteAsPrometheus(PrometheusWriter* w, const std::string& prefix)
const OVERRIDE;
// Returns a snapshot of this histogram including the bucketed values and
counts.
Status GetHistogramSnapshotPB(HistogramSnapshotPB* snapshot_pb,
diff --git a/src/kudu/server/default_path_handlers.h
b/src/kudu/util/prometheus_writer.cc
similarity index 51%
copy from src/kudu/server/default_path_handlers.h
copy to src/kudu/util/prometheus_writer.cc
index 1a5891546..d68f88b9e 100644
--- a/src/kudu/server/default_path_handlers.h
+++ b/src/kudu/util/prometheus_writer.cc
@@ -15,25 +15,10 @@
// specific language governing permissions and limitations
// under the License.
-#ifndef KUDU_SERVER_DEFAULT_PATH_HANDLERS_H
-#define KUDU_SERVER_DEFAULT_PATH_HANDLERS_H
+#include "kudu/util/prometheus_writer.h"
-namespace kudu {
+#include <sstream>
-class MetricRegistry;
-class Webserver;
-
-// Adds a set of default path handlers to the webserver to display
-// logs and configuration flags before the server is initialized.
-void AddPreInitializedDefaultPathHandlers(Webserver* webserver);
-
-// Adds a set of default path handlers to the webserver to display
-// stacks and support pprof profiling after the server is initialized.
-void AddPostInitializedDefaultPathHandlers(Webserver* webserver);
-
-// Adds an endpoint to get metrics in JSON format.
-void RegisterMetricsJsonHandler(Webserver* webserver, const MetricRegistry*
const metrics);
-
-} // namespace kudu
-
-#endif // KUDU_SERVER_DEFAULT_PATH_HANDLERS_H
+void PrometheusWriter::WriteEntry(const std::string& data) {
+ *output_ << data;
+}
diff --git a/src/kudu/server/default_path_handlers.h
b/src/kudu/util/prometheus_writer.h
similarity index 51%
copy from src/kudu/server/default_path_handlers.h
copy to src/kudu/util/prometheus_writer.h
index 1a5891546..528438f6a 100644
--- a/src/kudu/server/default_path_handlers.h
+++ b/src/kudu/util/prometheus_writer.h
@@ -14,26 +14,18 @@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
+#pragma once
-#ifndef KUDU_SERVER_DEFAULT_PATH_HANDLERS_H
-#define KUDU_SERVER_DEFAULT_PATH_HANDLERS_H
+#include <iosfwd>
+#include <string>
-namespace kudu {
+class PrometheusWriter {
-class MetricRegistry;
-class Webserver;
+ public:
+ explicit PrometheusWriter(std::ostringstream* output): output_(output) {}
-// Adds a set of default path handlers to the webserver to display
-// logs and configuration flags before the server is initialized.
-void AddPreInitializedDefaultPathHandlers(Webserver* webserver);
+ void WriteEntry(const std::string& data);
-// Adds a set of default path handlers to the webserver to display
-// stacks and support pprof profiling after the server is initialized.
-void AddPostInitializedDefaultPathHandlers(Webserver* webserver);
-
-// Adds an endpoint to get metrics in JSON format.
-void RegisterMetricsJsonHandler(Webserver* webserver, const MetricRegistry*
const metrics);
-
-} // namespace kudu
-
-#endif // KUDU_SERVER_DEFAULT_PATH_HANDLERS_H
+ private:
+ std::ostringstream* output_;
+};
