Repository: impala Updated Branches: refs/heads/master 36e301db4 -> 315142190
IMPALA-6075: Add Impala daemon metric for catalog version. This patch adds new metrics for current version of catalog, current catalog topic version and catalog service id which are currently used by impala daemon. Testing: Verified manually that the new metrics for catalog version, catalog topic version, catalog service id are displayed and they corresponds to the latest version in catalogd.INFO log file. Change-Id: Id2307eb434561ed74ff058106541c0ebda017d97 Reviewed-on: http://gerrit.cloudera.org:8080/8949 Reviewed-by: Tim Armstrong <tarmstr...@cloudera.com> Tested-by: Impala Public Jenkins Project: http://git-wip-us.apache.org/repos/asf/impala/repo Commit: http://git-wip-us.apache.org/repos/asf/impala/commit/572af614 Tree: http://git-wip-us.apache.org/repos/asf/impala/tree/572af614 Diff: http://git-wip-us.apache.org/repos/asf/impala/diff/572af614 Branch: refs/heads/master Commit: 572af614d6d2af21dc4d261a3c53ed9b2b05c833 Parents: 36e301d Author: Pranay <psi...@cloudera.com> Authored: Fri Jan 5 09:31:02 2018 -0800 Committer: Impala Public Jenkins <impala-public-jenk...@gerrit.cloudera.org> Committed: Wed Feb 7 23:03:40 2018 +0000 ---------------------------------------------------------------------- be/src/service/impala-server.cc | 8 ++++++++ be/src/service/impala-server.h | 4 +++- be/src/util/impalad-metrics.cc | 9 +++++++++ be/src/util/impalad-metrics.h | 12 ++++++++++++ common/thrift/metrics.json | 30 ++++++++++++++++++++++++++++++ 5 files changed, 62 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/impala/blob/572af614/be/src/service/impala-server.cc ---------------------------------------------------------------------- diff --git a/be/src/service/impala-server.cc b/be/src/service/impala-server.cc index cf5f5fb..ee8405b 100644 --- a/be/src/service/impala-server.cc +++ b/be/src/service/impala-server.cc @@ -1317,6 +1317,13 @@ Status ImpalaServer::AuthorizeProxyUser(const string& user, const string& do_as_ return Status(error_msg.str()); } +void ImpalaServer::CatalogUpdateVersionInfo::UpdateCatalogVersionMetrics() +{ + ImpaladMetrics::CATALOG_VERSION->SetValue(catalog_version); + ImpaladMetrics::CATALOG_TOPIC_VERSION->SetValue(catalog_topic_version); + ImpaladMetrics::CATALOG_SERVICE_ID->SetValue(PrintId(catalog_service_id)); +} + void ImpalaServer::CatalogUpdateCallback( const StatestoreSubscriber::TopicDeltaMap& incoming_topic_deltas, vector<TTopicDelta>* subscriber_topic_updates) { @@ -1352,6 +1359,7 @@ void ImpalaServer::CatalogUpdateCallback( LOG(INFO) << "Catalog topic update applied with version: " << resp.new_catalog_version << " new min catalog object version: " << resp.min_catalog_object_version; + catalog_update_info_.UpdateCatalogVersionMetrics(); } ImpaladMetrics::CATALOG_READY->SetValue(resp.new_catalog_version > 0); // TODO: deal with an error status http://git-wip-us.apache.org/repos/asf/impala/blob/572af614/be/src/service/impala-server.h ---------------------------------------------------------------------- diff --git a/be/src/service/impala-server.h b/be/src/service/impala-server.h index dad1ebf..237b0cb 100644 --- a/be/src/service/impala-server.h +++ b/be/src/service/impala-server.h @@ -997,7 +997,9 @@ class ImpalaServer : public ImpalaServiceIf, catalog_topic_version(0L), min_catalog_object_version(0L) { } - + /// Update the metrics to store the current version of catalog, current topic and + /// current service id used by impalad. + void UpdateCatalogVersionMetrics(); /// The last catalog version returned from UpdateCatalog() int64_t catalog_version; /// The CatalogService ID that this catalog version is from. http://git-wip-us.apache.org/repos/asf/impala/blob/572af614/be/src/util/impalad-metrics.cc ---------------------------------------------------------------------- diff --git a/be/src/util/impalad-metrics.cc b/be/src/util/impalad-metrics.cc index 18e96a8..8f5f1be 100644 --- a/be/src/util/impalad-metrics.cc +++ b/be/src/util/impalad-metrics.cc @@ -76,6 +76,9 @@ const char* ImpaladMetricKeys::CATALOG_NUM_DBS = "catalog.num-databases"; const char* ImpaladMetricKeys::CATALOG_NUM_TABLES = "catalog.num-tables"; +const char* ImpaladMetricKeys::CATALOG_VERSION = "catalog.curr-version"; +const char* ImpaladMetricKeys::CATALOG_TOPIC_VERSION = "catalog.curr-topic"; +const char* ImpaladMetricKeys::CATALOG_SERVICE_ID = "catalog.curr-serviceid"; const char* ImpaladMetricKeys::CATALOG_READY = "catalog.ready"; const char* ImpaladMetricKeys::NUM_FILES_OPEN_FOR_INSERT = @@ -126,6 +129,8 @@ IntCounter* ImpaladMetrics::HEDGED_READ_OPS_WIN = NULL; // Gauges IntGauge* ImpaladMetrics::CATALOG_NUM_DBS = NULL; IntGauge* ImpaladMetrics::CATALOG_NUM_TABLES = NULL; +IntGauge* ImpaladMetrics::CATALOG_VERSION = NULL; +IntGauge* ImpaladMetrics::CATALOG_TOPIC_VERSION = NULL; IntGauge* ImpaladMetrics::IMPALA_SERVER_NUM_OPEN_BEESWAX_SESSIONS = NULL; IntGauge* ImpaladMetrics::IMPALA_SERVER_NUM_OPEN_HS2_SESSIONS = NULL; IntGauge* ImpaladMetrics::IO_MGR_NUM_BUFFERS = NULL; @@ -146,6 +151,7 @@ IntGauge* ImpaladMetrics::RESULTSET_CACHE_TOTAL_BYTES = NULL; BooleanProperty* ImpaladMetrics::CATALOG_READY = NULL; BooleanProperty* ImpaladMetrics::IMPALA_SERVER_READY = NULL; StringProperty* ImpaladMetrics::IMPALA_SERVER_VERSION = NULL; +StringProperty* ImpaladMetrics::CATALOG_SERVICE_ID = NULL; // Histograms HistogramMetric* ImpaladMetrics::QUERY_DURATIONS = NULL; @@ -235,6 +241,9 @@ void ImpaladMetrics::CreateMetrics(MetricGroup* m) { // Initialize catalog metrics CATALOG_NUM_DBS = m->AddGauge(ImpaladMetricKeys::CATALOG_NUM_DBS, 0); CATALOG_NUM_TABLES = m->AddGauge(ImpaladMetricKeys::CATALOG_NUM_TABLES, 0); + CATALOG_VERSION = m->AddGauge(ImpaladMetricKeys::CATALOG_VERSION, 0); + CATALOG_TOPIC_VERSION = m->AddGauge(ImpaladMetricKeys::CATALOG_TOPIC_VERSION, 0); + CATALOG_SERVICE_ID = m->AddProperty<string>(ImpaladMetricKeys::CATALOG_SERVICE_ID, ""); CATALOG_READY = m->AddProperty<bool>(ImpaladMetricKeys::CATALOG_READY, false); // Maximum duration to be tracked by the query durations metric. No particular reasoning http://git-wip-us.apache.org/repos/asf/impala/blob/572af614/be/src/util/impalad-metrics.h ---------------------------------------------------------------------- diff --git a/be/src/util/impalad-metrics.h b/be/src/util/impalad-metrics.h index 65ddf1e..f32e3fa 100644 --- a/be/src/util/impalad-metrics.h +++ b/be/src/util/impalad-metrics.h @@ -109,6 +109,15 @@ class ImpaladMetricKeys { /// Number of DBs in the catalog static const char* CATALOG_NUM_DBS; + /// Current version of catalog with impalad. + static const char* CATALOG_VERSION; + + /// Catalog topic version with impalad. + static const char* CATALOG_TOPIC_VERSION; + + /// ServiceID of Catalog with impalad. + static const char* CATALOG_SERVICE_ID; + /// Number of tables in the catalog static const char* CATALOG_NUM_TABLES; @@ -180,6 +189,8 @@ class ImpaladMetrics { // Gauges static IntGauge* CATALOG_NUM_DBS; static IntGauge* CATALOG_NUM_TABLES; + static IntGauge* CATALOG_VERSION; + static IntGauge* CATALOG_TOPIC_VERSION; static IntGauge* IMPALA_SERVER_NUM_OPEN_BEESWAX_SESSIONS; static IntGauge* IMPALA_SERVER_NUM_OPEN_HS2_SESSIONS; static IntGauge* IO_MGR_NUM_BUFFERS; @@ -199,6 +210,7 @@ class ImpaladMetrics { static BooleanProperty* CATALOG_READY; static BooleanProperty* IMPALA_SERVER_READY; static StringProperty* IMPALA_SERVER_VERSION; + static StringProperty* CATALOG_SERVICE_ID; // Histograms static HistogramMetric* QUERY_DURATIONS; static HistogramMetric* DDL_DURATIONS; http://git-wip-us.apache.org/repos/asf/impala/blob/572af614/common/thrift/metrics.json ---------------------------------------------------------------------- diff --git a/common/thrift/metrics.json b/common/thrift/metrics.json index f493d33..df78a0b 100644 --- a/common/thrift/metrics.json +++ b/common/thrift/metrics.json @@ -230,6 +230,36 @@ "key": "catalog.num-tables" }, { + "description": "Catalog topic update version.", + "contexts": [ + "IMPALAD" + ], + "label": "Catalog topic update version", + "units": "NONE", + "kind": "GAUGE", + "key": "catalog.curr-version" + }, + { + "description": "Statestore topic update version.", + "contexts": [ + "IMPALAD" + ], + "label": "Statestore topic update version", + "units": "NONE", + "kind": "GAUGE", + "key": "catalog.curr-topic" + }, + { + "description": "Catalog service id.", + "contexts": [ + "IMPALAD" + ], + "label": "Catalog service id", + "units": "NONE", + "kind": "PROPERTY", + "key": "catalog.curr-serviceid" + }, + { "description": "Indicates if the catalog is ready.", "contexts": [ "IMPALAD"
