This is an automated email from the ASF dual-hosted git repository. granthenke pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/kudu.git
commit b4c1775ed269cd86df81708a67f202127dfa9161 Author: Grant Henke <[email protected]> AuthorDate: Thu Oct 15 14:28:09 2020 -0500 [docs] Automatically generate a metrics reference page This patch adds an generated metrics page that is very similar to the existing command line and configuration references. A new `—dump_metrics_xml` flag was added so I could dump the metrics and then translate them to an adoc using xslt. Change-Id: I052a043a10a258fd52ff6cc7779e37becb0a3a35 Reviewed-on: http://gerrit.cloudera.org:8080/16604 Reviewed-by: Attila Bukor <[email protected]> Tested-by: Kudu Jenkins Reviewed-by: Alexey Serbin <[email protected]> --- docs/administration.adoc | 6 +- docs/metrics_reference.adoc | 37 ++++++ docs/support/jekyll-templates/document.html.erb | 1 + docs/support/scripts/make_docs.sh | 27 ++++- docs/support/xsl/metrics_to_asciidoc.xsl | 149 ++++++++++++++++++++++++ src/kudu/util/flags.cc | 8 ++ src/kudu/util/metrics.cc | 29 +++++ src/kudu/util/metrics.h | 4 + 8 files changed, 254 insertions(+), 7 deletions(-) diff --git a/docs/administration.adoc b/docs/administration.adoc index 72fca5b..9ad27a8 100644 --- a/docs/administration.adoc +++ b/docs/administration.adoc @@ -115,7 +115,8 @@ The levels are ordered and lower levels include the levels above them. If no lev during normal operation. * `info` - Generally useful metrics that operators always want to have available but may not be monitored under normal circumstances. - * `warn` - Metrics which can often indicate operational oddities, which may need more investigation. + * `warn` - Metrics which can often indicate operational oddities that may need + more investigation. For example: @@ -124,6 +125,9 @@ For example: $ curl -s 'http://example-ts:8050/metrics?include_schema=1&metrics=connections_accepted' ---- +NOTE: See the `link:metrics_reference.html[metrics reference]` +page for more information on the available metrics. + [source,json] ---- [ diff --git a/docs/metrics_reference.adoc b/docs/metrics_reference.adoc new file mode 100644 index 0000000..5de8b04 --- /dev/null +++ b/docs/metrics_reference.adoc @@ -0,0 +1,37 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +[[metrics_reference]] += Apache Kudu Metrics Reference + +:author: Kudu Team +:imagesdir: ./images +:icons: font +:toc: left +:toclevels: 2 +:doctype: book +:backend: html5 +:sectlinks: +:experimental: + +// The contents of this file are generated from the output of the +// `--dump_metrics_xml` flag for each binary, during the build of +// the documentation. Do not edit this file or the included files +// manually. + +// This gets replaced by the script that builds the docs +@@METRICS_REFERENCE@@ \ No newline at end of file diff --git a/docs/support/jekyll-templates/document.html.erb b/docs/support/jekyll-templates/document.html.erb index e6dcfa7..e36041a 100644 --- a/docs/support/jekyll-templates/document.html.erb +++ b/docs/support/jekyll-templates/document.html.erb @@ -101,6 +101,7 @@ end %> :background_tasks, "Background Maintenance Tasks", :configuration_reference, "Kudu Configuration Reference", :command_line_tools_reference, "Kudu Command Line Tools Reference", + :metrics_reference, "Kudu Metrics Reference", :known_issues, "Known Issues and Limitations", :contributing, "Contributing to Kudu", :export_control, "Export Control Notice" diff --git a/docs/support/scripts/make_docs.sh b/docs/support/scripts/make_docs.sh index 7738374..24f0077 100755 --- a/docs/support/scripts/make_docs.sh +++ b/docs/support/scripts/make_docs.sh @@ -154,8 +154,6 @@ binaries=("kudu-master" \ "kudu-tserver") for binary in ${binaries[@]}; do - echo "Running $binary --helpxml" - ( # Reset environment to avoid affecting the default flag values. for var in $(env | awk -F= '{print $1}' | egrep -i 'KUDU|GLOG'); do @@ -163,9 +161,14 @@ for binary in ${binaries[@]}; do eval "unset $var" done - # Create the XML file. + echo "Running $binary --helpxml" + # Create the help XML file. # This command exits with a nonzero value. $BUILD_ROOT/bin/$binary --helpxml > ${GEN_DOC_DIR}/$(basename $binary).xml || true + + echo "Running $binary --dump_metrics_xml" + # Create the metrics XML file. + $BUILD_ROOT/bin/$binary --dump_metrics_xml > ${GEN_DOC_DIR}/$(basename $binary)-metrics.xml ) # Create the supported config reference @@ -174,7 +177,7 @@ for binary in ${binaries[@]}; do --stringparam support-level supported \ -o $GEN_DOC_DIR/${binary}_configuration_reference.adoc \ $SOURCE_ROOT/docs/support/xsl/gflags_to_asciidoc.xsl \ - ${GEN_DOC_DIR}/$binary.xml + ${GEN_DOC_DIR}/${binary}.xml INCLUSIONS_SUPPORTED+="include::${binary}_configuration_reference.adoc[leveloffset=+1]\\"$'\n' # Create the unsupported config reference @@ -183,8 +186,16 @@ for binary in ${binaries[@]}; do --stringparam support-level unsupported \ -o $GEN_DOC_DIR/${binary}_configuration_reference_unsupported.adoc \ $SOURCE_ROOT/docs/support/xsl/gflags_to_asciidoc.xsl \ - ${GEN_DOC_DIR}/$binary.xml + ${GEN_DOC_DIR}/${binary}.xml INCLUSIONS_UNSUPPORTED+="include::${binary}_configuration_reference_unsupported.adoc[leveloffset=+1]\\"$'\n' + + # Create the metrics reference + xsltproc \ + --stringparam binary $binary \ + -o $GEN_DOC_DIR/${binary}_metrics_reference.adoc \ + $SOURCE_ROOT/docs/support/xsl/metrics_to_asciidoc.xsl \ + ${GEN_DOC_DIR}/${binary}-metrics.xml + INCLUSIONS_METRICS+="include::${binary}_metrics_reference.adoc[leveloffset=+1]\\"$'\n' done # Add the includes to the configuration reference files, replacing the template lines @@ -192,6 +203,10 @@ cp $SOURCE_ROOT/docs/configuration_reference* $GEN_DOC_DIR/ $SED "s#@@CONFIGURATION_REFERENCE@@#${INCLUSIONS_SUPPORTED}#" ${GEN_DOC_DIR}/configuration_reference.adoc $SED "s#@@CONFIGURATION_REFERENCE@@#${INCLUSIONS_UNSUPPORTED}#" ${GEN_DOC_DIR}/configuration_reference_unsupported.adoc +# Add the includes to the metrics reference files, replacing the template lines +cp $SOURCE_ROOT/docs/metrics_reference.adoc $GEN_DOC_DIR/ +$SED "s#@@METRICS_REFERENCE@@#${INCLUSIONS_METRICS}#" ${GEN_DOC_DIR}/metrics_reference.adoc + # Create tool references echo "Running kudu --helpxml" ( @@ -206,7 +221,7 @@ echo "Running kudu --helpxml" $BUILD_ROOT/bin/kudu --helpxml > ${GEN_DOC_DIR}/kudu.xml || true ) -# Create the supported config reference +# Create the command line tool reference xsltproc \ -o $GEN_DOC_DIR/command_line_tools.adoc \ $SOURCE_ROOT/docs/support/xsl/tool_to_asciidoc.xsl \ diff --git a/docs/support/xsl/metrics_to_asciidoc.xsl b/docs/support/xsl/metrics_to_asciidoc.xsl new file mode 100644 index 0000000..2738472 --- /dev/null +++ b/docs/support/xsl/metrics_to_asciidoc.xsl @@ -0,0 +1,149 @@ +<?xml version="1.0"?> +<!-- + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +--> + +<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"; version="1.0"> +<xsl:param name="binary"/> +<xsl:output method="text"/> + +<!-- Normalize space --> +<xsl:template match="text()"> + <xsl:if test="normalize-space(.)"> + <xsl:value-of select="normalize-space(.)"/> + </xsl:if> +</xsl:template> + +<!-- Grab nodes of the <metric> elements --> +<xsl:template match="AllMetrics"> +<!-- Inject the license text into the header of each file --> +//// +// +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +//// + +:author: Kudu Team +:imagesdir: ./images +:icons: font +:toc: left +:toclevels: 1 +:doctype: book +:backend: html5 +:sectlinks: +:experimental: + +[[<xsl:value-of select="$binary"/>_metrics]] += `<xsl:value-of select="$binary"/>` Metrics + +[[<xsl:value-of select="$binary"/>_warn]] +== Warning Metrics + +Metrics tagged as 'warn' are metrics which can often indicate operational oddities +that may need more investigation. + +<xsl:for-each select="metric"> + <xsl:sort select="name"/> + <xsl:if test="contains(level, 'warn')"> +[[<xsl:value-of select="$binary"/>_<xsl:value-of select="name"/>]] +=== `<xsl:value-of select="name"/>` + +<xsl:value-of select="label"/> +{nbsp} +<xsl:value-of select="description"/> + +[cols="1h,3d", width="50%"] +|=== +| Entity Type | <xsl:value-of select="entity_type"/> +| Unit | <xsl:value-of select="unit"/> +| Type | <xsl:value-of select="type"/> +| Level | <xsl:value-of select="level"/> +|=== +{nbsp} + + </xsl:if> +</xsl:for-each> + +[[<xsl:value-of select="$binary"/>_info]] +== Info Metrics + +Metrics tagged as 'info' are generally useful metrics that operators always want +to have available but may not be monitored under normal circumstances. + +<xsl:for-each select="metric"> + <xsl:sort select="name"/> + <xsl:if test="contains(level, 'info')"> +[[<xsl:value-of select="$binary"/>_<xsl:value-of select="name"/>]] +=== `<xsl:value-of select="name"/>` + +<xsl:value-of select="label"/> +{nbsp} +<xsl:value-of select="description"/> + +[cols="1h,3d", width="50%"] +|=== +| Entity Type | <xsl:value-of select="entity_type"/> +| Unit | <xsl:value-of select="unit"/> +| Type | <xsl:value-of select="type"/> +| Level | <xsl:value-of select="level"/> +|=== +{nbsp} + + </xsl:if> +</xsl:for-each> + +[[<xsl:value-of select="$binary"/>_debug]] +== Debug Metrics + +Metrics tagged as 'debug' are diagnostically helpful but generally not monitored +during normal operation. + +<xsl:for-each select="metric"> + <xsl:sort select="name"/> + <xsl:if test="contains(level, 'debug')"> +[[<xsl:value-of select="$binary"/>_<xsl:value-of select="name"/>]] +=== `<xsl:value-of select="name"/>` + +<xsl:value-of select="label"/> +{nbsp} +<xsl:value-of select="description"/> + +[cols="1h,3d", width="50%"] +|=== +| Entity Type | <xsl:value-of select="entity_type"/> +| Unit | <xsl:value-of select="unit"/> +| Type | <xsl:value-of select="type"/> +| Level | <xsl:value-of select="level"/> +|=== +{nbsp} + + </xsl:if> +</xsl:for-each> +''' +</xsl:template> +</xsl:stylesheet> diff --git a/src/kudu/util/flags.cc b/src/kudu/util/flags.cc index 4faa5be..4e6c98e 100644 --- a/src/kudu/util/flags.cc +++ b/src/kudu/util/flags.cc @@ -72,6 +72,11 @@ DEFINE_bool(dump_metrics_json, false, "by this binary."); TAG_FLAG(dump_metrics_json, hidden); +DEFINE_bool(dump_metrics_xml, false, + "Dump an XML document describing all of the metrics which may be emitted " + "by this binary."); +TAG_FLAG(dump_metrics_xml, hidden); + #ifdef TCMALLOC_ENABLED DEFINE_bool(enable_process_lifetime_heap_profiling, false, "Enables heap " "profiling for the lifetime of the process. Profile output will be stored in the " @@ -493,6 +498,9 @@ void HandleCommonFlags() { } else if (FLAGS_dump_metrics_json) { MetricPrototypeRegistry::get()->WriteAsJson(); exit(0); + } else if (FLAGS_dump_metrics_xml) { + MetricPrototypeRegistry::get()->WriteAsXML(); + exit(0); } else if (FLAGS_version) { cout << VersionInfo::GetAllVersionInfo() << endl; exit(0); diff --git a/src/kudu/util/metrics.cc b/src/kudu/util/metrics.cc index 39bc450..663ffe6 100644 --- a/src/kudu/util/metrics.cc +++ b/src/kudu/util/metrics.cc @@ -615,6 +615,35 @@ void MetricPrototypeRegistry::WriteAsJson() const { std::cout << s.str() << std::endl; } +void MetricPrototypeRegistry::WriteAsXML() const { + std::lock_guard<simple_spinlock> l(lock_); + std::cout << "<?xml version=\"1.0\"?>" << "\n"; + // Add a root node for the document. + std::cout << "<AllMetrics>" << "\n"; + + // Dump metric prototypes. + for (const MetricPrototype* p : metrics_) { + std::cout << "<metric>"; + std::cout << "<name>" << p->name() << "</name>"; + std::cout << "<label>" << p->label() << "</label>"; + std::cout << "<type>" << MetricType::Name(p->type()) << "</type>"; + std::cout << "<unit>" << MetricUnit::Name(p->unit()) << "</unit>"; + std::cout << "<description>" << p->description() << "</description>"; + std::cout << "<level>" << MetricLevelName(p->level()) << "</level>"; + std::cout << "<entity_type>" << p->entity_type() << "</entity_type>"; + std::cout << "</metric>" << "\n"; + } + + // Dump entity prototypes. + for (const MetricEntityPrototype* e : entities_) { + std::cout << "<entity>"; + std::cout << "<name>" << e->name() << "</name>"; + std::cout << "</entity>" << "\n"; + } + + std::cout << "</AllMetrics>" << "\n"; +} + // // MetricPrototype // diff --git a/src/kudu/util/metrics.h b/src/kudu/util/metrics.h index 3d1b98a..db8bccb 100644 --- a/src/kudu/util/metrics.h +++ b/src/kudu/util/metrics.h @@ -917,6 +917,10 @@ class MetricPrototypeRegistry { // Convenience wrapper around WriteAsJson(...). This dumps the JSON information // to stdout. void WriteAsJson() const; + + // This dumps the XML information to stdout. + void WriteAsXML() const; + private: friend class Singleton<MetricPrototypeRegistry>; friend class MetricPrototype;
