This is an automated email from the ASF dual-hosted git repository.
vinoth pushed a commit to branch asf-site
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/asf-site by this push:
new 4328d28 Travis CI build asf-site
4328d28 is described below
commit 4328d2878ad4e4ef5a675262b7b53f6b42716837
Author: CI <[email protected]>
AuthorDate: Thu Jul 23 05:21:38 2020 +0000
Travis CI build asf-site
---
content/assets/images/hudi_graphite_metrics.png | Bin 0 -> 165146 bytes
content/assets/images/hudi_jxm_metrics.png | Bin 0 -> 243761 bytes
content/assets/js/lunr/lunr-store.js | 5 +
content/docs/0.5.3-cloud.html | 11 +
content/docs/azure_hoodie.html | 11 +
content/docs/cloud.html | 11 +
content/docs/comparison.html | 11 +
content/docs/concepts.html | 11 +
content/docs/configurations.html | 11 +
content/docs/deployment.html | 30 +-
content/docs/docker_demo.html | 11 +
content/docs/docs-versions.html | 11 +
content/docs/gcs_hoodie.html | 11 +
content/docs/metrics.html | 536 ++++++++++++++++++++++++
content/docs/migration_guide.html | 11 +
content/docs/oss_hoodie.html | 11 +
content/docs/performance.html | 11 +
content/docs/powered_by.html | 11 +
content/docs/privacy.html | 11 +
content/docs/querying_data.html | 11 +
content/docs/quick-start-guide.html | 11 +
content/docs/s3_hoodie.html | 11 +
content/docs/structure.html | 11 +
content/docs/use_cases.html | 11 +
content/docs/writing_data.html | 11 +
content/sitemap.xml | 4 +
26 files changed, 776 insertions(+), 19 deletions(-)
diff --git a/content/assets/images/hudi_graphite_metrics.png
b/content/assets/images/hudi_graphite_metrics.png
new file mode 100644
index 0000000..7617a87
Binary files /dev/null and b/content/assets/images/hudi_graphite_metrics.png
differ
diff --git a/content/assets/images/hudi_jxm_metrics.png
b/content/assets/images/hudi_jxm_metrics.png
new file mode 100644
index 0000000..67c2718
Binary files /dev/null and b/content/assets/images/hudi_jxm_metrics.png differ
diff --git a/content/assets/js/lunr/lunr-store.js
b/content/assets/js/lunr/lunr-store.js
index d0e1f6c..4314edf 100644
--- a/content/assets/js/lunr/lunr-store.js
+++ b/content/assets/js/lunr/lunr-store.js
@@ -864,6 +864,11 @@ var store = [{
"tags": [],
"url": "https://hudi.apache.org/docs/cloud.html";,
"teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
+ "title": "HoodieMetrics Guide",
+ "excerpt":"In this section, we will introduce the MetricsReporter and
HoodieMetrics in Hudi. You can view the metrics-related configurations here.
MetricsReporter MetricsReporter provides APIs for reporting HoodieMetrics to
user-specified backends. Currently, the implementations include
InMemoryMetricsReporter, JmxMetricsReporter, MetricsGraphiteReporter and
DatadogMetricsReporter. Since InMemoryMetricsReporter is only used for testing,
we will introduce the other...","categories": [],
+ "tags": [],
+ "url": "https://hudi.apache.org/docs/metrics.html";,
+ "teaser":"https://hudi.apache.org/assets/images/500x300.png"},{
"title": "Privacy Policy",
"excerpt":"Information about your use of this website is collected
using server access logs and a tracking cookie. The collected information
consists of the following: The IP address from which you access the website;
The type of browser and operating system you use to access our site; The date
and time...","categories": [],
"tags": [],
diff --git a/content/docs/0.5.3-cloud.html b/content/docs/0.5.3-cloud.html
index 528ac61..5c48972 100644
--- a/content/docs/0.5.3-cloud.html
+++ b/content/docs/0.5.3-cloud.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/azure_hoodie.html b/content/docs/azure_hoodie.html
index cc344b3..092250d 100644
--- a/content/docs/azure_hoodie.html
+++ b/content/docs/azure_hoodie.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/cloud.html b/content/docs/cloud.html
index 8355f6d..e8d11d6 100644
--- a/content/docs/cloud.html
+++ b/content/docs/cloud.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/comparison.html b/content/docs/comparison.html
index 0bc1559..3fd1343 100644
--- a/content/docs/comparison.html
+++ b/content/docs/comparison.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/concepts.html b/content/docs/concepts.html
index a223ada..62c9f45 100644
--- a/content/docs/concepts.html
+++ b/content/docs/concepts.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/configurations.html b/content/docs/configurations.html
index d7d3814..db94025 100644
--- a/content/docs/configurations.html
+++ b/content/docs/configurations.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/deployment.html b/content/docs/deployment.html
index b335148..023675b 100644
--- a/content/docs/deployment.html
+++ b/content/docs/deployment.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
@@ -376,7 +387,6 @@
<li><a href="#repair-compaction">Repair Compaction</a></li>
</ul>
</li>
- <li><a href="#metrics">Metrics</a></li>
<li><a href="#troubleshooting">Troubleshooting</a>
<ul>
<li><a href="#missing-records">Missing records</a></li>
@@ -881,24 +891,6 @@ consistent with the compaction plan</p>
<span class="o">.....</span>
</code></pre></div></div>
-<h2 id="metrics"><a
href="docs/configurations.html#metrics-configs">Metrics</a></h2>
-
-<p>Once the Hudi writer is configured with the right table and environment for
metrics, it produces the following graphite metrics, that aid in debugging hudi
tables</p>
-
-<ul>
- <li><strong>Commit Duration</strong> - This is amount of time it took to
successfully commit a batch of records</li>
- <li><strong>Rollback Duration</strong> - Similarly, amount of time taken to
undo partial data left over by a failed commit (happens everytime automatically
after a failing write)</li>
- <li><strong>File Level metrics</strong> - Shows the amount of new files
added, versions, deleted (cleaned) in each commit</li>
- <li><strong>Record Level Metrics</strong> - Total records inserted/updated
etc per commit</li>
- <li><strong>Partition Level metrics</strong> - number of partitions upserted
(super useful to understand sudden spikes in commit duration)</li>
-</ul>
-
-<p>These metrics can then be plotted on a standard tool like grafana. Below is
a sample commit duration chart.</p>
-
-<figure>
- <img class="docimage" src="/assets/images/hudi_commit_duration.png"
alt="hudi_commit_duration.png" style="max-width: 100%" />
-</figure>
-
<h2 id="troubleshooting">Troubleshooting</h2>
<p>Section below generally aids in debugging Hudi failures. Off the bat, the
following metadata is added to every record to help triage issues easily using
standard Hadoop SQL engines (Hive/Presto/Spark)</p>
diff --git a/content/docs/docker_demo.html b/content/docs/docker_demo.html
index f1645f2..6f23ab8 100644
--- a/content/docs/docker_demo.html
+++ b/content/docs/docker_demo.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/docs-versions.html b/content/docs/docs-versions.html
index fa8d19e..d441ab4 100644
--- a/content/docs/docs-versions.html
+++ b/content/docs/docs-versions.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/gcs_hoodie.html b/content/docs/gcs_hoodie.html
index 25d5d49..66397c9 100644
--- a/content/docs/gcs_hoodie.html
+++ b/content/docs/gcs_hoodie.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/metrics.html b/content/docs/metrics.html
new file mode 100644
index 0000000..4eb3020
--- /dev/null
+++ b/content/docs/metrics.html
@@ -0,0 +1,536 @@
+<!doctype html>
+<html lang="en" class="no-js">
+ <head>
+ <meta charset="utf-8">
+
+<!-- begin _includes/seo.html --><title>HoodieMetrics Guide - Apache
Hudi</title>
+<meta name="description" content="In this section, we will introduce the
MetricsReporter and HoodieMetrics in Hudi. You can view the metrics-related
configurations here.">
+
+<meta property="og:type" content="article">
+<meta property="og:locale" content="en_US">
+<meta property="og:site_name" content="">
+<meta property="og:title" content="HoodieMetrics Guide">
+<meta property="og:url" content="https://hudi.apache.org/docs/metrics.html";>
+
+
+ <meta property="og:description" content="In this section, we will introduce
the MetricsReporter and HoodieMetrics in Hudi. You can view the metrics-related
configurations here.">
+
+
+
+
+
+ <meta property="article:modified_time" content="2020-06-20T15:59:57-04:00">
+
+
+
+
+
+
+
+<!-- end _includes/seo.html -->
+
+
+<!--<link href="/feed.xml" type="application/atom+xml" rel="alternate" title="
Feed">-->
+
+<!-- https://t.co/dKP3o1e -->
+<meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+<script>
+ document.documentElement.className =
document.documentElement.className.replace(/\bno-js\b/g, '') + ' js ';
+</script>
+
+<!-- For all browsers -->
+<link rel="stylesheet" href="/assets/css/main.css">
+
+<!--[if IE]>
+ <style>
+ /* old IE unsupported flexbox fixes */
+ .greedy-nav .site-title {
+ padding-right: 3em;
+ }
+ .greedy-nav button {
+ position: absolute;
+ top: 0;
+ right: 0;
+ height: 100%;
+ }
+ </style>
+<![endif]-->
+
+
+
+<link rel="icon" type="image/x-icon" href="/assets/images/favicon.ico">
+<link rel="stylesheet" href="/assets/css/font-awesome.min.css">
+<script src="/assets/js/jquery.min.js"></script>
+
+
+<script src="/assets/js/main.min.js"></script>
+
+ </head>
+
+ <body class="layout--single">
+ <!--[if lt IE 9]>
+<div class="notice--danger align-center" style="margin: 0;">You are using an
<strong>outdated</strong> browser. Please <a
href="https://browsehappy.com/";>upgrade your browser</a> to improve your
experience.</div>
+<![endif]-->
+
+ <div class="masthead">
+ <div class="masthead__inner-wrap" id="masthead__inner-wrap">
+ <div class="masthead__menu">
+ <nav id="site-nav" class="greedy-nav">
+
+ <a class="site-logo" href="/">
+ <div style="width: 150px; height: 40px">
+ </div>
+ </a>
+
+ <a class="site-title" href="/">
+
+ </a>
+ <ul class="visible-links"><li class="masthead__menu-item">
+ <a href="/docs/quick-start-guide.html" target="_self"
>Documentation</a>
+ </li><li class="masthead__menu-item">
+ <a href="/community.html" target="_self" >Community</a>
+ </li><li class="masthead__menu-item">
+ <a href="/blog.html" target="_self" >Blog</a>
+ </li><li class="masthead__menu-item">
+ <a href="https://cwiki.apache.org/confluence/display/HUDI/FAQ";
target="_blank" >FAQ</a>
+ </li><li class="masthead__menu-item">
+ <a href="/releases.html" target="_self" >Releases</a>
+ </li></ul>
+ <button class="greedy-nav__toggle hidden" type="button">
+ <span class="visually-hidden">Toggle menu</span>
+ <div class="navicon"></div>
+ </button>
+ <ul class="hidden-links hidden"></ul>
+ </nav>
+ </div>
+ </div>
+</div>
+<!--
+<p class="notice--warning" style="margin: 0 !important; text-align: center
!important;"><strong>Note:</strong> This site is work in progress, if you
notice any issues, please <a target="_blank"
href="https://github.com/apache/hudi/issues";>Report on Issue</a>.
+ Click <a href="/"> here</a> back to old site.</p>
+-->
+
+ <div class="initial-content">
+ <div id="main" role="main">
+
+
+ <div class="sidebar sticky">
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+<nav class="nav__list">
+
+ <input id="ac-toc" name="accordion-toc" type="checkbox" />
+ <label for="ac-toc">Toggle Menu</label>
+ <ul class="nav__items">
+
+ <li>
+
+ <span class="nav__sub-title">Getting Started</span>
+
+
+
+ <ul>
+
+
+
+
+
+
+
+
+ <li><a href="/docs/quick-start-guide.html" class="">Quick
Start</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/use_cases.html" class="">Use Cases</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/powered_by.html" class="">Talks & Powered
By</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/comparison.html" class="">Comparison</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/docker_demo.html" class="">Docker
Demo</a></li>
+
+
+
+ </ul>
+
+ </li>
+
+ <li>
+
+ <span class="nav__sub-title">Documentation</span>
+
+
+
+ <ul>
+
+
+
+
+
+
+
+
+ <li><a href="/docs/concepts.html" class="">Concepts</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/writing_data.html" class="">Writing
Data</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/querying_data.html" class="">Querying
Data</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/configurations.html"
class="">Configuration</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/performance.html"
class="">Performance</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/deployment.html" class="">Deployment</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/cloud.html" class="">Cloud Storage</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="active">Metrics</a></li>
+
+
+
+ </ul>
+
+ </li>
+
+ <li>
+
+ <span class="nav__sub-title">INFO</span>
+
+
+
+ <ul>
+
+
+
+
+
+
+
+
+ <li><a href="/docs/docs-versions.html" class="">Docs
Versions</a></li>
+
+
+
+
+
+
+
+
+
+
+ <li><a href="/docs/privacy.html" class="">Privacy Policy</a></li>
+
+
+
+ </ul>
+
+ </li>
+
+ </ul>
+</nav>
+
+
+
+
+ </div>
+
+
+ <article class="page" itemscope itemtype="https://schema.org/CreativeWork";>
+ <!-- Look the author details up from the site config. -->
+
+
+ <div class="page__inner-wrap">
+
+ <header>
+ <h1 id="page-title" class="page__title"
itemprop="headline">HoodieMetrics Guide
+</h1>
+ <!-- Output author details if some exist. -->
+
+ </header>
+
+
+ <section class="page__content" itemprop="text">
+
+ <aside class="sidebar__right sticky">
+ <nav class="toc">
+ <header><h4 class="nav__title"><i class="fas fa-file-alt"></i> IN
THIS PAGE</h4></header>
+ <ul class="toc__menu">
+ <li><a href="#metricsreporter">MetricsReporter</a>
+ <ul>
+ <li><a href="#jmxmetricsreporter">JmxMetricsReporter</a></li>
+ <li><a href="#metricsgraphitereporter">MetricsGraphiteReporter</a></li>
+ <li><a href="#datadogmetricsreporter">DatadogMetricsReporter</a></li>
+ </ul>
+ </li>
+ <li><a href="#hoodiemetrics">HoodieMetrics</a></li>
+</ul>
+ </nav>
+ </aside>
+
+ <p>In this section, we will introduce the <code
class="highlighter-rouge">MetricsReporter</code> and <code
class="highlighter-rouge">HoodieMetrics</code> in Hudi. You can view the
metrics-related configurations <a
href="configurations.html#metrics-configs">here</a>.</p>
+
+<h2 id="metricsreporter">MetricsReporter</h2>
+
+<p>MetricsReporter provides APIs for reporting <code
class="highlighter-rouge">HoodieMetrics</code> to user-specified backends.
Currently, the implementations include InMemoryMetricsReporter,
JmxMetricsReporter, MetricsGraphiteReporter and DatadogMetricsReporter. Since
InMemoryMetricsReporter is only used for testing, we will introduce the other
three implementations.</p>
+
+<h3 id="jmxmetricsreporter">JmxMetricsReporter</h3>
+
+<p>JmxMetricsReporter is an implementation of JMX reporter, which used to
report JMX metrics.</p>
+
+<h4 id="configurations">Configurations</h4>
+<p>The following is an example of <code
class="highlighter-rouge">JmxMetricsReporter</code>. More detaile
configurations can be referenced <a href="configurations.html#jmx">here</a>.</p>
+
+<div class="language-properties highlighter-rouge"><div class="highlight"><pre
class="highlight"><code> <span class="py">hoodie.metrics.on</span><span
class="p">=</span><span class="s">true</span>
+ <span class="py">hoodie.metrics.reporter.type</span><span
class="p">=</span><span class="s">JMX</span>
+ <span class="py">hoodie.metrics.jmx.host</span><span class="p">=</span><span
class="s">192.168.0.106</span>
+ <span class="py">hoodie.metrics.jmx.port</span><span class="p">=</span><span
class="s">4001</span>
+</code></pre></div></div>
+
+<h4 id="demo">Demo</h4>
+<p>As configured above, JmxMetricsReporter will started JMX server on port
4001. We can start a jconsole to connect to 192.168.0.106:4001. Below is an
illustration of monitoring Hudi JMX metrics through jconsole.</p>
+<figure>
+ <img class="docimage" src="/assets/images/hudi_jxm_metrics.png"
alt="hudi_jxm_metrics.png" style="max-width: 100%" />
+</figure>
+
+<h3 id="metricsgraphitereporter">MetricsGraphiteReporter</h3>
+
+<p>MetricsGraphiteReporter is an implementation of Graphite reporter, which
connects to a Graphite server, and send <code
class="highlighter-rouge">HoodieMetrics</code> to it.</p>
+
+<h4 id="configurations-1">Configurations</h4>
+<p>The following is an example of <code
class="highlighter-rouge">MetricsGraphiteReporter</code>. More detaile
configurations can be referenced <a
href="configurations.html#graphite">here</a>.</p>
+
+<div class="language-properties highlighter-rouge"><div class="highlight"><pre
class="highlight"><code> <span class="py">hoodie.metrics.on</span><span
class="p">=</span><span class="s">true</span>
+ <span class="py">hoodie.metrics.reporter.type</span><span
class="p">=</span><span class="s">GRAPHITE</span>
+ <span class="py">hoodie.metrics.graphite.host</span><span
class="p">=</span><span class="s">192.168.0.106</span>
+ <span class="py">hoodie.metrics.graphite.port</span><span
class="p">=</span><span class="s">2003</span>
+ <span class="py">hoodie.metrics.graphite.metric.prefix</span><span
class="p">=</span><span class="s"><your metrics prefix></span>
+</code></pre></div></div>
+<h4 id="demo-1">Demo</h4>
+<p>As configured above, assuming a Graphite server is running on host
192.168.0.106 and port 2003, a running Hudi job will connect and report metrics
data to it. Below is an illustration of monitoring hudi metrics through
Graphite.</p>
+<figure>
+ <img class="docimage" src="/assets/images/hudi_graphite_metrics.png"
alt="hudi_graphite_metrics.png" style="max-width: 100%" />
+ </figure>
+
+<h3 id="datadogmetricsreporter">DatadogMetricsReporter</h3>
+
+<p>DatadogMetricsReporter is an implementation of Datadog reporter.
+A reporter which publishes metric values to Datadog monitoring service via
Datadog HTTP API.</p>
+
+<h4 id="configurations-2">Configurations</h4>
+<p>The following is an example of <code
class="highlighter-rouge">DatadogMetricsReporter</code>. More detailed
configurations can be referenced <a
href="configurations.html#datadog">here</a>.</p>
+
+<div class="language-properties highlighter-rouge"><div class="highlight"><pre
class="highlight"><code><span class="py">hoodie.metrics.on</span><span
class="p">=</span><span class="s">true</span>
+<span class="py">hoodie.metrics.reporter.type</span><span
class="p">=</span><span class="s">DATADOG</span>
+<span class="py">hoodie.metrics.datadog.api.site</span><span
class="p">=</span><span class="s">EU # or US</span>
+<span class="py">hoodie.metrics.datadog.api.key</span><span
class="p">=</span><span class="s"><your api key></span>
+<span class="py">hoodie.metrics.datadog.metric.prefix</span><span
class="p">=</span><span class="s"><your metrics prefix></span>
+</code></pre></div></div>
+
+<ul>
+ <li><code class="highlighter-rouge">hoodie.metrics.datadog.api.site</code>
will set the Datadog API site, which determines whether the requests will be
sent to api.datadoghq.eu (EU) or api.datadoghq.com (US). Set this according to
your Datadog account settings.</li>
+ <li><code class="highlighter-rouge">hoodie.metrics.datadog.api.key</code>
will set the api key.</li>
+ <li><code
class="highlighter-rouge">hoodie.metrics.datadog.metric.prefix</code> will help
segregate metrics by setting different prefixes for different jobs. Note that
it will use <code class="highlighter-rouge">.</code> to delimit the prefix and
the metric name. For example, if the prefix is set to <code
class="highlighter-rouge">foo</code>, then <code
class="highlighter-rouge">foo.</code> will be prepended to the metric name.</li>
+</ul>
+
+<h4 id="demo-2">Demo</h4>
+<p>In this demo, we ran a <code
class="highlighter-rouge">HoodieDeltaStreamer</code> job with <code
class="highlighter-rouge">HoodieMetrics</code> turned on and other
configurations set properly.</p>
+
+<figure>
+ <img class="docimage"
src="/assets/images/blog/2020-05-28-datadog-metrics-demo.png"
alt="hudi_datadog_metrics.png" style="max-width: 100%" />
+</figure>
+
+<p>As shown above, we were able to collect Hudi’s action-related metrics
like</p>
+
+<ul>
+ <li><code class="highlighter-rouge"><prefix>.<table
name>.commit.totalScanTime</code></li>
+ <li><code class="highlighter-rouge"><prefix>.<table
name>.clean.duration</code></li>
+ <li><code class="highlighter-rouge"><prefix>.<table
name>.index.lookup.duration</code></li>
+</ul>
+
+<p>as well as <code
class="highlighter-rouge">HoodieDeltaStreamer</code>-specific metrics</p>
+
+<ul>
+ <li><code class="highlighter-rouge"><prefix>.<table
name>.deltastreamer.duration</code></li>
+ <li><code class="highlighter-rouge"><prefix>.<table
name>.deltastreamer.hiveSyncDuration</code></li>
+</ul>
+
+<h2 id="hoodiemetrics">HoodieMetrics</h2>
+
+<p>Once the Hudi writer is configured with the right table and environment for
<code class="highlighter-rouge">HoodieMetrics</code>, it produces the following
<code class="highlighter-rouge">HoodieMetrics</code>, that aid in debugging
hudi tables</p>
+
+<ul>
+ <li><strong>Commit Duration</strong> - The amount of time it took to
successfully commit a batch of records</li>
+ <li><strong>Rollback Duration</strong> - Similarly, the amount of time taken
to undo partial data left over by a failed commit (rollback happens
automatically after a failing write)</li>
+ <li><strong>File Level metrics</strong> - Shows the amount of new files
added, versions, deleted (cleaned) in each commit</li>
+ <li><strong>Record Level Metrics</strong> - Total records inserted/updated
etc per commit</li>
+ <li><strong>Partition Level metrics</strong> - number of partitions upserted
(super useful to understand sudden spikes in commit duration)</li>
+</ul>
+
+<p>These <code class="highlighter-rouge">HoodieMetrics</code> can then be
plotted on a standard tool like grafana. Below is a sample commit duration
chart.</p>
+
+<figure>
+ <img class="docimage" src="/assets/images/hudi_commit_duration.png"
alt="hudi_commit_duration.png" style="max-width: 100%" />
+</figure>
+
+ </section>
+
+ <a href="#masthead__inner-wrap" class="back-to-top">Back to top
↑</a>
+
+
+
+
+ </div>
+
+ </article>
+
+</div>
+
+ </div>
+
+ <div class="page__footer">
+ <footer>
+
+<div class="row">
+ <div class="col-lg-12 footer">
+ <p>
+ <table class="table-apache-info">
+ <tr>
+ <td>
+ <a class="footer-link-img" href="https://apache.org";>
+ <img width="250px" src="/assets/images/asf_logo.svg" alt="The
Apache Software Foundation">
+ </a>
+ </td>
+ <td>
+ <a style="float: right"
href="https://www.apache.org/events/current-event.html";>
+ <img
src="https://www.apache.org/events/current-event-234x60.png"; />
+ </a>
+ </td>
+ </tr>
+ </table>
+ </p>
+ <p>
+ <a href="https://www.apache.org/licenses/";>License</a> | <a
href="https://www.apache.org/security/";>Security</a> | <a
href="https://www.apache.org/foundation/thanks.html";>Thanks</a> | <a
href="https://www.apache.org/foundation/sponsorship.html";>Sponsorship</a>
+ </p>
+ <p>
+ Copyright © <span id="copyright-year">2019</span> <a
href="https://apache.org";>The Apache Software Foundation</a>, Licensed under
the <a href="https://www.apache.org/licenses/LICENSE-2.0";> Apache License,
Version 2.0</a>.
+ Hudi, Apache and the Apache feather logo are trademarks of The Apache
Software Foundation. <a href="/docs/privacy">Privacy Policy</a>
+ </p>
+ </div>
+</div>
+ </footer>
+ </div>
+
+
+ </body>
+</html>
\ No newline at end of file
diff --git a/content/docs/migration_guide.html
b/content/docs/migration_guide.html
index b5eb556..58ec7c2 100644
--- a/content/docs/migration_guide.html
+++ b/content/docs/migration_guide.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/oss_hoodie.html b/content/docs/oss_hoodie.html
index cfb3e22..40ba008 100644
--- a/content/docs/oss_hoodie.html
+++ b/content/docs/oss_hoodie.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/performance.html b/content/docs/performance.html
index c9eceb5..5ff7af4 100644
--- a/content/docs/performance.html
+++ b/content/docs/performance.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/powered_by.html b/content/docs/powered_by.html
index e824aee..4ab6547 100644
--- a/content/docs/powered_by.html
+++ b/content/docs/powered_by.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/privacy.html b/content/docs/privacy.html
index 309891c..cfdb1a0 100644
--- a/content/docs/privacy.html
+++ b/content/docs/privacy.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/querying_data.html b/content/docs/querying_data.html
index 86f195b..a280ece 100644
--- a/content/docs/querying_data.html
+++ b/content/docs/querying_data.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/quick-start-guide.html
b/content/docs/quick-start-guide.html
index 85bfb64..ba3f188 100644
--- a/content/docs/quick-start-guide.html
+++ b/content/docs/quick-start-guide.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/s3_hoodie.html b/content/docs/s3_hoodie.html
index 2759c06..e03dce1 100644
--- a/content/docs/s3_hoodie.html
+++ b/content/docs/s3_hoodie.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/structure.html b/content/docs/structure.html
index 66ef1e6..e68514f 100644
--- a/content/docs/structure.html
+++ b/content/docs/structure.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/use_cases.html b/content/docs/use_cases.html
index 06cc0c8..6b3e408 100644
--- a/content/docs/use_cases.html
+++ b/content/docs/use_cases.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/docs/writing_data.html b/content/docs/writing_data.html
index 6452694..e229106 100644
--- a/content/docs/writing_data.html
+++ b/content/docs/writing_data.html
@@ -288,6 +288,17 @@
+
+
+
+
+
+
+
+ <li><a href="/docs/metrics.html" class="">Metrics</a></li>
+
+
+
</ul>
</li>
diff --git a/content/sitemap.xml b/content/sitemap.xml
index c99a3a2..1d0b7e4 100644
--- a/content/sitemap.xml
+++ b/content/sitemap.xml
@@ -693,6 +693,10 @@
<lastmod>2019-06-16T21:59:57-04:00</lastmod>
</url>
<url>
+<loc>https://hudi.apache.org/docs/metrics.html</loc>
+<lastmod>2020-06-20T15:59:57-04:00</lastmod>
+</url>
+<url>
<loc>https://hudi.apache.org/cn/docs/privacy.html</loc>
<lastmod>2019-12-30T14:59:57-05:00</lastmod>
</url>
![https://hudi.apache.org/assets/images/500x300.png"},{](https://hudi.apache.org/assets/images/500x300.png"},{){kind=link}
![https://www.apache.org/events/current-event-234x60.png"](https://www.apache.org/events/current-event-234x60.png"){kind=link}