danny0405 commented on code in PR #20165:
URL: https://github.com/apache/hudi/pull/20165#discussion_r4300869414
##########
hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkHiveSyncMetrics.java:
##########
@@ -56,15 +67,54 @@ public class FlinkHiveSyncMetrics extends
HoodieFlinkMetrics {
* Duration of each sync in milliseconds, failed ones included: a sync that
fails after retrying
* an unreachable metastore is the slow case this is meant to show.
*/
- private final Histogram syncDurationMs;
+ private final Histogram syncDurationMs = newHistogram();
/** Wall-clock time of the last completed sync in epoch milliseconds, or 0
until the first one. */
private volatile long lastSyncSuccessTimeMs = 0L;
- public FlinkHiveSyncMetrics(MetricGroup metricGroup) {
+ /**
+ * Time to build the sync tool, which creates the Hudi meta client and
connects to the metastore.
+ * Recorded whether building it succeeds or fails, so the time spent failing
to reach the
+ * metastore shows here.
+ */
+ private final Histogram initDurationMs = newHistogram();
+
+ /** Time a sync spent reading the table schema from storage, for syncs that
read it. */
+ private final Histogram schemaReadDurationMs = newHistogram();
+
+ /**
+ * Time a sync spent finding the partitions to sync, from the timeline or by
listing storage, for
+ * syncs that looked for them. This grows with the number of commits a sync
has to catch up on.
+ */
+ private final Histogram partitionScanDurationMs = newHistogram();
+
+ /**
+ * Time a sync spent on everything but reading the schema and finding
partitions: mostly metastore
+ * calls, plus the local work between them, such as comparing the partitions
and schemas found.
+ */
+ private final Histogram remainingDurationMs = newHistogram();
+
+ /**
+ * Number of partitions added to the metastore. A MERGE_ON_READ table adds
the same partitions to
+ * each of its Hive tables, and they are counted once. It staying flat while
commits land means
+ * the syncs register nothing.
+ */
+ private final Counter partitionsAddedCount = new ThreadSafeSimpleCounter();
+
+ /**
+ * Number of syncs that found the storage schema changed and pushed it to
the metastore, counted
+ * once per sync however many Hive tables the table is registered as.
+ */
+ private final Counter schemaEvolvedCount = new ThreadSafeSimpleCounter();
+
+ private final boolean statsEnabled;
+
+ /**
+ * @param statsEnabled whether to register the breakdown of each sync, which
the sync records only when its stats are enabled
+ */
+ public FlinkHiveSyncMetrics(MetricGroup metricGroup, boolean statsEnabled) {
Review Comment:
statsEnabled -> detailedStatsEnabled?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
