This is an automated email from the ASF dual-hosted git repository. yihua pushed a commit to branch branch-0.x in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 8eb7518711d9d8c9d705b1d91cfaba8cca81cddb Author: Jon Vexler <[email protected]> AuthorDate: Thu Apr 25 19:46:17 2024 -0400 [HUDI-7658] Add time to meta sync failure log (#11080) Co-authored-by: Jonathan Vexler <=> --- .../apache/hudi/utilities/streamer/StreamSync.java | 26 +++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java index 3c6c36d2a3e..90f2e712b51 100644 --- a/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java +++ b/hudi-utilities/src/main/java/org/apache/hudi/utilities/streamer/StreamSync.java @@ -1035,20 +1035,13 @@ public class StreamSync implements Serializable, Closeable { Map<String, HoodieException> failedMetaSyncs = new HashMap<>(); for (String impl : syncClientToolClasses) { Timer.Context syncContext = metrics.getMetaSyncTimerContext(); - boolean success = false; + Option<HoodieMetaSyncException> metaSyncException = Option.empty(); try { SyncUtilHelpers.runHoodieMetaSync(impl.trim(), metaProps, conf, fs, cfg.targetBasePath, cfg.baseFileFormat); - success = true; } catch (HoodieMetaSyncException e) { - LOG.error("SyncTool class {} failed with exception {}", impl.trim(), e); - failedMetaSyncs.put(impl, e); - } - long metaSyncTimeNanos = syncContext != null ? syncContext.stop() : 0; - metrics.updateStreamerMetaSyncMetrics(getSyncClassShortName(impl), metaSyncTimeNanos); - if (success) { - long timeMs = metaSyncTimeNanos / 1000000L; - LOG.info("[MetaSync] SyncTool class {} completed successfully and took {} s {} ms ", impl.trim(), timeMs / 1000L, timeMs % 1000L); + metaSyncException = Option.of(e); } + logMetaSync(impl, syncContext, failedMetaSyncs, metaSyncException); } if (!failedMetaSyncs.isEmpty()) { throw getHoodieMetaSyncException(failedMetaSyncs); @@ -1056,6 +1049,19 @@ public class StreamSync implements Serializable, Closeable { } } + private void logMetaSync(String impl, Timer.Context syncContext, Map<String, HoodieException> failedMetaSyncs, Option<HoodieMetaSyncException> metaSyncException) { + long metaSyncTimeNanos = syncContext != null ? syncContext.stop() : 0; + metrics.updateStreamerMetaSyncMetrics(getSyncClassShortName(impl), metaSyncTimeNanos); + long timeMs = metaSyncTimeNanos / 1000000L; + String timeString = String.format("and took %d s %d ms ", timeMs / 1000L, timeMs % 1000L); + if (metaSyncException.isPresent()) { + LOG.error("[MetaSync] SyncTool class {} failed with exception {} {}", impl.trim(), metaSyncException.get(), timeString); + failedMetaSyncs.put(impl, metaSyncException.get()); + } else { + LOG.info("[MetaSync] SyncTool class {} completed successfully {}", impl.trim(), timeString); + } + } + /** * Note that depending on configs and source-type, schemaProvider could either be eagerly or lazily created. * SchemaProvider creation is a precursor to HoodieWriteClient and AsyncCompactor creation. This method takes care of
