This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new fff794d4b384 fix(flink): keep reporting hive sync metrics after a
coordinator restart (#20499)
fff794d4b384 is described below
commit fff794d4b384bf45fb8f5d396434be010fdf4833
Author: Shihuan Liu <[email protected]>
AuthorDate: Wed Nov 4 18:32:19 2026 -0800
fix(flink): keep reporting hive sync metrics after a coordinator restart
(#20499)
* fix(flink): keep reporting hive sync metrics after a coordinator restart
* fix(flink): hold shared hive sync metrics weakly so finished jobs release
them
---
.../apache/hudi/metrics/FlinkHiveSyncMetrics.java | 71 ++++++++++++---
.../hudi/sink/StreamWriteOperatorCoordinator.java | 3 +-
.../hudi/metrics/TestFlinkHiveSyncMetrics.java | 59 ++++++++++++
.../sink/TestStreamWriteOperatorCoordinator.java | 19 ++++
.../hudi/utils/SharedCoordinatorMetricGroup.java | 100 +++++++++++++++++++++
5 files changed, 239 insertions(+), 13 deletions(-)
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkHiveSyncMetrics.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkHiveSyncMetrics.java
index dd1a6d99127d..9ae6fc074bb0 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkHiveSyncMetrics.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/metrics/FlinkHiveSyncMetrics.java
@@ -21,11 +21,16 @@ package org.apache.hudi.metrics;
import org.apache.hudi.hive.HoodieHiveSyncStats;
import com.codahale.metrics.SlidingWindowReservoir;
+import com.github.benmanes.caffeine.cache.Cache;
+import com.github.benmanes.caffeine.cache.Caffeine;
import org.apache.flink.dropwizard.metrics.DropwizardHistogramWrapper;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Histogram;
import org.apache.flink.metrics.MetricGroup;
import org.apache.flink.metrics.ThreadSafeSimpleCounter;
+import org.apache.flink.runtime.metrics.scope.ScopeFormat;
+
+import java.util.Map;
/**
* Metrics for the Hive sync that {@link
org.apache.hudi.sink.StreamWriteOperatorCoordinator} runs
@@ -37,10 +42,27 @@ import org.apache.flink.metrics.ThreadSafeSimpleCounter;
*
* <p>The breakdown of each sync, from the init duration to the schema evolved
count, is only
* registered when {@code
hoodie.datasource.hive_sync.detailed_metrics.enabled} is set.
+ *
+ * <p>Obtain an instance with {@link #register}. Flink keeps one coordinator
metric group per operator
+ * for the life of the job, but creates a new coordinator whenever it rebuilds
the execution graph,
+ * as the adaptive scheduler does on every restart and rescale. That group
ignores a metric registered
+ * under a name it already holds, so metrics owned by each coordinator would
stay frozen at the first
+ * coordinator's values. Every coordinator of the same operator in the same
job therefore shares one
+ * instance.
*/
public class FlinkHiveSyncMetrics extends HoodieFlinkMetrics {
private static final int HISTOGRAM_WINDOW_SIZE = 100;
+ /**
+ * The instance shared by the coordinators of each operator, keyed by job
and operator ID.
+ *
+ * <p>Values are weak: between coordinators, the only strong reference to an
instance is the
+ * {@link #HIVE_SYNC_LAST_SUCCESS_TIME_MS} gauge registered on the
operator's group, so an instance
+ * lives as long as that group reports it and its entry goes once the job's
group is closed. That
+ * gauge must keep referencing the instance, and the cache must not be
bounded by size.
+ */
+ private static final Cache<String, FlinkHiveSyncMetrics> SHARED_METRICS =
Caffeine.newBuilder().weakValues().build();
+
static final String HIVE_SYNC_SUCCESS_COUNT = "hiveSyncSuccessCount";
static final String HIVE_SYNC_FAILURE_COUNT = "hiveSyncFailureCount";
static final String HIVE_SYNC_DURATION_MS = "hiveSyncDurationMs";
@@ -114,7 +136,7 @@ public class FlinkHiveSyncMetrics extends
HoodieFlinkMetrics {
* @param detailedMetricsEnabled whether to register and update the
breakdown of each sync, which the sync records
* only when its detailed metrics are enabled
*/
- public FlinkHiveSyncMetrics(MetricGroup metricGroup, boolean
detailedMetricsEnabled) {
+ FlinkHiveSyncMetrics(MetricGroup metricGroup, boolean
detailedMetricsEnabled) {
super(metricGroup);
this.detailedMetricsEnabled = detailedMetricsEnabled;
// Detailed histograms are only accessed when detailed metrics are enabled.
@@ -124,19 +146,46 @@ public class FlinkHiveSyncMetrics extends
HoodieFlinkMetrics {
this.remainingDurationMs = detailedMetricsEnabled ? newHistogram() : null;
}
+ /**
+ * Returns the metrics of the coordinator that owns {@code metricGroup},
registered on it.
+ *
+ * <p>A coordinator that replaces an earlier one of the same operator and
job gets the earlier
+ * one's instance, so its syncs update the metrics the group already
reports, and the counts carry
+ * on across the restart. The instance is still registered on {@code
metricGroup}, which covers a
+ * group that is not the one it was first registered on, such as the group
of a JobMaster
+ * recreated for the same job.
+ *
+ * <p>A group that does not identify its job and operator, as in tests, gets
an instance of its own.
+ */
+ public static FlinkHiveSyncMetrics register(MetricGroup metricGroup, boolean
detailedMetricsEnabled) {
+ Map<String, String> variables = metricGroup.getAllVariables();
+ String jobId = variables.get(ScopeFormat.SCOPE_JOB_ID);
+ String operatorId = variables.get(ScopeFormat.SCOPE_OPERATOR_ID);
+ FlinkHiveSyncMetrics metrics = jobId == null || operatorId == null
+ ? new FlinkHiveSyncMetrics(metricGroup, detailedMetricsEnabled)
+ : SHARED_METRICS.get(jobId + "/" + operatorId,
+ key -> new FlinkHiveSyncMetrics(metricGroup,
detailedMetricsEnabled));
+ metrics.registerMetrics(metricGroup);
+ return metrics;
+ }
+
@Override
public void registerMetrics() {
- metricGroup.counter(HIVE_SYNC_SUCCESS_COUNT, syncSuccessCount);
- metricGroup.counter(HIVE_SYNC_FAILURE_COUNT, syncFailureCount);
- metricGroup.histogram(HIVE_SYNC_DURATION_MS, syncDurationMs);
- metricGroup.gauge(HIVE_SYNC_LAST_SUCCESS_TIME_MS, () ->
lastSyncSuccessTimeMs);
+ registerMetrics(metricGroup);
+ }
+
+ private void registerMetrics(MetricGroup group) {
+ group.counter(HIVE_SYNC_SUCCESS_COUNT, syncSuccessCount);
+ group.counter(HIVE_SYNC_FAILURE_COUNT, syncFailureCount);
+ group.histogram(HIVE_SYNC_DURATION_MS, syncDurationMs);
+ group.gauge(HIVE_SYNC_LAST_SUCCESS_TIME_MS, () -> lastSyncSuccessTimeMs);
if (detailedMetricsEnabled) {
- metricGroup.histogram(HIVE_SYNC_INIT_DURATION_MS, initDurationMs);
- metricGroup.histogram(HIVE_SYNC_SCHEMA_READ_DURATION_MS,
schemaReadDurationMs);
- metricGroup.histogram(HIVE_SYNC_PARTITION_SCAN_DURATION_MS,
partitionScanDurationMs);
- metricGroup.histogram(HIVE_SYNC_REMAINING_DURATION_MS,
remainingDurationMs);
- metricGroup.counter(HIVE_SYNC_PARTITIONS_ADDED_COUNT,
partitionsAddedCount);
- metricGroup.counter(HIVE_SYNC_SCHEMA_EVOLVED_COUNT, schemaEvolvedCount);
+ group.histogram(HIVE_SYNC_INIT_DURATION_MS, initDurationMs);
+ group.histogram(HIVE_SYNC_SCHEMA_READ_DURATION_MS, schemaReadDurationMs);
+ group.histogram(HIVE_SYNC_PARTITION_SCAN_DURATION_MS,
partitionScanDurationMs);
+ group.histogram(HIVE_SYNC_REMAINING_DURATION_MS, remainingDurationMs);
+ group.counter(HIVE_SYNC_PARTITIONS_ADDED_COUNT, partitionsAddedCount);
+ group.counter(HIVE_SYNC_SCHEMA_EVOLVED_COUNT, schemaEvolvedCount);
}
}
diff --git
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
index 595578c05d7f..87c944780d3a 100644
---
a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
+++
b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/sink/StreamWriteOperatorCoordinator.java
@@ -475,8 +475,7 @@ public class StreamWriteOperatorCoordinator
.threadFactory(getThreadFactory("hive-sync"))
.waitForTasksFinish(true).build();
this.hiveSyncContext = HiveSyncContext.create(conf, this.storageConf);
- this.hiveSyncMetrics = new FlinkHiveSyncMetrics(context.metricGroup(),
hiveSyncContext.isDetailedMetricsEnabled());
- this.hiveSyncMetrics.registerMetrics();
+ this.hiveSyncMetrics =
FlinkHiveSyncMetrics.register(context.metricGroup(),
hiveSyncContext.isDetailedMetricsEnabled());
}
private void syncHiveAsync() {
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkHiveSyncMetrics.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkHiveSyncMetrics.java
index 9026ea0d2219..149053e5b5e9 100644
---
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkHiveSyncMetrics.java
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/metrics/TestFlinkHiveSyncMetrics.java
@@ -20,11 +20,14 @@ package org.apache.hudi.metrics;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.hive.HoodieHiveSyncStats;
+import org.apache.hudi.utils.SharedCoordinatorMetricGroup;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Gauge;
import org.apache.flink.metrics.Histogram;
import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.jobgraph.OperatorID;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -48,7 +51,9 @@ import static
org.apache.hudi.metrics.FlinkHiveSyncMetrics.HIVE_SYNC_SCHEMA_READ
import static
org.apache.hudi.metrics.FlinkHiveSyncMetrics.HIVE_SYNC_SUCCESS_COUNT;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -235,6 +240,60 @@ class TestFlinkHiveSyncMetrics {
assertEquals(2L * syncsPerThread, duration().getCount());
}
+ @Test
+ void testRestartedCoordinatorUpdatesRegisteredMetrics() {
+ SharedCoordinatorMetricGroup group = new SharedCoordinatorMetricGroup(new
JobID().toString(), new OperatorID().toString());
+ FlinkHiveSyncMetrics first = FlinkHiveSyncMetrics.register(group, true);
+ first.markSyncSucceeded(10L);
+
+ FlinkHiveSyncMetrics second =
FlinkHiveSyncMetrics.register(group.newView(), true);
+ second.markSyncSucceeded(20L);
+ second.markSyncFailed(30L);
+ second.updateSyncMetrics(stats(Option.of(1L), Option.of(2L),
Option.of(3L), 5, true));
+
+ assertSame(first, second);
+ assertEquals(2, group.getCounter(HIVE_SYNC_SUCCESS_COUNT).getCount());
+ assertEquals(1, group.getCounter(HIVE_SYNC_FAILURE_COUNT).getCount());
+ assertEquals(3, group.getHistogram(HIVE_SYNC_DURATION_MS).getCount());
+ assertTrue((Long)
group.getGauge(HIVE_SYNC_LAST_SUCCESS_TIME_MS).getValue() > 0L);
+ assertEquals(5,
group.getCounter(HIVE_SYNC_PARTITIONS_ADDED_COUNT).getCount());
+ assertEquals(1,
group.getCounter(HIVE_SYNC_SCHEMA_EVOLVED_COUNT).getCount());
+ }
+
+ @Test
+ void testSharedMetricsRegisteredOnNewGroupOfSameOperator() {
+ String jobId = new JobID().toString();
+ String operatorId = new OperatorID().toString();
+ FlinkHiveSyncMetrics first = FlinkHiveSyncMetrics.register(new
SharedCoordinatorMetricGroup(jobId, operatorId), true);
+ first.markSyncSucceeded(10L);
+
+ SharedCoordinatorMetricGroup newGroup = new
SharedCoordinatorMetricGroup(jobId, operatorId);
+ FlinkHiveSyncMetrics second = FlinkHiveSyncMetrics.register(newGroup,
true);
+
+ assertSame(first, second);
+ assertEquals(1, newGroup.getCounter(HIVE_SYNC_SUCCESS_COUNT).getCount());
+ assertNotNull(newGroup.getHistogram(HIVE_SYNC_INIT_DURATION_MS));
+ }
+
+ @Test
+ void testMetricsNotSharedAcrossOperatorsOrJobs() {
+ String jobId = new JobID().toString();
+ String operatorId = new OperatorID().toString();
+ FlinkHiveSyncMetrics metrics = FlinkHiveSyncMetrics.register(new
SharedCoordinatorMetricGroup(jobId, operatorId), true);
+
+ assertNotSame(metrics,
+ FlinkHiveSyncMetrics.register(new SharedCoordinatorMetricGroup(jobId,
new OperatorID().toString()), true));
+ assertNotSame(metrics,
+ FlinkHiveSyncMetrics.register(new SharedCoordinatorMetricGroup(new
JobID().toString(), operatorId), true));
+ }
+
+ @Test
+ void testMetricsNotSharedWhenGroupDoesNotIdentifyOperator() {
+ CapturingMetricGroup group = new CapturingMetricGroup();
+
+ assertNotSame(FlinkHiveSyncMetrics.register(group, true),
FlinkHiveSyncMetrics.register(group, true));
+ }
+
private long successCount() {
return metricGroup.counters.get(HIVE_SYNC_SUCCESS_COUNT).getCount();
}
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
index 5664dcbb9c60..7adfa975ef59 100644
---
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/sink/TestStreamWriteOperatorCoordinator.java
@@ -65,9 +65,11 @@ import org.apache.hudi.storage.StoragePath;
import org.apache.hudi.storage.StoragePathInfo;
import org.apache.hudi.storage.hadoop.HadoopStorageConfiguration;
import org.apache.hudi.util.StreamerUtil;
+import org.apache.hudi.utils.SharedCoordinatorMetricGroup;
import org.apache.hudi.utils.TestConfigurations;
import org.apache.hudi.utils.TestUtils;
+import org.apache.flink.api.common.JobID;
import org.apache.flink.configuration.Configuration;
import org.apache.flink.metrics.Counter;
import org.apache.flink.metrics.Gauge;
@@ -776,6 +778,23 @@ public class TestStreamWriteOperatorCoordinator {
"A sync that never ran records no remaining time");
}
+ @Test
+ void testHiveSyncMetricsReportSyncsAfterCoordinatorRestart() throws
Exception {
+ coordinator.close();
+ Configuration conf = getUnreachableMetastoreConf();
+ SharedCoordinatorMetricGroup metricGroup = new
SharedCoordinatorMetricGroup(new JobID().toString(), new
OperatorID().toString());
+ coordinator = startCoordinator(conf, metricGroup);
+ assertThrows(HoodieHiveSyncException.class, () ->
coordinator.doSyncHive());
+ coordinator.close();
+
+ // a restart that rebuilds the execution graph creates a new coordinator
on the same metric group
+ coordinator = startCoordinator(conf, metricGroup.newView());
+ assertThrows(HoodieHiveSyncException.class, () ->
coordinator.doSyncHive());
+
+ assertEquals(2, metricGroup.getCounter("hiveSyncFailureCount").getCount());
+ assertEquals(2, metricGroup.getHistogram("hiveSyncDurationMs").getCount());
+ }
+
private Configuration getUnreachableMetastoreConf() {
Configuration conf =
TestConfigurations.getDefaultConf(tempFile.getAbsolutePath());
conf.set(FlinkOptions.HIVE_SYNC_ENABLED, true);
diff --git
a/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/SharedCoordinatorMetricGroup.java
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/SharedCoordinatorMetricGroup.java
new file mode 100644
index 000000000000..4c4ae782b23d
--- /dev/null
+++
b/hudi-flink-datasource/hudi-flink/src/test/java/org/apache/hudi/utils/SharedCoordinatorMetricGroup.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.utils;
+
+import org.apache.flink.metrics.Counter;
+import org.apache.flink.metrics.Gauge;
+import org.apache.flink.metrics.Histogram;
+import org.apache.flink.metrics.Metric;
+import org.apache.flink.metrics.groups.OperatorCoordinatorMetricGroup;
+import org.apache.flink.metrics.groups.UnregisteredMetricsGroup;
+import org.apache.flink.runtime.metrics.scope.ScopeFormat;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+
+/**
+ * A coordinator metric group that behaves like the one Flink gives each
coordinator of an operator.
+ *
+ * <p>It identifies its job and operator, and {@link #newView()} returns
another group over the same
+ * registrations, as Flink gives every coordinator it creates for the operator
during the job. Like
+ * Flink's, it keeps the first metric registered under a name and ignores
later ones.
+ */
+public class SharedCoordinatorMetricGroup extends UnregisteredMetricsGroup
+ implements OperatorCoordinatorMetricGroup {
+ private final Map<String, Metric> registered;
+ private final Map<String, String> variables;
+
+ public SharedCoordinatorMetricGroup(String jobId, String operatorId) {
+ this(new ConcurrentHashMap<>(), variables(jobId, operatorId));
+ }
+
+ private SharedCoordinatorMetricGroup(Map<String, Metric> registered,
Map<String, String> variables) {
+ this.registered = registered;
+ this.variables = variables;
+ }
+
+ /** Returns another group over the same registrations, as a coordinator
created by a restart gets. */
+ public SharedCoordinatorMetricGroup newView() {
+ return new SharedCoordinatorMetricGroup(registered, variables);
+ }
+
+ @Override
+ public <C extends Counter> C counter(String name, C counter) {
+ registered.putIfAbsent(name, counter);
+ return counter;
+ }
+
+ @Override
+ public <H extends Histogram> H histogram(String name, H histogram) {
+ registered.putIfAbsent(name, histogram);
+ return histogram;
+ }
+
+ @Override
+ public <T, G extends Gauge<T>> G gauge(String name, G gauge) {
+ registered.putIfAbsent(name, gauge);
+ return gauge;
+ }
+
+ @Override
+ public Map<String, String> getAllVariables() {
+ return variables;
+ }
+
+ public Counter getCounter(String name) {
+ return (Counter) registered.get(name);
+ }
+
+ public Histogram getHistogram(String name) {
+ return (Histogram) registered.get(name);
+ }
+
+ public Gauge<?> getGauge(String name) {
+ return (Gauge<?>) registered.get(name);
+ }
+
+ private static Map<String, String> variables(String jobId, String
operatorId) {
+ Map<String, String> variables = new HashMap<>();
+ variables.put(ScopeFormat.SCOPE_JOB_ID, jobId);
+ variables.put(ScopeFormat.SCOPE_OPERATOR_ID, operatorId);
+ return variables;
+ }
+}