This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new cb0f9b18c3da feat(hive-sync): add config to force-recreate the
metastore table on every sync (#19426)
cb0f9b18c3da is described below
commit cb0f9b18c3da23c6ddb9d6d45f64a216f11fb122
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri Jul 31 02:06:22 2026 -0700
feat(hive-sync): add config to force-recreate the metastore table on every
sync (#19426)
* feat(hive-sync): add config to force-recreate the metastore table on
every sync
Add hoodie.meta.sync.force.recreate.table to HoodieSyncConfig. When true,
HiveSyncTool always drops and recreates the metastore table via the existing
createOrReplaceTable path on every sync of a table that already exists,
regardless of whether the incremental sync would otherwise succeed or be a
no-op. This gives operators a way to force a full resync of the table schema
and properties, e.g. after the table drifted from the Hoodie table
definition out-of-band.
* Address review comments: republish hudi_writer_version on recreate,
rename shouldRecreateTable to shouldRecreateTableBeforeSync
* Address review: assert hudi_writer_version is republished by
force-recreate
---
.../java/org/apache/hudi/hive/HiveSyncTool.java | 25 ++++++++++++--
.../org/apache/hudi/hive/TestHiveSyncTool.java | 38 +++++++++++++++++++++-
.../apache/hudi/sync/common/HoodieSyncConfig.java | 10 ++++++
3 files changed, 69 insertions(+), 4 deletions(-)
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
index 1c6360db3580..21b361d16dfc 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
@@ -73,6 +73,7 @@ import static
org.apache.hudi.hive.util.HiveSchemaUtil.getSchemaDifference;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_FILE_FORMAT;
import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_PATH;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_CONDITIONAL_SYNC;
+import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_FORCE_RECREATE_TABLE;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_INCREMENTAL;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_SNAPSHOT_WITH_TABLE_NAME;
@@ -239,9 +240,9 @@ public class HiveSyncTool extends HoodieSyncTool implements
AutoCloseable {
log.info("Trying to sync hoodie table {} with base path {} of type {}",
tableName, syncClient.getBasePath(), syncClient.getTableType());
final boolean tableExists = syncClient.tableExists(tableName);
- // if table exists and location of the metastore table doesn't match the
hoodie base path, recreate the table
- if (tableExists &&
!FSUtils.comparePathsWithoutScheme(syncClient.getBasePath(),
syncClient.getTableLocation(tableName))) {
- log.info("basepath is updated for the table {}", tableName);
+ // recreate the table if it exists and either its metastore location no
longer matches the hoodie base path,
+ // or a full recreate was explicitly requested
+ if (tableExists && shouldRecreateTableBeforeSync(tableName)) {
recreateAndSyncHiveTable(tableName, useRealtimeInputFormat,
readAsOptimized);
return;
}
@@ -360,6 +361,23 @@ public class HiveSyncTool extends HoodieSyncTool
implements AutoCloseable {
return config.getBooleanOrDefault(RECREATE_HIVE_TABLE_ON_ERROR);
}
+ /**
+ * Whether the metastore table (assumed to already exist) should be dropped
and recreated before the normal
+ * sync runs, either because a full recreate was explicitly requested, or
because its stored location no
+ * longer matches the hoodie base path.
+ */
+ private boolean shouldRecreateTableBeforeSync(String tableName) {
+ if (config.getBooleanOrDefault(META_SYNC_FORCE_RECREATE_TABLE)) {
+ log.info("Force recreating the table {} since {} is set to true",
tableName, META_SYNC_FORCE_RECREATE_TABLE.key());
+ return true;
+ }
+ if (!FSUtils.comparePathsWithoutScheme(syncClient.getBasePath(),
syncClient.getTableLocation(tableName))) {
+ log.info("basepath is updated for the table {}", tableName);
+ return true;
+ }
+ return false;
+ }
+
private void recreateAndSyncHiveTable(String tableName, boolean
useRealtimeInputFormat, boolean readAsOptimized) {
log.info("recreating and syncing the table {}", tableName);
Timer.Context timerContext = metrics.getRecreateAndSyncTimer();
@@ -368,6 +386,7 @@ public class HiveSyncTool extends HoodieSyncTool implements
AutoCloseable {
createOrReplaceTable(tableName, useRealtimeInputFormat, readAsOptimized,
schema);
syncAllPartitions(tableName);
syncClient.updateLastCommitTimeSynced(tableName);
+ syncClient.updateHoodieWriterVersion(tableName);
if (Objects.nonNull(timerContext)) {
long durationInNs = timerContext.stop();
metrics.updateRecreateAndSyncDurationInMs(durationInNs);
diff --git
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
index 4f109c9cdd64..50384913fb82 100644
---
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
+++
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
@@ -116,6 +116,7 @@ import static
org.apache.hudi.hive.testutils.HiveTestUtil.hiveSyncProps;
import static org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_BASE_PATH;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_CONDITIONAL_SYNC;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_DATABASE_NAME;
+import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_FORCE_RECREATE_TABLE;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_INCREMENTAL;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_EXTRACTOR_CLASS;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS;
@@ -271,7 +272,42 @@ public class TestHiveSyncTool {
throw new HoodieHiveSyncException("Failed to get the metastore location
from the table " + tableName, e);
}
}
-
+
+ @ParameterizedTest
+ @MethodSource("syncMode")
+ void testForceRecreateTable(String syncMode) throws Exception {
+ hiveSyncProps.setProperty(HIVE_SYNC_MODE.key(), syncMode);
+ String instantTime = "100";
+ HiveTestUtil.createCOWTable(instantTime, 1, true);
+ reInitHiveSyncClient();
+ reSyncHiveTable();
+ assertTrue(hiveClient.tableExists(HiveTestUtil.TABLE_NAME));
+
+ // simulate manual drift on the metastore table that a fully-synced
incremental sync would never touch
+ ddlExecutor.runSQL("ALTER TABLE `" + HiveTestUtil.TABLE_NAME + "` SET
TBLPROPERTIES ('drift_marker'='true')");
+ IMetaStoreClient client = IMetaStoreClientUtil.getMSC(getHiveConf());
+ assertTrue(client.getTable(HiveTestUtil.DB_NAME,
HiveTestUtil.TABLE_NAME).getParameters().containsKey("drift_marker"));
+
+ // nothing changed on the Hoodie timeline, so a normal sync is a no-op and
leaves the drift behind
+ reInitHiveSyncClient();
+ reSyncHiveTable();
+ client.reconnect();
+ assertTrue(client.getTable(HiveTestUtil.DB_NAME,
HiveTestUtil.TABLE_NAME).getParameters().containsKey("drift_marker"),
+ "a no-op incremental sync must not touch the table");
+
+ // forcing recreation drops and rebuilds the table from scratch, even
though nothing else changed
+ hiveSyncProps.setProperty(META_SYNC_FORCE_RECREATE_TABLE.key(), "true");
+ reInitHiveSyncClient();
+ reSyncHiveTable();
+ client.reconnect();
+ assertFalse(client.getTable(HiveTestUtil.DB_NAME,
HiveTestUtil.TABLE_NAME).getParameters().containsKey("drift_marker"),
+ "force-recreate must drop and recreate the table even when the
incremental sync would otherwise be a no-op");
+ assertEquals(HoodieVersion.get(), client.getTable(HiveTestUtil.DB_NAME,
HiveTestUtil.TABLE_NAME).getParameters().get(HoodieVersion.HOODIE_WRITER_VERSION),
+ "force-recreate must republish the hudi writer version on the
recreated table");
+ assertEquals(instantTime,
hiveClient.getLastCommitTimeSynced(HiveTestUtil.TABLE_NAME).get());
+ client.close();
+ }
+
@ParameterizedTest
@MethodSource("syncMode")
public void testSyncAllPartition() throws Exception {
diff --git
a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java
b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java
index 2e46a3007b72..6952d245711f 100644
---
a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java
+++
b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieSyncConfig.java
@@ -182,6 +182,16 @@ public class HoodieSyncConfig extends HoodieConfig {
.withDocumentation("If true, TOUCH partition events will be emitted
during meta sync. "
+ "TOUCH events indicate partitions that exist in both storage and
metastore, no schema or location change, but the partition has received data.");
+ public static final ConfigProperty<Boolean> META_SYNC_FORCE_RECREATE_TABLE =
ConfigProperty
+ .key("hoodie.meta.sync.force.recreate.table")
+ .defaultValue(false)
+ .sinceVersion("1.3.0")
+ .markAdvanced()
+ .withDocumentation("If true, always drop and recreate the table on every
sync, regardless of whether "
+ + "the incremental sync would otherwise succeed. Useful for forcing
a full resync of the table schema "
+ + "and properties to the metastore, e.g. after the table drifted
from the Hoodie table definition "
+ + "out-of-band.");
+
@Getter
@Setter
private Configuration hadoopConf;
