This is an automated email from the ASF dual-hosted git repository.
voonhous pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 8ee40b4fb54f fix(hive-sync): make skip_ro_suffix take precedence over
sync_snapshot_with_table_name (#19427)
8ee40b4fb54f is described below
commit 8ee40b4fb54fff3beb924134af2367438913062a
Author: Y Ethan Guo <[email protected]>
AuthorDate: Fri Jul 31 02:24:32 2026 -0700
fix(hive-sync): make skip_ro_suffix take precedence over
sync_snapshot_with_table_name (#19427)
* fix(hive-sync): make skip_ro_suffix take precedence over
sync_snapshot_with_table_name
hoodie.datasource.hive_sync.skip_ro_suffix=true claims the bare table name
for the read-optimized (RO) view of a MOR table. Hudi 1.x flipped the
default
of hoodie.meta.sync.sync_snapshot_with_table_name from false to true
(HUDI-7415), so under the default ALL hive-sync-table-strategy,
HiveSyncTool.doSync() now syncs the bare table name twice in the same run
when both configs are true: once as the RO table, then again as the RT
table, with the RT sync silently winning. This makes skip_ro_suffix a no-op
and breaks read-optimized queries against the bare table name in case the
engine can't read with HoodieParquetRealtimeInputFormat (e.g., Presto).
This change skips the redundant bare-name RT sync when skip_ro_suffix is
set, and logs a WARN naming the table and stating which config wins. The
real-time view remains available at <table>_rt.
* Address review comments: fix bare-table-name wording, remove
now-redundant comment, strengthen regression test to a second sync round
* Drop the second-round explanation comment per review feedback
---
.../java/org/apache/hudi/hive/HiveSyncTool.java | 9 ++++-
.../org/apache/hudi/hive/TestHiveSyncTool.java | 47 ++++++++++++++++++++++
2 files changed, 55 insertions(+), 1 deletion(-)
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
index 21b361d16dfc..80b83816be8b 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
@@ -212,7 +212,14 @@ public class HiveSyncTool extends HoodieSyncTool
implements AutoCloseable {
syncHoodieTable(snapshotTableName, true, false);
// sync origin table for MOR
if (config.getBoolean(META_SYNC_SNAPSHOT_WITH_TABLE_NAME)) {
- syncHoodieTable(tableName, true, false);
+ if
(config.getBoolean(HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE)) {
+ log.warn("{}=true claims the bare table name '{}' for the
read-optimized view; "
+ + "ignoring {} for this table (the real-time view
remains registered as '{}').",
+ HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE.key(),
tableId(databaseName, tableName),
+ META_SYNC_SNAPSHOT_WITH_TABLE_NAME.key(),
snapshotTableName);
+ } else {
+ syncHoodieTable(tableName, true, false);
+ }
}
}
break;
diff --git
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
index 50384913fb82..9f5211ba5efe 100644
---
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
+++
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
@@ -99,6 +99,7 @@ import static
org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_AUTO_CREATE_DATABAS
import static
org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_BATCH_SYNC_PARTITION_NUM;
import static
org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_CREATE_MANAGED_TABLE;
import static org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_IGNORE_EXCEPTIONS;
+import static
org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE;
import static
org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_AS_DATA_SOURCE_TABLE;
import static
org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BATCHING_ENABLED;
import static
org.apache.hudi.hive.HiveSyncConfigHolder.HIVE_SYNC_BATCHING_THREADS;
@@ -120,6 +121,7 @@ import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_FORCE_RECRE
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_INCREMENTAL;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_EXTRACTOR_CLASS;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_PARTITION_FIELDS;
+import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_SNAPSHOT_WITH_TABLE_NAME;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_TABLE_NAME;
import static
org.apache.hudi.sync.common.HoodieSyncConfig.META_SYNC_TOUCH_PARTITIONS_ENABLED;
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
@@ -1593,6 +1595,51 @@ public class TestHiveSyncTool {
}
}
+ @Test
+ void testSkipRoSuffixTakesPrecedenceOverSnapshotWithTableName() throws
Exception {
+ // skip_ro_suffix explicitly claims the bare table name for the RO view;
the now-default-true
+ // sync_snapshot_with_table_name must not be allowed to flip it to RT.
+ hiveSyncProps.setProperty(HIVE_SYNC_TABLE_STRATEGY.key(),
HoodieSyncTableStrategy.ALL.name());
+
hiveSyncProps.setProperty(HIVE_SKIP_RO_SUFFIX_FOR_READ_OPTIMIZED_TABLE.key(),
"true");
+ hiveSyncProps.setProperty(META_SYNC_SNAPSHOT_WITH_TABLE_NAME.key(),
"true");
+ hiveSyncProps.setProperty(HIVE_SYNC_AS_DATA_SOURCE_TABLE.key(), "true");
+
+ String instantTime = "100";
+ String deltaCommitTime = "101";
+ HiveTestUtil.createMORTable(instantTime, deltaCommitTime, 5, true, true);
+
+ reInitHiveSyncClient();
+ reSyncHiveTable();
+
+ // a second sync round with a new commit reproduces the flip; a single
round does not
+ ZonedDateTime dateTime = ZonedDateTime.now().plusDays(6);
+ String commitTime2 = "102";
+ String deltaCommitTime2 = "103";
+ HiveTestUtil.addMORPartitions(1, true, false, true, dateTime, commitTime2,
deltaCommitTime2);
+ reInitHiveSyncClient();
+ reSyncHiveTable();
+
+ String snapshotTableName = HiveTestUtil.TABLE_NAME +
HiveSyncTool.SUFFIX_SNAPSHOT_TABLE;
+ String roSuffixTableName = HiveTestUtil.TABLE_NAME +
HiveSyncTool.SUFFIX_READ_OPTIMIZED_TABLE;
+
+ // the bare table name must stay registered as the read-optimized view
+ StorageDescriptor bareTableSd =
hiveClient.getMetastoreStorageDescriptor(HiveTestUtil.TABLE_NAME);
+ assertEquals(HoodieParquetInputFormat.class.getName(),
bareTableSd.getInputFormat(),
+ "Bare table name should remain the RO view (skip_ro_suffix=true)
despite sync_snapshot_with_table_name=true");
+ assertEquals("true",
bareTableSd.getSerdeInfo().getParameters().get(ConfigUtils.IS_QUERY_AS_RO_TABLE),
+ "Bare table name should still be marked as the RO table");
+
+ // the real-time/snapshot view remains available, and only there
+ assertTrue(hiveClient.tableExists(snapshotTableName), "Table " +
snapshotTableName + " should exist after sync completes");
+ StorageDescriptor snapshotTableSd =
hiveClient.getMetastoreStorageDescriptor(snapshotTableName);
+ assertEquals(HoodieParquetRealtimeInputFormat.class.getName(),
snapshotTableSd.getInputFormat(),
+ "Table " + snapshotTableName + " should use the realtime input
format");
+
+ // no separate "<t>_ro" table should be created when skip_ro_suffix is set
+ assertFalse(hiveClient.tableExists(roSuffixTableName),
+ "Table " + roSuffixTableName + " should not exist when skip_ro_suffix
is set");
+ }
+
@ParameterizedTest
@EnumSource(value = HoodieSyncTableStrategy.class, names = {"RO", "RT"})
public void
testSyncMergeOnReadWithStrategyWhenTableExist(HoodieSyncTableStrategy strategy)
throws Exception {
