This is an automated email from the ASF dual-hosted git repository.
sivabalan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new a16d43171da9 feat(hudi-sync): Publish HUDI version to Hive metastore
(#18307)
a16d43171da9 is described below
commit a16d43171da9c312cedfbe9b126b81394ca74bd3
Author: Krishen <[email protected]>
AuthorDate: Fri Mar 13 14:43:26 2026 -0700
feat(hudi-sync): Publish HUDI version to Hive metastore (#18307)
feat(hudi-sync): Publish HUDI version to Hive metastore (allowing users to
infer which HUDI client jar to use for a given dataset)
Co-authored-by: Krishen Bhan <“[email protected]”>
---
.../src/main/java/org/apache/hudi/hive/HiveSyncTool.java | 1 +
.../java/org/apache/hudi/hive/HoodieHiveSyncClient.java | 13 +++++++++++++
.../test/java/org/apache/hudi/hive/TestHiveSyncTool.java | 9 ++++++++-
.../apache/hudi/sync/common/HoodieMetaSyncOperations.java | 7 +++++++
4 files changed, 29 insertions(+), 1 deletion(-)
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
index 8c827d4dc1fc..2a130d8ee797 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HiveSyncTool.java
@@ -271,6 +271,7 @@ public class HiveSyncTool extends HoodieSyncTool implements
AutoCloseable {
if (!config.getBoolean(META_SYNC_CONDITIONAL_SYNC) ||
meetSyncConditions) {
syncClient.updateLastCommitTimeSynced(tableName);
}
+ syncClient.updateHoodieWriterVersion(tableName);
log.info("Sync complete for {}", tableName);
} catch (HoodieHiveSyncException ex) {
if (shouldRecreateAndSyncTable()) {
diff --git
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java
index 2935e433365b..467f2a17cc59 100644
---
a/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java
+++
b/hudi-sync/hudi-hive-sync/src/main/java/org/apache/hudi/hive/HoodieHiveSyncClient.java
@@ -19,6 +19,7 @@
package org.apache.hudi.hive;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hudi.HoodieVersion;
import org.apache.hudi.common.model.HoodieFileFormat;
import org.apache.hudi.common.schema.HoodieSchema;
import org.apache.hudi.common.table.HoodieTableMetaClient;
@@ -486,6 +487,18 @@ public class HoodieHiveSyncClient extends HoodieSyncClient
{
}
}
+ @Override
+ public void updateHoodieWriterVersion(String tableName) {
+ try {
+ Table table = client.getTable(databaseName, tableName);
+ table.putToParameters(HoodieVersion.HOODIE_WRITER_VERSION,
HoodieVersion.get());
+ client.alter_table(databaseName, tableName, table);
+ } catch (Exception e) {
+ throw new HoodieHiveSyncException(String.format("Failed to update hudi
writer major version %s for %s",
+ HoodieVersion.get(), tableName), e);
+ }
+ }
+
@VisibleForTesting
StorageDescriptor getMetastoreStorageDescriptor(String tableName) throws
TException {
return client.getTable(databaseName, tableName).getSd();
diff --git
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
index 01ff1daea6da..ec9d4b0406de 100644
---
a/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
+++
b/hudi-sync/hudi-hive-sync/src/test/java/org/apache/hudi/hive/TestHiveSyncTool.java
@@ -18,6 +18,7 @@
package org.apache.hudi.hive;
+import org.apache.hudi.HoodieVersion;
import org.apache.hudi.common.config.HoodieCommonConfig;
import org.apache.hudi.common.fs.FSUtils;
import org.apache.hudi.common.model.HoodieCommitMetadata;
@@ -569,6 +570,8 @@ public class TestHiveSyncTool {
String sparkTableProperties =
getSparkTableProperties(syncAsDataSourceTable, useSchemaFromCommitMetadata);
assertEquals(
"EXTERNAL\tTRUE\n"
+ + String.format("%s\t%s\n", HoodieVersion.HOODIE_WRITER_VERSION,
+ HoodieVersion.get())
+ "last_commit_completion_time_sync\t" +
getLastCommitCompletionTimeSynced() + "\n"
+ "last_commit_time_sync\t100\n"
+ sparkTableProperties
@@ -607,8 +610,10 @@ public class TestHiveSyncTool {
hiveDriver.getResults(results);
assertEquals(
-
String.format("%slast_commit_completion_time_sync\t%s\nlast_commit_time_sync\t%s\n%s",
+
String.format("%s%s\t%s\nlast_commit_completion_time_sync\t%s\nlast_commit_time_sync\t%s\n%s",
createManagedTable ? StringUtils.EMPTY_STRING : "EXTERNAL\tTRUE\n",
+ HoodieVersion.HOODIE_WRITER_VERSION,
+ HoodieVersion.get(),
getLastCommitCompletionTimeSynced(),
instantTime,
getSparkTableProperties(true, true)),
@@ -702,6 +707,8 @@ public class TestHiveSyncTool {
results.subList(0, results.size() - 1));
assertEquals(
"EXTERNAL\tTRUE\n"
+ + String.format("%s\t%s\n", HoodieVersion.HOODIE_WRITER_VERSION,
+ HoodieVersion.get())
+ "last_commit_completion_time_sync\t" +
getLastCommitCompletionTimeSynced() + "\n"
+ "last_commit_time_sync\t101\n"
+ sparkTableProperties
diff --git
a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieMetaSyncOperations.java
b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieMetaSyncOperations.java
index 24a974536ef3..503db0459237 100644
---
a/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieMetaSyncOperations.java
+++
b/hudi-sync/hudi-sync-common/src/main/java/org/apache/hudi/sync/common/HoodieMetaSyncOperations.java
@@ -279,4 +279,11 @@ public interface HoodieMetaSyncOperations {
default String generatePushDownFilter(List<String> writtenPartitions,
List<FieldSchema> partitionFields) {
throw new UnsupportedOperationException();
}
+
+ /**
+ * Publish version of HUDI client library used
+ */
+ default void updateHoodieWriterVersion(String tableName) {
+ throw new UnsupportedOperationException();
+ }
}
