codope commented on code in PR #10153:
URL: https://github.com/apache/hudi/pull/10153#discussion_r1416836312
##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/spark/sql/hudi/command/index/TestFunctionalIndex.scala:
##########
@@ -140,6 +146,68 @@ class TestFunctionalIndex extends HoodieSparkSqlTestBase {
}
}
+ test("Test Functional Index With Hive Sync Non Partitioned Table") {
+ if (HoodieSparkUtils.gteqSpark3_2) {
+ withTempDir { tmp =>
+ Seq("mor").foreach { tableType =>
+ val databaseName = "default"
+ val tableName = generateTableName
+ val basePath = s"${tmp.getCanonicalPath}/$tableName"
+ spark.sql(
+ s"""
+ |create table $tableName (
+ | id int,
+ | name string,
+ | price double,
+ | ts long
+ |) using hudi
+ | options (
+ | primaryKey ='id',
+ | type = '$tableType',
+ | preCombineField = 'ts'
+ | )
+ | partitioned by(ts)
+ | location '$basePath'
+ """.stripMargin)
+ spark.sql(s"insert into $tableName values(1, 'a1', 10, 1000)")
+ spark.sql(s"insert into $tableName values(2, 'a2', 10, 1001)")
+ spark.sql(s"insert into $tableName values(3, 'a3', 10, 1002)")
+
+ val createIndexSql = s"create index idx_datestr on $tableName using
column_stats(ts) options(func='from_unixtime', format='yyyy-MM-dd')"
+ spark.sql(createIndexSql)
+ val metaClient = HoodieTableMetaClient.builder()
+ .setBasePath(basePath)
+ .setConf(spark.sessionState.newHadoopConf())
+ .build()
+ assertTrue(metaClient.getFunctionalIndexMetadata.isPresent)
+ val functionalIndexMetadata =
metaClient.getFunctionalIndexMetadata.get()
+ assertEquals(1, functionalIndexMetadata.getIndexDefinitions.size())
+ assertEquals("func_index_idx_datestr",
functionalIndexMetadata.getIndexDefinitions.get("func_index_idx_datestr").getIndexName)
+
+ // sync to hive without partition metadata
+ val hiveSyncProps = new TypedProperties()
+ hiveSyncProps.setProperty(HIVE_URL.key,
"jdbc:hive2://localhost:10000")
+ hiveSyncProps.setProperty(HIVE_USER.key, "")
+ hiveSyncProps.setProperty(HIVE_PASS.key, "")
+ hiveSyncProps.setProperty(META_SYNC_DATABASE_NAME.key, databaseName)
+ hiveSyncProps.setProperty(META_SYNC_TABLE_NAME.key, tableName)
+ hiveSyncProps.setProperty(META_SYNC_BASE_PATH.key, basePath)
+ hiveSyncProps.setProperty(HIVE_USE_PRE_APACHE_INPUT_FORMAT.key,
"false")
+ // hiveSyncProps.setProperty(META_SYNC_PARTITION_FIELDS.key,
"datestr")
+ hiveSyncProps.setProperty(META_SYNC_PARTITION_EXTRACTOR_CLASS.key,
classOf[SlashEncodedDayPartitionValueExtractor].getName)
+ hiveSyncProps.setProperty(HIVE_BATCH_SYNC_PARTITION_NUM.key, "3")
+ HiveTestUtil.setUp(Option.of(hiveSyncProps))
Review Comment:
It's fixed now.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
