nsivabalan commented on a change in pull request #4714:
URL: https://github.com/apache/hudi/pull/4714#discussion_r796026389
##########
File path:
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableConfig.java
##########
@@ -183,6 +185,14 @@
public static final ConfigProperty<String> URL_ENCODE_PARTITIONING =
KeyGeneratorOptions.URL_ENCODE_PARTITIONING;
public static final ConfigProperty<String> HIVE_STYLE_PARTITIONING_ENABLE =
KeyGeneratorOptions.HIVE_STYLE_PARTITIONING_ENABLE;
+ public static final List<String> PERSISTED_CONFIG_LIST = Arrays.asList(
+ Config.DATE_TIME_PARSER_PROP,
Review comment:
does this mean, we are going to persist all key gen props to table
config?
Also, do we also impose any constraints that one can't change the key gen
props once the table is created?
##########
File path:
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
##########
@@ -282,6 +296,41 @@ object HoodieFileIndex {
properties
}
+ def convertFilterForTimestampKeyGenerator(metaClient: HoodieTableMetaClient,
+ partitionFilters: Seq[Expression]): Seq[Expression] = {
+
+ val tableConfig = metaClient.getTableConfig
+ val keyGenerator = tableConfig.getKeyGeneratorClassName
+
+ if
(keyGenerator.equals(classOf[TimestampBasedKeyGenerator].getCanonicalName) ||
+
keyGenerator.equals(classOf[TimestampBasedAvroKeyGenerator].getCanonicalName)) {
+ val inputFormat =
tableConfig.getString(KeyGeneratorOptions.Config.TIMESTAMP_INPUT_DATE_FORMAT_PROP)
+ val outputFormat =
tableConfig.getString(KeyGeneratorOptions.Config.TIMESTAMP_OUTPUT_DATE_FORMAT_PROP)
+ if (StringUtils.isNullOrEmpty(inputFormat) ||
StringUtils.isNullOrEmpty(outputFormat) ||
+ inputFormat.equals(outputFormat)) {
+ partitionFilters
+ } else {
+ try {
+ val inDateFormat = new SimpleDateFormat(inputFormat)
+ val outDateFormat = new SimpleDateFormat(outputFormat)
+ partitionFilters.toArray.map {
Review comment:
do we know how this might pan out if input format is EPOCHMILLISECONDS?
InputDateformat will be empty is it and so we will fallback to using original
partition filters?
##########
File path:
hudi-common/src/main/java/org/apache/hudi/common/table/HoodieTableMetaClient.java
##########
@@ -759,6 +780,11 @@ public PropertyBuilder
fromMetaClient(HoodieTableMetaClient metaClient) {
public PropertyBuilder fromProperties(Properties properties) {
HoodieConfig hoodieConfig = new HoodieConfig(properties);
+
+ for (String key : hoodieConfig.getProps().stringPropertyNames()) {
Review comment:
might be easier if we go through PERSISTED_CONFIG_LIST instead of all
hoodieWriteConfigs
##########
File path:
hudi-spark-datasource/hudi-spark-common/src/main/scala/org/apache/hudi/HoodieFileIndex.scala
##########
@@ -282,6 +296,41 @@ object HoodieFileIndex {
properties
}
+ def convertFilterForTimestampKeyGenerator(metaClient: HoodieTableMetaClient,
+ partitionFilters: Seq[Expression]): Seq[Expression] = {
+
+ val tableConfig = metaClient.getTableConfig
+ val keyGenerator = tableConfig.getKeyGeneratorClassName
+
+ if
(keyGenerator.equals(classOf[TimestampBasedKeyGenerator].getCanonicalName) ||
Review comment:
lets account for NullPointerException. key gen property may not be set
in all code paths.
##########
File path:
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestMORDataSource.scala
##########
@@ -770,4 +775,79 @@ class TestMORDataSource extends HoodieClientTestBase {
.load(basePath + "/*/*/*/*")
assertEquals(numRecords - numRecordsToDelete, snapshotDF2.count())
}
+
+ /**
+ * This tests the case that query by with a specified partition condition on
hudi table which is
+ * different between the value of the partition field and the actual
partition path,
+ * like hudi table written by TimestampBasedKeyGenerator.
+ *
+ * For MOR table, test all the three query modes.
+ */
+ @Test
+ def testPrunePartitionForTimestampBasedKeyGenerator(): Unit = {
+ val options = commonOpts ++ Map(
+ "hoodie.compact.inline" -> "false",
+ DataSourceWriteOptions.TABLE_TYPE.key ->
DataSourceWriteOptions.MOR_TABLE_TYPE_OPT_VAL,
+ DataSourceWriteOptions.KEYGENERATOR_CLASS_NAME.key ->
"org.apache.hudi.keygen.TimestampBasedKeyGenerator",
+ Config.TIMESTAMP_TYPE_FIELD_PROP -> "DATE_STRING",
+ Config.TIMESTAMP_OUTPUT_DATE_FORMAT_PROP -> "yyyy/MM/dd",
+ Config.TIMESTAMP_TIMEZONE_FORMAT_PROP -> "GMT+8:00",
+ Config.TIMESTAMP_INPUT_DATE_FORMAT_PROP -> "yyyy-MM-dd"
+ )
+
+ val dataGen1 = new HoodieTestDataGenerator(Array("2022-01-01"))
+ val records1 = recordsToStrings(dataGen1.generateInserts("001", 50)).toList
+ val inputDF1 = spark.read.json(spark.sparkContext.parallelize(records1, 2))
+ inputDF1.write.format("org.apache.hudi")
+ .options(options)
+ .mode(SaveMode.Overwrite)
+ .save(basePath)
+ metaClient = HoodieTableMetaClient.builder()
+ .setBasePath(basePath)
+ .setConf(spark.sessionState.newHadoopConf)
+ .build()
+ val commit1Time =
metaClient.getActiveTimeline.lastInstant().get().getTimestamp
+
+ val dataGen2 = new HoodieTestDataGenerator(Array("2022-01-02"))
+ val records2 = recordsToStrings(dataGen2.generateInserts("002", 50)).toList
Review comment:
can we ingest diff no of records in each batch. just so we our
assertions are intact. right now, we assert 50 for 2022-01-01 and for
2022-01-02 too. may be, 40 and 60.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
