nsivabalan commented on code in PR #20604:
URL: https://github.com/apache/hudi/pull/20604#discussion_r4608307893


##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")

Review Comment:
   This only reads `tableConfig.getMetadataPartitions`. Please also assert, 
after commit 1, that (a) the MDT `record_index` partition has no file slices / 
nothing on storage, (b) the MDT timeline has no RLI writes, and (c) the index 
metadata has no `record_index` definition. (a) fails today in the partitioned 
case: `maybeInitializeNewFileGroupsForPartitionedRLI` only checks 
`isRecordLevelIndexEnabled()`, so min-count file groups get created during the 
deferral and are cleared at build time. After commit 2, assert the definition 
exists with the right `isPartitioned` option.



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // What the index should be sized from once it is built.
+    val rowsPerPartition = 
spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect()
+      .map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    // Commit 2: updates of records tagged through the fallback index, plus 
new records. The index is built here.
+    val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300))
+    val newRecords = dataGen.generateInserts("002", 10)
+    val secondBatch = new util.ArrayList[HoodieRecord[_]](updates)
+    secondBatch.addAll(newRecords)
+    allRecords.addAll(newRecords)
+    write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    // Commit 3: upserts tagged through the record index itself.
+    val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 
300))

Review Comment:
   This updates only first-batch keys again. To exercise the switch, commit 3 
should update the keys that commit 2 *inserted*, and re-insert a key that 
commit 2 deleted. Also assert that the RLI entry count equals the distinct key 
count after commits 2 and 3. Same ask for the SQL, structured streaming and 
Streamer cases, which currently have no write after the build.



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // What the index should be sized from once it is built.
+    val rowsPerPartition = 
spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect()
+      .map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    // Commit 2: updates of records tagged through the fallback index, plus 
new records. The index is built here.
+    val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300))
+    val newRecords = dataGen.generateInserts("002", 10)
+    val secondBatch = new util.ArrayList[HoodieRecord[_]](updates)
+    secondBatch.addAll(newRecords)
+    allRecords.addAll(newRecords)
+    write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    // Commit 3: upserts tagged through the record index itself.
+    val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 
300))
+    write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], 
UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+
+    // No duplicates across the fallback to record index switch.
+    val keys = 
spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0))
+    assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, 
keys.sorted.toSeq)
+
+    // The index is sized from the data the first commit left behind, not from 
an empty table.
+    val fileGroupIds = recordIndexFileGroupIds()
+    if (testCase.partitioned) {
+      val perPartition = 
fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName)
+        .mapValues(_.size).toMap
+      assertEquals(rowsPerPartition.mapValues(rows => 
expectedFileGroupCount(rows)).toMap, perPartition)
+    } else {
+      assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size)

Review Comment:
   2000 and 2010 rows both give 20 file groups (and roughly 667 vs 670 per 
partition both give 6), so this can't tell "sized from commit 1's data" from 
"sized after commit 2". It does catch the built-from-empty regression. Make 
commit 2 add enough rows to move the count, e.g. +2000.



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala:
##########
@@ -228,6 +229,94 @@ class TestStructuredStreaming extends 
HoodieSparkClientTestBase {
       tableType, addCompactionConfigs = false, isAsyncCompaction = false)
   }
 
+  /**
+   * Deferred record index initialization through the streaming sink: the 
first micro-batch leaves the index unbuilt
+   * and the second one builds it, with no duplicates across the switch from 
the fallback index.
+   */
+  @Tag("functional")

Review Comment:
   nit: this is the only tagged method in an untagged class. Please confirm the 
CI profiles run it where you expect.



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // What the index should be sized from once it is built.
+    val rowsPerPartition = 
spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect()
+      .map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    // Commit 2: updates of records tagged through the fallback index, plus 
new records. The index is built here.
+    val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300))
+    val newRecords = dataGen.generateInserts("002", 10)
+    val secondBatch = new util.ArrayList[HoodieRecord[_]](updates)
+    secondBatch.addAll(newRecords)
+    allRecords.addAll(newRecords)
+    write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    // Commit 3: upserts tagged through the record index itself.
+    val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 
300))
+    write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], 
UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+
+    // No duplicates across the fallback to record index switch.
+    val keys = 
spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0))
+    assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, 
keys.sorted.toSeq)
+
+    // The index is sized from the data the first commit left behind, not from 
an empty table.
+    val fileGroupIds = recordIndexFileGroupIds()
+    if (testCase.partitioned) {
+      val perPartition = 
fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName)
+        .mapValues(_.size).toMap
+      assertEquals(rowsPerPartition.mapValues(rows => 
expectedFileGroupCount(rows)).toMap, perPartition)
+    } else {
+      assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size)
+    }
+  }
+
+  /**
+   * The commit that builds the record index replaces the data the index was 
just sized from, so the index has to
+   * carry the replacing records and none of the replaced ones.
+   */
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForInsertOverwrite"))
+  def testRecordIndexBuiltByInsertOverwrite(testCase: DeferredInitTestCase): 
Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // Commit 2 builds the record index and then replaces every partition the 
first commit wrote.
+    val replacingBatch = dataGen.generateInserts("002", 500)
+    write(replacingBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
INSERT_OVERWRITE_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    val replacingKeys = replacingBatch.asScala.map(_.getRecordKey).sorted.toSeq
+    assertEquals(replacingKeys, 
spark.read.format("hudi").load(basePath).select("_row_key")
+      .collect().map(_.getString(0)).sorted.toSeq)
+
+    // Commit 3 is tagged through the record index. Replaced records must not 
come back and survivors must not double.
+    val laterUpdates = dataGen.generateUpdates("003", 
replacingBatch.subList(0, 100))
+    write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], 
UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertEquals(replacingKeys, 
spark.read.format("hudi").load(basePath).select("_row_key")
+      .collect().map(_.getString(0)).sorted.toSeq)
+  }
+
+  /**
+   * The same deferral through Spark SQL: INSERT INTO leaves the record index 
unbuilt and the following MERGE INTO
+   * builds it, sized from the rows the first statement wrote.
+   */
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForSql"))
+  def testRecordIndexDeferredUntilSecondStatement(testCase: 
DeferredInitTestCase): Unit = {
+    val tableName = s"rli_deferred_sql_${if (testCase.partitioned) 
"partitioned" else "global"}_v${testCase.tableVersion.versionCode()}"
+    val tablePath = tempDir.resolve(tableName).toAbsolutePath.toString
+    val indexEnableProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key
+    } else {
+      HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key
+    }
+    val minFileGroupProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key
+    } else {
+      
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key
+    }
+    val maxFileGroupProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key
+    } else {
+      
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key
+    }
+    val indexType = if (testCase.partitioned) "RECORD_LEVEL_INDEX" else 
"GLOBAL_RECORD_LEVEL_INDEX"
+    spark.sql(
+      s"""
+         |create table $tableName (
+         |  id string,
+         |  ts long,
+         |  name string,
+         |  part string
+         |) using hudi
+         | options (
+         |  primaryKey = 'id',
+         |  orderingFields = 'ts',
+         |  hoodie.metadata.enable = 'true',
+         |  ${HoodieWriteConfig.WRITE_TABLE_VERSION.key} = 
'${testCase.tableVersion.versionCode()}',
+         |  ${HoodieWriteConfig.AUTO_UPGRADE_VERSION.key} = 'false',
+         |  ${HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key} = 
'${testCase.streamingWrites}',
+         |  $indexEnableProp = 'true',
+         |  hoodie.index.type = '$indexType',
+         |  ${HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key} = 
'true',
+         |  
${HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key} = 
'9600',
+         |  $minFileGroupProp = '1',
+         |  $maxFileGroupProp = '1000'
+         | )
+         | partitioned by (part)
+         | location '$tablePath'
+       """.stripMargin)
+    val sqlMetaClient = 
HoodieTableMetaClient.builder().setBasePath(tablePath).setConf(storageConf).build()
+    val recordIndexBuilt = () => 
HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getMetadataPartitions
+      .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath)
+    val assertSqlTableVersion = () => assertEquals(testCase.tableVersion,
+      
HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getTableVersion,
+      "Table was written at an unexpected version")
+    val rowsOf = (from: Int, to: Int, ts: Long, name: String) =>
+      s"select cast(id as string) as id, ${ts}L as ts, '$name' as name, 
concat('p', cast(id % 3 as string)) as part from range($from, $to)"
+
+    try {
+      spark.conf.set("hoodie.spark.sql.insert.into.operation", 
testCase.firstOperation)
+      spark.sql(s"insert into $tableName ${rowsOf(0, 2000, 1L, "v1")}")
+    } finally {
+      spark.conf.unset("hoodie.spark.sql.insert.into.operation")
+    }
+    assertSqlTableVersion()
+    assertFalse(recordIndexBuilt(), "Record index should not be built by the 
first statement")
+
+    // The index is sized from the rows the first statement wrote, so capture 
them before the merge adds more.
+    // Keyed by the partition path the table writes, which is Hive styled for 
a SQL created table.
+    val rowsPerPartition = spark
+      .sql(s"select _hoodie_partition_path, count(*) as rows from $tableName 
group by _hoodie_partition_path")
+      .collect().map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    spark.sql(
+      s"""
+         |merge into $tableName t
+         |using (${rowsOf(0, 300, 4L, "v4")} union all ${rowsOf(2000, 2010, 
4L, "v4")}) s
+         |on t.id = s.id
+         |when matched then update set *
+         |when not matched then insert *
+       """.stripMargin)
+    assertTrue(recordIndexBuilt(), "Record index should be built by the second 
statement")
+
+    // Every key appears once, and the merge updated exactly the rows it 
matched.
+    assertEquals(2010L, spark.sql(s"select id from $tableName").count())
+    assertEquals(2010L, spark.sql(s"select distinct id from 
$tableName").count())
+    assertEquals(310L, spark.sql(s"select id from $tableName where name = 
'v4'").count())
+
+    // The index is sized from the rows the first statement left behind, not 
from the table as the merge left it.
+    val fileGroupIds = recordIndexFileGroupIds(tablePath)
+    if (testCase.partitioned) {
+      val perPartition = 
fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName)
+        .mapValues(_.size).toMap
+      assertEquals(rowsPerPartition.mapValues(rows => 
expectedFileGroupCount(rows)).toMap, perPartition)
+    } else {
+      assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size)
+    }
+  }
+
+  private def assertTableVersion(expected: HoodieTableVersion): Unit = {
+    metaClient = HoodieTableMetaClient.reload(metaClient)
+    assertEquals(expected, metaClient.getTableConfig.getTableVersion, "Table 
was written at an unexpected version")
+  }
+
+  private def isRecordIndexAvailable: Boolean = {
+    metaClient = HoodieTableMetaClient.reload(metaClient)
+    
metaClient.getTableConfig.getMetadataPartitions.contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath)
+  }
+
+  private def recordIndexFileGroupIds(tablePath: String = basePath): 
Seq[String] = {
+    val metadataMetaClient = HoodieTableMetaClient.builder()
+      .setBasePath(HoodieTableMetadata.getMetadataTableBasePath(tablePath))
+      .setConf(storageConf)
+      .build()
+    HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataMetaClient, 
HOption.empty(),
+      
MetadataPartitionType.RECORD_INDEX.getPartitionPath).asScala.map(_.getFileId).toSeq
+  }
+}
+
+object TestDeferredRecordIndexInit {
+
+  case class DeferredInitTestCase(tableType: HoodieTableType,
+                                  partitioned: Boolean,
+                                  tableVersion: HoodieTableVersion,
+                                  streamingWrites: Boolean,
+                                  firstOperation: String) {
+    override def toString: String =
+      s"${tableType.name()}/${if (partitioned) "partitioned" else 
"global"}/v${tableVersion.versionCode()}" +
+        s"/streaming=$streamingWrites/$firstOperation"
+  }
+
+  // The partitioned record index and streaming metadata writes both need a 
table version above six.
+  def testArgsForDeferredInit: java.util.stream.Stream[Arguments] = {
+    java.util.stream.Stream.of(
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, 
INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, 
partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = false, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = true, HoodieTableVersion.TEN, streamingWrites = false, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, 
partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL))
+    )
+  }
+
+  def testArgsForSql: java.util.stream.Stream[Arguments] = {
+    java.util.stream.Stream.of(
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, 
"bulk_insert")),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
"bulk_insert")),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "insert")),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "upsert")),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, 
"bulk_insert"))
+    )
+  }
+
+  def testArgsForInsertOverwrite: java.util.stream.Stream[Arguments] = {
+    java.util.stream.Stream.of(
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, 
partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL)),
+      Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, 
partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, 
BULK_INSERT_OPERATION_OPT_VAL))
+    )
+  }
+
+  /** Mirrors HoodieTableMetadataUtil.estimateFileGroupCount with growth 
factor 2 and 200 entries per file group. */
+  def expectedFileGroupCount(records: Long): Int = {

Review Comment:
   This formula is duplicated in all three files with growth factor 2 and 48 
bytes/entry hard-coded. Suggest calling 
`HoodieTableMetadataUtil.estimateFileGroupCount(RECORD_INDEX, ...)` with the 
configured values.



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // What the index should be sized from once it is built.
+    val rowsPerPartition = 
spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect()
+      .map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    // Commit 2: updates of records tagged through the fallback index, plus 
new records. The index is built here.
+    val updates = dataGen.generateUpdates("002", allRecords.subList(0, 300))
+    val newRecords = dataGen.generateInserts("002", 10)
+    val secondBatch = new util.ArrayList[HoodieRecord[_]](updates)
+    secondBatch.addAll(newRecords)
+    allRecords.addAll(newRecords)
+    write(secondBatch, UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    // Commit 3: upserts tagged through the record index itself.
+    val laterUpdates = dataGen.generateUpdates("003", allRecords.subList(0, 
300))
+    write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], 
UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+
+    // No duplicates across the fallback to record index switch.
+    val keys = 
spark.read.format("hudi").load(basePath).select("_row_key").collect().map(_.getString(0))
+    assertEquals(allRecords.asScala.map(_.getRecordKey).sorted.toSeq, 
keys.sorted.toSeq)
+
+    // The index is sized from the data the first commit left behind, not from 
an empty table.
+    val fileGroupIds = recordIndexFileGroupIds()
+    if (testCase.partitioned) {
+      val perPartition = 
fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName)
+        .mapValues(_.size).toMap
+      assertEquals(rowsPerPartition.mapValues(rows => 
expectedFileGroupCount(rows)).toMap, perPartition)
+    } else {
+      assertEquals(expectedFileGroupCount(rowsBeforeBuild), fileGroupIds.size)
+    }
+  }
+
+  /**
+   * The commit that builds the record index replaces the data the index was 
just sized from, so the index has to
+   * carry the replacing records and none of the replaced ones.
+   */
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForInsertOverwrite"))
+  def testRecordIndexBuiltByInsertOverwrite(testCase: DeferredInitTestCase): 
Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // Commit 2 builds the record index and then replaces every partition the 
first commit wrote.
+    val replacingBatch = dataGen.generateInserts("002", 500)
+    write(replacingBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
INSERT_OVERWRITE_OPERATION_OPT_VAL, SaveMode.Append)
+    assertTrue(isRecordIndexAvailable, "Record index should be built by the 
second commit")
+
+    val replacingKeys = replacingBatch.asScala.map(_.getRecordKey).sorted.toSeq
+    assertEquals(replacingKeys, 
spark.read.format("hudi").load(basePath).select("_row_key")
+      .collect().map(_.getString(0)).sorted.toSeq)
+
+    // Commit 3 is tagged through the record index. Replaced records must not 
come back and survivors must not double.
+    val laterUpdates = dataGen.generateUpdates("003", 
replacingBatch.subList(0, 100))
+    write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], 
UPSERT_OPERATION_OPT_VAL, SaveMode.Append)
+    assertEquals(replacingKeys, 
spark.read.format("hudi").load(basePath).select("_row_key")
+      .collect().map(_.getString(0)).sorted.toSeq)
+  }
+
+  /**
+   * The same deferral through Spark SQL: INSERT INTO leaves the record index 
unbuilt and the following MERGE INTO
+   * builds it, sized from the rows the first statement wrote.
+   */
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForSql"))
+  def testRecordIndexDeferredUntilSecondStatement(testCase: 
DeferredInitTestCase): Unit = {
+    val tableName = s"rli_deferred_sql_${if (testCase.partitioned) 
"partitioned" else "global"}_v${testCase.tableVersion.versionCode()}"
+    val tablePath = tempDir.resolve(tableName).toAbsolutePath.toString
+    val indexEnableProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key
+    } else {
+      HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key
+    }
+    val minFileGroupProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key
+    } else {
+      
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key
+    }
+    val maxFileGroupProp = if (testCase.partitioned) {
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key
+    } else {
+      
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key
+    }
+    val indexType = if (testCase.partitioned) "RECORD_LEVEL_INDEX" else 
"GLOBAL_RECORD_LEVEL_INDEX"
+    spark.sql(
+      s"""
+         |create table $tableName (
+         |  id string,
+         |  ts long,
+         |  name string,
+         |  part string
+         |) using hudi
+         | options (
+         |  primaryKey = 'id',
+         |  orderingFields = 'ts',
+         |  hoodie.metadata.enable = 'true',
+         |  ${HoodieWriteConfig.WRITE_TABLE_VERSION.key} = 
'${testCase.tableVersion.versionCode()}',
+         |  ${HoodieWriteConfig.AUTO_UPGRADE_VERSION.key} = 'false',
+         |  ${HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key} = 
'${testCase.streamingWrites}',
+         |  $indexEnableProp = 'true',
+         |  hoodie.index.type = '$indexType',
+         |  ${HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key} = 
'true',
+         |  
${HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key} = 
'9600',
+         |  $minFileGroupProp = '1',
+         |  $maxFileGroupProp = '1000'
+         | )
+         | partitioned by (part)
+         | location '$tablePath'
+       """.stripMargin)
+    val sqlMetaClient = 
HoodieTableMetaClient.builder().setBasePath(tablePath).setConf(storageConf).build()
+    val recordIndexBuilt = () => 
HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getMetadataPartitions
+      .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath)
+    val assertSqlTableVersion = () => assertEquals(testCase.tableVersion,
+      
HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getTableVersion,
+      "Table was written at an unexpected version")
+    val rowsOf = (from: Int, to: Int, ts: Long, name: String) =>
+      s"select cast(id as string) as id, ${ts}L as ts, '$name' as name, 
concat('p', cast(id % 3 as string)) as part from range($from, $to)"
+
+    try {
+      spark.conf.set("hoodie.spark.sql.insert.into.operation", 
testCase.firstOperation)
+      spark.sql(s"insert into $tableName ${rowsOf(0, 2000, 1L, "v1")}")
+    } finally {
+      spark.conf.unset("hoodie.spark.sql.insert.into.operation")
+    }
+    assertSqlTableVersion()
+    assertFalse(recordIndexBuilt(), "Record index should not be built by the 
first statement")
+
+    // The index is sized from the rows the first statement wrote, so capture 
them before the merge adds more.
+    // Keyed by the partition path the table writes, which is Hive styled for 
a SQL created table.
+    val rowsPerPartition = spark
+      .sql(s"select _hoodie_partition_path, count(*) as rows from $tableName 
group by _hoodie_partition_path")
+      .collect().map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    spark.sql(
+      s"""
+         |merge into $tableName t
+         |using (${rowsOf(0, 300, 4L, "v4")} union all ${rowsOf(2000, 2010, 
4L, "v4")}) s
+         |on t.id = s.id
+         |when matched then update set *
+         |when not matched then insert *
+       """.stripMargin)
+    assertTrue(recordIndexBuilt(), "Record index should be built by the second 
statement")
+
+    // Every key appears once, and the merge updated exactly the rows it 
matched.
+    assertEquals(2010L, spark.sql(s"select id from $tableName").count())
+    assertEquals(2010L, spark.sql(s"select distinct id from 
$tableName").count())
+    assertEquals(310L, spark.sql(s"select id from $tableName where name = 
'v4'").count())
+
+    // The index is sized from the rows the first statement left behind, not 
from the table as the merge left it.

Review Comment:
   Same as above: the numbers here give the same count either way, so this 
comment claims more than the assertion can show.



##########
hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestDeferredRecordIndexOnStreamerPath.java:
##########
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hudi.utilities.deltastreamer;
+
+import org.apache.hudi.common.config.HoodieMetadataConfig;
+import org.apache.hudi.common.model.FileSlice;
+import org.apache.hudi.common.model.WriteOperationType;
+import org.apache.hudi.common.table.HoodieTableMetaClient;
+import org.apache.hudi.common.table.HoodieTableVersion;
+import org.apache.hudi.common.testutils.HoodieTestUtils;
+import org.apache.hudi.common.util.Option;
+import org.apache.hudi.config.HoodieIndexConfig;
+import org.apache.hudi.config.HoodieWriteConfig;
+import org.apache.hudi.metadata.HoodieTableMetadata;
+import org.apache.hudi.metadata.HoodieTableMetadataUtil;
+import org.apache.hudi.metadata.MetadataPartitionType;
+
+import org.junit.jupiter.api.Tag;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
+
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Deferred record index initialization on the HoodieStreamer path: the first 
sync leaves the index unbuilt and the
+ * second one builds it, with no duplicates across the switch from the 
fallback index.
+ */
+@Tag("functional")
+public class TestDeferredRecordIndexOnStreamerPath extends 
HoodieDeltaStreamerTestBase {
+
+  private static void enableDeferredRecordIndex(HoodieDeltaStreamer.Config 
cfg, boolean partitioned, int tableVersion) {
+    cfg.configs.add(HoodieMetadataConfig.ENABLE.key() + "=true");
+    cfg.configs.add(HoodieWriteConfig.WRITE_TABLE_VERSION.key() + "=" + 
tableVersion);
+    cfg.configs.add(HoodieWriteConfig.AUTO_UPGRADE_VERSION.key() + "=false");
+    
cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key()
 + "=" + !partitioned);
+    cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() 
+ "=" + partitioned);
+    cfg.configs.add(HoodieIndexConfig.INDEX_TYPE.key() + "="
+        + (partitioned ? "RECORD_LEVEL_INDEX" : "GLOBAL_RECORD_LEVEL_INDEX"));
+    cfg.configs.add(HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key() 
+ "=true");
+    // Pin the sizing inputs so the expected file group count follows from the 
record count alone: 9600 bytes over
+    // 48 bytes per entry is 200 entries per file group, and the floor is kept 
out of the way.
+    
cfg.configs.add(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key()
 + "=9600");
+    
cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key()
 + "=1");
+    
cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key()
 + "=1");
+    
cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key()
 + "=1000");
+    
cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key()
 + "=1000");
+  }
+
+  /**
+   * The file group count the record index is sized to for the given number of 
records, applying the growth factor and
+   * the entries that fit in one file group.
+   */
+  private static int expectedFileGroupCount(long records) {
+    long estimated = (long) Math.ceil(records * 2.0) / 200;
+    return (int) Math.max(1, Math.min(1000, estimated));
+  }
+
+  private static List<String> recordIndexFileGroupIds(String tableBasePath) {
+    HoodieTableMetaClient metadataMetaClient = HoodieTableMetaClient.builder()
+        
.setBasePath(HoodieTableMetadata.getMetadataTableBasePath(tableBasePath))
+        .setConf(HoodieTestUtils.getDefaultStorageConf())
+        .build();
+    return 
HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataMetaClient, 
Option.empty(),
+            MetadataPartitionType.RECORD_INDEX.getPartitionPath())
+        .stream().map(FileSlice::getFileId).collect(Collectors.toList());
+  }
+
+  private static boolean isRecordIndexBuilt(String tableBasePath) {
+    return HoodieTableMetaClient.builder()
+        .setBasePath(tableBasePath)
+        .setConf(HoodieTestUtils.getDefaultStorageConf())
+        .build()
+        .getTableConfig().getMetadataPartitions()
+        .contains(MetadataPartitionType.RECORD_INDEX.getPartitionPath());
+  }
+
+  static Stream<Arguments> testArgs() {
+    return Stream.of(
+        Arguments.arguments(false, HoodieTableVersion.SIX.versionCode()),
+        Arguments.arguments(false, HoodieTableVersion.TEN.versionCode()),
+        Arguments.arguments(true, HoodieTableVersion.TEN.versionCode()));
+  }
+
+  @ParameterizedTest
+  @MethodSource("testArgs")
+  public void testRecordIndexDeferredUntilSecondSync(boolean partitioned, int 
tableVersion) throws Exception {
+    String tableBasePath = basePath + "/test_rli_deferred_streamer_"
+        + (partitioned ? "partitioned" : "global") + "_v" + tableVersion;
+
+    HoodieDeltaStreamer.Config insertCfg =
+        TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, 
WriteOperationType.INSERT);
+    enableDeferredRecordIndex(insertCfg, partitioned, tableVersion);
+    new HoodieDeltaStreamer(insertCfg, jsc).sync();
+    assertEquals(tableVersion, 
HoodieTableMetaClient.builder().setBasePath(tableBasePath)
+        
.setConf(HoodieTestUtils.getDefaultStorageConf()).build().getTableConfig().getTableVersion().versionCode());
+    assertFalse(isRecordIndexBuilt(tableBasePath), "Record index should not be 
built by the first sync");
+
+    // The index is sized from the data the first sync left behind, so capture 
it before the second sync adds to it.
+    assertRecordCount(1000, tableBasePath, sqlContext);
+    sqlContext.clearCache();
+    Map<String, Long> rowsPerPartition = 
sqlContext.read().format("hudi").load(tableBasePath)
+        .groupBy("_hoodie_partition_path").count()
+        .collectAsList().stream().collect(Collectors.toMap(row -> 
row.getString(0), row -> row.getLong(1)));
+    long rowsBeforeBuild = 
rowsPerPartition.values().stream().mapToLong(Long::longValue).sum();
+
+    HoodieDeltaStreamer.Config upsertCfg =
+        TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, 
WriteOperationType.UPSERT);
+    enableDeferredRecordIndex(upsertCfg, partitioned, tableVersion);
+    new HoodieDeltaStreamer(upsertCfg, jsc).sync();
+    assertTrue(isRecordIndexBuilt(tableBasePath), "Record index should be 
built by the second sync");
+
+    // The second sync carries 1000 records from the test source: 500 inserts, 
450 updates and 50 deletes, so the
+    // 1000 the first sync wrote become 1450, one row per key, and the deletes 
make this cover a key removed through
+    // the fallback index as well. The same 1450 comes out with the deferral 
turned off, so this pins what the two

Review Comment:
   This case does discriminate on sizing (1000 vs 1450), which is good. It 
still needs a third sync that updates the second sync's inserts and re-inserts 
its deletes, plus the RLI entry count check.



##########
hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala:
##########
@@ -0,0 +1,331 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hudi.functional
+
+import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin}
+import org.apache.hudi.DataSourceWriteOptions._
+import org.apache.hudi.common.config.HoodieMetadataConfig
+import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType}
+import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, 
HoodieTableVersion}
+import org.apache.hudi.common.testutils.HoodieTestDataGenerator
+import org.apache.hudi.common.util.{Option => HOption}
+import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
+import 
org.apache.hudi.functional.TestDeferredRecordIndexInit.{expectedFileGroupCount, 
DeferredInitTestCase}
+import org.apache.hudi.metadata.{HoodieTableMetadata, HoodieTableMetadataUtil, 
MetadataPartitionType}
+
+import org.apache.spark.sql.SaveMode
+import org.junit.jupiter.api.Tag
+import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue}
+import org.junit.jupiter.params.ParameterizedTest
+import org.junit.jupiter.params.provider.{Arguments, MethodSource}
+
+import java.util
+
+import scala.collection.JavaConverters._
+
+/**
+ * End to end coverage of hoodie.metadata.record.level.index.defer.init on its 
own, with no record threshold:
+ * the record index is not built by the first commit, is built by the second 
one and sized from the first commit's
+ * data, and the switch from the fallback index produces no duplicates.
+ */
+@Tag("functional")
+class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
+
+  private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, 
String] = Map(
+    HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+    DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(),
+    RECORDKEY_FIELD.key -> "_row_key",
+    PARTITIONPATH_FIELD.key -> "partition_path",
+    HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+    HoodieWriteConfig.WRITE_TABLE_VERSION.key -> 
testCase.tableVersion.versionCode().toString,
+    HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false",
+    HoodieMetadataConfig.ENABLE.key -> "true",
+    HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
(!testCase.partitioned).toString,
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> 
testCase.partitioned.toString,
+    HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> 
testCase.streamingWrites.toString,
+    // Pinned low so the sizing result is not simply the floor.
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    
HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> 
"1",
+    HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> 
"1000",
+    // 9600 bytes / 48 bytes per entry = 200 entries per file group.
+    HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> 
"9600",
+    HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true",
+    HoodieCompactionConfig.INLINE_COMPACT.key -> "false",
+    HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) 
"RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"))
+
+  @ParameterizedTest
+  @MethodSource(Array("testArgsForDeferredInit"))
+  def testRecordIndexDeferredUntilSecondCommit(testCase: 
DeferredInitTestCase): Unit = {
+    val options = deferredInitOptions(testCase)
+    val dataGen = new HoodieTestDataGenerator()
+    val allRecords = new util.ArrayList[HoodieRecord[_]]()
+    val write = (records: util.List[HoodieRecord[_]], operation: String, mode: 
SaveMode) =>
+      toDataset(spark, records).write.format("hudi").options(options)
+        .option(DataSourceWriteOptions.OPERATION.key, 
operation).mode(mode).save(basePath)
+
+    // Commit 1: the record index must not be built, since the table had no 
completed commit when the write started.
+    val firstBatch = dataGen.generateInserts("001", 2000)
+    allRecords.addAll(firstBatch)
+    write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], 
testCase.firstOperation, SaveMode.Overwrite)
+    assertTableVersion(testCase.tableVersion)
+    assertFalse(isRecordIndexAvailable, "Record index should not be built by 
the first commit")
+
+    // What the index should be sized from once it is built.
+    val rowsPerPartition = 
spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect()
+      .map(row => row.getString(0) -> row.getLong(1)).toMap
+    val rowsBeforeBuild = rowsPerPartition.values.sum
+
+    // Commit 2: updates of records tagged through the fallback index, plus 
new records. The index is built here.

Review Comment:
   This comment isn't accurate. `doInitTable` runs `initMetadataTable` before 
the write, so the index is built at the start of commit 2 and these updates are 
tagged through the new record index, not the fallback. The fallback only ever 
tags commit 1's inserts.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to