This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20604-06dc47c2d4666d36c64fa39de4980a90bb64cde9 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 46aa241723f8a82c9b3c625cfd87489ebf56ce93 Author: Naga Surya Prathik Vunnava <[email protected]> AuthorDate: Thu Nov 19 04:41:40 2026 +0000 fix(metadata): keep deferred record index initialization from leaking or misfiring, with end-to-end tests (#20604) * feat(metadata): defer record index initialization until the table holds enough records On a fresh table the record index is initialized at the first write, when the table holds no data, so its file group count is clamped to the configured minimum for the life of the table. Add hoodie.metadata.record.level.index.defer.init.min.records (default 0, used with hoodie.metadata.record.level.index.defer.init): the record index is deferred until the latest base files hold that many records, and is then sized from them. Until then, lookups use the fallback index. - The record count is read from base file footers and reused across the metadata writers of a commit. - Once over the threshold, the build waits while another instant is pending, and only a non-streaming metadata writer builds the index, so no writer that started while the index was deferred misses it. - The threshold is rejected with multi-writer concurrency modes, the Flink engine and negative values, and is ignored with async metadata indexing. An index that is already built or inflight is never deferred. - In Spark, a new data partition of the partitioned record index is sized from the existing partitions (p90) with the creating commit's inserts as a floor. No file groups are created for a deferred index. - While deferred, the global record index fallback moves records across partitions as the record index would. * Drop the record count threshold and cover the existing deferral end to end The deferral of record index initialization is covered for the Spark datasource, Spark SQL, the structured streaming sink and HoodieStreamer, over the global and the partitioned record index, table versions six and ten, and bulk_insert, insert, upsert and insert_overwrite as the first operation. Each case asserts that the first commit leaves the index unbuilt, that the second builds it, that its file group count follows from the records the first commit wrote, and that no keys are duplicated across the switch from the fallback index. The record count threshold, the p90 sizing of new partitions and the fallback index change are left for follow ups so that this change carries only the coverage. * Tighten the deferral assertions The streamer test went through the base class read helpers, which clear the Spark cache before reading, and pins the record and distinct record counts the two syncs leave rather than comparing a count against itself. The Spark SQL and insert overwrite cases assert the table version they are parameterized on, the SQL cases set the streaming metadata write flag their parameter names carry, the insert overwrite case writes the operation it is given, and the datasource case groups by the partition path the table writes rather than the partition column. The streaming case is tagged functional alongside its siblings. * Say what the second sync writes The test source emits 500 inserts, 450 updates and 50 deletes, so the count covers a key removed through the fallback index as well. * Keep the deferral from creating partitioned record index file groups or ending on a rollback The partitioned record index now gets file groups for new partitions only once it is initialized or being built by the async indexer, and the deferral counts completed commits only, so a rolled back first commit no longer ends it. The end-to-end tests now check what the metadata table holds after commit 1 (no record index files, commits or definition), the index definition and its sizing from commit 1's data after commit 2, and that the index holds one entry per row in the row's partition and file group after commits 2 and 3, where commit 3 updates commit 2's inserts and brings back its deletes. They add a failed first commit, a failed building commit, a building commit that starts a new partition, and continuous-mode HoodieStreamer, and share their checks through DeferredRecordIndexTestUtils. --- .../metadata/HoodieBackedTableMetadataWriter.java | 18 +- .../apache/hudi/metadata/index/IndexerFactory.java | 5 +- .../functional/DeferredRecordIndexTestUtils.scala | 159 +++++++++ .../functional/TestDeferredRecordIndexInit.scala | 374 +++++++++++++++++++++ .../hudi/functional/TestStructuredStreaming.scala | 82 ++++- .../TestDeferredRecordIndexOnStreamerPath.java | 160 +++++++++ 6 files changed, 790 insertions(+), 8 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 5cc35cc85d2e..e36f9b14b709 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -1095,25 +1095,37 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab * @param instantTime Timestamp for the mdt commit */ private void maybeInitializeNewFileGroupsForPartitionedRLI(HoodieCommitMetadata commitMetadata, String instantTime) { - if (dataWriteConfig.isRecordLevelIndexEnabled()) { + if (isPartitionedRecordIndexInitialized()) { Set<String> partitionsTouchedByInflightCommit = new HashSet<>(commitMetadata.getPartitionToWriteStats().keySet()); initializeNewFileGroupsForPartitionedRLIHelper(partitionsTouchedByInflightCommit, instantTime); } } private void maybeInitializeNewFileGroupsForPartitionedRLI(HoodieData<WriteStatus> writeStatus, String instantTime) { - if (dataWriteConfig.isRecordLevelIndexEnabled()) { + if (isPartitionedRecordIndexInitialized()) { Set<String> partitionsTouchedByInflightCommit = new HashSet<>(writeStatus.map(WriteStatus::getPartitionPath).collectAsList()); initializeNewFileGroupsForPartitionedRLIHelper(partitionsTouchedByInflightCommit, instantTime); } } private void maybeInitializeNewFileGroupsForPartitionedRLI(Set<String> partitionsTouchedByInflightCommit, String instantTime) { - if (dataWriteConfig.isRecordLevelIndexEnabled()) { + if (isPartitionedRecordIndexInitialized()) { initializeNewFileGroupsForPartitionedRLIHelper(partitionsTouchedByInflightCommit, instantTime); } } + /** + * New partitions get record index file groups only once the partitioned record index exists or is being built by the + * async indexer. Until then, for instance while its initialization is deferred, there is nothing to add them to, and + * the build sizes every partition itself. + */ + private boolean isPartitionedRecordIndexInitialized() { + HoodieTableConfig tableConfig = dataMetaClient.getTableConfig(); + return dataWriteConfig.isRecordLevelIndexEnabled() + && (tableConfig.isMetadataPartitionAvailable(RECORD_INDEX) + || tableConfig.getMetadataPartitionsInflight().contains(RECORD_INDEX.getPartitionPath())); + } + private void initializeNewFileGroupsForPartitionedRLIHelper(Set<String> partitionsTouchedByInflightCommit, String instantTime) { try { Set<String> partitionsFromFilesIndex = new HashSet<>(metadata.getAllPartitionPaths()); diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/index/IndexerFactory.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/index/IndexerFactory.java index 2aa0e330687f..92f3f785154b 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/index/IndexerFactory.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/index/IndexerFactory.java @@ -94,12 +94,13 @@ public class IndexerFactory { } /** - * Returns whether the initialization of record index should be deferred, true if the table is a fresh table. + * Returns whether the initialization of record index should be deferred, true if the table has no completed commit. + * Completed rollbacks and cleans do not count, since a table whose only commit failed and was rolled back has no data. */ private static boolean shouldDeferRecordIndexInit( HoodieWriteConfig dataTableWriteConfig, HoodieTableMetaClient dataTableMetaClient) { return dataTableWriteConfig.getMetadataConfig().shouldDeferRliInitForFreshTable() - && dataTableMetaClient.getActiveTimeline().filterCompletedInstants().countInstants() == 0; + && dataTableMetaClient.getCommitsTimeline().filterCompletedInstants().empty(); } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/DeferredRecordIndexTestUtils.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/DeferredRecordIndexTestUtils.scala new file mode 100644 index 000000000000..aa5816c123bc --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/DeferredRecordIndexTestUtils.scala @@ -0,0 +1,159 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.client.common.HoodieSparkEngineContext +import org.apache.hudi.common.config.{HoodieMetadataConfig, TypedProperties} +import org.apache.hudi.common.function.SerializableFunctionUnchecked +import org.apache.hudi.common.model.FileSlice +import org.apache.hudi.common.table.HoodieTableMetaClient +import org.apache.hudi.common.util.{Option => HOption} +import org.apache.hudi.core.index.record.HoodieRecordIndex +import org.apache.hudi.metadata.{HoodieBackedTableMetadata, HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} +import org.apache.hudi.storage.{StorageConfiguration, StoragePath} + +import org.apache.spark.api.java.JavaSparkContext +import org.apache.spark.sql.SparkSession +import org.junit.jupiter.api.Assertions.{assertEquals, assertFalse, assertTrue} + +import java.util.Properties + +import scala.collection.JavaConverters._ + +/** + * Assertions shared by the end to end tests of deferred record index initialization, so every writer is held to the + * same checks: what the metadata table holds before and after the build, whether the index agrees with the table, and + * how the index was sized. + */ +object DeferredRecordIndexTestUtils { + + private val RecordIndexPath: String = MetadataPartitionType.RECORD_INDEX.getPartitionPath + + /** + * Asserts the record index left nothing behind: the table config does not list it, it has no index definition, the + * metadata table holds no files under its partition, and no completed metadata table commit wrote to it. + */ + def assertRecordIndexNotBuilt(basePath: String, storageConf: StorageConfiguration[_]): Unit = { + val metaClient = buildMetaClient(basePath, storageConf) + assertFalse(metaClient.getTableConfig.getMetadataPartitions.contains(RecordIndexPath), + "The table config should not list the record index") + assertTrue(metaClient.getIndexForMetadataPartition(RecordIndexPath).isEmpty, + "The record index should have no index definition") + + val metadataBasePath = HoodieTableMetadata.getMetadataTableBasePath(basePath) + val partitionPath = new StoragePath(metadataBasePath, RecordIndexPath) + val storage = metaClient.getStorage + if (storage.exists(partitionPath)) { + assertEquals(Seq.empty, storage.listFiles(partitionPath).asScala.map(_.getPath.toString).toSeq, + "The metadata table should hold no record index files") + } + + val metadataTimeline = buildMetaClient(metadataBasePath, storageConf).getActiveTimeline + val commitsWritingRecordIndex = metadataTimeline.getCommitsTimeline.filterCompletedInstants().getInstants.asScala + .filter(instant => metadataTimeline.readCommitMetadata(instant).getPartitionToWriteStats.containsKey(RecordIndexPath)) + .map(_.requestedTime()).toSeq + assertEquals(Seq.empty, commitsWritingRecordIndex, "No metadata table commit should write to the record index") + } + + /** + * Asserts the record index is listed in the table config and has an index definition that says whether it is + * partitioned, which is what readers go by. + */ + def assertRecordIndexBuilt(basePath: String, storageConf: StorageConfiguration[_], partitioned: Boolean): Unit = { + val metaClient = buildMetaClient(basePath, storageConf) + assertTrue(metaClient.getTableConfig.getMetadataPartitions.contains(RecordIndexPath), + "The table config should list the record index") + val definition = metaClient.getIndexForMetadataPartition(RecordIndexPath) + assertTrue(definition.isPresent, "The record index should have an index definition") + assertEquals(partitioned, HoodieRecordIndex.isPartitioned(definition.get), + "The record index definition should say whether the index is partitioned") + } + + /** + * Asserts the record index holds exactly one entry per row of the table, pointing at the partition and the file group + * the row is in. A key indexed twice, a key missing from the index, or an entry left for a deleted row fails here. + */ + def assertRecordIndexMatchesTable(spark: SparkSession, basePath: String, storageConf: StorageConfiguration[_]): Unit = { + val metaClient = buildMetaClient(basePath, storageConf) + val engineContext = new HoodieSparkEngineContext(JavaSparkContext.fromSparkContext(spark.sparkContext)) + val metadata = new HoodieBackedTableMetadata(engineContext, metaClient.getStorage, + HoodieMetadataConfig.newBuilder().enable(true).build(), basePath) + val allFileSlices: SerializableFunctionUnchecked[java.util.List[FileSlice], java.util.List[FileSlice]] = fileSlices => fileSlices + val indexed = try { + metadata.readRecordIndexLocations(allFileSlices).collectAsList().asScala + .map(pair => (pair.getKey, pair.getValue.getPartitionPath, pair.getValue.getFileId)).toSeq + } finally { + metadata.close() + } + val stored = spark.read.format("hudi").load(basePath) + .select("_hoodie_record_key", "_hoodie_partition_path", "_hoodie_file_name").collect() + .map(row => (row.getString(0), row.getString(1), row.getString(2))).toSeq + + assertEquals(stored.map(row => (row._1, row._2)).sorted, indexed.map(entry => (entry._1, entry._2)).sorted, + "The record index should hold one entry per row, in the row's partition") + val indexedFileIds = indexed.map(entry => entry._1 -> entry._3).toMap + val rowsInAnotherFileGroup = stored.filterNot(row => row._3.contains(indexedFileIds(row._1))).map(_._1) + assertEquals(Seq.empty, rowsInAnotherFileGroup, "The record index should point every row at its file group") + } + + /** + * Asserts the record index was sized from the given rows, per partition for the partitioned index and over the whole + * table for the global one, with the formula and the sizing configs the writer used. + */ + def assertRecordIndexSizedFrom(basePath: String, + storageConf: StorageConfiguration[_], + writeProps: Properties, + partitioned: Boolean, + rowsPerPartition: java.util.Map[String, java.lang.Long]): Unit = { + val metadataConfig = HoodieMetadataConfig.newBuilder().fromProperties(writeProps).build() + val expectedFileGroupCount = (records: Long) => HoodieTableMetadataUtil.estimateFileGroupCount( + MetadataPartitionType.RECORD_INDEX, () => records, HoodieTableMetadataUtil.RECORD_INDEX_AVERAGE_RECORD_SIZE, + if (partitioned) metadataConfig.getRecordLevelIndexMinFileGroupCount else metadataConfig.getGlobalRecordLevelIndexMinFileGroupCount, + if (partitioned) metadataConfig.getRecordLevelIndexMaxFileGroupCount else metadataConfig.getGlobalRecordLevelIndexMaxFileGroupCount, + metadataConfig.getRecordIndexGrowthFactor, metadataConfig.getRecordIndexMaxFileGroupSizeBytes) + + val metadataMetaClient = buildMetaClient(HoodieTableMetadata.getMetadataTableBasePath(basePath), storageConf) + val fileGroupIds = HoodieTableMetadataUtil.getPartitionLatestFileSlices(metadataMetaClient, HOption.empty(), RecordIndexPath) + .asScala.map(_.getFileId).toSeq + if (partitioned) { + val expected = rowsPerPartition.asScala.map { case (partition, rows) => partition -> expectedFileGroupCount(rows) }.toMap + val actual = fileGroupIds.groupBy(HoodieTableMetadataUtil.getDataTablePartitionNameFromFileGroupName).mapValues(_.size).toMap + assertEquals(expected, actual, "Each partition's record index should be sized from that partition's rows") + } else { + assertEquals(expectedFileGroupCount(rowsPerPartition.asScala.values.map(_.longValue).sum), fileGroupIds.size, + "The record index should be sized from the table's rows") + } + } + + /** Builds the write properties for {@link assertRecordIndexSizedFrom} from writer options. */ + def toProperties(options: Map[String, String]): Properties = { + val props = new TypedProperties() + options.foreach { case (key, value) => props.setProperty(key, value) } + props + } + + /** Counts the rows of the table in each of its partitions. */ + def rowsPerPartition(spark: SparkSession, basePath: String): java.util.Map[String, java.lang.Long] = + spark.read.format("hudi").load(basePath).groupBy("_hoodie_partition_path").count().collect() + .map(row => row.getString(0) -> java.lang.Long.valueOf(row.getLong(1))).toMap.asJava + + private def buildMetaClient(basePath: String, storageConf: StorageConfiguration[_]): HoodieTableMetaClient = + HoodieTableMetaClient.builder().setBasePath(basePath).setConf(storageConf.newInstance()).build() +} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala new file mode 100644 index 000000000000..3f95b6bbd8c5 --- /dev/null +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestDeferredRecordIndexInit.scala @@ -0,0 +1,374 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hudi.functional + +import org.apache.hudi.{DataSourceWriteOptions, SparkDatasetMixin} +import org.apache.hudi.DataSourceWriteOptions._ +import org.apache.hudi.common.config.HoodieMetadataConfig +import org.apache.hudi.common.model.{HoodieRecord, HoodieTableType} +import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, HoodieTableVersion} +import org.apache.hudi.common.testutils.HoodieTestDataGenerator +import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} +import org.apache.hudi.functional.DeferredRecordIndexTestUtils._ +import org.apache.hudi.functional.TestDeferredRecordIndexInit.DeferredInitTestCase + +import org.apache.spark.sql.{DataFrame, SaveMode} +import org.apache.spark.sql.functions.lit +import org.junit.jupiter.api.Assertions.assertEquals +import org.junit.jupiter.api.Tag +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.{Arguments, MethodSource, ValueSource} + +import java.util + +import scala.collection.JavaConverters._ + +/** + * End to end coverage of hoodie.metadata.record.level.index.defer.init on its own, with no record threshold: the + * first commit leaves nothing of the record index in the metadata table, the second builds it sized from the first + * commit's data, and the index agrees with the table after the second commit and after the third, the first one + * tagged through the index alone. + */ +@Tag("functional") +class TestDeferredRecordIndexInit extends RecordLevelIndexTestBase with SparkDatasetMixin { + + private def deferredInitOptions(testCase: DeferredInitTestCase): Map[String, String] = Map( + HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> testCase.tableType.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieWriteConfig.WRITE_TABLE_VERSION.key -> testCase.tableVersion.versionCode().toString, + HoodieWriteConfig.AUTO_UPGRADE_VERSION.key -> "false", + HoodieMetadataConfig.ENABLE.key -> "true", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key -> (!testCase.partitioned).toString, + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key -> testCase.partitioned.toString, + HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key -> testCase.streamingWrites.toString, + // Pinned low so the sizing result is not simply the floor. + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key -> "1", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key -> "1000", + // 9600 bytes / 48 bytes per entry = 200 entries per file group. + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieCompactionConfig.INLINE_COMPACT.key -> "false", + HoodieIndexConfig.INDEX_TYPE.key -> (if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX")) + + @ParameterizedTest + @MethodSource(Array("testArgsForDeferredInit")) + def testRecordIndexDeferredUntilSecondCommit(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val write = (batch: DataFrame, operation: String, mode: SaveMode) => + batch.write.format("hudi").options(options).option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1: the record index must not be built, since the table had no completed commit when the write started. + val firstBatch = dataGen.generateInserts("001", 2000) + write(toDataset(spark, firstBatch), testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertRecordIndexNotBuilt(basePath, storageConf) + val rowsBeforeBuild = rowsPerPartition(spark, basePath) + + // Commit 2 builds the index from what commit 1 left, then writes updates, deletes and enough new rows that an + // index sized after this write would get a different file group count. + // It also starts a partition commit 1 did not write, which the build cannot have sized from commit 1's data. + val newRecords = dataGen.generateInserts("002", 2000) + val firstNewPartition = "2026/02/01" + val firstNewPartitionRecords = dataGen.generateInsertsForPartition("002", 20, firstNewPartition) + val deletedRecords = firstBatch.subList(300, 350) + write(toDataset(spark, dataGen.generateUpdates("002", firstBatch.subList(0, 300))) + .union(toDataset(spark, dataGen.generateUpdates("002", deletedRecords)).withColumn("_hoodie_is_deleted", lit(true))) + .union(toDataset(spark, newRecords)) + .union(toDataset(spark, firstNewPartitionRecords)), UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertRecordIndexBuilt(basePath, storageConf, testCase.partitioned) + val rowsSizedFrom = new util.HashMap[String, java.lang.Long](rowsBeforeBuild) + rowsSizedFrom.put(firstNewPartition, 0L) + assertRecordIndexSizedFrom(basePath, storageConf, toProperties(options), testCase.partitioned, rowsSizedFrom) + val deletedKeys = deletedRecords.asScala.map(_.getRecordKey).toSet + val keysAfterBuild = (firstBatch.asScala ++ newRecords.asScala ++ firstNewPartitionRecords.asScala) + .map(_.getRecordKey).filterNot(deletedKeys) + assertTableKeys(keysAfterBuild) + assertRecordIndexMatchesTable(spark, basePath, storageConf) + + // Commit 3 is the first write tagged through the index alone: it updates rows commit 2 inserted, brings back the + // keys commit 2 deleted, and starts a partition no earlier commit wrote. + val newPartitionRecords = dataGen.generateInsertsForPartition("003", 20, "2026/01/01") + write(toDataset(spark, dataGen.generateUpdates("003", newRecords.subList(0, 300))) + .union(toDataset(spark, dataGen.generateUpdates("003", deletedRecords))) + .union(toDataset(spark, newPartitionRecords)), UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTableKeys(keysAfterBuild ++ deletedKeys ++ newPartitionRecords.asScala.map(_.getRecordKey)) + assertRecordIndexMatchesTable(spark, basePath, storageConf) + } + + /** + * The commit that builds the record index replaces the data the index was just sized from, so the index has to + * carry the replacing records and none of the replaced ones. + */ + @ParameterizedTest + @MethodSource(Array("testArgsForInsertOverwrite")) + def testRecordIndexBuiltByInsertOverwrite(testCase: DeferredInitTestCase): Unit = { + val options = deferredInitOptions(testCase) + val dataGen = new HoodieTestDataGenerator() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + val firstBatch = dataGen.generateInserts("001", 2000) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], testCase.firstOperation, SaveMode.Overwrite) + assertTableVersion(testCase.tableVersion) + assertRecordIndexNotBuilt(basePath, storageConf) + + // Commit 2 builds the record index and then replaces every partition the first commit wrote. + val replacingBatch = dataGen.generateInserts("002", 500) + write(replacingBatch.asInstanceOf[util.List[HoodieRecord[_]]], INSERT_OVERWRITE_OPERATION_OPT_VAL, SaveMode.Append) + assertRecordIndexBuilt(basePath, storageConf, testCase.partitioned) + val replacingKeys = replacingBatch.asScala.map(_.getRecordKey) + assertTableKeys(replacingKeys) + assertRecordIndexMatchesTable(spark, basePath, storageConf) + + // Commit 3 is tagged through the record index. Replaced records must not come back and survivors must not double. + val laterUpdates = dataGen.generateUpdates("003", replacingBatch.subList(0, 100)) + write(laterUpdates.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTableKeys(replacingKeys) + assertRecordIndexMatchesTable(spark, basePath, storageConf) + } + + /** + * A first commit that fails is rolled back by the next write and leaves no data behind, so the retry must defer the + * build as well rather than build the index from an empty table. + */ + @ParameterizedTest + @ValueSource(booleans = Array(false, true)) + def testRecordIndexDeferredPastFailedFirstCommit(partitioned: Boolean): Unit = { + val options = deferredInitOptions(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned, + HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)) + val dataGen = new HoodieTestDataGenerator() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + // Commit 1 is turned back into an inflight commit, as a writer that failed before completing it would leave it. + write(dataGen.generateInserts("001", 2000).asInstanceOf[util.List[HoodieRecord[_]]], BULK_INSERT_OPERATION_OPT_VAL, + SaveMode.Overwrite) + metaClient = HoodieTableMetaClient.reload(metaClient) + metaClient.getActiveTimeline.revertToInflight(metaClient.getCommitsTimeline.filterCompletedInstants().lastInstant().get()) + + // The retry rolls commit 1 back first, so it starts on a table with no data and must not build the index. + val retriedBatch = dataGen.generateInserts("002", 1000) + write(retriedBatch.asInstanceOf[util.List[HoodieRecord[_]]], BULK_INSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertTableKeys(retriedBatch.asScala.map(_.getRecordKey)) + assertRecordIndexNotBuilt(basePath, storageConf) + val rowsBeforeBuild = rowsPerPartition(spark, basePath) + + val newRecords = dataGen.generateInserts("003", 1000) + write(newRecords.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertRecordIndexBuilt(basePath, storageConf, partitioned) + assertRecordIndexSizedFrom(basePath, storageConf, toProperties(options), partitioned, rowsBeforeBuild) + assertTableKeys((retriedBatch.asScala ++ newRecords.asScala).map(_.getRecordKey)) + assertRecordIndexMatchesTable(spark, basePath, storageConf) + } + + /** + * The commit that builds the record index fails after the build and is rolled back by the retry. The index must still + * agree with the table, holding commit 1's rows and the retry's, and none of what the failed commit wrote. + */ + @ParameterizedTest + @ValueSource(booleans = Array(false, true)) + def testRecordIndexAfterFailedBuildingCommit(partitioned: Boolean): Unit = { + val options = deferredInitOptions(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned, + HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)) + val dataGen = new HoodieTestDataGenerator() + val write = (records: util.List[HoodieRecord[_]], operation: String, mode: SaveMode) => + toDataset(spark, records).write.format("hudi").options(options) + .option(DataSourceWriteOptions.OPERATION.key, operation).mode(mode).save(basePath) + + val firstBatch = dataGen.generateInserts("001", 2000) + write(firstBatch.asInstanceOf[util.List[HoodieRecord[_]]], BULK_INSERT_OPERATION_OPT_VAL, SaveMode.Overwrite) + assertRecordIndexNotBuilt(basePath, storageConf) + + // Commit 2 builds the index, then is turned back into an inflight commit as a writer that failed would leave it. + write(dataGen.generateInserts("002", 500).asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, + SaveMode.Append) + assertRecordIndexBuilt(basePath, storageConf, partitioned) + metaClient = HoodieTableMetaClient.reload(metaClient) + metaClient.getActiveTimeline.revertToInflight(metaClient.getCommitsTimeline.filterCompletedInstants().lastInstant().get()) + + // The retry rolls commit 2 back first, then updates commit 1's rows and adds its own. + val retriedRecords = dataGen.generateInserts("003", 500) + write(dataGen.generateUpdates("003", firstBatch.subList(0, 300)).asInstanceOf[util.List[HoodieRecord[_]]], + UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + write(retriedRecords.asInstanceOf[util.List[HoodieRecord[_]]], UPSERT_OPERATION_OPT_VAL, SaveMode.Append) + assertRecordIndexBuilt(basePath, storageConf, partitioned) + assertTableKeys((firstBatch.asScala ++ retriedRecords.asScala).map(_.getRecordKey)) + assertRecordIndexMatchesTable(spark, basePath, storageConf) + } + + /** + * The same deferral through Spark SQL: INSERT INTO leaves the record index unbuilt and the following MERGE INTO + * builds it, sized from the rows the first statement wrote. + */ + @ParameterizedTest + @MethodSource(Array("testArgsForSql")) + def testRecordIndexDeferredUntilSecondStatement(testCase: DeferredInitTestCase): Unit = { + val tableName = s"rli_deferred_sql_${if (testCase.partitioned) "partitioned" else "global"}_v${testCase.tableVersion.versionCode()}" + val tablePath = tempDir.resolve(tableName).toAbsolutePath.toString + val indexEnableProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key + } + val minFileGroupProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } + val maxFileGroupProp = if (testCase.partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } + val indexType = if (testCase.partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX" + spark.sql( + s""" + |create table $tableName ( + | id string, + | ts long, + | name string, + | part string + |) using hudi + | options ( + | primaryKey = 'id', + | orderingFields = 'ts', + | hoodie.metadata.enable = 'true', + | ${HoodieWriteConfig.WRITE_TABLE_VERSION.key} = '${testCase.tableVersion.versionCode()}', + | ${HoodieWriteConfig.AUTO_UPGRADE_VERSION.key} = 'false', + | ${HoodieMetadataConfig.STREAMING_WRITE_ENABLED.key} = '${testCase.streamingWrites}', + | $indexEnableProp = 'true', + | hoodie.index.type = '$indexType', + | ${HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key} = 'true', + | ${HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key} = '9600', + | $minFileGroupProp = '1', + | $maxFileGroupProp = '1000' + | ) + | partitioned by (part) + | location '$tablePath' + """.stripMargin) + val sqlMetaClient = HoodieTableMetaClient.builder().setBasePath(tablePath).setConf(storageConf).build() + val rowsOf = (from: Int, to: Int, ts: Long, name: String) => + s"select cast(id as string) as id, ${ts}L as ts, '$name' as name, concat('p', cast(id % 3 as string)) as part from range($from, $to)" + val merge = (source: String) => spark.sql( + s""" + |merge into $tableName t + |using ($source) s + |on t.id = s.id + |when matched and s.name != 'delete' then update set * + |when matched and s.name = 'delete' then delete + |when not matched then insert * + """.stripMargin) + val assertSqlTable = (expectedIds: Seq[Int], updatedName: String, updatedRows: Long) => { + assertEquals(expectedIds.map(_.toString).sorted, spark.sql(s"select id from $tableName").collect().map(_.getString(0)).toSeq.sorted) + assertEquals(updatedRows, spark.sql(s"select id from $tableName where name = '$updatedName'").count()) + assertRecordIndexMatchesTable(spark, tablePath, storageConf) + } + + try { + spark.conf.set("hoodie.spark.sql.insert.into.operation", testCase.firstOperation) + spark.sql(s"insert into $tableName ${rowsOf(0, 2000, 1L, "v1")}") + } finally { + spark.conf.unset("hoodie.spark.sql.insert.into.operation") + } + assertEquals(testCase.tableVersion, HoodieTableMetaClient.reload(sqlMetaClient).getTableConfig.getTableVersion, + "Table was written at an unexpected version") + assertRecordIndexNotBuilt(tablePath, storageConf) + // Keyed by the partition path the table writes, which is Hive styled for a SQL created table. + val rowsBeforeBuild = rowsPerPartition(spark, tablePath) + + // The merge builds the index from what the insert left, then updates, deletes and adds enough rows that an index + // sized after the merge would get a different file group count. + merge(s"${rowsOf(0, 300, 4L, "v4")} union all ${rowsOf(300, 350, 4L, "delete")} union all ${rowsOf(2000, 4000, 4L, "v4")}") + assertRecordIndexBuilt(tablePath, storageConf, testCase.partitioned) + assertRecordIndexSizedFrom(tablePath, storageConf, toProperties(Map( + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + minFileGroupProp -> "1", + maxFileGroupProp -> "1000")), testCase.partitioned, rowsBeforeBuild) + val idsAfterBuild = (0 until 300) ++ (350 until 4000) + assertSqlTable(idsAfterBuild, "v4", 2300L) + + // The next merge is tagged through the index alone: it updates rows the first merge inserted and brings back the + // rows it deleted. + merge(s"${rowsOf(2000, 2300, 5L, "v5")} union all ${rowsOf(300, 350, 5L, "v5")}") + assertSqlTable(idsAfterBuild ++ (300 until 350), "v5", 350L) + } + + private def assertTableVersion(expected: HoodieTableVersion): Unit = { + metaClient = HoodieTableMetaClient.reload(metaClient) + assertEquals(expected, metaClient.getTableConfig.getTableVersion, "Table was written at an unexpected version") + } + + private def assertTableKeys(expectedKeys: Seq[String]): Unit = + assertEquals(expectedKeys.sorted, spark.read.format("hudi").load(basePath).select("_row_key").collect() + .map(_.getString(0)).toSeq.sorted) +} + +object TestDeferredRecordIndexInit { + + case class DeferredInitTestCase(tableType: HoodieTableType, + partitioned: Boolean, + tableVersion: HoodieTableVersion, + streamingWrites: Boolean, + firstOperation: String) { + override def toString: String = + s"${tableType.name()}/${if (partitioned) "partitioned" else "global"}/v${tableVersion.versionCode()}" + + s"/streaming=$streamingWrites/$firstOperation" + } + + // The partitioned record index and streaming metadata writes both need a table version above six, and one case + // each here and on the streamer path keeps version six covered. + def testArgsForDeferredInit: java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.SIX, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = false, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)) + ) + } + + def testArgsForSql: java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "bulk_insert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "insert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, "upsert")), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, "bulk_insert")) + ) + } + + def testArgsForInsertOverwrite: java.util.stream.Stream[Arguments] = { + java.util.stream.Stream.of( + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.COPY_ON_WRITE, partitioned = true, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)), + Arguments.arguments(DeferredInitTestCase(HoodieTableType.MERGE_ON_READ, partitioned = false, HoodieTableVersion.TEN, streamingWrites = true, BULK_INSERT_OPERATION_OPT_VAL)) + ) + } +} diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala index 59d97929ffb0..e8b134981d1e 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestStructuredStreaming.scala @@ -21,9 +21,9 @@ import org.apache.hudi.{DataSourceReadOptions, DataSourceUtils, DataSourceWriteO import org.apache.hudi.DataSourceWriteOptions.{STREAMING_CHECKPOINT_IDENTIFIER, UPSERT_OPERATION_OPT_VAL} import org.apache.hudi.HoodieStreamingSink.SINK_CHECKPOINT_KEY import org.apache.hudi.client.SparkRDDWriteClient -import org.apache.hudi.common.config.{HoodieStorageConfig, TypedProperties} +import org.apache.hudi.common.config.{HoodieMetadataConfig, HoodieStorageConfig, TypedProperties} import org.apache.hudi.common.engine.RecordContext -import org.apache.hudi.common.model.{FileSlice, HoodieFailedWritesCleaningPolicy, HoodieTableType, WriteConcurrencyMode} +import org.apache.hudi.common.model.{FileSlice, HoodieFailedWritesCleaningPolicy, HoodieRecord, HoodieTableType, WriteConcurrencyMode} import org.apache.hudi.common.table.{HoodieTableConfig, HoodieTableMetaClient, TableSchemaResolver} import org.apache.hudi.common.table.marker.MarkerType import org.apache.hudi.common.table.read.BufferedRecord @@ -32,9 +32,10 @@ import org.apache.hudi.common.table.view.{FileSystemViewStorageConfig, FileSyste import org.apache.hudi.common.testutils.{HoodieTestDataGenerator, HoodieTestTable, HoodieTestUtils} import org.apache.hudi.common.testutils.HoodieTestDataGenerator.recordsToStrings import org.apache.hudi.common.util.{CollectionUtils, CommitUtils, MarkerUtils, Option => HOption} -import org.apache.hudi.config.{HoodieCleanConfig, HoodieClusteringConfig, HoodieCompactionConfig, HoodieLockConfig, HoodieWriteConfig} +import org.apache.hudi.config.{HoodieCleanConfig, HoodieClusteringConfig, HoodieCompactionConfig, HoodieIndexConfig, HoodieLockConfig, HoodieWriteConfig} import org.apache.hudi.core.transaction.lock.InProcessLockProvider import org.apache.hudi.exception.{HoodieException, TableNotFoundException} +import org.apache.hudi.functional.DeferredRecordIndexTestUtils.{assertRecordIndexBuilt, assertRecordIndexMatchesTable, assertRecordIndexNotBuilt, assertRecordIndexSizedFrom, rowsPerPartition, toProperties} import org.apache.hudi.storage.{HoodieStorage, StoragePath} import org.apache.hudi.testutils.{DataSourceTestUtils, HoodieSparkClientTestBase, HoodieSparkDeleteRecordMerger} @@ -228,6 +229,81 @@ class TestStructuredStreaming extends HoodieSparkClientTestBase { tableType, addCompactionConfigs = false, isAsyncCompaction = false) } + /** + * Deferred record index initialization through the streaming sink: the first micro-batch leaves the index unbuilt, + * the second builds it, and the index agrees with the table after the second and the third. + */ + @ParameterizedTest + @ValueSource(booleans = Array(false, true)) + def testRecordIndexDeferredUntilSecondMicroBatch(partitioned: Boolean): Unit = { + val (sourcePath, destPath) = initStreamingSourceAndDestPath("source", "dest") + val indexEnableProp = if (partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key + } + val minFileGroupProp = if (partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key + } + val maxFileGroupProp = if (partitioned) { + HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } else { + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key + } + val hudiOptions = getOptsWithTableType(HoodieTableType.COPY_ON_WRITE) ++ Map( + HoodieMetadataConfig.ENABLE.key -> "true", + indexEnableProp -> "true", + HoodieIndexConfig.INDEX_TYPE.key -> (if (partitioned) "RECORD_LEVEL_INDEX" else "GLOBAL_RECORD_LEVEL_INDEX"), + HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key -> "true", + HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key -> "9600", + minFileGroupProp -> "1", + maxFileGroupProp -> "1000") + val toDf = (records: java.util.List[HoodieRecord[_]]) => + spark.read.json(spark.sparkContext.parallelize(recordsToStrings(records).asScala.toList, 2)) + val keysOf = (records: java.util.List[HoodieRecord[_]]) => records.asScala.map(_.getRecordKey).toSeq + val assertTableKeys = (expectedKeys: Seq[String]) => assertEquals(expectedKeys.sorted, + spark.read.format("org.apache.hudi").load(destPath).select("_row_key").collect().map(_.getString(0)).toSeq.sorted) + + val firstBatch = dataGen.generateInserts("001", 2000).asInstanceOf[java.util.List[HoodieRecord[_]]] + val streamingQuery = initWritingStreamingQuery(toDf(firstBatch).schema, sourcePath, destPath, hudiOptions) + try { + toDf(firstBatch).coalesce(1).write.mode(SaveMode.Append).json(sourcePath) + val numCommits = waitTillAtleastNCommits(storage, destPath, 1, 120, 1) + assertRecordIndexNotBuilt(destPath, storageConf) + val rowsBeforeBuild = rowsPerPartition(spark, destPath) + + // The second micro-batch builds the index from what the first left, then writes updates, deletes and enough new + // rows that an index sized after it would get a different file group count. + val deletedRecords = firstBatch.subList(300, 350) + val newRecords = dataGen.generateInserts("002", 2000).asInstanceOf[java.util.List[HoodieRecord[_]]] + toDf(dataGen.generateUpdates("002", firstBatch.subList(0, 300)).asInstanceOf[java.util.List[HoodieRecord[_]]]) + .union(toDf(dataGen.generateUpdates("002", deletedRecords).asInstanceOf[java.util.List[HoodieRecord[_]]]) + .withColumn("_hoodie_is_deleted", lit(true))) + .union(toDf(newRecords)) + .coalesce(1).write.mode(SaveMode.Append).json(sourcePath) + waitTillAtleastNCommits(storage, destPath, numCommits + 1, 120, 1) + assertRecordIndexBuilt(destPath, storageConf, partitioned) + assertRecordIndexSizedFrom(destPath, storageConf, toProperties(hudiOptions), partitioned, rowsBeforeBuild) + val deletedKeys = keysOf(deletedRecords).toSet + val keysAfterBuild = (keysOf(firstBatch) ++ keysOf(newRecords)).filterNot(deletedKeys) + assertTableKeys(keysAfterBuild) + assertRecordIndexMatchesTable(spark, destPath, storageConf) + + // The third micro-batch is tagged through the index alone: it updates rows the second inserted and brings back + // the keys the second deleted. + toDf(dataGen.generateUpdates("003", newRecords.subList(0, 300)).asInstanceOf[java.util.List[HoodieRecord[_]]]) + .union(toDf(dataGen.generateUpdates("003", deletedRecords).asInstanceOf[java.util.List[HoodieRecord[_]]])) + .coalesce(1).write.mode(SaveMode.Append).json(sourcePath) + waitTillAtleastNCommits(storage, destPath, numCommits + 2, 120, 1) + assertTableKeys(keysAfterBuild ++ deletedKeys) + assertRecordIndexMatchesTable(spark, destPath, storageConf) + } finally { + streamingQuery.stop() + } + } + @throws[InterruptedException] private def waitTillAtleastNCommits(storage: HoodieStorage, tablePath: String, numCommits: Int, timeoutSecs: Int, sleepSecsAfterEachRun: Int) = { diff --git a/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestDeferredRecordIndexOnStreamerPath.java b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestDeferredRecordIndexOnStreamerPath.java new file mode 100644 index 000000000000..eccea9bd96e6 --- /dev/null +++ b/hudi-utilities/src/test/java/org/apache/hudi/utilities/deltastreamer/TestDeferredRecordIndexOnStreamerPath.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hudi.utilities.deltastreamer; + +import org.apache.hudi.common.config.HoodieMetadataConfig; +import org.apache.hudi.common.model.HoodieWriteStat; +import org.apache.hudi.common.model.WriteOperationType; +import org.apache.hudi.common.table.HoodieTableMetaClient; +import org.apache.hudi.common.table.HoodieTableVersion; +import org.apache.hudi.common.table.timeline.HoodieTimeline; +import org.apache.hudi.common.testutils.HoodieTestUtils; +import org.apache.hudi.config.HoodieIndexConfig; +import org.apache.hudi.config.HoodieWriteConfig; +import org.apache.hudi.storage.StorageConfiguration; + +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.util.Map; +import java.util.Properties; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.apache.hudi.functional.DeferredRecordIndexTestUtils.assertRecordIndexBuilt; +import static org.apache.hudi.functional.DeferredRecordIndexTestUtils.assertRecordIndexMatchesTable; +import static org.apache.hudi.functional.DeferredRecordIndexTestUtils.assertRecordIndexNotBuilt; +import static org.apache.hudi.functional.DeferredRecordIndexTestUtils.assertRecordIndexSizedFrom; +import static org.apache.hudi.functional.DeferredRecordIndexTestUtils.rowsPerPartition; +import static org.junit.jupiter.api.Assertions.assertEquals; + +/** + * Deferred record index initialization on the HoodieStreamer path: the first sync leaves the index unbuilt, the second + * builds it, and the index agrees with the table after the second sync and the third. + */ +@Tag("functional") +public class TestDeferredRecordIndexOnStreamerPath extends HoodieDeltaStreamerTestBase { + + private static void enableDeferredRecordIndex(HoodieDeltaStreamer.Config cfg, boolean partitioned, int tableVersion) { + cfg.configs.add(HoodieMetadataConfig.ENABLE.key() + "=true"); + cfg.configs.add(HoodieWriteConfig.WRITE_TABLE_VERSION.key() + "=" + tableVersion); + cfg.configs.add(HoodieWriteConfig.AUTO_UPGRADE_VERSION.key() + "=false"); + cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() + "=" + !partitioned); + cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() + "=" + partitioned); + cfg.configs.add(HoodieIndexConfig.INDEX_TYPE.key() + "=" + + (partitioned ? "RECORD_LEVEL_INDEX" : "GLOBAL_RECORD_LEVEL_INDEX")); + cfg.configs.add(HoodieMetadataConfig.DEFER_RLI_INIT_FOR_FRESH_TABLE.key() + "=true"); + // A small file group size and a floor of one keep the sizing result away from the floor. + cfg.configs.add(HoodieMetadataConfig.RECORD_INDEX_MAX_FILE_GROUP_SIZE_BYTES_PROP.key() + "=9600"); + cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key() + "=1"); + cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MIN_FILE_GROUP_COUNT_PROP.key() + "=1"); + cfg.configs.add(HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key() + "=1000"); + cfg.configs.add(HoodieMetadataConfig.RECORD_LEVEL_INDEX_MAX_FILE_GROUP_COUNT_PROP.key() + "=1000"); + } + + static Stream<Arguments> testArgs() { + return Stream.of( + Arguments.arguments(false, HoodieTableVersion.SIX.versionCode()), + Arguments.arguments(false, HoodieTableVersion.TEN.versionCode()), + Arguments.arguments(true, HoodieTableVersion.TEN.versionCode())); + } + + @ParameterizedTest + @MethodSource("testArgs") + public void testRecordIndexDeferredUntilSecondSync(boolean partitioned, int tableVersion) throws Exception { + String tableBasePath = basePath + "/test_rli_deferred_streamer_" + + (partitioned ? "partitioned" : "global") + "_v" + tableVersion; + StorageConfiguration<?> storageConf = HoodieTestUtils.getDefaultStorageConf(); + + HoodieDeltaStreamer.Config insertCfg = + TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, WriteOperationType.INSERT); + enableDeferredRecordIndex(insertCfg, partitioned, tableVersion); + new HoodieDeltaStreamer(insertCfg, jsc).sync(); + assertEquals(tableVersion, HoodieTableMetaClient.builder().setBasePath(tableBasePath) + .setConf(storageConf).build().getTableConfig().getTableVersion().versionCode()); + assertRecordIndexNotBuilt(tableBasePath, storageConf); + assertRecordCount(1000, tableBasePath, sqlContext); + sqlContext.clearCache(); + Map<String, Long> rowsBeforeBuild = rowsPerPartition(sqlContext.sparkSession(), tableBasePath); + + // Every later sync carries 1000 records from the test source: 500 inserts, 450 updates and 50 deletes, so the + // 1000 rows the first sync wrote become 1450 after the second and 1900 after the third. The second sync builds + // the index from what the first left, then writes its own records; the third is tagged through the index alone. + HoodieDeltaStreamer.Config upsertCfg = + TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT); + enableDeferredRecordIndex(upsertCfg, partitioned, tableVersion); + new HoodieDeltaStreamer(upsertCfg, jsc).sync(); + assertRecordIndexBuilt(tableBasePath, storageConf, partitioned); + assertRecordIndexSizedFrom(tableBasePath, storageConf, writeProps(upsertCfg), partitioned, rowsBeforeBuild); + assertSyncedTable(tableBasePath, storageConf, 1450); + + new HoodieDeltaStreamer(upsertCfg, jsc).sync(); + assertSyncedTable(tableBasePath, storageConf, 1900); + } + + /** + * The same deferral in continuous mode, where one write client outlives the commit that defers the build, the commit + * that runs it and the commits tagged through it. + */ + @ParameterizedTest + @MethodSource("testArgs") + public void testRecordIndexDeferredInContinuousMode(boolean partitioned, int tableVersion) throws Exception { + String tableBasePath = basePath + "/test_rli_deferred_continuous_streamer_" + + (partitioned ? "partitioned" : "global") + "_v" + tableVersion; + StorageConfiguration<?> storageConf = HoodieTestUtils.getDefaultStorageConf(); + HoodieDeltaStreamer.Config cfg = TestHoodieDeltaStreamer.TestHelpers.makeConfig(tableBasePath, WriteOperationType.UPSERT); + cfg.continuousMode = true; + enableDeferredRecordIndex(cfg, partitioned, tableVersion); + TestHoodieDeltaStreamer.deltaStreamerTestRunner(new HoodieDeltaStreamer(cfg, jsc), cfg, ignored -> { + TestHoodieDeltaStreamer.TestHelpers.assertAtLeastNCommits(3, tableBasePath); + return true; + }); + + // The index must have been sized from the rows the first commit wrote, which a build in that commit could not do. + HoodieTimeline commits = HoodieTableMetaClient.builder().setBasePath(tableBasePath).setConf(storageConf).build() + .getCommitsTimeline().filterCompletedInstants(); + Map<String, Long> rowsOfFirstCommit = commits.readCommitMetadata(commits.firstInstant().get()) + .getPartitionToWriteStats().entrySet().stream() + .collect(Collectors.toMap(Map.Entry::getKey, entry -> entry.getValue().stream().mapToLong(HoodieWriteStat::getNumWrites).sum())); + assertRecordIndexBuilt(tableBasePath, storageConf, partitioned); + assertRecordIndexSizedFrom(tableBasePath, storageConf, writeProps(cfg), partitioned, rowsOfFirstCommit); + sqlContext.clearCache(); + long rows = sqlContext.read().format("hudi").load(tableBasePath).count(); + assertDistinctRecordCount(rows, tableBasePath, sqlContext); + assertRecordIndexMatchesTable(sqlContext.sparkSession(), tableBasePath, storageConf); + } + + private static Properties writeProps(HoodieDeltaStreamer.Config cfg) { + Properties props = new Properties(); + cfg.configs.forEach(config -> { + String[] keyAndValue = config.split("=", 2); + props.setProperty(keyAndValue[0], keyAndValue[1]); + }); + return props; + } + + private void assertSyncedTable(String tableBasePath, StorageConfiguration<?> storageConf, long expectedRows) { + sqlContext.clearCache(); + assertRecordCount(expectedRows, tableBasePath, sqlContext); + assertDistinctRecordCount(expectedRows, tableBasePath, sqlContext); + assertRecordIndexMatchesTable(sqlContext.sparkSession(), tableBasePath, storageConf); + } +}
