This is an automated email from the ASF dual-hosted git repository. github-merge-queue[bot] pushed a commit to branch gh-readonly-queue/master/pr-20545-a65a573a49ff5072c3bb4729a806de0fa5b41f50 in repository https://gitbox.apache.org/repos/asf/hudi.git
commit c2c6bfb83e158c53e7a3c3052b28cf6a179f4107 Author: voonhous <[email protected]> AuthorDate: Thu Nov 12 01:26:01 2026 +0000 fix(metadata): flatten partitioned record index init union (#20545) * test(metadata): reproduce deep UnionRDD chain in partitioned RLI init Partitioned record level index initialization unions one HoodieData per data partition in a loop. JavaRDD.union does not flatten nested unions, so the lineage handed to the bulk commit is a UnionRDD chain as deep as the number of data partitions, and Spark serializes that lineage recursively. Add a functional test that initializes partitioned RLI over 50 data partitions and measures the longest UnionRDD chain from the stage lineage reported to a SparkListener. The test fails on master with a depth of 50 and is expected to pass once the depth stays flat. * fix(metadata): flatten partitioned record index init union Partitioned record level index initialization unioned the index records of each data partition one at a time. On Spark every JavaRDD.union adds a UnionRDD level, so the lineage handed to the bulk commit was a chain as deep as the number of data partitions, and Spark serializes task lineage recursively: around 500 partitions overflow a 1 MB stack and fail the initialization with StackOverflowError. Collect the per-partition records in the loop and union them once through a new HoodieEngineContext.union(List). The Spark engine context implements it with a single JavaSparkContext.union, which yields one UnionRDD whose parents are the inputs. The default implementation flattens HoodieListData inputs in one pass through the new HoodieListData.unionAll, since chained Stream.concat has the same deep-nesting problem, and keeps the eager semantic of emptyHoodieData() that the old fold had. Engines with other HoodieData types fall back to folding. Fixes #20543 * fix(metadata): drop HoodieListData.unionAll The in-memory path never had the deep-nesting problem: every in-memory engine context's emptyHoodieData() is eager, so each step of the fold materializes a list and Stream.concat does not nest. Only Spark needs the flat union, so the default HoodieEngineContext.union goes back to folding and its javadoc says why that is safe. --- .../metadata/HoodieBackedTableMetadataWriter.java | 7 +- .../client/common/HoodieSparkEngineContext.java | 17 +++ .../common/TestHoodieSparkEngineContext.java | 30 +++++ .../hudi/common/engine/HoodieEngineContext.java | 21 ++++ .../engine/TestHoodieLocalEngineContext.java | 20 ++++ .../hudi/functional/TestRecordLevelIndex.scala | 125 ++++++++++++++++++++- 6 files changed, 217 insertions(+), 3 deletions(-) diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java index 6295a99f22dc..5cc35cc85d2e 100644 --- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java +++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java @@ -464,12 +464,15 @@ public abstract class HoodieBackedTableMetadataWriter<I, O> implements HoodieTab HoodieTimer partitionInitTimer = HoodieTimer.start(); clearExistingMetadataPartition(relativePartitionPath); - HoodieData<HoodieRecord> records = engineContext.emptyHoodieData(); + List<HoodieData<HoodieRecord>> recordsPerDataPartition = new ArrayList<>(); for (DataPartitionAndRecords dataPartitionAndRecords: initializationPlan.dataPartitionAndRecords()) { initializeFileGroups(dataMetaClient, partitionType, instantTimeForPartition, dataPartitionAndRecords.numFileGroups(), relativePartitionPath, dataPartitionAndRecords.dataPartition()); - records = records.union(dataPartitionAndRecords.indexRecords()); + recordsPerDataPartition.add(dataPartitionAndRecords.indexRecords()); } + // Union once: on Spark, folding with HoodieData#union nests one UnionRDD per data partition, + // and the resulting lineage overflows the stack on tables with many partitions (partitioned RLI). + HoodieData<HoodieRecord> records = engineContext.union(recordsPerDataPartition); bulkCommit(instantTimeForPartition, relativePartitionPath, records, initializationPlan.indexParser()); diff --git a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java index 77eff393353c..1dd951535ee8 100644 --- a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java +++ b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java @@ -135,6 +135,23 @@ public class HoodieSparkEngineContext extends HoodieEngineContext { return HoodieJavaRDD.of(javaSparkContext.parallelize(data, parallelism)); } + /** + * Unions all the inputs with a single {@link JavaSparkContext#union}, which yields one + * {@code UnionRDD} whose parents are the inputs, instead of a chain of nested ones. + */ + @Override + public <T> HoodieData<T> union(List<HoodieData<T>> dataList) { + if (dataList.isEmpty()) { + return emptyHoodieData(); + } + if (dataList.size() == 1) { + return dataList.get(0); + } + @SuppressWarnings("unchecked") + JavaRDD<T>[] rdds = dataList.stream().map(data -> HoodieJavaRDD.getJavaRDD(data)).toArray(JavaRDD[]::new); + return HoodieJavaRDD.of(javaSparkContext.union(rdds)); + } + @Override public <I, O> List<O> map(List<I> data, SerializableFunction<I, O> func, int parallelism) { final Map<String, Registry> registries = getDistributedRegistries(); diff --git a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java index b51a0d44cafb..08960e52e722 100644 --- a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java +++ b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java @@ -25,13 +25,18 @@ import org.apache.hudi.common.function.SerializableBiFunction; import org.apache.hudi.common.function.SerializablePairFlatMapFunction; import org.apache.hudi.common.util.collection.ImmutablePair; import org.apache.hudi.common.util.collection.Pair; +import org.apache.hudi.data.HoodieJavaRDD; import org.apache.hudi.testutils.SparkClientFunctionalTestHarness; +import org.apache.hudi.util.JavaScalaConverters; +import org.apache.spark.rdd.RDD; +import org.apache.spark.rdd.UnionRDD; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import java.util.List; import java.util.stream.Collectors; @@ -39,6 +44,8 @@ import java.util.stream.IntStream; import java.util.stream.Stream; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertInstanceOf; +import static org.junit.jupiter.api.Assertions.assertSame; import static org.junit.jupiter.api.Assertions.assertTrue; public class TestHoodieSparkEngineContext extends SparkClientFunctionalTestHarness { @@ -106,6 +113,29 @@ public class TestHoodieSparkEngineContext extends SparkClientFunctionalTestHarne }); } + @Test + void testUnion() { + assertTrue(context.union(Collections.<HoodieData<Integer>>emptyList()).isEmpty()); + + HoodieData<Integer> single = context.parallelize(Arrays.asList(1, 2), 2); + assertSame(single, context.union(Collections.singletonList(single))); + + int numInputs = 50; + List<HoodieData<Integer>> inputs = IntStream.range(0, numInputs) + .mapToObj(i -> context.parallelize(Collections.singletonList(i), 1)) + .collect(Collectors.toList()); + HoodieData<Integer> union = context.union(inputs); + assertEquals(IntStream.range(0, numInputs).boxed().collect(Collectors.toList()), union.collectAsList()); + + // A single UnionRDD whose parents are the inputs, not a chain of nested UnionRDDs + RDD<Integer> unionRdd = HoodieJavaRDD.getJavaRDD(union).rdd(); + assertInstanceOf(UnionRDD.class, unionRdd); + List<Integer> expectedParentIds = inputs.stream().map(HoodieData::getId).collect(Collectors.toList()); + List<Integer> parentIds = JavaScalaConverters.convertScalaListToJavaList(((UnionRDD<Integer>) unionRdd).rdds()) + .stream().map(RDD::id).collect(Collectors.toList()); + assertEquals(expectedParentIds, parentIds); + } + static class SerializablePairFlatMapTestFunc implements SerializablePairFlatMapFunction<Iterator<java.lang.Integer>, java.lang.Integer, java.lang.Integer> { @Override public Stream<Pair<Integer, Integer>> call(Iterator<Integer> t) throws Exception { diff --git a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java index 43bc9cc78b35..3f2ad0298dc2 100644 --- a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java +++ b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java @@ -82,6 +82,27 @@ public abstract class HoodieEngineContext { public abstract <T> HoodieData<T> parallelize(List<T> data, int parallelism); + /** + * Unions all the given {@link HoodieData} into a single {@link HoodieData}. + * + * <p>The default implementation folds the inputs with {@link HoodieData#union(HoodieData)}, which + * does not nest when {@link #emptyHoodieData()} is eager, since every step materializes its result. + * Engines whose union nests per call should override this with a single flat union: on Spark, a + * chain of a few hundred {@code UnionRDD} levels is enough to overflow the stack during task + * serialization. + * + * @param dataList the {@link HoodieData} to union + * @param <T> type of object + * @return the union of all the inputs, or {@link #emptyHoodieData()} if there is none + */ + public <T> HoodieData<T> union(List<HoodieData<T>> dataList) { + HoodieData<T> result = emptyHoodieData(); + for (HoodieData<T> data : dataList) { + result = result.union(data); + } + return result; + } + public abstract <I, O> List<O> map(List<I> data, SerializableFunction<I, O> func, int parallelism); public abstract <I, K, V> List<V> mapToPairAndReduceByKey(List<I> data, SerializablePairFunction<I, K, V> mapToPairFunc, diff --git a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java index c4fc1e285f7c..1dcdbfc45171 100644 --- a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java +++ b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java @@ -19,6 +19,7 @@ package org.apache.hudi.common.engine; import org.apache.hudi.common.data.HoodieData; +import org.apache.hudi.common.data.HoodieListData; import org.apache.hudi.common.data.HoodieListPairData; import org.apache.hudi.common.data.HoodiePairData; import org.apache.hudi.common.function.SerializableFunction; @@ -31,6 +32,7 @@ import org.junit.jupiter.api.Test; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.Iterator; import java.util.List; @@ -170,4 +172,22 @@ public class TestHoodieLocalEngineContext { assertTrue(resultList.contains(42)); assertTrue(resultList.contains(17)); } + + @Test + void testUnion() { + assertTrue(context.union(Collections.<HoodieData<Integer>>emptyList()).isEmpty()); + + HoodieData<Integer> single = context.union(Collections.singletonList(HoodieListData.lazy(Arrays.asList(1, 2)))); + assertEquals(Arrays.asList(1, 2), single.collectAsList()); + + HoodieData<Integer> union = context.union(Arrays.asList( + context.parallelize(Arrays.asList(1, 2)), + context.emptyHoodieData(), + HoodieListData.lazy(Arrays.asList(3, 4)), + context.parallelize(Collections.singletonList(5)))); + assertEquals(Arrays.asList(1, 2, 3, 4, 5), union.collectAsList()); + // The result bears the eager semantic of emptyHoodieData(), even over lazy inputs + assertEquals(5, union.count()); + assertEquals(Arrays.asList(1, 2, 3, 4, 5), union.collectAsList()); + } } diff --git a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala index 6c4875fc3eef..1669aceeab1d 100644 --- a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala +++ b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala @@ -34,12 +34,14 @@ import org.apache.hudi.common.util.{Option => HOption} import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, HoodieWriteConfig} import org.apache.hudi.core.index.record.HoodieRecordIndex import org.apache.hudi.exception.{HoodieException, HoodieMetadataException} -import org.apache.hudi.functional.TestRecordLevelIndex.TestPartitionedRecordLevelIndexTestCase +import org.apache.hudi.functional.TestRecordLevelIndex.{TestPartitionedRecordLevelIndexTestCase, UnionDepthListener} import org.apache.hudi.index.HoodieIndex.IndexType.RECORD_LEVEL_INDEX import org.apache.hudi.metadata.{HoodieBackedTableMetadata, HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType} import org.apache.hudi.storage.{StoragePath, StoragePathInfo} import org.apache.hudi.table.action.compact.strategy.UnBoundedCompactionStrategy +import org.apache.spark.SparkContext +import org.apache.spark.scheduler.{SparkListener, SparkListenerJobEnd, SparkListenerJobStart, SparkListenerStageSubmitted} import org.apache.spark.sql.{Row, SaveMode} import org.apache.spark.sql.functions.lit import org.junit.jupiter.api.{Tag, Test} @@ -49,10 +51,12 @@ import org.junit.jupiter.params.provider.{Arguments, EnumSource, MethodSource, V import java.io.{PrintWriter, StringWriter} import java.util +import java.util.concurrent.{CountDownLatch, TimeUnit} import java.util.stream.Collectors import scala.collection.JavaConverters import scala.collection.JavaConverters._ +import scala.collection.mutable @Tag("functional") class TestRecordLevelIndex extends RecordLevelIndexTestBase with SparkDatasetMixin { @@ -680,6 +684,67 @@ class TestRecordLevelIndex extends RecordLevelIndexTestBase with SparkDatasetMix assertTrue(HoodieRecordIndex.isPartitioned(metaClient.getIndexMetadata.get().getIndexDefinitions.get(HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX))) } + /** + * Partitioned RLI initialization builds one HoodieData of index records per data partition and + * unions them one at a time. JavaRDD.union does not flatten nested unions, so the lineage handed to + * the bulk commit becomes a chain of UnionRDDs as deep as the number of data partitions. Spark + * serializes task lineage recursively, so a table with enough partitions fails the initialization + * with a StackOverflowError. The union depth must stay flat regardless of the partition count. + */ + @Test + def testPartitionedRecordLevelIndexInitializationUnionDepth(): Unit = { + val numDataPartitions = 50 + initMetaClient(HoodieTableType.COPY_ON_WRITE) + val options = Map(HoodieWriteConfig.TBL_NAME.key -> "hoodie_test", + DataSourceWriteOptions.TABLE_TYPE.key -> HoodieTableType.COPY_ON_WRITE.name(), + RECORDKEY_FIELD.key -> "_row_key", + PARTITIONPATH_FIELD.key -> "data_partition_path", + HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp", + HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "false", + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "false", + HoodieMetadataConfig.SECONDARY_INDEX_ENABLE_PROP.key() -> "false", + HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key() -> "false", + HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key() -> "false") + // One record per data partition: p0 .. p49 + spark.range(numDataPartitions) + .selectExpr("cast(id as string) as _row_key", "id as timestamp", + "concat('p', cast(id as string)) as data_partition_path") + .write.format("hudi") + .options(options) + .mode(SaveMode.Overwrite) + .save(basePath) + assertEquals(numDataPartitions, spark.read.format("hudi").load(basePath).count()) + + metaClient.reloadActiveTimeline() + val tableSchema = new TableSchemaResolver(metaClient).getTableSchemaFromLatestCommit(false).get().toString + val writeConfig = getWriteConfig(options ++ Map( + HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "true", + HoodieWriteConfig.AVRO_SCHEMA_STRING.key() -> tableSchema)) + + // Initialize the partitioned RLI and record the lineage of every stage it runs. + val listener = new UnionDepthListener + spark.sparkContext.addSparkListener(listener) + val metadata = try { + val writer = metadataWriter(writeConfig) + listener.awaitDelivered(spark.sparkContext) + writer.getTableMetadata + } finally { + spark.sparkContext.removeSparkListener(listener) + } + + metaClient = HoodieTableMetaClient.reload(metaClient) + assertTrue(HoodieRecordIndex.isPartitioned(metaClient.getIndexMetadata.get().getIndexDefinitions + .get(HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX))) + val locations = (0 until numDataPartitions).flatMap(i => + readRecordIndex(metadata, util.Collections.singletonList(i.toString), HOption.of(s"p$i"))) + assertEquals(numDataPartitions, locations.size) + + val unionDepth = listener.maxUnionDepth + assertTrue(unionDepth <= 1, + s"Partitioned RLI initialization over $numDataPartitions data partitions built a UnionRDD " + + s"chain of depth $unionDepth; the depth must not grow with the number of data partitions") + } + def readRecordIndex(metadata: HoodieBackedTableMetadata, recordKeys: java.util.List[String], dataTablePartition: HOption[String]): Map[String, HoodieRecordGlobalLocation] = { metadata.readRecordIndexLocationsWithKeys(HoodieListData.eager(recordKeys), dataTablePartition) .collectAsList().asScala.map(p => p.getKey -> p.getValue).toMap @@ -931,6 +996,64 @@ object TestRecordLevelIndex { case class TestPartitionedRecordLevelIndexTestCase(tableType: HoodieTableType, streamingWriteEnabled: Boolean) + /** + * Tracks the longest chain of directly nested UnionRDDs across the narrow lineage of every + * submitted stage. + */ + class UnionDepthListener extends SparkListener { + private val sentinelGroup = "union-depth-listener-sentinel" + private val rddNames = mutable.Map[Int, String]() + private val rddParents = mutable.Map[Int, Seq[Int]]() + @volatile private var sentinelJobId = -1 + private val sentinelDone = new CountDownLatch(1) + + override def onStageSubmitted(stageSubmitted: SparkListenerStageSubmitted): Unit = synchronized { + stageSubmitted.stageInfo.rddInfos.foreach { info => + rddNames(info.id) = info.name + rddParents(info.id) = info.parentIds + } + } + + override def onJobStart(jobStart: SparkListenerJobStart): Unit = { + if (Option(jobStart.properties).exists(p => sentinelGroup == p.getProperty("spark.jobGroup.id"))) { + sentinelJobId = jobStart.jobId + } + } + + override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = { + if (jobEnd.jobId == sentinelJobId) { + sentinelDone.countDown() + } + } + + /** + * Runs a marker job and waits for its end event. The listener bus delivers events in order, + * so every stage submitted before the marker has been seen once this returns. + */ + def awaitDelivered(sc: SparkContext): Unit = { + sc.setJobGroup(sentinelGroup, sentinelGroup) + try { + sc.parallelize(Seq(1), 1).count() + } finally { + sc.clearJobGroup() + } + assertTrue(sentinelDone.await(60, TimeUnit.SECONDS), "Timed out waiting for Spark listener events") + } + + def maxUnionDepth: Int = synchronized { + // RDD ids grow monotonically, so parents are visited before children. + val depth = mutable.Map[Int, Int]() + rddNames.keys.toSeq.sorted.foreach { id => + depth(id) = if (rddNames(id) == "UnionRDD") { + 1 + (rddParents(id).map(p => depth.getOrElse(p, 0)) :+ 0).max + } else { + 0 + } + } + (depth.values.toSeq :+ 0).max + } + } + def testArgsForPartitionedRecordLevelIndex: java.util.stream.Stream[Arguments] = { java.util.stream.Stream.of( Arguments.arguments(TestPartitionedRecordLevelIndexTestCase(HoodieTableType.COPY_ON_WRITE, streamingWriteEnabled = true)),
