This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new c2c6bfb83e15 fix(metadata): flatten partitioned record index init
union (#20545)
c2c6bfb83e15 is described below
commit c2c6bfb83e158c53e7a3c3052b28cf6a179f4107
Author: voonhous <[email protected]>
AuthorDate: Thu Nov 12 01:26:01 2026 +0000
fix(metadata): flatten partitioned record index init union (#20545)
* test(metadata): reproduce deep UnionRDD chain in partitioned RLI init
Partitioned record level index initialization unions one HoodieData
per data partition in a loop. JavaRDD.union does not flatten nested
unions, so the lineage handed to the bulk commit is a UnionRDD chain
as deep as the number of data partitions, and Spark serializes that
lineage recursively.
Add a functional test that initializes partitioned RLI over 50 data
partitions and measures the longest UnionRDD chain from the stage
lineage reported to a SparkListener. The test fails on master with a
depth of 50 and is expected to pass once the depth stays flat.
* fix(metadata): flatten partitioned record index init union
Partitioned record level index initialization unioned the index
records of each data partition one at a time. On Spark every
JavaRDD.union adds a UnionRDD level, so the lineage handed to the bulk
commit was a chain as deep as the number of data partitions, and Spark
serializes task lineage recursively: around 500 partitions overflow a
1 MB stack and fail the initialization with StackOverflowError.
Collect the per-partition records in the loop and union them once
through a new HoodieEngineContext.union(List). The Spark engine
context implements it with a single JavaSparkContext.union, which
yields one UnionRDD whose parents are the inputs. The default
implementation flattens HoodieListData inputs in one pass through the
new HoodieListData.unionAll, since chained Stream.concat has the same
deep-nesting problem, and keeps the eager semantic of
emptyHoodieData() that the old fold had. Engines with other HoodieData
types fall back to folding.
Fixes #20543
* fix(metadata): drop HoodieListData.unionAll
The in-memory path never had the deep-nesting problem: every in-memory
engine context's emptyHoodieData() is eager, so each step of the fold
materializes a list and Stream.concat does not nest. Only Spark needs
the flat union, so the default HoodieEngineContext.union goes back to
folding and its javadoc says why that is safe.
---
.../metadata/HoodieBackedTableMetadataWriter.java | 7 +-
.../client/common/HoodieSparkEngineContext.java | 17 +++
.../common/TestHoodieSparkEngineContext.java | 30 +++++
.../hudi/common/engine/HoodieEngineContext.java | 21 ++++
.../engine/TestHoodieLocalEngineContext.java | 20 ++++
.../hudi/functional/TestRecordLevelIndex.scala | 125 ++++++++++++++++++++-
6 files changed, 217 insertions(+), 3 deletions(-)
diff --git
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 6295a99f22dc..5cc35cc85d2e 100644
---
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -464,12 +464,15 @@ public abstract class HoodieBackedTableMetadataWriter<I,
O> implements HoodieTab
HoodieTimer partitionInitTimer = HoodieTimer.start();
clearExistingMetadataPartition(relativePartitionPath);
- HoodieData<HoodieRecord> records = engineContext.emptyHoodieData();
+ List<HoodieData<HoodieRecord>> recordsPerDataPartition = new
ArrayList<>();
for (DataPartitionAndRecords dataPartitionAndRecords:
initializationPlan.dataPartitionAndRecords()) {
initializeFileGroups(dataMetaClient, partitionType,
instantTimeForPartition,
dataPartitionAndRecords.numFileGroups(), relativePartitionPath,
dataPartitionAndRecords.dataPartition());
- records = records.union(dataPartitionAndRecords.indexRecords());
+ recordsPerDataPartition.add(dataPartitionAndRecords.indexRecords());
}
+ // Union once: on Spark, folding with HoodieData#union nests one
UnionRDD per data partition,
+ // and the resulting lineage overflows the stack on tables with many
partitions (partitioned RLI).
+ HoodieData<HoodieRecord> records =
engineContext.union(recordsPerDataPartition);
bulkCommit(instantTimeForPartition, relativePartitionPath, records,
initializationPlan.indexParser());
diff --git
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java
index 77eff393353c..1dd951535ee8 100644
---
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java
+++
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java
@@ -135,6 +135,23 @@ public class HoodieSparkEngineContext extends
HoodieEngineContext {
return HoodieJavaRDD.of(javaSparkContext.parallelize(data, parallelism));
}
+ /**
+ * Unions all the inputs with a single {@link JavaSparkContext#union}, which
yields one
+ * {@code UnionRDD} whose parents are the inputs, instead of a chain of
nested ones.
+ */
+ @Override
+ public <T> HoodieData<T> union(List<HoodieData<T>> dataList) {
+ if (dataList.isEmpty()) {
+ return emptyHoodieData();
+ }
+ if (dataList.size() == 1) {
+ return dataList.get(0);
+ }
+ @SuppressWarnings("unchecked")
+ JavaRDD<T>[] rdds = dataList.stream().map(data ->
HoodieJavaRDD.getJavaRDD(data)).toArray(JavaRDD[]::new);
+ return HoodieJavaRDD.of(javaSparkContext.union(rdds));
+ }
+
@Override
public <I, O> List<O> map(List<I> data, SerializableFunction<I, O> func, int
parallelism) {
final Map<String, Registry> registries = getDistributedRegistries();
diff --git
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java
index b51a0d44cafb..08960e52e722 100644
---
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java
+++
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java
@@ -25,13 +25,18 @@ import
org.apache.hudi.common.function.SerializableBiFunction;
import org.apache.hudi.common.function.SerializablePairFlatMapFunction;
import org.apache.hudi.common.util.collection.ImmutablePair;
import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.data.HoodieJavaRDD;
import org.apache.hudi.testutils.SparkClientFunctionalTestHarness;
+import org.apache.hudi.util.JavaScalaConverters;
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.rdd.UnionRDD;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
import java.util.stream.Collectors;
@@ -39,6 +44,8 @@ import java.util.stream.IntStream;
import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertTrue;
public class TestHoodieSparkEngineContext extends
SparkClientFunctionalTestHarness {
@@ -106,6 +113,29 @@ public class TestHoodieSparkEngineContext extends
SparkClientFunctionalTestHarne
});
}
+ @Test
+ void testUnion() {
+
assertTrue(context.union(Collections.<HoodieData<Integer>>emptyList()).isEmpty());
+
+ HoodieData<Integer> single = context.parallelize(Arrays.asList(1, 2), 2);
+ assertSame(single, context.union(Collections.singletonList(single)));
+
+ int numInputs = 50;
+ List<HoodieData<Integer>> inputs = IntStream.range(0, numInputs)
+ .mapToObj(i -> context.parallelize(Collections.singletonList(i), 1))
+ .collect(Collectors.toList());
+ HoodieData<Integer> union = context.union(inputs);
+ assertEquals(IntStream.range(0,
numInputs).boxed().collect(Collectors.toList()), union.collectAsList());
+
+ // A single UnionRDD whose parents are the inputs, not a chain of nested
UnionRDDs
+ RDD<Integer> unionRdd = HoodieJavaRDD.getJavaRDD(union).rdd();
+ assertInstanceOf(UnionRDD.class, unionRdd);
+ List<Integer> expectedParentIds =
inputs.stream().map(HoodieData::getId).collect(Collectors.toList());
+ List<Integer> parentIds =
JavaScalaConverters.convertScalaListToJavaList(((UnionRDD<Integer>)
unionRdd).rdds())
+ .stream().map(RDD::id).collect(Collectors.toList());
+ assertEquals(expectedParentIds, parentIds);
+ }
+
static class SerializablePairFlatMapTestFunc implements
SerializablePairFlatMapFunction<Iterator<java.lang.Integer>, java.lang.Integer,
java.lang.Integer> {
@Override
public Stream<Pair<Integer, Integer>> call(Iterator<Integer> t) throws
Exception {
diff --git
a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
index 43bc9cc78b35..3f2ad0298dc2 100644
---
a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
+++
b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
@@ -82,6 +82,27 @@ public abstract class HoodieEngineContext {
public abstract <T> HoodieData<T> parallelize(List<T> data, int parallelism);
+ /**
+ * Unions all the given {@link HoodieData} into a single {@link HoodieData}.
+ *
+ * <p>The default implementation folds the inputs with {@link
HoodieData#union(HoodieData)}, which
+ * does not nest when {@link #emptyHoodieData()} is eager, since every step
materializes its result.
+ * Engines whose union nests per call should override this with a single
flat union: on Spark, a
+ * chain of a few hundred {@code UnionRDD} levels is enough to overflow the
stack during task
+ * serialization.
+ *
+ * @param dataList the {@link HoodieData} to union
+ * @param <T> type of object
+ * @return the union of all the inputs, or {@link #emptyHoodieData()} if
there is none
+ */
+ public <T> HoodieData<T> union(List<HoodieData<T>> dataList) {
+ HoodieData<T> result = emptyHoodieData();
+ for (HoodieData<T> data : dataList) {
+ result = result.union(data);
+ }
+ return result;
+ }
+
public abstract <I, O> List<O> map(List<I> data, SerializableFunction<I, O>
func, int parallelism);
public abstract <I, K, V> List<V> mapToPairAndReduceByKey(List<I> data,
SerializablePairFunction<I, K, V> mapToPairFunc,
diff --git
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java
index c4fc1e285f7c..1dcdbfc45171 100644
---
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java
+++
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java
@@ -19,6 +19,7 @@
package org.apache.hudi.common.engine;
import org.apache.hudi.common.data.HoodieData;
+import org.apache.hudi.common.data.HoodieListData;
import org.apache.hudi.common.data.HoodieListPairData;
import org.apache.hudi.common.data.HoodiePairData;
import org.apache.hudi.common.function.SerializableFunction;
@@ -31,6 +32,7 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Collections;
import java.util.Iterator;
import java.util.List;
@@ -170,4 +172,22 @@ public class TestHoodieLocalEngineContext {
assertTrue(resultList.contains(42));
assertTrue(resultList.contains(17));
}
+
+ @Test
+ void testUnion() {
+
assertTrue(context.union(Collections.<HoodieData<Integer>>emptyList()).isEmpty());
+
+ HoodieData<Integer> single =
context.union(Collections.singletonList(HoodieListData.lazy(Arrays.asList(1,
2))));
+ assertEquals(Arrays.asList(1, 2), single.collectAsList());
+
+ HoodieData<Integer> union = context.union(Arrays.asList(
+ context.parallelize(Arrays.asList(1, 2)),
+ context.emptyHoodieData(),
+ HoodieListData.lazy(Arrays.asList(3, 4)),
+ context.parallelize(Collections.singletonList(5))));
+ assertEquals(Arrays.asList(1, 2, 3, 4, 5), union.collectAsList());
+ // The result bears the eager semantic of emptyHoodieData(), even over
lazy inputs
+ assertEquals(5, union.count());
+ assertEquals(Arrays.asList(1, 2, 3, 4, 5), union.collectAsList());
+ }
}
diff --git
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala
index 6c4875fc3eef..1669aceeab1d 100644
---
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala
+++
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala
@@ -34,12 +34,14 @@ import org.apache.hudi.common.util.{Option => HOption}
import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig,
HoodieWriteConfig}
import org.apache.hudi.core.index.record.HoodieRecordIndex
import org.apache.hudi.exception.{HoodieException, HoodieMetadataException}
-import
org.apache.hudi.functional.TestRecordLevelIndex.TestPartitionedRecordLevelIndexTestCase
+import
org.apache.hudi.functional.TestRecordLevelIndex.{TestPartitionedRecordLevelIndexTestCase,
UnionDepthListener}
import org.apache.hudi.index.HoodieIndex.IndexType.RECORD_LEVEL_INDEX
import org.apache.hudi.metadata.{HoodieBackedTableMetadata,
HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType}
import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
import
org.apache.hudi.table.action.compact.strategy.UnBoundedCompactionStrategy
+import org.apache.spark.SparkContext
+import org.apache.spark.scheduler.{SparkListener, SparkListenerJobEnd,
SparkListenerJobStart, SparkListenerStageSubmitted}
import org.apache.spark.sql.{Row, SaveMode}
import org.apache.spark.sql.functions.lit
import org.junit.jupiter.api.{Tag, Test}
@@ -49,10 +51,12 @@ import org.junit.jupiter.params.provider.{Arguments,
EnumSource, MethodSource, V
import java.io.{PrintWriter, StringWriter}
import java.util
+import java.util.concurrent.{CountDownLatch, TimeUnit}
import java.util.stream.Collectors
import scala.collection.JavaConverters
import scala.collection.JavaConverters._
+import scala.collection.mutable
@Tag("functional")
class TestRecordLevelIndex extends RecordLevelIndexTestBase with
SparkDatasetMixin {
@@ -680,6 +684,67 @@ class TestRecordLevelIndex extends
RecordLevelIndexTestBase with SparkDatasetMix
assertTrue(HoodieRecordIndex.isPartitioned(metaClient.getIndexMetadata.get().getIndexDefinitions.get(HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX)))
}
+ /**
+ * Partitioned RLI initialization builds one HoodieData of index records per
data partition and
+ * unions them one at a time. JavaRDD.union does not flatten nested unions,
so the lineage handed to
+ * the bulk commit becomes a chain of UnionRDDs as deep as the number of
data partitions. Spark
+ * serializes task lineage recursively, so a table with enough partitions
fails the initialization
+ * with a StackOverflowError. The union depth must stay flat regardless of
the partition count.
+ */
+ @Test
+ def testPartitionedRecordLevelIndexInitializationUnionDepth(): Unit = {
+ val numDataPartitions = 50
+ initMetaClient(HoodieTableType.COPY_ON_WRITE)
+ val options = Map(HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+ DataSourceWriteOptions.TABLE_TYPE.key ->
HoodieTableType.COPY_ON_WRITE.name(),
+ RECORDKEY_FIELD.key -> "_row_key",
+ PARTITIONPATH_FIELD.key -> "data_partition_path",
+ HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+ HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() ->
"false",
+ HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "false",
+ HoodieMetadataConfig.SECONDARY_INDEX_ENABLE_PROP.key() -> "false",
+ HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key() -> "false",
+ HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key() ->
"false")
+ // One record per data partition: p0 .. p49
+ spark.range(numDataPartitions)
+ .selectExpr("cast(id as string) as _row_key", "id as timestamp",
+ "concat('p', cast(id as string)) as data_partition_path")
+ .write.format("hudi")
+ .options(options)
+ .mode(SaveMode.Overwrite)
+ .save(basePath)
+ assertEquals(numDataPartitions,
spark.read.format("hudi").load(basePath).count())
+
+ metaClient.reloadActiveTimeline()
+ val tableSchema = new
TableSchemaResolver(metaClient).getTableSchemaFromLatestCommit(false).get().toString
+ val writeConfig = getWriteConfig(options ++ Map(
+ HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "true",
+ HoodieWriteConfig.AVRO_SCHEMA_STRING.key() -> tableSchema))
+
+ // Initialize the partitioned RLI and record the lineage of every stage it
runs.
+ val listener = new UnionDepthListener
+ spark.sparkContext.addSparkListener(listener)
+ val metadata = try {
+ val writer = metadataWriter(writeConfig)
+ listener.awaitDelivered(spark.sparkContext)
+ writer.getTableMetadata
+ } finally {
+ spark.sparkContext.removeSparkListener(listener)
+ }
+
+ metaClient = HoodieTableMetaClient.reload(metaClient)
+
assertTrue(HoodieRecordIndex.isPartitioned(metaClient.getIndexMetadata.get().getIndexDefinitions
+ .get(HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX)))
+ val locations = (0 until numDataPartitions).flatMap(i =>
+ readRecordIndex(metadata, util.Collections.singletonList(i.toString),
HOption.of(s"p$i")))
+ assertEquals(numDataPartitions, locations.size)
+
+ val unionDepth = listener.maxUnionDepth
+ assertTrue(unionDepth <= 1,
+ s"Partitioned RLI initialization over $numDataPartitions data partitions
built a UnionRDD "
+ + s"chain of depth $unionDepth; the depth must not grow with the
number of data partitions")
+ }
+
def readRecordIndex(metadata: HoodieBackedTableMetadata, recordKeys:
java.util.List[String], dataTablePartition: HOption[String]): Map[String,
HoodieRecordGlobalLocation] = {
metadata.readRecordIndexLocationsWithKeys(HoodieListData.eager(recordKeys),
dataTablePartition)
.collectAsList().asScala.map(p => p.getKey -> p.getValue).toMap
@@ -931,6 +996,64 @@ object TestRecordLevelIndex {
case class TestPartitionedRecordLevelIndexTestCase(tableType:
HoodieTableType, streamingWriteEnabled: Boolean)
+ /**
+ * Tracks the longest chain of directly nested UnionRDDs across the narrow
lineage of every
+ * submitted stage.
+ */
+ class UnionDepthListener extends SparkListener {
+ private val sentinelGroup = "union-depth-listener-sentinel"
+ private val rddNames = mutable.Map[Int, String]()
+ private val rddParents = mutable.Map[Int, Seq[Int]]()
+ @volatile private var sentinelJobId = -1
+ private val sentinelDone = new CountDownLatch(1)
+
+ override def onStageSubmitted(stageSubmitted:
SparkListenerStageSubmitted): Unit = synchronized {
+ stageSubmitted.stageInfo.rddInfos.foreach { info =>
+ rddNames(info.id) = info.name
+ rddParents(info.id) = info.parentIds
+ }
+ }
+
+ override def onJobStart(jobStart: SparkListenerJobStart): Unit = {
+ if (Option(jobStart.properties).exists(p => sentinelGroup ==
p.getProperty("spark.jobGroup.id"))) {
+ sentinelJobId = jobStart.jobId
+ }
+ }
+
+ override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = {
+ if (jobEnd.jobId == sentinelJobId) {
+ sentinelDone.countDown()
+ }
+ }
+
+ /**
+ * Runs a marker job and waits for its end event. The listener bus
delivers events in order,
+ * so every stage submitted before the marker has been seen once this
returns.
+ */
+ def awaitDelivered(sc: SparkContext): Unit = {
+ sc.setJobGroup(sentinelGroup, sentinelGroup)
+ try {
+ sc.parallelize(Seq(1), 1).count()
+ } finally {
+ sc.clearJobGroup()
+ }
+ assertTrue(sentinelDone.await(60, TimeUnit.SECONDS), "Timed out waiting
for Spark listener events")
+ }
+
+ def maxUnionDepth: Int = synchronized {
+ // RDD ids grow monotonically, so parents are visited before children.
+ val depth = mutable.Map[Int, Int]()
+ rddNames.keys.toSeq.sorted.foreach { id =>
+ depth(id) = if (rddNames(id) == "UnionRDD") {
+ 1 + (rddParents(id).map(p => depth.getOrElse(p, 0)) :+ 0).max
+ } else {
+ 0
+ }
+ }
+ (depth.values.toSeq :+ 0).max
+ }
+ }
+
def testArgsForPartitionedRecordLevelIndex:
java.util.stream.Stream[Arguments] = {
java.util.stream.Stream.of(
Arguments.arguments(TestPartitionedRecordLevelIndexTestCase(HoodieTableType.COPY_ON_WRITE,
streamingWriteEnabled = true)),