This is an automated email from the ASF dual-hosted git repository.

github-merge-queue[bot] pushed a commit to branch 
gh-readonly-queue/master/pr-20545-9a7728c5862a8939f58f47c35abb1d838fc2dcb3
in repository https://gitbox.apache.org/repos/asf/hudi.git

commit a5b4e0430d9f41cd2a6cac83c17d201e6f5219aa
Author: voonhous <[email protected]>
AuthorDate: Wed Nov 11 06:43:27 2026 +0000

    fix(metadata): flatten partitioned record index init union (#20545)
    
    * test(metadata): reproduce deep UnionRDD chain in partitioned RLI init
    
    Partitioned record level index initialization unions one HoodieData
    per data partition in a loop. JavaRDD.union does not flatten nested
    unions, so the lineage handed to the bulk commit is a UnionRDD chain
    as deep as the number of data partitions, and Spark serializes that
    lineage recursively.
    
    Add a functional test that initializes partitioned RLI over 50 data
    partitions and measures the longest UnionRDD chain from the stage
    lineage reported to a SparkListener. The test fails on master with a
    depth of 50 and is expected to pass once the depth stays flat.
    
    * fix(metadata): flatten partitioned record index init union
    
    Partitioned record level index initialization unioned the index
    records of each data partition one at a time. On Spark every
    JavaRDD.union adds a UnionRDD level, so the lineage handed to the bulk
    commit was a chain as deep as the number of data partitions, and Spark
    serializes task lineage recursively: around 500 partitions overflow a
    1 MB stack and fail the initialization with StackOverflowError.
    
    Collect the per-partition records in the loop and union them once
    through a new HoodieEngineContext.union(List). The Spark engine
    context implements it with a single JavaSparkContext.union, which
    yields one UnionRDD whose parents are the inputs. The default
    implementation flattens HoodieListData inputs in one pass through the
    new HoodieListData.unionAll, since chained Stream.concat has the same
    deep-nesting problem, and keeps the eager semantic of
    emptyHoodieData() that the old fold had. Engines with other HoodieData
    types fall back to folding.
    
    Fixes #20543
    
    * fix(metadata): drop HoodieListData.unionAll
    
    The in-memory path never had the deep-nesting problem: every in-memory
    engine context's emptyHoodieData() is eager, so each step of the fold
    materializes a list and Stream.concat does not nest. Only Spark needs
    the flat union, so the default HoodieEngineContext.union goes back to
    folding and its javadoc says why that is safe.
---
 .../metadata/HoodieBackedTableMetadataWriter.java  |   7 +-
 .../client/common/HoodieSparkEngineContext.java    |  17 +++
 .../common/TestHoodieSparkEngineContext.java       |  30 +++++
 .../hudi/common/engine/HoodieEngineContext.java    |  21 ++++
 .../engine/TestHoodieLocalEngineContext.java       |  20 ++++
 .../hudi/functional/TestRecordLevelIndex.scala     | 125 ++++++++++++++++++++-
 6 files changed, 217 insertions(+), 3 deletions(-)

diff --git 
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
 
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
index 6295a99f22dc..5cc35cc85d2e 100644
--- 
a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
+++ 
b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/metadata/HoodieBackedTableMetadataWriter.java
@@ -464,12 +464,15 @@ public abstract class HoodieBackedTableMetadataWriter<I, 
O> implements HoodieTab
 
       HoodieTimer partitionInitTimer = HoodieTimer.start();
       clearExistingMetadataPartition(relativePartitionPath);
-      HoodieData<HoodieRecord> records = engineContext.emptyHoodieData();
+      List<HoodieData<HoodieRecord>> recordsPerDataPartition = new 
ArrayList<>();
       for (DataPartitionAndRecords dataPartitionAndRecords: 
initializationPlan.dataPartitionAndRecords()) {
         initializeFileGroups(dataMetaClient, partitionType, 
instantTimeForPartition,
             dataPartitionAndRecords.numFileGroups(), relativePartitionPath, 
dataPartitionAndRecords.dataPartition());
-        records = records.union(dataPartitionAndRecords.indexRecords());
+        recordsPerDataPartition.add(dataPartitionAndRecords.indexRecords());
       }
+      // Union once: on Spark, folding with HoodieData#union nests one 
UnionRDD per data partition,
+      // and the resulting lineage overflows the stack on tables with many 
partitions (partitioned RLI).
+      HoodieData<HoodieRecord> records = 
engineContext.union(recordsPerDataPartition);
 
       bulkCommit(instantTimeForPartition, relativePartitionPath, records, 
initializationPlan.indexParser());
 
diff --git 
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java
 
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java
index 77eff393353c..1dd951535ee8 100644
--- 
a/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java
+++ 
b/hudi-client/hudi-spark-client/src/main/java/org/apache/hudi/client/common/HoodieSparkEngineContext.java
@@ -135,6 +135,23 @@ public class HoodieSparkEngineContext extends 
HoodieEngineContext {
     return HoodieJavaRDD.of(javaSparkContext.parallelize(data, parallelism));
   }
 
+  /**
+   * Unions all the inputs with a single {@link JavaSparkContext#union}, which 
yields one
+   * {@code UnionRDD} whose parents are the inputs, instead of a chain of 
nested ones.
+   */
+  @Override
+  public <T> HoodieData<T> union(List<HoodieData<T>> dataList) {
+    if (dataList.isEmpty()) {
+      return emptyHoodieData();
+    }
+    if (dataList.size() == 1) {
+      return dataList.get(0);
+    }
+    @SuppressWarnings("unchecked")
+    JavaRDD<T>[] rdds = dataList.stream().map(data -> 
HoodieJavaRDD.getJavaRDD(data)).toArray(JavaRDD[]::new);
+    return HoodieJavaRDD.of(javaSparkContext.union(rdds));
+  }
+
   @Override
   public <I, O> List<O> map(List<I> data, SerializableFunction<I, O> func, int 
parallelism) {
     final Map<String, Registry> registries = getDistributedRegistries();
diff --git 
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java
 
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java
index b51a0d44cafb..08960e52e722 100644
--- 
a/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java
+++ 
b/hudi-client/hudi-spark-client/src/test/java/org/apache/hudi/client/common/TestHoodieSparkEngineContext.java
@@ -25,13 +25,18 @@ import 
org.apache.hudi.common.function.SerializableBiFunction;
 import org.apache.hudi.common.function.SerializablePairFlatMapFunction;
 import org.apache.hudi.common.util.collection.ImmutablePair;
 import org.apache.hudi.common.util.collection.Pair;
+import org.apache.hudi.data.HoodieJavaRDD;
 import org.apache.hudi.testutils.SparkClientFunctionalTestHarness;
+import org.apache.hudi.util.JavaScalaConverters;
 
+import org.apache.spark.rdd.RDD;
+import org.apache.spark.rdd.UnionRDD;
 import org.junit.jupiter.api.BeforeEach;
 import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 import java.util.stream.Collectors;
@@ -39,6 +44,8 @@ import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertInstanceOf;
+import static org.junit.jupiter.api.Assertions.assertSame;
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 public class TestHoodieSparkEngineContext extends 
SparkClientFunctionalTestHarness {
@@ -106,6 +113,29 @@ public class TestHoodieSparkEngineContext extends 
SparkClientFunctionalTestHarne
     });
   }
 
+  @Test
+  void testUnion() {
+    
assertTrue(context.union(Collections.<HoodieData<Integer>>emptyList()).isEmpty());
+
+    HoodieData<Integer> single = context.parallelize(Arrays.asList(1, 2), 2);
+    assertSame(single, context.union(Collections.singletonList(single)));
+
+    int numInputs = 50;
+    List<HoodieData<Integer>> inputs = IntStream.range(0, numInputs)
+        .mapToObj(i -> context.parallelize(Collections.singletonList(i), 1))
+        .collect(Collectors.toList());
+    HoodieData<Integer> union = context.union(inputs);
+    assertEquals(IntStream.range(0, 
numInputs).boxed().collect(Collectors.toList()), union.collectAsList());
+
+    // A single UnionRDD whose parents are the inputs, not a chain of nested 
UnionRDDs
+    RDD<Integer> unionRdd = HoodieJavaRDD.getJavaRDD(union).rdd();
+    assertInstanceOf(UnionRDD.class, unionRdd);
+    List<Integer> expectedParentIds = 
inputs.stream().map(HoodieData::getId).collect(Collectors.toList());
+    List<Integer> parentIds = 
JavaScalaConverters.convertScalaListToJavaList(((UnionRDD<Integer>) 
unionRdd).rdds())
+        .stream().map(RDD::id).collect(Collectors.toList());
+    assertEquals(expectedParentIds, parentIds);
+  }
+
   static class SerializablePairFlatMapTestFunc implements 
SerializablePairFlatMapFunction<Iterator<java.lang.Integer>, java.lang.Integer, 
java.lang.Integer> {
     @Override
     public Stream<Pair<Integer, Integer>> call(Iterator<Integer> t) throws 
Exception {
diff --git 
a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
 
b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
index 43bc9cc78b35..3f2ad0298dc2 100644
--- 
a/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
+++ 
b/hudi-common/src/main/java/org/apache/hudi/common/engine/HoodieEngineContext.java
@@ -82,6 +82,27 @@ public abstract class HoodieEngineContext {
 
   public abstract <T> HoodieData<T> parallelize(List<T> data, int parallelism);
 
+  /**
+   * Unions all the given {@link HoodieData} into a single {@link HoodieData}.
+   *
+   * <p>The default implementation folds the inputs with {@link 
HoodieData#union(HoodieData)}, which
+   * does not nest when {@link #emptyHoodieData()} is eager, since every step 
materializes its result.
+   * Engines whose union nests per call should override this with a single 
flat union: on Spark, a
+   * chain of a few hundred {@code UnionRDD} levels is enough to overflow the 
stack during task
+   * serialization.
+   *
+   * @param dataList the {@link HoodieData} to union
+   * @param <T>      type of object
+   * @return the union of all the inputs, or {@link #emptyHoodieData()} if 
there is none
+   */
+  public <T> HoodieData<T> union(List<HoodieData<T>> dataList) {
+    HoodieData<T> result = emptyHoodieData();
+    for (HoodieData<T> data : dataList) {
+      result = result.union(data);
+    }
+    return result;
+  }
+
   public abstract <I, O> List<O> map(List<I> data, SerializableFunction<I, O> 
func, int parallelism);
 
   public abstract <I, K, V> List<V> mapToPairAndReduceByKey(List<I> data, 
SerializablePairFunction<I, K, V> mapToPairFunc,
diff --git 
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java
 
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java
index c4fc1e285f7c..1dcdbfc45171 100644
--- 
a/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java
+++ 
b/hudi-hadoop-common/src/test/java/org/apache/hudi/common/engine/TestHoodieLocalEngineContext.java
@@ -19,6 +19,7 @@
 package org.apache.hudi.common.engine;
 
 import org.apache.hudi.common.data.HoodieData;
+import org.apache.hudi.common.data.HoodieListData;
 import org.apache.hudi.common.data.HoodieListPairData;
 import org.apache.hudi.common.data.HoodiePairData;
 import org.apache.hudi.common.function.SerializableFunction;
@@ -31,6 +32,7 @@ import org.junit.jupiter.api.Test;
 
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.Iterator;
 import java.util.List;
 
@@ -170,4 +172,22 @@ public class TestHoodieLocalEngineContext {
     assertTrue(resultList.contains(42));
     assertTrue(resultList.contains(17));
   }
+
+  @Test
+  void testUnion() {
+    
assertTrue(context.union(Collections.<HoodieData<Integer>>emptyList()).isEmpty());
+
+    HoodieData<Integer> single = 
context.union(Collections.singletonList(HoodieListData.lazy(Arrays.asList(1, 
2))));
+    assertEquals(Arrays.asList(1, 2), single.collectAsList());
+
+    HoodieData<Integer> union = context.union(Arrays.asList(
+        context.parallelize(Arrays.asList(1, 2)),
+        context.emptyHoodieData(),
+        HoodieListData.lazy(Arrays.asList(3, 4)),
+        context.parallelize(Collections.singletonList(5))));
+    assertEquals(Arrays.asList(1, 2, 3, 4, 5), union.collectAsList());
+    // The result bears the eager semantic of emptyHoodieData(), even over 
lazy inputs
+    assertEquals(5, union.count());
+    assertEquals(Arrays.asList(1, 2, 3, 4, 5), union.collectAsList());
+  }
 }
diff --git 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala
 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala
index 6c4875fc3eef..1669aceeab1d 100644
--- 
a/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala
+++ 
b/hudi-spark-datasource/hudi-spark/src/test/scala/org/apache/hudi/functional/TestRecordLevelIndex.scala
@@ -34,12 +34,14 @@ import org.apache.hudi.common.util.{Option => HOption}
 import org.apache.hudi.config.{HoodieCompactionConfig, HoodieIndexConfig, 
HoodieWriteConfig}
 import org.apache.hudi.core.index.record.HoodieRecordIndex
 import org.apache.hudi.exception.{HoodieException, HoodieMetadataException}
-import 
org.apache.hudi.functional.TestRecordLevelIndex.TestPartitionedRecordLevelIndexTestCase
+import 
org.apache.hudi.functional.TestRecordLevelIndex.{TestPartitionedRecordLevelIndexTestCase,
 UnionDepthListener}
 import org.apache.hudi.index.HoodieIndex.IndexType.RECORD_LEVEL_INDEX
 import org.apache.hudi.metadata.{HoodieBackedTableMetadata, 
HoodieTableMetadata, HoodieTableMetadataUtil, MetadataPartitionType}
 import org.apache.hudi.storage.{StoragePath, StoragePathInfo}
 import 
org.apache.hudi.table.action.compact.strategy.UnBoundedCompactionStrategy
 
+import org.apache.spark.SparkContext
+import org.apache.spark.scheduler.{SparkListener, SparkListenerJobEnd, 
SparkListenerJobStart, SparkListenerStageSubmitted}
 import org.apache.spark.sql.{Row, SaveMode}
 import org.apache.spark.sql.functions.lit
 import org.junit.jupiter.api.{Tag, Test}
@@ -49,10 +51,12 @@ import org.junit.jupiter.params.provider.{Arguments, 
EnumSource, MethodSource, V
 
 import java.io.{PrintWriter, StringWriter}
 import java.util
+import java.util.concurrent.{CountDownLatch, TimeUnit}
 import java.util.stream.Collectors
 
 import scala.collection.JavaConverters
 import scala.collection.JavaConverters._
+import scala.collection.mutable
 
 @Tag("functional")
 class TestRecordLevelIndex extends RecordLevelIndexTestBase with 
SparkDatasetMixin {
@@ -680,6 +684,67 @@ class TestRecordLevelIndex extends 
RecordLevelIndexTestBase with SparkDatasetMix
     
assertTrue(HoodieRecordIndex.isPartitioned(metaClient.getIndexMetadata.get().getIndexDefinitions.get(HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX)))
   }
 
+  /**
+   * Partitioned RLI initialization builds one HoodieData of index records per 
data partition and
+   * unions them one at a time. JavaRDD.union does not flatten nested unions, 
so the lineage handed to
+   * the bulk commit becomes a chain of UnionRDDs as deep as the number of 
data partitions. Spark
+   * serializes task lineage recursively, so a table with enough partitions 
fails the initialization
+   * with a StackOverflowError. The union depth must stay flat regardless of 
the partition count.
+   */
+  @Test
+  def testPartitionedRecordLevelIndexInitializationUnionDepth(): Unit = {
+    val numDataPartitions = 50
+    initMetaClient(HoodieTableType.COPY_ON_WRITE)
+    val options = Map(HoodieWriteConfig.TBL_NAME.key -> "hoodie_test",
+      DataSourceWriteOptions.TABLE_TYPE.key -> 
HoodieTableType.COPY_ON_WRITE.name(),
+      RECORDKEY_FIELD.key -> "_row_key",
+      PARTITIONPATH_FIELD.key -> "data_partition_path",
+      HoodieTableConfig.ORDERING_FIELDS.key -> "timestamp",
+      HoodieMetadataConfig.GLOBAL_RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> 
"false",
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "false",
+      HoodieMetadataConfig.SECONDARY_INDEX_ENABLE_PROP.key() -> "false",
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_COLUMN_STATS.key() -> "false",
+      HoodieMetadataConfig.ENABLE_METADATA_INDEX_PARTITION_STATS.key() -> 
"false")
+    // One record per data partition: p0 .. p49
+    spark.range(numDataPartitions)
+      .selectExpr("cast(id as string) as _row_key", "id as timestamp",
+        "concat('p', cast(id as string)) as data_partition_path")
+      .write.format("hudi")
+      .options(options)
+      .mode(SaveMode.Overwrite)
+      .save(basePath)
+    assertEquals(numDataPartitions, 
spark.read.format("hudi").load(basePath).count())
+
+    metaClient.reloadActiveTimeline()
+    val tableSchema = new 
TableSchemaResolver(metaClient).getTableSchemaFromLatestCommit(false).get().toString
+    val writeConfig = getWriteConfig(options ++ Map(
+      HoodieMetadataConfig.RECORD_LEVEL_INDEX_ENABLE_PROP.key() -> "true",
+      HoodieWriteConfig.AVRO_SCHEMA_STRING.key() -> tableSchema))
+
+    // Initialize the partitioned RLI and record the lineage of every stage it 
runs.
+    val listener = new UnionDepthListener
+    spark.sparkContext.addSparkListener(listener)
+    val metadata = try {
+      val writer = metadataWriter(writeConfig)
+      listener.awaitDelivered(spark.sparkContext)
+      writer.getTableMetadata
+    } finally {
+      spark.sparkContext.removeSparkListener(listener)
+    }
+
+    metaClient = HoodieTableMetaClient.reload(metaClient)
+    
assertTrue(HoodieRecordIndex.isPartitioned(metaClient.getIndexMetadata.get().getIndexDefinitions
+      .get(HoodieTableMetadataUtil.PARTITION_NAME_RECORD_INDEX)))
+    val locations = (0 until numDataPartitions).flatMap(i =>
+      readRecordIndex(metadata, util.Collections.singletonList(i.toString), 
HOption.of(s"p$i")))
+    assertEquals(numDataPartitions, locations.size)
+
+    val unionDepth = listener.maxUnionDepth
+    assertTrue(unionDepth <= 1,
+      s"Partitioned RLI initialization over $numDataPartitions data partitions 
built a UnionRDD "
+        + s"chain of depth $unionDepth; the depth must not grow with the 
number of data partitions")
+  }
+
   def readRecordIndex(metadata: HoodieBackedTableMetadata, recordKeys: 
java.util.List[String], dataTablePartition: HOption[String]): Map[String, 
HoodieRecordGlobalLocation] = {
     
metadata.readRecordIndexLocationsWithKeys(HoodieListData.eager(recordKeys), 
dataTablePartition)
       .collectAsList().asScala.map(p => p.getKey -> p.getValue).toMap
@@ -931,6 +996,64 @@ object TestRecordLevelIndex {
 
   case class TestPartitionedRecordLevelIndexTestCase(tableType: 
HoodieTableType, streamingWriteEnabled: Boolean)
 
+  /**
+   * Tracks the longest chain of directly nested UnionRDDs across the narrow 
lineage of every
+   * submitted stage.
+   */
+  class UnionDepthListener extends SparkListener {
+    private val sentinelGroup = "union-depth-listener-sentinel"
+    private val rddNames = mutable.Map[Int, String]()
+    private val rddParents = mutable.Map[Int, Seq[Int]]()
+    @volatile private var sentinelJobId = -1
+    private val sentinelDone = new CountDownLatch(1)
+
+    override def onStageSubmitted(stageSubmitted: 
SparkListenerStageSubmitted): Unit = synchronized {
+      stageSubmitted.stageInfo.rddInfos.foreach { info =>
+        rddNames(info.id) = info.name
+        rddParents(info.id) = info.parentIds
+      }
+    }
+
+    override def onJobStart(jobStart: SparkListenerJobStart): Unit = {
+      if (Option(jobStart.properties).exists(p => sentinelGroup == 
p.getProperty("spark.jobGroup.id"))) {
+        sentinelJobId = jobStart.jobId
+      }
+    }
+
+    override def onJobEnd(jobEnd: SparkListenerJobEnd): Unit = {
+      if (jobEnd.jobId == sentinelJobId) {
+        sentinelDone.countDown()
+      }
+    }
+
+    /**
+     * Runs a marker job and waits for its end event. The listener bus 
delivers events in order,
+     * so every stage submitted before the marker has been seen once this 
returns.
+     */
+    def awaitDelivered(sc: SparkContext): Unit = {
+      sc.setJobGroup(sentinelGroup, sentinelGroup)
+      try {
+        sc.parallelize(Seq(1), 1).count()
+      } finally {
+        sc.clearJobGroup()
+      }
+      assertTrue(sentinelDone.await(60, TimeUnit.SECONDS), "Timed out waiting 
for Spark listener events")
+    }
+
+    def maxUnionDepth: Int = synchronized {
+      // RDD ids grow monotonically, so parents are visited before children.
+      val depth = mutable.Map[Int, Int]()
+      rddNames.keys.toSeq.sorted.foreach { id =>
+        depth(id) = if (rddNames(id) == "UnionRDD") {
+          1 + (rddParents(id).map(p => depth.getOrElse(p, 0)) :+ 0).max
+        } else {
+          0
+        }
+      }
+      (depth.values.toSeq :+ 0).max
+    }
+  }
+
   def testArgsForPartitionedRecordLevelIndex: 
java.util.stream.Stream[Arguments] = {
     java.util.stream.Stream.of(
       
Arguments.arguments(TestPartitionedRecordLevelIndexTestCase(HoodieTableType.COPY_ON_WRITE,
 streamingWriteEnabled = true)),

Reply via email to