This is an automated email from the ASF dual-hosted git repository.

sunchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 808d79616144 [SPARK-60098][SQL] Fix memory leaks in ParquetColumnVector
808d79616144 is described below

commit 808d79616144fbf209244e9c7a27927907908768
Author: Filipe Oliveira <[email protected]>
AuthorDate: Mon Oct 26 09:50:22 2026 -0700

    [SPARK-60098][SQL] Fix memory leaks in ParquetColumnVector
    
    ### What changes were proposed in this pull request?
    
    This tries to ensure that off-heap memory allocated by the Parquet reader 
is deallocated. The class `ParquetColumnVector` uses the method 
`WritableColumnVector.reserveNewColumn` in 4 scenarios. Given these column 
vectors don't belong to ColumnarBatches and the class is not calling 
`WritableColumnVector.close` when memory mode is OFF_HEAP these column vectors 
are leaking.
    
    To fix, the class `WritableColumnVector` now holds references to these 
column vectors and closes them on `WritableColumnVector.close`.
    
    ### Why are the changes needed?
    
    This will prevent OOMs for long running applications.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    The 4 scenarios using `WritableColumnVector.reserveNewColumn` are tested in 
`ParquetVectorizedSuite`. Definition levels and repetition levels are tested by 
existing tests. 2 new tests were added for the other 2 scenarios.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    No.
    
    Closes #59657 from frroliveira/fix-parquet-vector-closure.
    
    Authored-by: Filipe Oliveira <[email protected]>
    Signed-off-by: Chao Sun <[email protected]>
---
 .../execution/vectorized/OffHeapColumnVector.java  |   7 +-
 .../execution/vectorized/OnHeapColumnVector.java   |  15 +-
 .../execution/vectorized/WritableColumnVector.java |  60 ++++++--
 .../datasources/parquet/ParquetTestAccess.scala    |   5 +
 .../parquet/ParquetVectorizedSuite.scala           | 151 ++++++++++++++++++++-
 5 files changed, 221 insertions(+), 17 deletions(-)

diff --git 
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
 
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
index 1f3c2ac2e608..fc902090acc0 100644
--- 
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
+++ 
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
@@ -110,6 +110,11 @@ public final class OffHeapColumnVector extends 
WritableColumnVector {
     super.close();
   }
 
+  @Override
+  public boolean isFree() {
+    return nulls == 0 && data == 0 && lengthData == 0 && offsetData == 0;
+  }
+
   //
   // APIs dealing with nulls
   //
@@ -699,7 +704,7 @@ public final class OffHeapColumnVector extends 
WritableColumnVector {
   }
 
   @Override
-  public OffHeapColumnVector reserveNewColumn(int capacity, DataType type) {
+  protected OffHeapColumnVector reserveNewInternalColumn(int capacity, 
DataType type) {
     return new OffHeapColumnVector(capacity, type);
   }
 }
diff --git 
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
 
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
index eb12f86858f4..1a6017de5234 100644
--- 
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
+++ 
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
@@ -98,6 +98,19 @@ public final class OnHeapColumnVector extends 
WritableColumnVector {
     super.close();
   }
 
+  @Override
+  public boolean isFree() {
+    return nulls == null &&
+        byteData == null &&
+        shortData == null &&
+        intData == null &&
+        longData == null &&
+        floatData == null &&
+        doubleData == null &&
+        arrayLengths == null &&
+        arrayOffsets == null;
+  }
+
   //
   // APIs dealing with nulls
   //
@@ -665,7 +678,7 @@ public final class OnHeapColumnVector extends 
WritableColumnVector {
   }
 
   @Override
-  public OnHeapColumnVector reserveNewColumn(int capacity, DataType type) {
+  protected OnHeapColumnVector reserveNewInternalColumn(int capacity, DataType 
type) {
     return new OnHeapColumnVector(capacity, type);
   }
 }
diff --git 
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
 
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
index fab67965d701..dd1d67d3348e 100644
--- 
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
+++ 
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
@@ -19,6 +19,8 @@ package org.apache.spark.sql.execution.vectorized;
 import java.math.BigDecimal;
 import java.math.BigInteger;
 import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
 import java.util.Optional;
 
 import com.google.common.annotations.VisibleForTesting;
@@ -100,6 +102,13 @@ public abstract class WritableColumnVector extends 
ColumnVector {
 
   @Override
   public void close() {
+    if (reservedColumns != null) {
+      for (WritableColumnVector column : reservedColumns) {
+        column.close();
+      }
+      reservedColumns.clear();
+      reservedColumns = null;
+    }
     if (childColumns != null) {
       for (int i = 0; i < childColumns.length; i++) {
         childColumns[i].close();
@@ -120,6 +129,12 @@ public abstract class WritableColumnVector extends 
ColumnVector {
     // no-op
   }
 
+  /**
+   * Returns true if this vector's own data buffers are released; vectors it 
owns are not checked.
+   */
+  @VisibleForTesting
+  public abstract boolean isFree();
+
   public void reserveAdditional(int additionalCapacity) {
     reserve(elementsAppended + additionalCapacity);
   }
@@ -211,7 +226,7 @@ public abstract class WritableColumnVector extends 
ColumnVector {
    */
   public WritableColumnVector reserveDictionaryIds(int capacity) {
     if (dictionaryIds == null) {
-      dictionaryIds = reserveNewColumn(capacity, DataTypes.IntegerType);
+      dictionaryIds = reserveNewInternalColumn(capacity, 
DataTypes.IntegerType);
     } else {
       dictionaryIds.reset();
       dictionaryIds.reserve(capacity);
@@ -1033,9 +1048,26 @@ public abstract class WritableColumnVector extends 
ColumnVector {
   protected WritableColumnVector[] childColumns;
 
   /**
-   * Reserve a new column.
+   * Publicly reserved columns.
+   */
+  private List<WritableColumnVector> reservedColumns;
+
+  /**
+   * Reserve a new column owned by this column vector.
+   */
+  public WritableColumnVector reserveNewColumn(int capacity, DataType type) {
+    WritableColumnVector column = reserveNewInternalColumn(capacity, type);
+    if (reservedColumns == null) {
+      reservedColumns = new ArrayList<>(1);
+    }
+    reservedColumns.add(column);
+    return column;
+  }
+
+  /**
+   * Reserve a new column for internal usage.
    */
-  public abstract WritableColumnVector reserveNewColumn(int capacity, DataType 
type);
+  protected abstract WritableColumnVector reserveNewInternalColumn(int 
capacity, DataType type);
 
   protected boolean isArray() {
     return type instanceof ArrayType || type instanceof BinaryType || type 
instanceof StringType ||
@@ -1072,31 +1104,31 @@ public abstract class WritableColumnVector extends 
ColumnVector {
         childCapacity *= DEFAULT_ARRAY_LENGTH;
       }
       this.childColumns = new WritableColumnVector[1];
-      this.childColumns[0] = reserveNewColumn(childCapacity, childType);
+      this.childColumns[0] = reserveNewInternalColumn(childCapacity, 
childType);
     } else if (type instanceof StructType st) {
       this.childColumns = new WritableColumnVector[st.fields().length];
       for (int i = 0; i < childColumns.length; ++i) {
-        this.childColumns[i] = reserveNewColumn(capacity, 
st.fields()[i].dataType());
+        this.childColumns[i] = reserveNewInternalColumn(capacity, 
st.fields()[i].dataType());
       }
     } else if (type instanceof MapType mapType) {
       this.childColumns = new WritableColumnVector[2];
-      this.childColumns[0] = reserveNewColumn(capacity, mapType.keyType());
-      this.childColumns[1] = reserveNewColumn(capacity, mapType.valueType());
+      this.childColumns[0] = reserveNewInternalColumn(capacity, 
mapType.keyType());
+      this.childColumns[1] = reserveNewInternalColumn(capacity, 
mapType.valueType());
     } else if (type instanceof CalendarIntervalType) {
       // Three columns. Months as int. Days as Int. Microseconds as Long.
       this.childColumns = new WritableColumnVector[3];
-      this.childColumns[0] = reserveNewColumn(capacity, DataTypes.IntegerType);
-      this.childColumns[1] = reserveNewColumn(capacity, DataTypes.IntegerType);
-      this.childColumns[2] = reserveNewColumn(capacity, DataTypes.LongType);
+      this.childColumns[0] = reserveNewInternalColumn(capacity, 
DataTypes.IntegerType);
+      this.childColumns[1] = reserveNewInternalColumn(capacity, 
DataTypes.IntegerType);
+      this.childColumns[2] = reserveNewInternalColumn(capacity, 
DataTypes.LongType);
     } else if (type instanceof AnyTimestampNanoType) {
       // Two columns. EpochMicros as Long. NanosWithinMicro as Short.
       this.childColumns = new WritableColumnVector[2];
-      this.childColumns[0] = reserveNewColumn(capacity, DataTypes.LongType);
-      this.childColumns[1] = reserveNewColumn(capacity, DataTypes.ShortType);
+      this.childColumns[0] = reserveNewInternalColumn(capacity, 
DataTypes.LongType);
+      this.childColumns[1] = reserveNewInternalColumn(capacity, 
DataTypes.ShortType);
     } else if (type instanceof VariantType) {
       this.childColumns = new WritableColumnVector[2];
-      this.childColumns[0] = reserveNewColumn(capacity, DataTypes.BinaryType);
-      this.childColumns[1] = reserveNewColumn(capacity, DataTypes.BinaryType);
+      this.childColumns[0] = reserveNewInternalColumn(capacity, 
DataTypes.BinaryType);
+      this.childColumns[1] = reserveNewInternalColumn(capacity, 
DataTypes.BinaryType);
     } else {
       this.childColumns = null;
     }
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
index 1f0a261019dd..987a7c286a79 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
@@ -48,6 +48,7 @@ import org.apache.spark.util.SparkClassUtils
  *   - `LateMaterializationParquetRecordReader`'s scratch vectors and queued 
survivor sets, the
  *     column readers and row indexes a reader still holds, and a vector's 
byte-child capacity
  *     (private state a test checks the reader's memory by)
+ *   - `VectorizedParquetRecordReader.columnVectors` (field).
  */
 object ParquetTestAccess {
 
@@ -254,6 +255,10 @@ object ParquetTestAccess {
     f
   }
 
+  def columnVectors(reader: VectorizedParquetRecordReader): Array[AnyRef] = {
+    columnVectorsField.get(reader).asInstanceOf[Array[AnyRef]]
+  }
+
   private lazy val columnVectorCls = SparkClassUtils.classForName[Any](
     "org.apache.spark.sql.execution.datasources.parquet.ParquetColumnVector")
 
diff --git 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
index 5749b602a635..783d8282f2bb 100644
--- 
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
+++ 
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
@@ -17,23 +17,30 @@
 
 package org.apache.spark.sql.execution.datasources.parquet
 
-import java.io.IOException
+import java.io.{File, IOException}
 import java.util.{Optional, PrimitiveIterator}
 
 import scala.collection.mutable.ArrayBuffer
 import scala.language.implicitConversions
+import scala.util.control.NonFatal
 
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.mapred.FileSplit
+import org.apache.hadoop.mapreduce.TaskAttemptID
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
 import org.apache.parquet.bytes.BytesInput
 import org.apache.parquet.column.{ColumnDescriptor, Encoding, 
ParquetProperties}
 import org.apache.parquet.column.impl.ColumnWriteStoreV1
 import org.apache.parquet.column.page._
 import org.apache.parquet.column.page.mem.MemPageStore
 import org.apache.parquet.column.statistics.Statistics
+import org.apache.parquet.hadoop.ParquetInputFormat
 import org.apache.parquet.io.ParquetDecodingException
 import org.apache.parquet.io.api.Binary
 import org.apache.parquet.schema.{MessageType, MessageTypeParser}
 import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
 
+import org.apache.spark.TestUtils
 import org.apache.spark.memory.MemoryMode
 import org.apache.spark.sql.Row
 import org.apache.spark.sql.catalyst.InternalRow
@@ -41,8 +48,10 @@ import 
org.apache.spark.sql.catalyst.expressions.{RowOrdering, SpecificInternalR
 import org.apache.spark.sql.catalyst.util.DateTimeUtils
 import 
org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase.ParquetRowGroupReader
 import org.apache.spark.sql.execution.vectorized.ColumnVectorUtils
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types._
+import org.apache.spark.types.variant.VariantBuilder
 import org.apache.spark.util.ArrayImplicits._
 import org.apache.spark.util.collection.Utils.createArray
 
@@ -801,6 +810,81 @@ class ParquetVectorizedSuite extends ParquetTest with 
SharedSparkSession {
       .add("_2", IntegerType)
   )
 
+  test("SPARK-60098: Vectorized reader closes column vectors when missing all 
struct fields") {
+    withTempPath { dir =>
+      val data = Seq(
+        Tuple1((1, "a")),
+        Tuple1((2, null)),
+        Tuple1(null)
+      )
+      
spark.createDataFrame(data).coalesce(1).write.parquet(dir.getCanonicalPath)
+
+      Seq(true, false).foreach { offheapEnabled =>
+        val readSchema = new StructType().add("_1",
+          new StructType()
+            .add("_3", IntegerType, nullable = true)
+            .add("_4", LongType, nullable = true),
+          nullable = true
+        )
+        val file = TestUtils.listDirectory(dir).head
+        val reader = createVectorizedReader(file, readSchema, offheapEnabled)
+        try {
+          while (reader.nextKeyValue()) {}
+        } finally {
+          reader.close()
+          checkColumnVectorsAreClosed(reader)
+        }
+      }
+    }
+  }
+
+  test("SPARK-60098: Vectorized reader closes variant type column vectors") {
+    withTempPath { dir =>
+      val scalarTypes = Array(IntegerType, StringType)
+      val variantBuilder = new VariantBuilder(false)
+      scalarTypes.indices.map(_.toString).foreach(variantBuilder.addKey)
+      val rows = Seq(
+        Row(
+          Row(
+            variantBuilder.result().getMetadata,
+            null,
+            Row.fromSeq(Seq[Any](3, "str10").map(Row(null, _)))
+          )
+        )
+      )
+      val writeSchema =
+        StructType(
+          Array(
+            StructField(
+              "v",
+              SparkShreddingUtils.variantShreddingSchema(
+                StructType(
+                  scalarTypes.zipWithIndex.map { case (t, i) => 
StructField(i.toString, t) }
+                )
+              )
+            )
+          )
+        )
+
+      spark
+        .createDataFrame(spark.sparkContext.parallelize(rows, numSlices = 1), 
writeSchema)
+        .write
+        .parquet(dir.getCanonicalPath)
+
+      Seq(true, false).foreach { offheapEnabled =>
+        val readSchema = new StructType().add("v", VariantType, nullable = 
true)
+        val file = TestUtils.listDirectory(dir).head
+        val reader = createVectorizedReader(file, readSchema, offheapEnabled)
+        try {
+          while (reader.nextKeyValue()) {}
+        } finally {
+          reader.close()
+          checkColumnVectorsAreClosed(reader)
+        }
+      }
+    }
+  }
+
   private def truncateTypeTest(
     testName: String,
     readType: DataType,
@@ -1015,6 +1099,10 @@ class ParquetVectorizedSuite extends ParquetTest with 
SharedSparkSession {
       })
       i += 1
     }
+
+    // SPARK-60098: Verify correct closure.
+    recordReader.close()
+    checkColumnVectorsAreClosed(recordReader)
   }
 
   private def toDebugString(schema: StructType, row: InternalRow): String = {
@@ -1040,6 +1128,67 @@ class ParquetVectorizedSuite extends ParquetTest with 
SharedSparkSession {
     }
   }
 
+  private def createVectorizedReader(
+      file: String,
+      readSchema: StructType,
+      offheapEnabled: Boolean): VectorizedParquetRecordReader = {
+    val conf = spark.sessionState.conf
+    val hadoopConf = spark.sessionState.newHadoopConf()
+    hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS, 
classOf[ParquetReadSupport].getName)
+    hadoopConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA, 
readSchema.json)
+    Seq(
+      SQLConf.CASE_SENSITIVE,
+      SQLConf.PARQUET_BINARY_AS_STRING,
+      SQLConf.PARQUET_INT96_AS_TIMESTAMP,
+      SQLConf.PARQUET_INFER_TIMESTAMP_NTZ_ENABLED,
+      SQLConf.LEGACY_PARQUET_NANOS_AS_LONG
+    ).foreach { entry =>
+      hadoopConf.set(entry.key, conf.getConfString(entry.key))
+    }
+
+    val split = new FileSplit(new Path(file), 0, new File(file).length(), 
Array.empty[String])
+    val context = new TaskAttemptContextImpl(hadoopConf, new TaskAttemptID())
+    val reader = new VectorizedParquetRecordReader(
+      offheapEnabled,
+      conf.parquetVectorizedReaderBatchSize
+    )
+
+    try {
+      reader.enableReturningBatches()
+      reader.initialize(split, context)
+    } catch {
+      case NonFatal(e) =>
+        reader.close()
+        throw e
+    }
+
+    reader
+  }
+
+  private def checkColumnVectorsAreClosed(reader: 
VectorizedParquetRecordReader): Unit = {
+    val columnVectors = ParquetTestAccess.columnVectors(reader)
+    if (columnVectors != null) {
+      columnVectors.foreach(v => 
checkIsClosed(v.asInstanceOf[ParquetColumnVector]))
+    }
+  }
+
+  private def checkIsClosed(parquetColumnVector: ParquetColumnVector): Unit = {
+    val vector = parquetColumnVector.getValueVector
+    assert(vector.isFree)
+
+    val definitionsVector = parquetColumnVector.getDefinitionLevelVector
+    if (definitionsVector != null) {
+      assert(definitionsVector.isFree)
+    }
+
+    val repetitionsVector = parquetColumnVector.getRepetitionLevelVector
+    if (repetitionsVector != null) {
+      assert(repetitionsVector.isFree)
+    }
+
+    parquetColumnVector.getChildren.forEach(checkIsClosed)
+  }
+
   case class TestParquetRowGroupReader(groups: Seq[PageReadStore]) extends 
ParquetRowGroupReader {
     private var index: Int = 0
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to