This is an automated email from the ASF dual-hosted git repository.
sunchao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 808d79616144 [SPARK-60098][SQL] Fix memory leaks in ParquetColumnVector
808d79616144 is described below
commit 808d79616144fbf209244e9c7a27927907908768
Author: Filipe Oliveira <[email protected]>
AuthorDate: Mon Oct 26 09:50:22 2026 -0700
[SPARK-60098][SQL] Fix memory leaks in ParquetColumnVector
### What changes were proposed in this pull request?
This tries to ensure that off-heap memory allocated by the Parquet reader
is deallocated. The class `ParquetColumnVector` uses the method
`WritableColumnVector.reserveNewColumn` in 4 scenarios. Given these column
vectors don't belong to ColumnarBatches and the class is not calling
`WritableColumnVector.close` when memory mode is OFF_HEAP these column vectors
are leaking.
To fix, the class `WritableColumnVector` now holds references to these
column vectors and closes them on `WritableColumnVector.close`.
### Why are the changes needed?
This will prevent OOMs for long running applications.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
The 4 scenarios using `WritableColumnVector.reserveNewColumn` are tested in
`ParquetVectorizedSuite`. Definition levels and repetition levels are tested by
existing tests. 2 new tests were added for the other 2 scenarios.
### Was this patch authored or co-authored using generative AI tooling?
No.
Closes #59657 from frroliveira/fix-parquet-vector-closure.
Authored-by: Filipe Oliveira <[email protected]>
Signed-off-by: Chao Sun <[email protected]>
---
.../execution/vectorized/OffHeapColumnVector.java | 7 +-
.../execution/vectorized/OnHeapColumnVector.java | 15 +-
.../execution/vectorized/WritableColumnVector.java | 60 ++++++--
.../datasources/parquet/ParquetTestAccess.scala | 5 +
.../parquet/ParquetVectorizedSuite.scala | 151 ++++++++++++++++++++-
5 files changed, 221 insertions(+), 17 deletions(-)
diff --git
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
index 1f3c2ac2e608..fc902090acc0 100644
---
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
+++
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OffHeapColumnVector.java
@@ -110,6 +110,11 @@ public final class OffHeapColumnVector extends
WritableColumnVector {
super.close();
}
+ @Override
+ public boolean isFree() {
+ return nulls == 0 && data == 0 && lengthData == 0 && offsetData == 0;
+ }
+
//
// APIs dealing with nulls
//
@@ -699,7 +704,7 @@ public final class OffHeapColumnVector extends
WritableColumnVector {
}
@Override
- public OffHeapColumnVector reserveNewColumn(int capacity, DataType type) {
+ protected OffHeapColumnVector reserveNewInternalColumn(int capacity,
DataType type) {
return new OffHeapColumnVector(capacity, type);
}
}
diff --git
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
index eb12f86858f4..1a6017de5234 100644
---
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
+++
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/OnHeapColumnVector.java
@@ -98,6 +98,19 @@ public final class OnHeapColumnVector extends
WritableColumnVector {
super.close();
}
+ @Override
+ public boolean isFree() {
+ return nulls == null &&
+ byteData == null &&
+ shortData == null &&
+ intData == null &&
+ longData == null &&
+ floatData == null &&
+ doubleData == null &&
+ arrayLengths == null &&
+ arrayOffsets == null;
+ }
+
//
// APIs dealing with nulls
//
@@ -665,7 +678,7 @@ public final class OnHeapColumnVector extends
WritableColumnVector {
}
@Override
- public OnHeapColumnVector reserveNewColumn(int capacity, DataType type) {
+ protected OnHeapColumnVector reserveNewInternalColumn(int capacity, DataType
type) {
return new OnHeapColumnVector(capacity, type);
}
}
diff --git
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
index fab67965d701..dd1d67d3348e 100644
---
a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
+++
b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/WritableColumnVector.java
@@ -19,6 +19,8 @@ package org.apache.spark.sql.execution.vectorized;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Optional;
import com.google.common.annotations.VisibleForTesting;
@@ -100,6 +102,13 @@ public abstract class WritableColumnVector extends
ColumnVector {
@Override
public void close() {
+ if (reservedColumns != null) {
+ for (WritableColumnVector column : reservedColumns) {
+ column.close();
+ }
+ reservedColumns.clear();
+ reservedColumns = null;
+ }
if (childColumns != null) {
for (int i = 0; i < childColumns.length; i++) {
childColumns[i].close();
@@ -120,6 +129,12 @@ public abstract class WritableColumnVector extends
ColumnVector {
// no-op
}
+ /**
+ * Returns true if this vector's own data buffers are released; vectors it
owns are not checked.
+ */
+ @VisibleForTesting
+ public abstract boolean isFree();
+
public void reserveAdditional(int additionalCapacity) {
reserve(elementsAppended + additionalCapacity);
}
@@ -211,7 +226,7 @@ public abstract class WritableColumnVector extends
ColumnVector {
*/
public WritableColumnVector reserveDictionaryIds(int capacity) {
if (dictionaryIds == null) {
- dictionaryIds = reserveNewColumn(capacity, DataTypes.IntegerType);
+ dictionaryIds = reserveNewInternalColumn(capacity,
DataTypes.IntegerType);
} else {
dictionaryIds.reset();
dictionaryIds.reserve(capacity);
@@ -1033,9 +1048,26 @@ public abstract class WritableColumnVector extends
ColumnVector {
protected WritableColumnVector[] childColumns;
/**
- * Reserve a new column.
+ * Publicly reserved columns.
+ */
+ private List<WritableColumnVector> reservedColumns;
+
+ /**
+ * Reserve a new column owned by this column vector.
+ */
+ public WritableColumnVector reserveNewColumn(int capacity, DataType type) {
+ WritableColumnVector column = reserveNewInternalColumn(capacity, type);
+ if (reservedColumns == null) {
+ reservedColumns = new ArrayList<>(1);
+ }
+ reservedColumns.add(column);
+ return column;
+ }
+
+ /**
+ * Reserve a new column for internal usage.
*/
- public abstract WritableColumnVector reserveNewColumn(int capacity, DataType
type);
+ protected abstract WritableColumnVector reserveNewInternalColumn(int
capacity, DataType type);
protected boolean isArray() {
return type instanceof ArrayType || type instanceof BinaryType || type
instanceof StringType ||
@@ -1072,31 +1104,31 @@ public abstract class WritableColumnVector extends
ColumnVector {
childCapacity *= DEFAULT_ARRAY_LENGTH;
}
this.childColumns = new WritableColumnVector[1];
- this.childColumns[0] = reserveNewColumn(childCapacity, childType);
+ this.childColumns[0] = reserveNewInternalColumn(childCapacity,
childType);
} else if (type instanceof StructType st) {
this.childColumns = new WritableColumnVector[st.fields().length];
for (int i = 0; i < childColumns.length; ++i) {
- this.childColumns[i] = reserveNewColumn(capacity,
st.fields()[i].dataType());
+ this.childColumns[i] = reserveNewInternalColumn(capacity,
st.fields()[i].dataType());
}
} else if (type instanceof MapType mapType) {
this.childColumns = new WritableColumnVector[2];
- this.childColumns[0] = reserveNewColumn(capacity, mapType.keyType());
- this.childColumns[1] = reserveNewColumn(capacity, mapType.valueType());
+ this.childColumns[0] = reserveNewInternalColumn(capacity,
mapType.keyType());
+ this.childColumns[1] = reserveNewInternalColumn(capacity,
mapType.valueType());
} else if (type instanceof CalendarIntervalType) {
// Three columns. Months as int. Days as Int. Microseconds as Long.
this.childColumns = new WritableColumnVector[3];
- this.childColumns[0] = reserveNewColumn(capacity, DataTypes.IntegerType);
- this.childColumns[1] = reserveNewColumn(capacity, DataTypes.IntegerType);
- this.childColumns[2] = reserveNewColumn(capacity, DataTypes.LongType);
+ this.childColumns[0] = reserveNewInternalColumn(capacity,
DataTypes.IntegerType);
+ this.childColumns[1] = reserveNewInternalColumn(capacity,
DataTypes.IntegerType);
+ this.childColumns[2] = reserveNewInternalColumn(capacity,
DataTypes.LongType);
} else if (type instanceof AnyTimestampNanoType) {
// Two columns. EpochMicros as Long. NanosWithinMicro as Short.
this.childColumns = new WritableColumnVector[2];
- this.childColumns[0] = reserveNewColumn(capacity, DataTypes.LongType);
- this.childColumns[1] = reserveNewColumn(capacity, DataTypes.ShortType);
+ this.childColumns[0] = reserveNewInternalColumn(capacity,
DataTypes.LongType);
+ this.childColumns[1] = reserveNewInternalColumn(capacity,
DataTypes.ShortType);
} else if (type instanceof VariantType) {
this.childColumns = new WritableColumnVector[2];
- this.childColumns[0] = reserveNewColumn(capacity, DataTypes.BinaryType);
- this.childColumns[1] = reserveNewColumn(capacity, DataTypes.BinaryType);
+ this.childColumns[0] = reserveNewInternalColumn(capacity,
DataTypes.BinaryType);
+ this.childColumns[1] = reserveNewInternalColumn(capacity,
DataTypes.BinaryType);
} else {
this.childColumns = null;
}
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
index 1f0a261019dd..987a7c286a79 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetTestAccess.scala
@@ -48,6 +48,7 @@ import org.apache.spark.util.SparkClassUtils
* - `LateMaterializationParquetRecordReader`'s scratch vectors and queued
survivor sets, the
* column readers and row indexes a reader still holds, and a vector's
byte-child capacity
* (private state a test checks the reader's memory by)
+ * - `VectorizedParquetRecordReader.columnVectors` (field).
*/
object ParquetTestAccess {
@@ -254,6 +255,10 @@ object ParquetTestAccess {
f
}
+ def columnVectors(reader: VectorizedParquetRecordReader): Array[AnyRef] = {
+ columnVectorsField.get(reader).asInstanceOf[Array[AnyRef]]
+ }
+
private lazy val columnVectorCls = SparkClassUtils.classForName[Any](
"org.apache.spark.sql.execution.datasources.parquet.ParquetColumnVector")
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
index 5749b602a635..783d8282f2bb 100644
---
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetVectorizedSuite.scala
@@ -17,23 +17,30 @@
package org.apache.spark.sql.execution.datasources.parquet
-import java.io.IOException
+import java.io.{File, IOException}
import java.util.{Optional, PrimitiveIterator}
import scala.collection.mutable.ArrayBuffer
import scala.language.implicitConversions
+import scala.util.control.NonFatal
+import org.apache.hadoop.fs.Path
+import org.apache.hadoop.mapred.FileSplit
+import org.apache.hadoop.mapreduce.TaskAttemptID
+import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl
import org.apache.parquet.bytes.BytesInput
import org.apache.parquet.column.{ColumnDescriptor, Encoding,
ParquetProperties}
import org.apache.parquet.column.impl.ColumnWriteStoreV1
import org.apache.parquet.column.page._
import org.apache.parquet.column.page.mem.MemPageStore
import org.apache.parquet.column.statistics.Statistics
+import org.apache.parquet.hadoop.ParquetInputFormat
import org.apache.parquet.io.ParquetDecodingException
import org.apache.parquet.io.api.Binary
import org.apache.parquet.schema.{MessageType, MessageTypeParser}
import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
+import org.apache.spark.TestUtils
import org.apache.spark.memory.MemoryMode
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.InternalRow
@@ -41,8 +48,10 @@ import
org.apache.spark.sql.catalyst.expressions.{RowOrdering, SpecificInternalR
import org.apache.spark.sql.catalyst.util.DateTimeUtils
import
org.apache.spark.sql.execution.datasources.parquet.SpecificParquetRecordReaderBase.ParquetRowGroupReader
import org.apache.spark.sql.execution.vectorized.ColumnVectorUtils
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types._
+import org.apache.spark.types.variant.VariantBuilder
import org.apache.spark.util.ArrayImplicits._
import org.apache.spark.util.collection.Utils.createArray
@@ -801,6 +810,81 @@ class ParquetVectorizedSuite extends ParquetTest with
SharedSparkSession {
.add("_2", IntegerType)
)
+ test("SPARK-60098: Vectorized reader closes column vectors when missing all
struct fields") {
+ withTempPath { dir =>
+ val data = Seq(
+ Tuple1((1, "a")),
+ Tuple1((2, null)),
+ Tuple1(null)
+ )
+
spark.createDataFrame(data).coalesce(1).write.parquet(dir.getCanonicalPath)
+
+ Seq(true, false).foreach { offheapEnabled =>
+ val readSchema = new StructType().add("_1",
+ new StructType()
+ .add("_3", IntegerType, nullable = true)
+ .add("_4", LongType, nullable = true),
+ nullable = true
+ )
+ val file = TestUtils.listDirectory(dir).head
+ val reader = createVectorizedReader(file, readSchema, offheapEnabled)
+ try {
+ while (reader.nextKeyValue()) {}
+ } finally {
+ reader.close()
+ checkColumnVectorsAreClosed(reader)
+ }
+ }
+ }
+ }
+
+ test("SPARK-60098: Vectorized reader closes variant type column vectors") {
+ withTempPath { dir =>
+ val scalarTypes = Array(IntegerType, StringType)
+ val variantBuilder = new VariantBuilder(false)
+ scalarTypes.indices.map(_.toString).foreach(variantBuilder.addKey)
+ val rows = Seq(
+ Row(
+ Row(
+ variantBuilder.result().getMetadata,
+ null,
+ Row.fromSeq(Seq[Any](3, "str10").map(Row(null, _)))
+ )
+ )
+ )
+ val writeSchema =
+ StructType(
+ Array(
+ StructField(
+ "v",
+ SparkShreddingUtils.variantShreddingSchema(
+ StructType(
+ scalarTypes.zipWithIndex.map { case (t, i) =>
StructField(i.toString, t) }
+ )
+ )
+ )
+ )
+ )
+
+ spark
+ .createDataFrame(spark.sparkContext.parallelize(rows, numSlices = 1),
writeSchema)
+ .write
+ .parquet(dir.getCanonicalPath)
+
+ Seq(true, false).foreach { offheapEnabled =>
+ val readSchema = new StructType().add("v", VariantType, nullable =
true)
+ val file = TestUtils.listDirectory(dir).head
+ val reader = createVectorizedReader(file, readSchema, offheapEnabled)
+ try {
+ while (reader.nextKeyValue()) {}
+ } finally {
+ reader.close()
+ checkColumnVectorsAreClosed(reader)
+ }
+ }
+ }
+ }
+
private def truncateTypeTest(
testName: String,
readType: DataType,
@@ -1015,6 +1099,10 @@ class ParquetVectorizedSuite extends ParquetTest with
SharedSparkSession {
})
i += 1
}
+
+ // SPARK-60098: Verify correct closure.
+ recordReader.close()
+ checkColumnVectorsAreClosed(recordReader)
}
private def toDebugString(schema: StructType, row: InternalRow): String = {
@@ -1040,6 +1128,67 @@ class ParquetVectorizedSuite extends ParquetTest with
SharedSparkSession {
}
}
+ private def createVectorizedReader(
+ file: String,
+ readSchema: StructType,
+ offheapEnabled: Boolean): VectorizedParquetRecordReader = {
+ val conf = spark.sessionState.conf
+ val hadoopConf = spark.sessionState.newHadoopConf()
+ hadoopConf.set(ParquetInputFormat.READ_SUPPORT_CLASS,
classOf[ParquetReadSupport].getName)
+ hadoopConf.set(ParquetReadSupport.SPARK_ROW_REQUESTED_SCHEMA,
readSchema.json)
+ Seq(
+ SQLConf.CASE_SENSITIVE,
+ SQLConf.PARQUET_BINARY_AS_STRING,
+ SQLConf.PARQUET_INT96_AS_TIMESTAMP,
+ SQLConf.PARQUET_INFER_TIMESTAMP_NTZ_ENABLED,
+ SQLConf.LEGACY_PARQUET_NANOS_AS_LONG
+ ).foreach { entry =>
+ hadoopConf.set(entry.key, conf.getConfString(entry.key))
+ }
+
+ val split = new FileSplit(new Path(file), 0, new File(file).length(),
Array.empty[String])
+ val context = new TaskAttemptContextImpl(hadoopConf, new TaskAttemptID())
+ val reader = new VectorizedParquetRecordReader(
+ offheapEnabled,
+ conf.parquetVectorizedReaderBatchSize
+ )
+
+ try {
+ reader.enableReturningBatches()
+ reader.initialize(split, context)
+ } catch {
+ case NonFatal(e) =>
+ reader.close()
+ throw e
+ }
+
+ reader
+ }
+
+ private def checkColumnVectorsAreClosed(reader:
VectorizedParquetRecordReader): Unit = {
+ val columnVectors = ParquetTestAccess.columnVectors(reader)
+ if (columnVectors != null) {
+ columnVectors.foreach(v =>
checkIsClosed(v.asInstanceOf[ParquetColumnVector]))
+ }
+ }
+
+ private def checkIsClosed(parquetColumnVector: ParquetColumnVector): Unit = {
+ val vector = parquetColumnVector.getValueVector
+ assert(vector.isFree)
+
+ val definitionsVector = parquetColumnVector.getDefinitionLevelVector
+ if (definitionsVector != null) {
+ assert(definitionsVector.isFree)
+ }
+
+ val repetitionsVector = parquetColumnVector.getRepetitionLevelVector
+ if (repetitionsVector != null) {
+ assert(repetitionsVector.isFree)
+ }
+
+ parquetColumnVector.getChildren.forEach(checkIsClosed)
+ }
+
case class TestParquetRowGroupReader(groups: Seq[PageReadStore]) extends
ParquetRowGroupReader {
private var index: Int = 0
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]