This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7864cd1c50bb [SPARK-58395][ML][CONNECT] Include LDA metadata in size
estimates
7864cd1c50bb is described below
commit 7864cd1c50bb776788459d241b437c10bf11fa25
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Mon Aug 3 16:29:12 2026 +0800
[SPARK-58395][ML][CONNECT] Include LDA metadata in size estimates
### What changes were proposed in this pull request?
Adds parameter metadata to `DistributedLDAModel.estimatedSize` while
retaining its existing Arrow-data estimates for the serialized LDA internals.
Adds a specialized `LocalLDAModel.estimatedSize` that charges its learned topic
matrix and document-concentration vector without traversing its Spark session.
`LDAParams.supportedOptimizers` remains an `Array[String]` field for ABI
compatibility. It directly references an equivalent companion-owned array,
which the optimizer validator uses so its closure does not retain an LDA model.
Adds regression coverage for both model types.
### Why are the changes needed?
The distributed-model estimate omitted parameter metadata. Local LDA relied
on the generic object-graph walk, which can include its direct `SparkSession`
reference.
Restoring the instance `supportedOptimizers` field for ABI compatibility
would cause the validator lambda to capture the model if it referenced that
field. `SizeEstimator` then follows the closure into unrelated Spark-session
state and significantly overestimates the model size. Using the companion-owned
array preserves the field while avoiding that capture.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added `LocalLDAModel estimated size` and `DistributedLDAModel estimated
size` coverage in `LDASuite`. The suite was not run locally.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57589 from zhengruifeng/lda-model-size-metadata-dev3.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../scala/org/apache/spark/ml/clustering/LDA.scala | 30 ++++++++++++++++++----
.../org/apache/spark/ml/clustering/LDASuite.scala | 18 +++++++++++++
2 files changed, 43 insertions(+), 5 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index e4736718430b..0fc7174cadbb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -157,7 +157,7 @@ private[clustering] trait LDAParams extends Params with
HasFeaturesCol with HasM
/** Supported values for Param [[optimizer]]. */
@Since("1.6.0")
- final val supportedOptimizers: Array[String] = Array("online", "em")
+ final val supportedOptimizers: Array[String] = LDA.supportedOptimizers
/**
* Optimizer or inference algorithm used to estimate the LDA model.
@@ -180,7 +180,7 @@ private[clustering] trait LDAParams extends Params with
HasFeaturesCol with HasM
@Since("1.6.0")
final val optimizer = new Param[String](this, "optimizer", "Optimizer or
inference" +
" algorithm used to estimate the LDA model. Supported: " +
supportedOptimizers.mkString(", "),
- (value: String) =>
supportedOptimizers.contains(value.toLowerCase(Locale.ROOT)))
+ (value: String) =>
LDA.supportedOptimizers.contains(value.toLowerCase(Locale.ROOT)))
/** @group getParam */
@Since("1.6.0")
@@ -640,6 +640,21 @@ class LocalLDAModel private[ml] (
override def toString: String = {
s"LocalLDAModel: uid=$uid, k=${$(k)}, numFeatures=$vocabSize"
}
+
+ private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
+ if (oldLocalModel != null) {
+ // topicsMatrix: Matrix
+ if (oldLocalModel.topicsMatrix != null) {
+ size += oldLocalModel.topicsMatrix.asML.getSizeInBytes
+ }
+ // docConcentration: Vector
+ if (oldLocalModel.docConcentration != null) {
+ size += oldLocalModel.docConcentration.asML.getSizeInBytes
+ }
+ }
+ size
+ }
}
@Since("1.6.0")
@@ -827,12 +842,15 @@ class DistributedLDAModel private[ml] (
}
private[spark] override def estimatedSize: Long = {
- this.oldDistributedModel.toInternals.map {
+ var size = estimateMatadataSize
+ // oldDistributedModel: metadata, global topic totals, graph vertices, and
graph edges.
+ oldDistributedModel.toInternals.foreach {
case df: org.apache.spark.sql.classic.DataFrame =>
- df.toArrowBatchRdd.map(_.length.toLong).reduce(_ + _)
+ size += df.toArrowBatchRdd.map(_.length.toLong).reduce(_ + _)
case o => throw new UnsupportedOperationException(
s"Unsupported dataframe type: ${o.getClass.getName}")
- }.sum
+ }
+ size
}
}
@@ -1056,6 +1074,8 @@ class LDA @Since("1.6.0") (
@Since("2.0.0")
object LDA extends MLReadable[LDA] {
+ private[clustering] val supportedOptimizers: Array[String] = Array("online",
"em")
+
/** Get dataset for spark.mllib LDA */
private[clustering] def getOldDataset(
dataset: Dataset[_],
diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala
index a0223396da31..0b8949534003 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala
@@ -236,6 +236,24 @@ class LDASuite extends MLTest with DefaultReadWriteTest {
assert(lp <= 0.0 && lp != Double.NegativeInfinity)
}
+ test("LocalLDAModel estimated size") {
+ val lda = new LDA().setK(2).setSeed(1).setOptimizer("online").setMaxIter(1)
+ val model = lda.fit(LDASuite.generateLDAData(spark, 3, 2, 3))
+ val maxSize = 1024 * 16
+ assert(
+ model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
+ test("DistributedLDAModel estimated size") {
+ val lda = new LDA().setK(2).setSeed(1).setOptimizer("em").setMaxIter(1)
+ val model = lda.fit(LDASuite.generateLDAData(spark, 3, 2, 3))
+ val maxSize = 1024 * 16
+ assert(
+ model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
test("read/write LocalLDAModel") {
def checkModelData(model: LDAModel, model2: LDAModel): Unit = {
assert(model.vocabSize === model2.vocabSize)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]