This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 7864cd1c50bb [SPARK-58395][ML][CONNECT] Include LDA metadata in size 
estimates
7864cd1c50bb is described below

commit 7864cd1c50bb776788459d241b437c10bf11fa25
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Mon Aug 3 16:29:12 2026 +0800

    [SPARK-58395][ML][CONNECT] Include LDA metadata in size estimates
    
    ### What changes were proposed in this pull request?
    
    Adds parameter metadata to `DistributedLDAModel.estimatedSize` while 
retaining its existing Arrow-data estimates for the serialized LDA internals. 
Adds a specialized `LocalLDAModel.estimatedSize` that charges its learned topic 
matrix and document-concentration vector without traversing its Spark session.
    
    `LDAParams.supportedOptimizers` remains an `Array[String]` field for ABI 
compatibility. It directly references an equivalent companion-owned array, 
which the optimizer validator uses so its closure does not retain an LDA model. 
Adds regression coverage for both model types.
    
    ### Why are the changes needed?
    
    The distributed-model estimate omitted parameter metadata. Local LDA relied 
on the generic object-graph walk, which can include its direct `SparkSession` 
reference.
    
    Restoring the instance `supportedOptimizers` field for ABI compatibility 
would cause the validator lambda to capture the model if it referenced that 
field. `SizeEstimator` then follows the closure into unrelated Spark-session 
state and significantly overestimates the model size. Using the companion-owned 
array preserves the field while avoiding that capture.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Added `LocalLDAModel estimated size` and `DistributedLDAModel estimated 
size` coverage in `LDASuite`. The suite was not run locally.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #57589 from zhengruifeng/lda-model-size-metadata-dev3.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
---
 .../scala/org/apache/spark/ml/clustering/LDA.scala | 30 ++++++++++++++++++----
 .../org/apache/spark/ml/clustering/LDASuite.scala  | 18 +++++++++++++
 2 files changed, 43 insertions(+), 5 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala 
b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
index e4736718430b..0fc7174cadbb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/LDA.scala
@@ -157,7 +157,7 @@ private[clustering] trait LDAParams extends Params with 
HasFeaturesCol with HasM
 
   /** Supported values for Param [[optimizer]]. */
   @Since("1.6.0")
-  final val supportedOptimizers: Array[String] = Array("online", "em")
+  final val supportedOptimizers: Array[String] = LDA.supportedOptimizers
 
   /**
    * Optimizer or inference algorithm used to estimate the LDA model.
@@ -180,7 +180,7 @@ private[clustering] trait LDAParams extends Params with 
HasFeaturesCol with HasM
   @Since("1.6.0")
   final val optimizer = new Param[String](this, "optimizer", "Optimizer or 
inference" +
     " algorithm used to estimate the LDA model. Supported: " + 
supportedOptimizers.mkString(", "),
-    (value: String) => 
supportedOptimizers.contains(value.toLowerCase(Locale.ROOT)))
+    (value: String) => 
LDA.supportedOptimizers.contains(value.toLowerCase(Locale.ROOT)))
 
   /** @group getParam */
   @Since("1.6.0")
@@ -640,6 +640,21 @@ class LocalLDAModel private[ml] (
   override def toString: String = {
     s"LocalLDAModel: uid=$uid, k=${$(k)}, numFeatures=$vocabSize"
   }
+
+  private[spark] override def estimatedSize: Long = {
+    var size = estimateMatadataSize
+    if (oldLocalModel != null) {
+      // topicsMatrix: Matrix
+      if (oldLocalModel.topicsMatrix != null) {
+        size += oldLocalModel.topicsMatrix.asML.getSizeInBytes
+      }
+      // docConcentration: Vector
+      if (oldLocalModel.docConcentration != null) {
+        size += oldLocalModel.docConcentration.asML.getSizeInBytes
+      }
+    }
+    size
+  }
 }
 
 @Since("1.6.0")
@@ -827,12 +842,15 @@ class DistributedLDAModel private[ml] (
   }
 
   private[spark] override def estimatedSize: Long = {
-    this.oldDistributedModel.toInternals.map {
+    var size = estimateMatadataSize
+    // oldDistributedModel: metadata, global topic totals, graph vertices, and 
graph edges.
+    oldDistributedModel.toInternals.foreach {
       case df: org.apache.spark.sql.classic.DataFrame =>
-        df.toArrowBatchRdd.map(_.length.toLong).reduce(_ + _)
+        size += df.toArrowBatchRdd.map(_.length.toLong).reduce(_ + _)
       case o => throw new UnsupportedOperationException(
         s"Unsupported dataframe type: ${o.getClass.getName}")
-    }.sum
+    }
+    size
   }
 }
 
@@ -1056,6 +1074,8 @@ class LDA @Since("1.6.0") (
 @Since("2.0.0")
 object LDA extends MLReadable[LDA] {
 
+  private[clustering] val supportedOptimizers: Array[String] = Array("online", 
"em")
+
   /** Get dataset for spark.mllib LDA */
   private[clustering] def getOldDataset(
        dataset: Dataset[_],
diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala 
b/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala
index a0223396da31..0b8949534003 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/LDASuite.scala
@@ -236,6 +236,24 @@ class LDASuite extends MLTest with DefaultReadWriteTest {
     assert(lp <= 0.0 && lp != Double.NegativeInfinity)
   }
 
+  test("LocalLDAModel estimated size") {
+    val lda = new LDA().setK(2).setSeed(1).setOptimizer("online").setMaxIter(1)
+    val model = lda.fit(LDASuite.generateLDAData(spark, 3, 2, 3))
+    val maxSize = 1024 * 16
+    assert(
+      model.estimatedSize < maxSize,
+      s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+  }
+
+  test("DistributedLDAModel estimated size") {
+    val lda = new LDA().setK(2).setSeed(1).setOptimizer("em").setMaxIter(1)
+    val model = lda.fit(LDASuite.generateLDAData(spark, 3, 2, 3))
+    val maxSize = 1024 * 16
+    assert(
+      model.estimatedSize < maxSize,
+      s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+  }
+
   test("read/write LocalLDAModel") {
     def checkModelData(model: LDAModel, model2: LDAModel): Unit = {
       assert(model.vocabSize === model2.vocabSize)


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to