This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new b2311a07c73f [SPARK-58509][ML][CONNECT] Exclude OneVsRest classifier 
from model size estimation
b2311a07c73f is described below

commit b2311a07c73fced09f20c2d2cd1acb5f63faddf1
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Mon Aug 3 19:54:02 2026 +0800

    [SPARK-58509][ML][CONNECT] Exclude OneVsRest classifier from model size 
estimation
    
    ### What changes were proposed in this pull request?
    
    This follow-up to SPARK-58250 excludes `OneVsRestModel.classifier` from 
parameter-metadata sizing. It continues to account explicitly for label 
metadata and the fitted child models.
    
    The regression test initializes the source classifier logger before fitting 
OneVsRest, ensuring the estimate does not retain shared runtime state through 
the classifier param.
    
    ### Why are the changes needed?
    
    The fitted OneVsRestModel retains the base classifier estimator in its 
param map. That estimator can retain shared runtime state, including 
SparkSession-related state, which must not contribute to Spark Connect ML 
model-cache accounting.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Updated the OneVsRestModel estimated-size regression test.
    
    Static checks: `git diff --check`, non-ASCII scan, and changed-Scala 
line-length scan.
    
    The targeted Scala suite was not run locally.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex GPT-5
    
    Closes #57715 from zhengruifeng/one-vs-rest-model-size-dev3.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
    (cherry picked from commit 51f1b648eec69b3e4e45b5eb11c31c507e1503f9)
    Signed-off-by: Ruifeng Zheng <[email protected]>
---
 .../scala/org/apache/spark/ml/classification/OneVsRest.scala   | 10 ++++++++--
 .../org/apache/spark/ml/classification/OneVsRestSuite.scala    |  9 ++++++---
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git 
a/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala 
b/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
index 38c68fd7000c..42c6cf69600b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
@@ -150,8 +150,14 @@ final class OneVsRestModel private[ml] (
   val numFeatures: Int = models.head.numFeatures
 
   private[spark] override def estimatedSize: Long = {
-    estimateMatadataSize + SizeEstimator.estimate(labelMetadata) +
-      models.iterator.map(_.estimatedSize).sum
+    var size = estimateMatadataSize(excluded = Seq(
+      // classifier: Param[ClassifierType]
+      classifier))
+    // labelMetadata: Metadata
+    size += SizeEstimator.estimate(labelMetadata)
+    // models: Array[_ <: ClassificationModel[_, _]]
+    size += models.iterator.map(_.estimatedSize).sum
+    size
   }
 
   /** @group setParam */
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala 
b/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala
index 70408731a20f..75839a81c1f8 100644
--- 
a/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala
@@ -68,7 +68,7 @@ class OneVsRestSuite extends MLTest with DefaultReadWriteTest 
{
     ParamsSuite.checkParams(model)
   }
 
-  test("SPARK-58250: OneVsRestModel estimated size") {
+  test("SPARK-58250 and SPARK-58509: OneVsRestModel estimated size") {
     val trainingData = Seq(
       (0.0, Vectors.dense(0.0, 0.0)),
       (0.0, Vectors.dense(0.0, 1.0)),
@@ -77,13 +77,16 @@ class OneVsRestSuite extends MLTest with 
DefaultReadWriteTest {
       (2.0, Vectors.dense(2.0, 0.0)),
       (2.0, Vectors.dense(2.0, 1.0))).toDF("label", "features")
 
+    val classifier = new LogisticRegression().setMaxIter(1)
+    // Initialize the classifier's logger before retaining it in the 
OneVsRestModel.
+    classifier.fit(trainingData)
     val model = new OneVsRest()
-      .setClassifier(new LogisticRegression().setMaxIter(1))
+      .setClassifier(classifier)
       .fit(trainingData)
 
     val maxSize = 32 * 1024
     assert(model.estimatedSize < maxSize,
-      s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+      s"Estimation (${model.estimatedSize}) should not include shared runtime 
state")
   }
 
   test("one-vs-rest: default params") {


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to