This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 51f1b648eec6 [SPARK-58509][ML][CONNECT] Exclude OneVsRest classifier
from model size estimation
51f1b648eec6 is described below
commit 51f1b648eec69b3e4e45b5eb11c31c507e1503f9
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Mon Aug 3 19:54:02 2026 +0800
[SPARK-58509][ML][CONNECT] Exclude OneVsRest classifier from model size
estimation
### What changes were proposed in this pull request?
This follow-up to SPARK-58250 excludes `OneVsRestModel.classifier` from
parameter-metadata sizing. It continues to account explicitly for label
metadata and the fitted child models.
The regression test initializes the source classifier logger before fitting
OneVsRest, ensuring the estimate does not retain shared runtime state through
the classifier param.
### Why are the changes needed?
The fitted OneVsRestModel retains the base classifier estimator in its
param map. That estimator can retain shared runtime state, including
SparkSession-related state, which must not contribute to Spark Connect ML
model-cache accounting.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Updated the OneVsRestModel estimated-size regression test.
Static checks: `git diff --check`, non-ASCII scan, and changed-Scala
line-length scan.
The targeted Scala suite was not run locally.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex GPT-5
Closes #57715 from zhengruifeng/one-vs-rest-model-size-dev3.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../scala/org/apache/spark/ml/classification/OneVsRest.scala | 10 ++++++++--
.../org/apache/spark/ml/classification/OneVsRestSuite.scala | 9 ++++++---
2 files changed, 14 insertions(+), 5 deletions(-)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
b/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
index 38c68fd7000c..42c6cf69600b 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/OneVsRest.scala
@@ -150,8 +150,14 @@ final class OneVsRestModel private[ml] (
val numFeatures: Int = models.head.numFeatures
private[spark] override def estimatedSize: Long = {
- estimateMatadataSize + SizeEstimator.estimate(labelMetadata) +
- models.iterator.map(_.estimatedSize).sum
+ var size = estimateMatadataSize(excluded = Seq(
+ // classifier: Param[ClassifierType]
+ classifier))
+ // labelMetadata: Metadata
+ size += SizeEstimator.estimate(labelMetadata)
+ // models: Array[_ <: ClassificationModel[_, _]]
+ size += models.iterator.map(_.estimatedSize).sum
+ size
}
/** @group setParam */
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala
index 70408731a20f..75839a81c1f8 100644
---
a/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/ml/classification/OneVsRestSuite.scala
@@ -68,7 +68,7 @@ class OneVsRestSuite extends MLTest with DefaultReadWriteTest
{
ParamsSuite.checkParams(model)
}
- test("SPARK-58250: OneVsRestModel estimated size") {
+ test("SPARK-58250 and SPARK-58509: OneVsRestModel estimated size") {
val trainingData = Seq(
(0.0, Vectors.dense(0.0, 0.0)),
(0.0, Vectors.dense(0.0, 1.0)),
@@ -77,13 +77,16 @@ class OneVsRestSuite extends MLTest with
DefaultReadWriteTest {
(2.0, Vectors.dense(2.0, 0.0)),
(2.0, Vectors.dense(2.0, 1.0))).toDF("label", "features")
+ val classifier = new LogisticRegression().setMaxIter(1)
+ // Initialize the classifier's logger before retaining it in the
OneVsRestModel.
+ classifier.fit(trainingData)
val model = new OneVsRest()
- .setClassifier(new LogisticRegression().setMaxIter(1))
+ .setClassifier(classifier)
.fit(trainingData)
val maxSize = 32 * 1024
assert(model.estimatedSize < maxSize,
- s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ s"Estimation (${model.estimatedSize}) should not include shared runtime
state")
}
test("one-vs-rest: default params") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]