This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new b0bd7cc8a0bc [SPARK-58448][CONNECT] Estimate IsotonicRegressionModel 
size
b0bd7cc8a0bc is described below

commit b0bd7cc8a0bcbb628e30a11030b0c58ee9ed0a3f
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Fri Jul 31 08:25:48 2026 +0800

    [SPARK-58448][CONNECT] Estimate IsotonicRegressionModel size
    
    ### What changes were proposed in this pull request?
    
    This PR adds an explicit `estimatedSize` implementation for 
`IsotonicRegressionModel`. It accounts for model metadata and the legacy 
model's boundary and prediction arrays.
    
    ### Why are the changes needed?
    
    ML Connect cache accounting needs a bounded, model-specific estimate of the 
state retained by `IsotonicRegressionModel`, rather than relying on generic 
object-graph traversal.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Added a focused `IsotonicRegressionSuite` assertion that verifies the 
estimate includes metadata and both learned arrays.
    
    ```
    build/sbt -java-home /usr/lib/jvm/java-17-openjdk-amd64 'mllib/testOnly 
*IsotonicRegressionSuite'
    ```
    
    Passed: 36 tests, 0 failures.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #57652 from zhengruifeng/isotonic-regression-model-size-metadata.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
---
 .../org/apache/spark/ml/regression/IsotonicRegression.scala  | 12 ++++++++++++
 .../apache/spark/ml/regression/IsotonicRegressionSuite.scala | 11 +++++++++++
 2 files changed, 23 insertions(+)

diff --git 
a/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala 
b/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
index 6eddcb416d8a..d6c7af65d841 100644
--- 
a/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
+++ 
b/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
@@ -38,6 +38,7 @@ import org.apache.spark.sql.{DataFrame, Dataset, Row}
 import org.apache.spark.sql.functions.{col, udf}
 import org.apache.spark.sql.types.{DoubleType, StructType}
 import org.apache.spark.storage.StorageLevel
+import org.apache.spark.util.SizeEstimator
 
 /**
  * Params for isotonic regression.
@@ -246,6 +247,17 @@ class IsotonicRegressionModel private[ml] (
     copyValues(new IsotonicRegressionModel(uid, oldModel), 
extra).setParent(parent)
   }
 
+  private[spark] override def estimatedSize: Long = {
+    var size = estimateMatadataSize
+    if (oldModel != null) {
+      // boundaries: Array[Double]
+      size += SizeEstimator.estimate(oldModel.boundaries)
+      // predictions: Array[Double]
+      size += SizeEstimator.estimate(oldModel.predictions)
+    }
+    size
+  }
+
   @Since("2.0.0")
   override def transform(dataset: Dataset[_]): DataFrame = {
     val outputSchema = transformSchema(dataset.schema, logging = true)
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/regression/IsotonicRegressionSuite.scala
 
b/mllib/src/test/scala/org/apache/spark/ml/regression/IsotonicRegressionSuite.scala
index 3077a60b56b7..ae779a51e599 100644
--- 
a/mllib/src/test/scala/org/apache/spark/ml/regression/IsotonicRegressionSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/regression/IsotonicRegressionSuite.scala
@@ -22,6 +22,7 @@ import org.apache.spark.ml.param.ParamsSuite
 import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
 import org.apache.spark.sql.{DataFrame, Row}
 import org.apache.spark.sql.functions.col
+import org.apache.spark.util.SizeEstimator
 
 class IsotonicRegressionSuite extends MLTest with DefaultReadWriteTest {
 
@@ -67,6 +68,16 @@ class IsotonicRegressionSuite extends MLTest with 
DefaultReadWriteTest {
     }
   }
 
+  test("model size estimation") {
+    val dataset = generateIsotonicInput(Seq(1, 2, 3, 1, 6, 17, 16, 17, 18))
+    val model = new IsotonicRegression().fit(dataset)
+
+    val expectedSize = model.estimateMatadataSize +
+      SizeEstimator.estimate(model.boundaries.toArray) +
+      SizeEstimator.estimate(model.predictions.toArray)
+    assert(model.estimatedSize === expectedSize)
+  }
+
   test("params validation") {
     val dataset = generateIsotonicInput(Seq(1, 2, 3))
     val ir = new IsotonicRegression


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to