This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b0bd7cc8a0bc [SPARK-58448][CONNECT] Estimate IsotonicRegressionModel
size
b0bd7cc8a0bc is described below
commit b0bd7cc8a0bcbb628e30a11030b0c58ee9ed0a3f
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Fri Jul 31 08:25:48 2026 +0800
[SPARK-58448][CONNECT] Estimate IsotonicRegressionModel size
### What changes were proposed in this pull request?
This PR adds an explicit `estimatedSize` implementation for
`IsotonicRegressionModel`. It accounts for model metadata and the legacy
model's boundary and prediction arrays.
### Why are the changes needed?
ML Connect cache accounting needs a bounded, model-specific estimate of the
state retained by `IsotonicRegressionModel`, rather than relying on generic
object-graph traversal.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added a focused `IsotonicRegressionSuite` assertion that verifies the
estimate includes metadata and both learned arrays.
```
build/sbt -java-home /usr/lib/jvm/java-17-openjdk-amd64 'mllib/testOnly
*IsotonicRegressionSuite'
```
Passed: 36 tests, 0 failures.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57652 from zhengruifeng/isotonic-regression-model-size-metadata.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../org/apache/spark/ml/regression/IsotonicRegression.scala | 12 ++++++++++++
.../apache/spark/ml/regression/IsotonicRegressionSuite.scala | 11 +++++++++++
2 files changed, 23 insertions(+)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
b/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
index 6eddcb416d8a..d6c7af65d841 100644
---
a/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
+++
b/mllib/src/main/scala/org/apache/spark/ml/regression/IsotonicRegression.scala
@@ -38,6 +38,7 @@ import org.apache.spark.sql.{DataFrame, Dataset, Row}
import org.apache.spark.sql.functions.{col, udf}
import org.apache.spark.sql.types.{DoubleType, StructType}
import org.apache.spark.storage.StorageLevel
+import org.apache.spark.util.SizeEstimator
/**
* Params for isotonic regression.
@@ -246,6 +247,17 @@ class IsotonicRegressionModel private[ml] (
copyValues(new IsotonicRegressionModel(uid, oldModel),
extra).setParent(parent)
}
+ private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
+ if (oldModel != null) {
+ // boundaries: Array[Double]
+ size += SizeEstimator.estimate(oldModel.boundaries)
+ // predictions: Array[Double]
+ size += SizeEstimator.estimate(oldModel.predictions)
+ }
+ size
+ }
+
@Since("2.0.0")
override def transform(dataset: Dataset[_]): DataFrame = {
val outputSchema = transformSchema(dataset.schema, logging = true)
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/regression/IsotonicRegressionSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/regression/IsotonicRegressionSuite.scala
index 3077a60b56b7..ae779a51e599 100644
---
a/mllib/src/test/scala/org/apache/spark/ml/regression/IsotonicRegressionSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/ml/regression/IsotonicRegressionSuite.scala
@@ -22,6 +22,7 @@ import org.apache.spark.ml.param.ParamsSuite
import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
import org.apache.spark.sql.{DataFrame, Row}
import org.apache.spark.sql.functions.col
+import org.apache.spark.util.SizeEstimator
class IsotonicRegressionSuite extends MLTest with DefaultReadWriteTest {
@@ -67,6 +68,16 @@ class IsotonicRegressionSuite extends MLTest with
DefaultReadWriteTest {
}
}
+ test("model size estimation") {
+ val dataset = generateIsotonicInput(Seq(1, 2, 3, 1, 6, 17, 16, 17, 18))
+ val model = new IsotonicRegression().fit(dataset)
+
+ val expectedSize = model.estimateMatadataSize +
+ SizeEstimator.estimate(model.boundaries.toArray) +
+ SizeEstimator.estimate(model.predictions.toArray)
+ assert(model.estimatedSize === expectedSize)
+ }
+
test("params validation") {
val dataset = generateIsotonicInput(Seq(1, 2, 3))
val ir = new IsotonicRegression
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]