This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new a17c08472af4 [SPARK-58360][ML][CONNECT] Avoid nested parent overcount
in RFormulaModel size estimate
a17c08472af4 is described below
commit a17c08472af451de6624a901da8ddcf6fb034369
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Jul 28 16:19:45 2026 +0800
[SPARK-58360][ML][CONNECT] Avoid nested parent overcount in RFormulaModel
size estimate
### What changes were proposed in this pull request?
This PR adds a specialized `RFormulaModel.estimatedSize` implementation. It
counts the model metadata, the resolved formula, and the nested `PipelineModel`
through its parent-safe estimate.
### Why are the changes needed?
The default model-size estimate clears only the outer model parent.
`RFormulaModel` retains its nested pipeline, whose stages can otherwise retain
estimator-parent graphs and overcount Spark session or context state in ML
Connect cache accounting.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added `RFormulaSuite` coverage that fits a categorical R formula and
verifies that the fitted model estimate stays below the expected upper bound.
Also ran `git diff --check`, ASCII, and source-line-length checks.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57552 from zhengruifeng/rformula-size-estimate-dev3.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit 6f1257366a04a5fe54e866962dd2cb2851e5e42c)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../src/main/scala/org/apache/spark/ml/feature/RFormula.scala | 10 ++++++++++
.../test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala | 9 +++++++++
2 files changed, 19 insertions(+)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
index d4d9aade5b25..844cedb4a302 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
@@ -34,6 +34,7 @@ import org.apache.spark.ml.util._
import org.apache.spark.sql.{DataFrame, Dataset}
import org.apache.spark.sql.functions.col
import org.apache.spark.sql.types._
+import org.apache.spark.util.SizeEstimator
/**
* Base trait for [[RFormula]] and [[RFormulaModel]].
@@ -354,6 +355,15 @@ class RFormulaModel private[feature](
// For ml connect only
private[ml] def this() = this("", null, null)
+ private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
+ // ResolvedRFormula(label: String, terms: Seq[Seq[String]], hasIntercept:
Boolean)
+ size += SizeEstimator.estimate(resolvedFormula)
+ // pipelineModel: PipelineModel
+ size += pipelineModel.estimatedSize
+ size
+ }
+
@Since("2.0.0")
override def transform(dataset: Dataset[_]): DataFrame = {
checkCanTransform(dataset.schema)
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
index d4dd4fa33a3e..3f90749d011e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
@@ -54,6 +54,15 @@ class RFormulaSuite extends MLTest with DefaultReadWriteTest
{
ParamsSuite.checkParams(new RFormula())
}
+ test("RFormulaModel estimated size") {
+ val dataset = Seq(("a", 1.0), ("b", 2.0), ("c", 3.0)).toDF("category",
"label")
+ val model = new RFormula().setFormula("label ~ category").fit(dataset)
+ val maxSize = 1024 * 16
+ assert(
+ model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
test("transform numeric data") {
val formula = new RFormula().setFormula("id ~ v1 + v2")
val original = Seq((0, 1.0, 3.0), (2, 2.0, 5.0)).toDF("id", "v1", "v2")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]