This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 6f1257366a04 [SPARK-58360][ML][CONNECT] Avoid nested parent overcount 
in RFormulaModel size estimate
6f1257366a04 is described below

commit 6f1257366a04a5fe54e866962dd2cb2851e5e42c
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Jul 28 16:19:45 2026 +0800

    [SPARK-58360][ML][CONNECT] Avoid nested parent overcount in RFormulaModel 
size estimate
    
    ### What changes were proposed in this pull request?
    
    This PR adds a specialized `RFormulaModel.estimatedSize` implementation. It 
counts the model metadata, the resolved formula, and the nested `PipelineModel` 
through its parent-safe estimate.
    
    ### Why are the changes needed?
    
    The default model-size estimate clears only the outer model parent. 
`RFormulaModel` retains its nested pipeline, whose stages can otherwise retain 
estimator-parent graphs and overcount Spark session or context state in ML 
Connect cache accounting.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Added `RFormulaSuite` coverage that fits a categorical R formula and 
verifies that the fitted model estimate stays below the expected upper bound.
    
    Also ran `git diff --check`, ASCII, and source-line-length checks.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #57552 from zhengruifeng/rformula-size-estimate-dev3.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
---
 .../src/main/scala/org/apache/spark/ml/feature/RFormula.scala  | 10 ++++++++++
 .../test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala |  9 +++++++++
 2 files changed, 19 insertions(+)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala 
b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
index d4d9aade5b25..844cedb4a302 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RFormula.scala
@@ -34,6 +34,7 @@ import org.apache.spark.ml.util._
 import org.apache.spark.sql.{DataFrame, Dataset}
 import org.apache.spark.sql.functions.col
 import org.apache.spark.sql.types._
+import org.apache.spark.util.SizeEstimator
 
 /**
  * Base trait for [[RFormula]] and [[RFormulaModel]].
@@ -354,6 +355,15 @@ class RFormulaModel private[feature](
   // For ml connect only
   private[ml] def this() = this("", null, null)
 
+  private[spark] override def estimatedSize: Long = {
+    var size = estimateMatadataSize
+    // ResolvedRFormula(label: String, terms: Seq[Seq[String]], hasIntercept: 
Boolean)
+    size += SizeEstimator.estimate(resolvedFormula)
+    // pipelineModel: PipelineModel
+    size += pipelineModel.estimatedSize
+    size
+  }
+
   @Since("2.0.0")
   override def transform(dataset: Dataset[_]): DataFrame = {
     checkCanTransform(dataset.schema)
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala 
b/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
index d4dd4fa33a3e..3f90749d011e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/RFormulaSuite.scala
@@ -54,6 +54,15 @@ class RFormulaSuite extends MLTest with DefaultReadWriteTest 
{
     ParamsSuite.checkParams(new RFormula())
   }
 
+  test("RFormulaModel estimated size") {
+    val dataset = Seq(("a", 1.0), ("b", 2.0), ("c", 3.0)).toDF("category", 
"label")
+    val model = new RFormula().setFormula("label ~ category").fit(dataset)
+    val maxSize = 1024 * 16
+    assert(
+      model.estimatedSize < maxSize,
+      s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+  }
+
   test("transform numeric data") {
     val formula = new RFormula().setFormula("id ~ v1 + v2")
     val original = Seq((0, 1.0, 3.0), (2, 2.0, 5.0)).toDF("id", "v1", "v2")


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to