This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 0b058ae346c3 [SPARK-58361][ML][CONNECT] Include clustering model 
metadata in size estimates
0b058ae346c3 is described below

commit 0b058ae346c31a1553a2d85be0fa2e1f01e4baa6
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Jul 28 08:45:27 2026 +0800

    [SPARK-58361][ML][CONNECT] Include clustering model metadata in size 
estimates
    
    ### What changes were proposed in this pull request?
    
    This PR updates `KMeansModel`, `BisectingKMeansModel`, and 
`GaussianMixtureModel` size estimates to include their parameter metadata in 
addition to learned clustering state.
    
    ### Why are the changes needed?
    
    These models override the default estimator and previously omitted their 
parameter-map and UID metadata. Counting this metadata makes their cache 
accounting consistent with other Spark ML models.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Added one fitted-model upper-bound size-estimation test in each model's 
existing clustering suite.
    
    Also ran `git diff --check`, ASCII, and source-line-length checks.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #57553 from zhengruifeng/clustering-model-size-metadata-dev3.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
    (cherry picked from commit f711fdf2d5c587d3dea3b7ed564acef1e2deb3f7)
    Signed-off-by: Ruifeng Zheng <[email protected]>
---
 .../org/apache/spark/ml/clustering/BisectingKMeans.scala    | 13 +++++++++++--
 .../org/apache/spark/ml/clustering/GaussianMixture.scala    |  9 +++++++--
 .../main/scala/org/apache/spark/ml/clustering/KMeans.scala  | 11 +++++++++--
 .../apache/spark/ml/clustering/BisectingKMeansSuite.scala   |  8 ++++++++
 .../apache/spark/ml/clustering/GaussianMixtureSuite.scala   |  8 ++++++++
 .../scala/org/apache/spark/ml/clustering/KMeansSuite.scala  |  8 ++++++++
 6 files changed, 51 insertions(+), 6 deletions(-)

diff --git 
a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala 
b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index c129da5f0d7e..e63a3eaa6455 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -179,8 +179,17 @@ class BisectingKMeansModel private[ml] (
   @Since("2.1.0")
   override def summary: BisectingKMeansSummary = super.summary
 
-  private[spark] override def estimatedSize: Long =
-    SizeEstimator.estimate(parentModel)
+  private[spark] override def estimatedSize: Long = {
+    var size = estimateMatadataSize
+    if (parentModel != null) {
+      // parentModel contains:
+      // - root: ClusteringTreeNode containing centers, costs, and children.
+      // - distanceMeasure: String and trainingCost: Double.
+      // - distanceMeasureInstance, derived from distanceMeasure.
+      size += SizeEstimator.estimate(parentModel)
+    }
+    size
+  }
 
   // BisectingKMeans model hasn't supported offloading, so put an empty 
`saveSummary` here for now
   override private[spark] def saveSummary(path: String): Unit = {}
diff --git 
a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala 
b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index ddcca167ff30..1b85f7f99875 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -222,8 +222,13 @@ class GaussianMixtureModel private[ml] (
   @Since("2.0.0")
   override def summary: GaussianMixtureSummary = super.summary
 
-  private[spark] override def estimatedSize: Long =
-    SizeEstimator.estimate((weights, gaussians))
+  private[spark] override def estimatedSize: Long = {
+    var size = estimateMatadataSize
+    // weights: Array[Double]
+    // gaussians: Array[MultivariateGaussian], each containing a mean Vector 
and covariance Matrix
+    size += SizeEstimator.estimate((weights, gaussians))
+    size
+  }
 
   private[spark] def createSummary(
     predictions: DataFrame, logLikelihood: Double, iteration: Int
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala 
b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
index 2de3eb531f56..243077374296 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
@@ -213,8 +213,15 @@ class KMeansModel private[ml] (
   @Since("2.0.0")
   override def summary: KMeansSummary = super.summary
 
-  private[spark] override def estimatedSize: Long =
-    SizeEstimator.estimate(parentModel.clusterCenters)
+  private[spark] override def estimatedSize: Long = {
+    var size = estimateMatadataSize
+    if (parentModel != null) {
+      // parentModel contains clusterCenters, distanceMeasure, trainingCost, 
and numIter.
+      // It also has transient derived fields for distance calculation and 
statistics.
+      size += SizeEstimator.estimate(parentModel)
+    }
+    size
+  }
 
   private[spark] def createSummary(
     predictions: DataFrame, numIter: Int, trainingCost: Double
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
 
b/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
index c9e28cb5f7e5..10c64250e649 100644
--- 
a/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
@@ -56,6 +56,14 @@ class BisectingKMeansSuite extends MLTest with 
DefaultReadWriteTest {
     assert(copiedModel.hasSummary)
   }
 
+  test("BisectingKMeansModel estimated size") {
+    val model = new BisectingKMeans().setK(2).fit(dataset)
+    val maxSize = 1024 * 16
+    assert(
+      model.estimatedSize < maxSize,
+      s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+  }
+
   test("BisectingKMeans validate input dataset") {
     testInvalidWeights(new BisectingKMeans().setWeightCol("weight").fit(_))
     testInvalidVectors(new BisectingKMeans().fit(_))
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
 
b/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
index 93064b8440be..9119b64d1c86 100644
--- 
a/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
@@ -87,6 +87,14 @@ class GaussianMixtureSuite extends MLTest with 
DefaultReadWriteTest {
     assert(copiedModel.hasSummary)
   }
 
+  test("GaussianMixtureModel estimated size") {
+    val model = new GaussianMixture().setK(2).fit(dataset)
+    val maxSize = 1024 * 16
+    assert(
+      model.estimatedSize < maxSize,
+      s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+  }
+
   test("GaussianMixture validate input dataset") {
     testInvalidWeights(new GaussianMixture().setWeightCol("weight").fit(_))
     testInvalidVectors(new GaussianMixture().fit(_))
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala 
b/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala
index 852cce6645aa..9b2d6b909e98 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala
@@ -70,6 +70,14 @@ class KMeansSuite extends MLTest with DefaultReadWriteTest 
with PMMLReadWriteTes
     assert(copiedModel.hasSummary)
   }
 
+  test("KMeansModel estimated size") {
+    val model = new KMeans().setK(2).fit(dataset)
+    val maxSize = 1024 * 16
+    assert(
+      model.estimatedSize < maxSize,
+      s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+  }
+
   test("KMeans validate input dataset") {
     testInvalidWeights(new KMeans().setWeightCol("weight").fit(_))
     testInvalidVectors(new KMeans().fit(_))


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to