This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new c121eb46fff8 [SPARK-58715][ML][CONNECT] Use explicit fields for
clustering model size estimation
c121eb46fff8 is described below
commit c121eb46fff841886cb77a1e992f51f6d3ee6d3f
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Wed Aug 12 08:13:02 2026 +0800
[SPARK-58715][ML][CONNECT] Use explicit fields for clustering model size
estimation
### What changes were proposed in this pull request?
This PR updates the ML `KMeansModel`, `BisectingKMeansModel`, and
`GaussianMixtureModel` size estimates to pass only their persisted MLlib data
fields to `SizeEstimator`.
For Gaussian mixture models, it estimates each Gaussian mean and covariance
rather than the `MultivariateGaussian` wrapper. For bisecting K-means, it makes
the MLlib model root available to Spark-internal code so the ML wrapper can
list the fields directly.
### Why are the changes needed?
`SizeEstimator` recursively follows object references. Estimating an entire
parent MLlib model can include transient or lazily initialized derived state,
such as distance structures, statistics, or logging fields, rather than only
persisted model data. Explicit fields make the accounting predictable and avoid
retaining those auxiliary object graphs during estimation.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
```
build/sbt -java-home /usr/lib/jvm/java-17-openjdk-amd64 'mllib / Compile /
compile'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57932 from zhengruifeng/SPARK-58715-clustering-size-estimation.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit afa9031aafbe50259ff56ff68d78c0bc8c7ac79c)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../org/apache/spark/ml/clustering/BisectingKMeans.scala | 12 +++++++-----
.../org/apache/spark/ml/clustering/GaussianMixture.scala | 5 +++--
.../main/scala/org/apache/spark/ml/clustering/KMeans.scala | 12 +++++++++---
.../apache/spark/mllib/clustering/BisectingKMeansModel.scala | 2 +-
4 files changed, 20 insertions(+), 11 deletions(-)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index e63a3eaa6455..6b149102e253 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -182,11 +182,13 @@ class BisectingKMeansModel private[ml] (
private[spark] override def estimatedSize: Long = {
var size = estimateMatadataSize
if (parentModel != null) {
- // parentModel contains:
- // - root: ClusteringTreeNode containing centers, costs, and children.
- // - distanceMeasure: String and trainingCost: Double.
- // - distanceMeasureInstance, derived from distanceMeasure.
- size += SizeEstimator.estimate(parentModel)
+ // root: ClusteringTreeNode
+ // distanceMeasure: String
+ // trainingCost: Double
+ size += SizeEstimator.estimate((
+ parentModel.root,
+ parentModel.distanceMeasure,
+ parentModel.trainingCost))
}
size
}
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index 272fce565a8f..817802746bbf 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -225,8 +225,9 @@ class GaussianMixtureModel private[ml] (
private[spark] override def estimatedSize: Long = {
var size = estimateMatadataSize
// weights: Array[Double]
- // gaussians: Array[MultivariateGaussian], each containing a mean Vector
and covariance Matrix
- size += SizeEstimator.estimate((weights, gaussians))
+ size += SizeEstimator.estimate(weights)
+ // gaussians: Array[MultivariateGaussian], each with mean: Vector and cov:
Matrix
+ gaussians.foreach(gaussian => size +=
SizeEstimator.estimate((gaussian.mean, gaussian.cov)))
size
}
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
index 243077374296..5d410f77c7eb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
@@ -216,9 +216,15 @@ class KMeansModel private[ml] (
private[spark] override def estimatedSize: Long = {
var size = estimateMatadataSize
if (parentModel != null) {
- // parentModel contains clusterCenters, distanceMeasure, trainingCost,
and numIter.
- // It also has transient derived fields for distance calculation and
statistics.
- size += SizeEstimator.estimate(parentModel)
+ // clusterCenters: Array[Vector]
+ // distanceMeasure: String
+ // trainingCost: Double
+ // numIter: Int
+ size += SizeEstimator.estimate((
+ parentModel.clusterCenters,
+ parentModel.distanceMeasure,
+ parentModel.trainingCost,
+ parentModel.numIter))
}
size
}
diff --git
a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
index 083c3e3e77a9..113445269ce7 100644
---
a/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
+++
b/mllib/src/main/scala/org/apache/spark/mllib/clustering/BisectingKMeansModel.scala
@@ -41,7 +41,7 @@ import org.apache.spark.util.ArrayImplicits._
*/
@Since("1.6.0")
class BisectingKMeansModel private[clustering] (
- private[clustering] val root: ClusteringTreeNode,
+ private[spark] val root: ClusteringTreeNode,
@Since("2.4.0") val distanceMeasure: String,
@Since("3.0.0") val trainingCost: Double
) extends Serializable with Saveable with Logging {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
