This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 0b058ae346c3 [SPARK-58361][ML][CONNECT] Include clustering model
metadata in size estimates
0b058ae346c3 is described below
commit 0b058ae346c31a1553a2d85be0fa2e1f01e4baa6
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Jul 28 08:45:27 2026 +0800
[SPARK-58361][ML][CONNECT] Include clustering model metadata in size
estimates
### What changes were proposed in this pull request?
This PR updates `KMeansModel`, `BisectingKMeansModel`, and
`GaussianMixtureModel` size estimates to include their parameter metadata in
addition to learned clustering state.
### Why are the changes needed?
These models override the default estimator and previously omitted their
parameter-map and UID metadata. Counting this metadata makes their cache
accounting consistent with other Spark ML models.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added one fitted-model upper-bound size-estimation test in each model's
existing clustering suite.
Also ran `git diff --check`, ASCII, and source-line-length checks.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57553 from zhengruifeng/clustering-model-size-metadata-dev3.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit f711fdf2d5c587d3dea3b7ed564acef1e2deb3f7)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../org/apache/spark/ml/clustering/BisectingKMeans.scala | 13 +++++++++++--
.../org/apache/spark/ml/clustering/GaussianMixture.scala | 9 +++++++--
.../main/scala/org/apache/spark/ml/clustering/KMeans.scala | 11 +++++++++--
.../apache/spark/ml/clustering/BisectingKMeansSuite.scala | 8 ++++++++
.../apache/spark/ml/clustering/GaussianMixtureSuite.scala | 8 ++++++++
.../scala/org/apache/spark/ml/clustering/KMeansSuite.scala | 8 ++++++++
6 files changed, 51 insertions(+), 6 deletions(-)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
index c129da5f0d7e..e63a3eaa6455 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/BisectingKMeans.scala
@@ -179,8 +179,17 @@ class BisectingKMeansModel private[ml] (
@Since("2.1.0")
override def summary: BisectingKMeansSummary = super.summary
- private[spark] override def estimatedSize: Long =
- SizeEstimator.estimate(parentModel)
+ private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
+ if (parentModel != null) {
+ // parentModel contains:
+ // - root: ClusteringTreeNode containing centers, costs, and children.
+ // - distanceMeasure: String and trainingCost: Double.
+ // - distanceMeasureInstance, derived from distanceMeasure.
+ size += SizeEstimator.estimate(parentModel)
+ }
+ size
+ }
// BisectingKMeans model hasn't supported offloading, so put an empty
`saveSummary` here for now
override private[spark] def saveSummary(path: String): Unit = {}
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
index ddcca167ff30..1b85f7f99875 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/GaussianMixture.scala
@@ -222,8 +222,13 @@ class GaussianMixtureModel private[ml] (
@Since("2.0.0")
override def summary: GaussianMixtureSummary = super.summary
- private[spark] override def estimatedSize: Long =
- SizeEstimator.estimate((weights, gaussians))
+ private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
+ // weights: Array[Double]
+ // gaussians: Array[MultivariateGaussian], each containing a mean Vector
and covariance Matrix
+ size += SizeEstimator.estimate((weights, gaussians))
+ size
+ }
private[spark] def createSummary(
predictions: DataFrame, logLikelihood: Double, iteration: Int
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
index 2de3eb531f56..243077374296 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
@@ -213,8 +213,15 @@ class KMeansModel private[ml] (
@Since("2.0.0")
override def summary: KMeansSummary = super.summary
- private[spark] override def estimatedSize: Long =
- SizeEstimator.estimate(parentModel.clusterCenters)
+ private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
+ if (parentModel != null) {
+ // parentModel contains clusterCenters, distanceMeasure, trainingCost,
and numIter.
+ // It also has transient derived fields for distance calculation and
statistics.
+ size += SizeEstimator.estimate(parentModel)
+ }
+ size
+ }
private[spark] def createSummary(
predictions: DataFrame, numIter: Int, trainingCost: Double
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
index c9e28cb5f7e5..10c64250e649 100644
---
a/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/ml/clustering/BisectingKMeansSuite.scala
@@ -56,6 +56,14 @@ class BisectingKMeansSuite extends MLTest with
DefaultReadWriteTest {
assert(copiedModel.hasSummary)
}
+ test("BisectingKMeansModel estimated size") {
+ val model = new BisectingKMeans().setK(2).fit(dataset)
+ val maxSize = 1024 * 16
+ assert(
+ model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
test("BisectingKMeans validate input dataset") {
testInvalidWeights(new BisectingKMeans().setWeightCol("weight").fit(_))
testInvalidVectors(new BisectingKMeans().fit(_))
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
index 93064b8440be..9119b64d1c86 100644
---
a/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/ml/clustering/GaussianMixtureSuite.scala
@@ -87,6 +87,14 @@ class GaussianMixtureSuite extends MLTest with
DefaultReadWriteTest {
assert(copiedModel.hasSummary)
}
+ test("GaussianMixtureModel estimated size") {
+ val model = new GaussianMixture().setK(2).fit(dataset)
+ val maxSize = 1024 * 16
+ assert(
+ model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
test("GaussianMixture validate input dataset") {
testInvalidWeights(new GaussianMixture().setWeightCol("weight").fit(_))
testInvalidVectors(new GaussianMixture().fit(_))
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala
index 852cce6645aa..9b2d6b909e98 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/KMeansSuite.scala
@@ -70,6 +70,14 @@ class KMeansSuite extends MLTest with DefaultReadWriteTest
with PMMLReadWriteTes
assert(copiedModel.hasSummary)
}
+ test("KMeansModel estimated size") {
+ val model = new KMeans().setK(2).fit(dataset)
+ val maxSize = 1024 * 16
+ assert(
+ model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
test("KMeans validate input dataset") {
testInvalidWeights(new KMeans().setWeightCol("weight").fit(_))
testInvalidVectors(new KMeans().fit(_))
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]