This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 50024a39100a [SPARK-58870][ML] Make KMeans distance measures singleton
50024a39100a is described below
commit 50024a39100a28f0909fc6752f33cd418c8b991b
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Thu Aug 20 08:27:47 2026 +0800
[SPARK-58870][ML] Make KMeans distance measures singleton
### What changes were proposed in this pull request?
This PR updates mllib KMeans distance measure implementations to use
singleton objects instead of allocating a new instance when decoding the
configured distance measure.
It also changes KMeansModel's decoded distanceMeasureInstance from a
transient lazy val to a private val, and updates internal/test call sites to
reference the Euclidean distance measure singleton directly.
### Why are the changes needed?
This is part of SPARK-58584. KMeans only has stateless distance measure
implementations, so allocating per decode is unnecessary. Reusing singleton
instances avoids per-model helper allocation and lets KMeansModel hold the
decoded helper directly.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- build/sbt -java-home /usr/lib/jvm/java-17-openjdk-amd64 mllib/compile
- build/sbt -java-home /usr/lib/jvm/java-17-openjdk-amd64 'mllib/testOnly
org.apache.spark.mllib.clustering.KMeansSuite'
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex
Closes #58122 from
zhengruifeng/spark-58870-mllib-kmeans-distance-measure-singleton.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit 6ae31964161f2a5843e9767c843113a80a45a92b)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../org/apache/spark/mllib/clustering/DistanceMeasure.scala | 11 ++++-------
.../scala/org/apache/spark/mllib/clustering/KMeansModel.scala | 2 +-
.../scala/org/apache/spark/mllib/clustering/LocalKMeans.scala | 2 +-
.../scala/org/apache/spark/mllib/clustering/KMeansSuite.scala | 2 +-
4 files changed, 7 insertions(+), 10 deletions(-)
diff --git
a/mllib/src/main/scala/org/apache/spark/mllib/clustering/DistanceMeasure.scala
b/mllib/src/main/scala/org/apache/spark/mllib/clustering/DistanceMeasure.scala
index 5b0fb5ef18c8..f6bec405554c 100644
---
a/mllib/src/main/scala/org/apache/spark/mllib/clustering/DistanceMeasure.scala
+++
b/mllib/src/main/scala/org/apache/spark/mllib/clustering/DistanceMeasure.scala
@@ -258,8 +258,8 @@ object DistanceMeasure {
private[spark] def decodeFromString(distanceMeasure: String):
DistanceMeasure =
distanceMeasure match {
- case EUCLIDEAN => new EuclideanDistanceMeasure
- case COSINE => new CosineDistanceMeasure
+ case EUCLIDEAN => EuclideanDistanceMeasure
+ case COSINE => CosineDistanceMeasure
case _ => throw new IllegalArgumentException(s"distanceMeasure must be
one of: " +
s"$EUCLIDEAN, $COSINE. $distanceMeasure provided.")
}
@@ -278,7 +278,7 @@ object DistanceMeasure {
k.toLong * k * numFeatures < 1000000
}
-private[spark] class EuclideanDistanceMeasure extends DistanceMeasure {
+private[spark] object EuclideanDistanceMeasure extends DistanceMeasure {
/**
* Statistics used in triangle inequality to obtain useful bounds to find
closest centers.
@@ -400,10 +400,7 @@ private[spark] class EuclideanDistanceMeasure extends
DistanceMeasure {
centroid: VectorWithNorm): Double = {
EuclideanDistanceMeasure.fastSquaredDistance(point, centroid)
}
-}
-
-private[spark] object EuclideanDistanceMeasure {
/**
* @return the squared Euclidean distance between two vectors computed by
* [[org.apache.spark.mllib.util.MLUtils#fastSquaredDistance]].
@@ -415,7 +412,7 @@ private[spark] object EuclideanDistanceMeasure {
}
}
-private[spark] class CosineDistanceMeasure extends DistanceMeasure {
+private[spark] object CosineDistanceMeasure extends DistanceMeasure {
/**
* Statistics used in triangle inequality to obtain useful bounds to find
closest centers.
diff --git
a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
index e5c0b27072d0..ad2b812684a0 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeansModel.scala
@@ -43,7 +43,7 @@ class KMeansModel (@Since("1.0.0") val clusterCenters:
Array[Vector],
private[spark] val numIter: Int)
extends Saveable with Serializable with PMMLExportable {
- @transient private lazy val distanceMeasureInstance: DistanceMeasure =
+ private val distanceMeasureInstance: DistanceMeasure =
DistanceMeasure.decodeFromString(distanceMeasure)
@transient private lazy val clusterCentersWithNorm =
diff --git
a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LocalKMeans.scala
b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LocalKMeans.scala
index ac31c0d3be47..2fe15a59f4c7 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/LocalKMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/LocalKMeans.scala
@@ -75,7 +75,7 @@ private[mllib] object LocalKMeans extends Logging {
}
- val distanceMeasureInstance = new EuclideanDistanceMeasure
+ val distanceMeasureInstance = EuclideanDistanceMeasure
// Run up to maxIterations iterations of Lloyd's algorithm
val oldClosest = Array.fill(points.length)(-1)
diff --git
a/mllib/src/test/scala/org/apache/spark/mllib/clustering/KMeansSuite.scala
b/mllib/src/test/scala/org/apache/spark/mllib/clustering/KMeansSuite.scala
index 94afcb8c8e5c..80ed707aec63 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/clustering/KMeansSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/clustering/KMeansSuite.scala
@@ -92,7 +92,7 @@ class KMeansSuite extends SparkFunSuite with
MLlibTestSparkContext {
.setInitializationSteps(10)
.setSeed(seed)
- val distanceMeasureInstance = new EuclideanDistanceMeasure
+ val distanceMeasureInstance = EuclideanDistanceMeasure
val initialCenters = km.initKMeansParallel(normedData,
distanceMeasureInstance).map(_.vector)
assert(initialCenters.length === initialCenters.distinct.length)
assert(initialCenters.length <= numDistinctPoints)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
