This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.3
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.3 by this push:
new 437d6bb8b4cb [SPARK-61076][ML][PYTHON] Accept intermediate storage
level keyword arguments in PySpark KMeans
437d6bb8b4cb is described below
commit 437d6bb8b4cb5e18afd6c615232870ab2dd6d9b1
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Sat Nov 7 19:06:02 2026 +0800
[SPARK-61076][ML][PYTHON] Accept intermediate storage level keyword
arguments in PySpark KMeans
### What changes were proposed in this pull request?
Accept `intermediateStorageLevel` as a keyword in PySpark KMeans's
`__init__` and `setParams`, and update their documented signatures. This
follows up #56949.
### Why are the changes needed?
Although KMeans exposes this shared parameter and its setter,
`KMeans(intermediateStorageLevel="DISK_ONLY")` and
`KMeans().setParams(intermediateStorageLevel="DISK_ONLY")` currently raise
`TypeError`.
### Does this PR introduce _any_ user-facing change?
Yes. Users can configure the intermediate storage level through constructor
and `setParams` keyword arguments. The default remains `MEMORY_AND_DISK`.
### How was this patch tested?
- Extended the existing KMeans test to check the default, constructor
configuration, `setParams` updates, and propagation to the fitted model.
- `ClusteringTests.test_kmeans` and Spark Connect's
`ClusteringParityTests.test_kmeans` both passed, using an existing Hive-enabled
assembly with identical KMeans JVM sources.
- Python lint and formatting checks passed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-6)
Closes #60194 from zhengruifeng/kmeans-intermediate-storage-keywords.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit c8f07d4fb1fd159b9a376a964630d94142327dd5)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
python/pyspark/ml/clustering.py | 8 ++++++--
python/pyspark/ml/tests/test_clustering.py | 8 ++++++++
2 files changed, 14 insertions(+), 2 deletions(-)
diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index c23cd238d32e..7491da0c68c9 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -795,12 +795,14 @@ class KMeans(JavaEstimator[KMeansModel], _KMeansParams,
JavaMLWritable, JavaMLRe
weightCol: Optional[str] = None,
solver: str = "auto",
maxBlockSizeInMB: float = 0.0,
+ intermediateStorageLevel: str = "MEMORY_AND_DISK",
):
"""
__init__(self, \\*, featuresCol="features",
predictionCol="prediction", k=2, \
initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20,
seed=None, \
distanceMeasure="euclidean", weightCol=None, solver="auto", \
- maxBlockSizeInMB=0.0)
+ maxBlockSizeInMB=0.0, \
+ intermediateStorageLevel="MEMORY_AND_DISK")
"""
super().__init__()
self._java_obj =
self._new_java_obj("org.apache.spark.ml.clustering.KMeans", self.uid)
@@ -827,12 +829,14 @@ class KMeans(JavaEstimator[KMeansModel], _KMeansParams,
JavaMLWritable, JavaMLRe
weightCol: Optional[str] = None,
solver: str = "auto",
maxBlockSizeInMB: float = 0.0,
+ intermediateStorageLevel: str = "MEMORY_AND_DISK",
) -> "KMeans":
"""
setParams(self, \\*, featuresCol="features",
predictionCol="prediction", k=2, \
initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20,
seed=None, \
distanceMeasure="euclidean", weightCol=None, solver="auto", \
- maxBlockSizeInMB=0.0)
+ maxBlockSizeInMB=0.0, \
+ intermediateStorageLevel="MEMORY_AND_DISK")
Sets params for KMeans.
"""
diff --git a/python/pyspark/ml/tests/test_clustering.py
b/python/pyspark/ml/tests/test_clustering.py
index e22e97a5e7f1..a4737829ed35 100644
--- a/python/pyspark/ml/tests/test_clustering.py
+++ b/python/pyspark/ml/tests/test_clustering.py
@@ -58,17 +58,25 @@ class ClusteringTestsMixin:
.select("weight", "features")
)
+ self.assertEqual(KMeans().getIntermediateStorageLevel(),
"MEMORY_AND_DISK")
km = KMeans(
k=2,
maxIter=2,
weightCol="weight",
+ intermediateStorageLevel="DISK_ONLY",
)
self.assertEqual(km.getK(), 2)
self.assertEqual(km.getMaxIter(), 2)
self.assertEqual(km.getWeightCol(), "weight")
+ self.assertEqual(km.getIntermediateStorageLevel(), "DISK_ONLY")
+ km.setParams(intermediateStorageLevel="MEMORY_ONLY")
+ self.assertEqual(km.getIntermediateStorageLevel(), "MEMORY_ONLY")
+ km.setParams(intermediateStorageLevel="DISK_ONLY")
+ self.assertEqual(km.getIntermediateStorageLevel(), "DISK_ONLY")
model = km.fit(df)
self.assertEqual(km.uid, model.uid)
+ self.assertEqual(model.getIntermediateStorageLevel(), "DISK_ONLY")
centers = model.clusterCenters()
self.assertEqual(len(centers), 2)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]