This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new c8f07d4fb1fd [SPARK-61076][ML][PYTHON] Accept intermediate storage 
level keyword arguments in PySpark KMeans
c8f07d4fb1fd is described below

commit c8f07d4fb1fd159b9a376a964630d94142327dd5
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Sat Nov 7 19:06:02 2026 +0800

    [SPARK-61076][ML][PYTHON] Accept intermediate storage level keyword 
arguments in PySpark KMeans
    
    ### What changes were proposed in this pull request?
    
    Accept `intermediateStorageLevel` as a keyword in PySpark KMeans's 
`__init__` and `setParams`, and update their documented signatures. This 
follows up #56949.
    
    ### Why are the changes needed?
    
    Although KMeans exposes this shared parameter and its setter, 
`KMeans(intermediateStorageLevel="DISK_ONLY")` and 
`KMeans().setParams(intermediateStorageLevel="DISK_ONLY")` currently raise 
`TypeError`.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Users can configure the intermediate storage level through constructor 
and `setParams` keyword arguments. The default remains `MEMORY_AND_DISK`.
    
    ### How was this patch tested?
    
    - Extended the existing KMeans test to check the default, constructor 
configuration, `setParams` updates, and propagation to the fitted model.
    - `ClusteringTests.test_kmeans` and Spark Connect's 
`ClusteringParityTests.test_kmeans` both passed, using an existing Hive-enabled 
assembly with identical KMeans JVM sources.
    - Python lint and formatting checks passed.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-6)
    
    Closes #60194 from zhengruifeng/kmeans-intermediate-storage-keywords.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
    
    
    skip-checks: true
---
 python/pyspark/ml/clustering.py            | 8 ++++++--
 python/pyspark/ml/tests/test_clustering.py | 8 ++++++++
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/python/pyspark/ml/clustering.py b/python/pyspark/ml/clustering.py
index 91600256e822..2423ea4c6cb6 100644
--- a/python/pyspark/ml/clustering.py
+++ b/python/pyspark/ml/clustering.py
@@ -796,12 +796,14 @@ class KMeans(JavaEstimator[KMeansModel], _KMeansParams, 
JavaMLWritable, JavaMLRe
         weightCol: Optional[str] = None,
         solver: str = "auto",
         maxBlockSizeInMB: float = 0.0,
+        intermediateStorageLevel: str = "MEMORY_AND_DISK",
     ):
         """
         __init__(self, \\*, featuresCol="features", 
predictionCol="prediction", k=2, \
                  initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, 
seed=None, \
                  distanceMeasure="euclidean", weightCol=None, solver="auto", \
-                 maxBlockSizeInMB=0.0)
+                 maxBlockSizeInMB=0.0, \
+                 intermediateStorageLevel="MEMORY_AND_DISK")
         """
         super().__init__()
         self._java_obj = 
self._new_java_obj("org.apache.spark.ml.clustering.KMeans", self.uid)
@@ -828,12 +830,14 @@ class KMeans(JavaEstimator[KMeansModel], _KMeansParams, 
JavaMLWritable, JavaMLRe
         weightCol: Optional[str] = None,
         solver: str = "auto",
         maxBlockSizeInMB: float = 0.0,
+        intermediateStorageLevel: str = "MEMORY_AND_DISK",
     ) -> "KMeans":
         """
         setParams(self, \\*, featuresCol="features", 
predictionCol="prediction", k=2, \
                   initMode="k-means||", initSteps=2, tol=1e-4, maxIter=20, 
seed=None, \
                   distanceMeasure="euclidean", weightCol=None, solver="auto", \
-                  maxBlockSizeInMB=0.0)
+                  maxBlockSizeInMB=0.0, \
+                  intermediateStorageLevel="MEMORY_AND_DISK")
 
         Sets params for KMeans.
         """
diff --git a/python/pyspark/ml/tests/test_clustering.py 
b/python/pyspark/ml/tests/test_clustering.py
index 6b753d591027..aa72a236552e 100644
--- a/python/pyspark/ml/tests/test_clustering.py
+++ b/python/pyspark/ml/tests/test_clustering.py
@@ -58,17 +58,25 @@ class ClusteringTestsMixin:
             .select("weight", "features")
         )
 
+        self.assertEqual(KMeans().getIntermediateStorageLevel(), 
"MEMORY_AND_DISK")
         km = KMeans(
             k=2,
             maxIter=2,
             weightCol="weight",
+            intermediateStorageLevel="DISK_ONLY",
         )
         self.assertEqual(km.getK(), 2)
         self.assertEqual(km.getMaxIter(), 2)
         self.assertEqual(km.getWeightCol(), "weight")
+        self.assertEqual(km.getIntermediateStorageLevel(), "DISK_ONLY")
+        km.setParams(intermediateStorageLevel="MEMORY_ONLY")
+        self.assertEqual(km.getIntermediateStorageLevel(), "MEMORY_ONLY")
+        km.setParams(intermediateStorageLevel="DISK_ONLY")
+        self.assertEqual(km.getIntermediateStorageLevel(), "DISK_ONLY")
 
         model = km.fit(df)
         self.assertEqual(km.uid, model.uid)
+        self.assertEqual(model.getIntermediateStorageLevel(), "DISK_ONLY")
 
         centers = model.clusterCenters()
         self.assertEqual(len(centers), 2)


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to