This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 4bfba2208658 [SPARK-61017][ML][PYTHON] Support configurable 
intermediate storage levels in iterative classifiers
4bfba2208658 is described below

commit 4bfba220865866eef30c83a0f155d7bf1bfa2705
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Sat Nov 7 19:08:33 2026 +0800

    [SPARK-61017][ML][PYTHON] Support configurable intermediate storage levels 
in iterative classifiers
    
    ### What changes were proposed in this pull request?
    
    Add the existing `intermediateStorageLevel` shared parameter and its setter 
to `LogisticRegression`, `LinearSVC`, and `FMClassifier` in Scala and PySpark. 
The PySpark estimators also accept it as a keyword in `__init__` and 
`setParams`. Apply the configured level when caching their intermediate 
training data.
    
    Support for `MultilayerPerceptronClassifier` appears more complicated 
because persistence is handled by its internal neural-network trainer. It will 
be considered in a separate PR.
    
    ### Why are the changes needed?
    
    These classifiers create and cache intermediate training datasets 
internally, so caching the input DataFrame does not control their storage 
level. Exposing this parameter lets users choose a level such as `DISK_ONLY` to 
reduce memory pressure.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. The three classifiers gain `intermediateStorageLevel` and 
`setIntermediateStorageLevel`. The default remains `MEMORY_AND_DISK`; the 
shared validator rejects `NONE` and invalid storage levels.
    
    ### How was this patch tested?
    
    - Added simple default-value and setter/getter checks to the existing 
`params` tests in `LogisticRegressionSuite`, `LinearSVCSuite`, and 
`FMClassifierSuite`.
    - The three focused Scala `params` tests and the Scala test-source style 
check passed.
    - PySpark constructor/`setParams` regression and Java/Python 
parameter-parity checks passed. Python lint, formatting, and MLlib MiMa also 
passed.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-6)
    
    Closes #60179 from zhengruifeng/classifier-intermediate-storage.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
    
    (cherry picked from commit fce8967e03dcd5572c0caab349921281220335a6)
    Signed-off-by: Ruifeng Zheng <[email protected]>
    
    
    skip-checks: true
---
 .../spark/ml/classification/FMClassifier.scala     | 14 ++++--
 .../apache/spark/ml/classification/LinearSVC.scala | 11 +++--
 .../ml/classification/LogisticRegression.scala     | 10 ++--
 .../ml/classification/FMClassifierSuite.scala      |  6 ++-
 .../spark/ml/classification/LinearSVCSuite.scala   |  6 ++-
 .../classification/LogisticRegressionSuite.scala   |  6 ++-
 python/pyspark/ml/classification.py                | 54 +++++++++++++++++++---
 python/pyspark/ml/tests/test_classification.py     | 23 +++++++++
 8 files changed, 112 insertions(+), 18 deletions(-)

diff --git 
a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala 
b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
index 29ca909f7930..5f419a0f6c69 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
@@ -25,6 +25,7 @@ import org.apache.spark.annotation.Since
 import org.apache.spark.internal.Logging
 import org.apache.spark.ml.linalg._
 import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared.HasIntermediateStorageLevel
 import org.apache.spark.ml.regression.{FactorizationMachines, 
FactorizationMachinesParams}
 import org.apache.spark.ml.regression.FactorizationMachines._
 import org.apache.spark.ml.util._
@@ -38,7 +39,7 @@ import org.apache.spark.storage.StorageLevel
  * Params for FMClassifier.
  */
 private[classification] trait FMClassifierParams extends 
ProbabilisticClassifierParams
-  with FactorizationMachinesParams {
+  with FactorizationMachinesParams with HasIntermediateStorageLevel {
 }
 
 // scalastyle:off line.size.limit
@@ -179,6 +180,10 @@ class FMClassifier @Since("3.0.0") (
   @Since("3.0.0")
   def setSeed(value: Long): this.type = set(seed, value)
 
+  /** @group expertSetParam */
+  @Since("4.4.0")
+  def setIntermediateStorageLevel(value: String): this.type = 
set(intermediateStorageLevel, value)
+
   override protected def train(
       dataset: Dataset[_]): FMClassificationModel = instrumented { instr =>
     val numClasses = 2
@@ -191,7 +196,8 @@ class FMClassifier @Since("3.0.0") (
     instr.logPipelineStage(this)
     instr.logDataset(dataset)
     instr.logParams(this, factorSize, fitIntercept, fitLinear, regParam,
-      miniBatchFraction, initStd, maxIter, stepSize, tol, solver, thresholds)
+      miniBatchFraction, initStd, maxIter, stepSize, tol, solver, thresholds,
+      intermediateStorageLevel)
     instr.logNumClasses(numClasses)
 
     val numFeatures = getNumFeatures(dataset, $(featuresCol))
@@ -205,7 +211,9 @@ class FMClassifier @Since("3.0.0") (
     ).rdd.map { case Row(l: Double, v: Vector) => (l, OldVectors.fromML(v))
     }.setName("training instances")
 
-    if (handlePersistence) data.persist(StorageLevel.MEMORY_AND_DISK)
+    if (handlePersistence) {
+      data.persist(StorageLevel.fromString($(intermediateStorageLevel)))
+    }
 
     val (coefficients, objectiveHistory) = trainImpl(data, numFeatures, 
LogisticLoss)
 
diff --git 
a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala 
b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
index e994a463bf66..71eeac90295a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
@@ -48,7 +48,8 @@ import org.apache.spark.storage.StorageLevel
 /** Params for linear SVM Classifier. */
 private[classification] trait LinearSVCParams extends ClassifierParams with 
HasRegParam
   with HasMaxIter with HasFitIntercept with HasTol with HasStandardization 
with HasWeightCol
-  with HasAggregationDepth with HasThreshold with HasMaxBlockSizeInMB {
+  with HasAggregationDepth with HasThreshold with HasMaxBlockSizeInMB
+  with HasIntermediateStorageLevel {
 
   /**
    * Param for threshold in binary classification prediction.
@@ -171,6 +172,10 @@ class LinearSVC @Since("2.2.0") (
   @Since("3.1.0")
   def setMaxBlockSizeInMB(value: Double): this.type = set(maxBlockSizeInMB, 
value)
 
+  /** @group expertSetParam */
+  @Since("4.4.0")
+  def setIntermediateStorageLevel(value: String): this.type = 
set(intermediateStorageLevel, value)
+
   private[spark] override def estimateModelSize(dataset: Dataset[_]): Long = {
     val numFeatures = DatasetUtils.getNumFeatures(dataset, $(featuresCol))
     var size = this.estimateMatadataSize
@@ -186,7 +191,7 @@ class LinearSVC @Since("2.2.0") (
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol, 
rawPredictionCol,
       regParam, maxIter, fitIntercept, tol, standardization, threshold, 
aggregationDepth,
-      maxBlockSizeInMB)
+      maxBlockSizeInMB, intermediateStorageLevel)
 
     if (dataset.storageLevel != StorageLevel.NONE) {
       instr.logWarning("Input instances will be standardized, blockified to 
blocks, and " +
@@ -304,7 +309,7 @@ class LinearSVC @Since("2.2.0") (
 
     val maxMemUsage = (actualBlockSizeInMB * 1024L * 1024L).ceil.toLong
     val blocks = InstanceBlock.blokifyWithMaxMemUsage(standardized, 
maxMemUsage)
-      .persist(StorageLevel.MEMORY_AND_DISK)
+      .persist(StorageLevel.fromString($(intermediateStorageLevel)))
       .setName(s"training blocks (blockSizeInMB=$actualBlockSizeInMB)")
 
     val getAggregatorFunc = new HingeBlockAggregator(bcInverseStd, 
bcScaledMean,
diff --git 
a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
 
b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index d91261cb304b..dec009bd5716 100644
--- 
a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++ 
b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -55,7 +55,7 @@ import org.apache.spark.util._
 private[classification] trait LogisticRegressionParams extends 
ProbabilisticClassifierParams
   with HasRegParam with HasElasticNetParam with HasMaxIter with 
HasFitIntercept with HasTol
   with HasStandardization with HasWeightCol with HasThreshold with 
HasAggregationDepth
-  with HasMaxBlockSizeInMB {
+  with HasMaxBlockSizeInMB with HasIntermediateStorageLevel {
 
   import 
org.apache.spark.ml.classification.LogisticRegression.supportedFamilyNames
 
@@ -451,6 +451,10 @@ class LogisticRegression @Since("1.2.0") (
   @Since("3.1.0")
   def setMaxBlockSizeInMB(value: Double): this.type = set(maxBlockSizeInMB, 
value)
 
+  /** @group expertSetParam */
+  @Since("4.4.0")
+  def setIntermediateStorageLevel(value: String): this.type = 
set(intermediateStorageLevel, value)
+
   private def assertBoundConstrainedOptimizationParamsValid(
       numCoefficientSets: Int,
       numFeatures: Int): Unit = {
@@ -510,7 +514,7 @@ class LogisticRegression @Since("1.2.0") (
     instr.logDataset(dataset)
     instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol, 
rawPredictionCol,
       probabilityCol, regParam, elasticNetParam, standardization, threshold, 
thresholds, maxIter,
-      tol, fitIntercept, maxBlockSizeInMB)
+      tol, fitIntercept, maxBlockSizeInMB, intermediateStorageLevel)
 
     if (dataset.storageLevel != StorageLevel.NONE) {
       instr.logWarning("Input instances will be standardized, blockified to 
blocks, and " +
@@ -962,7 +966,7 @@ class LogisticRegression @Since("1.2.0") (
 
     val maxMemUsage = (actualBlockSizeInMB * 1024L * 1024L).ceil.toLong
     val blocks = InstanceBlock.blokifyWithMaxMemUsage(scaled, maxMemUsage)
-      .persist(StorageLevel.MEMORY_AND_DISK)
+      .persist(StorageLevel.fromString($(intermediateStorageLevel)))
       .setName(s"$uid: training blocks (blockSizeInMB=$actualBlockSizeInMB)")
 
     val costFun = if (multinomial) {
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
 
b/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
index 7066beb39923..5614239bbc31 100644
--- 
a/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
@@ -42,7 +42,11 @@ class FMClassifierSuite extends MLTest with 
DefaultReadWriteTest {
   }
 
   test("params") {
-    ParamsSuite.checkParams(new FMClassifier)
+    val classifier = new FMClassifier
+    ParamsSuite.checkParams(classifier)
+    assert(classifier.getIntermediateStorageLevel === "MEMORY_AND_DISK")
+    classifier.setIntermediateStorageLevel("DISK_ONLY")
+    assert(classifier.getIntermediateStorageLevel === "DISK_ONLY")
     val model = new FMClassificationModel("fmc_test", 0.0, Vectors.dense(0.0),
       new DenseMatrix(1, 8, new Array[Double](8)))
     ParamsSuite.checkParams(model)
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala 
b/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
index 9dc03c147f6d..98908994bd7d 100644
--- 
a/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
@@ -93,7 +93,11 @@ class LinearSVCSuite extends MLTest with 
DefaultReadWriteTest {
   }
 
   test("params") {
-    ParamsSuite.checkParams(new LinearSVC)
+    val classifier = new LinearSVC
+    ParamsSuite.checkParams(classifier)
+    assert(classifier.getIntermediateStorageLevel === "MEMORY_AND_DISK")
+    classifier.setIntermediateStorageLevel("DISK_ONLY")
+    assert(classifier.getIntermediateStorageLevel === "DISK_ONLY")
     val model = new LinearSVCModel("linearSVC", Vectors.dense(0.0), 0.0)
     ParamsSuite.checkParams(model)
   }
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
 
b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
index 124a08db3de6..7c0929123261 100644
--- 
a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
+++ 
b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
@@ -179,7 +179,11 @@ class LogisticRegressionSuite extends MLTest with 
DefaultReadWriteTest {
   }
 
   test("params") {
-    ParamsSuite.checkParams(new LogisticRegression)
+    val classifier = new LogisticRegression
+    ParamsSuite.checkParams(classifier)
+    assert(classifier.getIntermediateStorageLevel === "MEMORY_AND_DISK")
+    classifier.setIntermediateStorageLevel("DISK_ONLY")
+    assert(classifier.getIntermediateStorageLevel === "DISK_ONLY")
     val model = new LogisticRegressionModel("logReg", Vectors.dense(0.0), 0.0)
     ParamsSuite.checkParams(model)
   }
diff --git a/python/pyspark/ml/classification.py 
b/python/pyspark/ml/classification.py
index 0006cbc619ee..fdd81c312ade 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -48,6 +48,7 @@ from pyspark.ml.param.shared import (
     HasBlockSize,
     HasElasticNetParam,
     HasFitIntercept,
+    HasIntermediateStorageLevel,
     HasMaxBlockSizeInMB,
     HasMaxIter,
     HasParallelism,
@@ -607,6 +608,7 @@ class _LinearSVCParams(
     HasAggregationDepth,
     HasThreshold,
     HasMaxBlockSizeInMB,
+    HasIntermediateStorageLevel,
 ):
     """
     Params for :py:class:`LinearSVC` and :py:class:`LinearSVCModel`.
@@ -738,12 +740,14 @@ class LinearSVC(
         weightCol: Optional[str] = None,
         aggregationDepth: int = 2,
         maxBlockSizeInMB: float = 0.0,
+        intermediateStorageLevel: str = "MEMORY_AND_DISK",
     ):
         """
         __init__(self, \\*, featuresCol="features", labelCol="label", 
predictionCol="prediction", \
                  maxIter=100, regParam=0.0, tol=1e-6, 
rawPredictionCol="rawPrediction", \
                  fitIntercept=True, standardization=True, threshold=0.0, 
weightCol=None, \
-                 aggregationDepth=2, maxBlockSizeInMB=0.0):
+                 aggregationDepth=2, maxBlockSizeInMB=0.0, \
+                 intermediateStorageLevel="MEMORY_AND_DISK"):
         """
         super().__init__()
         self._java_obj = self._new_java_obj(
@@ -770,12 +774,14 @@ class LinearSVC(
         weightCol: Optional[str] = None,
         aggregationDepth: int = 2,
         maxBlockSizeInMB: float = 0.0,
+        intermediateStorageLevel: str = "MEMORY_AND_DISK",
     ) -> "LinearSVC":
         """
         setParams(self, \\*, featuresCol="features", labelCol="label", 
predictionCol="prediction", \
                   maxIter=100, regParam=0.0, tol=1e-6, 
rawPredictionCol="rawPrediction", \
                   fitIntercept=True, standardization=True, threshold=0.0, 
weightCol=None, \
-                  aggregationDepth=2, maxBlockSizeInMB=0.0):
+                  aggregationDepth=2, maxBlockSizeInMB=0.0, \
+                  intermediateStorageLevel="MEMORY_AND_DISK"):
         Sets params for Linear SVM Classifier.
         """
         kwargs = self._input_kwargs
@@ -847,6 +853,13 @@ class LinearSVC(
         """
         return self._set(maxBlockSizeInMB=value)
 
+    @since("4.4.0")
+    def setIntermediateStorageLevel(self, value: str) -> "LinearSVC":
+        """
+        Sets the value of :py:attr:`intermediateStorageLevel`.
+        """
+        return self._set(intermediateStorageLevel=value)
+
 
 class LinearSVCModel(
     _JavaClassificationModel[Vector],
@@ -952,6 +965,7 @@ class _LogisticRegressionParams(
     HasAggregationDepth,
     HasThreshold,
     HasMaxBlockSizeInMB,
+    HasIntermediateStorageLevel,
 ):
     """
     Params for :py:class:`LogisticRegression` and 
:py:class:`LogisticRegressionModel`.
@@ -1263,6 +1277,7 @@ class LogisticRegression(
         lowerBoundsOnIntercepts: Optional[Vector] = ...,
         upperBoundsOnIntercepts: Optional[Vector] = ...,
         maxBlockSizeInMB: float = ...,
+        intermediateStorageLevel: str = ...,
     ): ...
 
     @overload
@@ -1289,6 +1304,7 @@ class LogisticRegression(
         lowerBoundsOnIntercepts: Optional[Vector] = ...,
         upperBoundsOnIntercepts: Optional[Vector] = ...,
         maxBlockSizeInMB: float = ...,
+        intermediateStorageLevel: str = ...,
     ): ...
 
     @keyword_only
@@ -1316,6 +1332,7 @@ class LogisticRegression(
         lowerBoundsOnIntercepts: Optional[Vector] = None,
         upperBoundsOnIntercepts: Optional[Vector] = None,
         maxBlockSizeInMB: float = 0.0,
+        intermediateStorageLevel: str = "MEMORY_AND_DISK",
     ):
         """
         __init__(self, \\*, featuresCol="features", labelCol="label", 
predictionCol="prediction", \
@@ -1325,7 +1342,8 @@ class LogisticRegression(
                  aggregationDepth=2, family="auto", \
                  lowerBoundsOnCoefficients=None, 
upperBoundsOnCoefficients=None, \
                  lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, \
-                 maxBlockSizeInMB=0.0):
+                 maxBlockSizeInMB=0.0, \
+                 intermediateStorageLevel="MEMORY_AND_DISK"):
         If the threshold and thresholds Params are both set, they must be 
equivalent.
         """
         super().__init__()
@@ -1360,6 +1378,7 @@ class LogisticRegression(
         lowerBoundsOnIntercepts: Optional[Vector] = ...,
         upperBoundsOnIntercepts: Optional[Vector] = ...,
         maxBlockSizeInMB: float = ...,
+        intermediateStorageLevel: str = ...,
     ) -> "LogisticRegression": ...
 
     @overload
@@ -1386,6 +1405,7 @@ class LogisticRegression(
         lowerBoundsOnIntercepts: Optional[Vector] = ...,
         upperBoundsOnIntercepts: Optional[Vector] = ...,
         maxBlockSizeInMB: float = ...,
+        intermediateStorageLevel: str = ...,
     ) -> "LogisticRegression": ...
 
     @keyword_only
@@ -1414,6 +1434,7 @@ class LogisticRegression(
         lowerBoundsOnIntercepts: Optional[Vector] = None,
         upperBoundsOnIntercepts: Optional[Vector] = None,
         maxBlockSizeInMB: float = 0.0,
+        intermediateStorageLevel: str = "MEMORY_AND_DISK",
     ) -> "LogisticRegression":
         """
         setParams(self, \\*, featuresCol="features", labelCol="label", 
predictionCol="prediction", \
@@ -1423,7 +1444,8 @@ class LogisticRegression(
                   aggregationDepth=2, family="auto", \
                   lowerBoundsOnCoefficients=None, 
upperBoundsOnCoefficients=None, \
                   lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, \
-                  maxBlockSizeInMB=0.0):
+                  maxBlockSizeInMB=0.0, \
+                  intermediateStorageLevel="MEMORY_AND_DISK"):
         Sets params for logistic regression.
         If the threshold and thresholds Params are both set, they must be 
equivalent.
         """
@@ -1525,6 +1547,13 @@ class LogisticRegression(
         """
         return self._set(maxBlockSizeInMB=value)
 
+    @since("4.4.0")
+    def setIntermediateStorageLevel(self, value: str) -> "LogisticRegression":
+        """
+        Sets the value of :py:attr:`intermediateStorageLevel`.
+        """
+        return self._set(intermediateStorageLevel=value)
+
 
 class LogisticRegressionModel(
     _JavaProbabilisticClassificationModel[Vector],
@@ -4042,6 +4071,7 @@ class OneVsRestModelWriter(MLWriter):
 class FMClassifier(
     _JavaProbabilisticClassifier["FMClassificationModel"],
     _FactorizationMachinesParams,
+    HasIntermediateStorageLevel,
     JavaMLWritable,
     JavaMLReadable["FMClassifier"],
 ):
@@ -4129,13 +4159,15 @@ class FMClassifier(
         solver: str = "adamW",
         thresholds: Optional[List[float]] = None,
         seed: Optional[int] = None,
+        intermediateStorageLevel: str = "MEMORY_AND_DISK",
     ):
         """
         __init__(self, \\*, featuresCol="features", labelCol="label", 
predictionCol="prediction", \
                  probabilityCol="probability", 
rawPredictionCol="rawPrediction", \
                  factorSize=8, fitIntercept=True, fitLinear=True, 
regParam=0.0, \
                  miniBatchFraction=1.0, initStd=0.01, maxIter=100, 
stepSize=1.0, \
-                 tol=1e-6, solver="adamW", thresholds=None, seed=None)
+                 tol=1e-6, solver="adamW", thresholds=None, seed=None, \
+                 intermediateStorageLevel="MEMORY_AND_DISK")
         """
         super().__init__()
         self._java_obj = self._new_java_obj(
@@ -4166,13 +4198,15 @@ class FMClassifier(
         solver: str = "adamW",
         thresholds: Optional[List[float]] = None,
         seed: Optional[int] = None,
+        intermediateStorageLevel: str = "MEMORY_AND_DISK",
     ) -> "FMClassifier":
         """
         setParams(self, \\*, featuresCol="features", labelCol="label", 
predictionCol="prediction", \
                   probabilityCol="probability", 
rawPredictionCol="rawPrediction", \
                   factorSize=8, fitIntercept=True, fitLinear=True, 
regParam=0.0, \
                   miniBatchFraction=1.0, initStd=0.01, maxIter=100, 
stepSize=1.0, \
-                  tol=1e-6, solver="adamW", thresholds=None, seed=None)
+                  tol=1e-6, solver="adamW", thresholds=None, seed=None, \
+                  intermediateStorageLevel="MEMORY_AND_DISK")
         Sets Params for FMClassifier.
         """
         kwargs = self._input_kwargs
@@ -4258,10 +4292,18 @@ class FMClassifier(
         """
         return self._set(regParam=value)
 
+    @since("4.4.0")
+    def setIntermediateStorageLevel(self, value: str) -> "FMClassifier":
+        """
+        Sets the value of :py:attr:`intermediateStorageLevel`.
+        """
+        return self._set(intermediateStorageLevel=value)
+
 
 class FMClassificationModel(
     _JavaProbabilisticClassificationModel[Vector],
     _FactorizationMachinesParams,
+    HasIntermediateStorageLevel,
     JavaMLWritable,
     JavaMLReadable["FMClassificationModel"],
     HasTrainingSummary,
diff --git a/python/pyspark/ml/tests/test_classification.py 
b/python/pyspark/ml/tests/test_classification.py
index 761ec650fac1..ad3428bb4c87 100644
--- a/python/pyspark/ml/tests/test_classification.py
+++ b/python/pyspark/ml/tests/test_classification.py
@@ -58,6 +58,29 @@ from pyspark.testing.sqlutils import ReusedSQLTestCase
 
 
 class ClassificationTestsMixin:
+    def test_intermediate_storage_level(self):
+        dataset = self.spark.createDataFrame(
+            [(0.0, Vectors.dense(0.0, 1.0)), (1.0, Vectors.dense(1.0, 0.0))],
+            ["label", "features"],
+        )
+        classifiers = [
+            LogisticRegression(maxIter=2, 
intermediateStorageLevel="DISK_ONLY"),
+            LinearSVC(maxIter=2, intermediateStorageLevel="DISK_ONLY"),
+            FMClassifier(maxIter=2, seed=42, 
intermediateStorageLevel="DISK_ONLY"),
+        ]
+        for classifier in classifiers:
+            with self.subTest(classifier=type(classifier).__name__):
+                self.assertEqual(
+                    type(classifier)().getIntermediateStorageLevel(), 
"MEMORY_AND_DISK"
+                )
+                self.assertEqual(classifier.getIntermediateStorageLevel(), 
"DISK_ONLY")
+                classifier.setParams(intermediateStorageLevel="MEMORY_ONLY")
+                self.assertEqual(classifier.getIntermediateStorageLevel(), 
"MEMORY_ONLY")
+                classifier.setIntermediateStorageLevel("DISK_ONLY")
+                self.assertEqual(classifier.getIntermediateStorageLevel(), 
"DISK_ONLY")
+                model = classifier.fit(dataset)
+                self.assertEqual(model.getIntermediateStorageLevel(), 
"DISK_ONLY")
+
     def test_naive_bayes(self):
         spark = self.spark
         df = spark.createDataFrame(


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to