This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 4bfba2208658 [SPARK-61017][ML][PYTHON] Support configurable
intermediate storage levels in iterative classifiers
4bfba2208658 is described below
commit 4bfba220865866eef30c83a0f155d7bf1bfa2705
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Sat Nov 7 19:08:33 2026 +0800
[SPARK-61017][ML][PYTHON] Support configurable intermediate storage levels
in iterative classifiers
### What changes were proposed in this pull request?
Add the existing `intermediateStorageLevel` shared parameter and its setter
to `LogisticRegression`, `LinearSVC`, and `FMClassifier` in Scala and PySpark.
The PySpark estimators also accept it as a keyword in `__init__` and
`setParams`. Apply the configured level when caching their intermediate
training data.
Support for `MultilayerPerceptronClassifier` appears more complicated
because persistence is handled by its internal neural-network trainer. It will
be considered in a separate PR.
### Why are the changes needed?
These classifiers create and cache intermediate training datasets
internally, so caching the input DataFrame does not control their storage
level. Exposing this parameter lets users choose a level such as `DISK_ONLY` to
reduce memory pressure.
### Does this PR introduce _any_ user-facing change?
Yes. The three classifiers gain `intermediateStorageLevel` and
`setIntermediateStorageLevel`. The default remains `MEMORY_AND_DISK`; the
shared validator rejects `NONE` and invalid storage levels.
### How was this patch tested?
- Added simple default-value and setter/getter checks to the existing
`params` tests in `LogisticRegressionSuite`, `LinearSVCSuite`, and
`FMClassifierSuite`.
- The three focused Scala `params` tests and the Scala test-source style
check passed.
- PySpark constructor/`setParams` regression and Java/Python
parameter-parity checks passed. Python lint, formatting, and MLlib MiMa also
passed.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-6)
Closes #60179 from zhengruifeng/classifier-intermediate-storage.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit fce8967e03dcd5572c0caab349921281220335a6)
Signed-off-by: Ruifeng Zheng <[email protected]>
skip-checks: true
---
.../spark/ml/classification/FMClassifier.scala | 14 ++++--
.../apache/spark/ml/classification/LinearSVC.scala | 11 +++--
.../ml/classification/LogisticRegression.scala | 10 ++--
.../ml/classification/FMClassifierSuite.scala | 6 ++-
.../spark/ml/classification/LinearSVCSuite.scala | 6 ++-
.../classification/LogisticRegressionSuite.scala | 6 ++-
python/pyspark/ml/classification.py | 54 +++++++++++++++++++---
python/pyspark/ml/tests/test_classification.py | 23 +++++++++
8 files changed, 112 insertions(+), 18 deletions(-)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
index 29ca909f7930..5f419a0f6c69 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/FMClassifier.scala
@@ -25,6 +25,7 @@ import org.apache.spark.annotation.Since
import org.apache.spark.internal.Logging
import org.apache.spark.ml.linalg._
import org.apache.spark.ml.param._
+import org.apache.spark.ml.param.shared.HasIntermediateStorageLevel
import org.apache.spark.ml.regression.{FactorizationMachines,
FactorizationMachinesParams}
import org.apache.spark.ml.regression.FactorizationMachines._
import org.apache.spark.ml.util._
@@ -38,7 +39,7 @@ import org.apache.spark.storage.StorageLevel
* Params for FMClassifier.
*/
private[classification] trait FMClassifierParams extends
ProbabilisticClassifierParams
- with FactorizationMachinesParams {
+ with FactorizationMachinesParams with HasIntermediateStorageLevel {
}
// scalastyle:off line.size.limit
@@ -179,6 +180,10 @@ class FMClassifier @Since("3.0.0") (
@Since("3.0.0")
def setSeed(value: Long): this.type = set(seed, value)
+ /** @group expertSetParam */
+ @Since("4.4.0")
+ def setIntermediateStorageLevel(value: String): this.type =
set(intermediateStorageLevel, value)
+
override protected def train(
dataset: Dataset[_]): FMClassificationModel = instrumented { instr =>
val numClasses = 2
@@ -191,7 +196,8 @@ class FMClassifier @Since("3.0.0") (
instr.logPipelineStage(this)
instr.logDataset(dataset)
instr.logParams(this, factorSize, fitIntercept, fitLinear, regParam,
- miniBatchFraction, initStd, maxIter, stepSize, tol, solver, thresholds)
+ miniBatchFraction, initStd, maxIter, stepSize, tol, solver, thresholds,
+ intermediateStorageLevel)
instr.logNumClasses(numClasses)
val numFeatures = getNumFeatures(dataset, $(featuresCol))
@@ -205,7 +211,9 @@ class FMClassifier @Since("3.0.0") (
).rdd.map { case Row(l: Double, v: Vector) => (l, OldVectors.fromML(v))
}.setName("training instances")
- if (handlePersistence) data.persist(StorageLevel.MEMORY_AND_DISK)
+ if (handlePersistence) {
+ data.persist(StorageLevel.fromString($(intermediateStorageLevel)))
+ }
val (coefficients, objectiveHistory) = trainImpl(data, numFeatures,
LogisticLoss)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
index e994a463bf66..71eeac90295a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/classification/LinearSVC.scala
@@ -48,7 +48,8 @@ import org.apache.spark.storage.StorageLevel
/** Params for linear SVM Classifier. */
private[classification] trait LinearSVCParams extends ClassifierParams with
HasRegParam
with HasMaxIter with HasFitIntercept with HasTol with HasStandardization
with HasWeightCol
- with HasAggregationDepth with HasThreshold with HasMaxBlockSizeInMB {
+ with HasAggregationDepth with HasThreshold with HasMaxBlockSizeInMB
+ with HasIntermediateStorageLevel {
/**
* Param for threshold in binary classification prediction.
@@ -171,6 +172,10 @@ class LinearSVC @Since("2.2.0") (
@Since("3.1.0")
def setMaxBlockSizeInMB(value: Double): this.type = set(maxBlockSizeInMB,
value)
+ /** @group expertSetParam */
+ @Since("4.4.0")
+ def setIntermediateStorageLevel(value: String): this.type =
set(intermediateStorageLevel, value)
+
private[spark] override def estimateModelSize(dataset: Dataset[_]): Long = {
val numFeatures = DatasetUtils.getNumFeatures(dataset, $(featuresCol))
var size = this.estimateMatadataSize
@@ -186,7 +191,7 @@ class LinearSVC @Since("2.2.0") (
instr.logDataset(dataset)
instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol,
rawPredictionCol,
regParam, maxIter, fitIntercept, tol, standardization, threshold,
aggregationDepth,
- maxBlockSizeInMB)
+ maxBlockSizeInMB, intermediateStorageLevel)
if (dataset.storageLevel != StorageLevel.NONE) {
instr.logWarning("Input instances will be standardized, blockified to
blocks, and " +
@@ -304,7 +309,7 @@ class LinearSVC @Since("2.2.0") (
val maxMemUsage = (actualBlockSizeInMB * 1024L * 1024L).ceil.toLong
val blocks = InstanceBlock.blokifyWithMaxMemUsage(standardized,
maxMemUsage)
- .persist(StorageLevel.MEMORY_AND_DISK)
+ .persist(StorageLevel.fromString($(intermediateStorageLevel)))
.setName(s"training blocks (blockSizeInMB=$actualBlockSizeInMB)")
val getAggregatorFunc = new HingeBlockAggregator(bcInverseStd,
bcScaledMean,
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
index d91261cb304b..dec009bd5716 100644
---
a/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
+++
b/mllib/src/main/scala/org/apache/spark/ml/classification/LogisticRegression.scala
@@ -55,7 +55,7 @@ import org.apache.spark.util._
private[classification] trait LogisticRegressionParams extends
ProbabilisticClassifierParams
with HasRegParam with HasElasticNetParam with HasMaxIter with
HasFitIntercept with HasTol
with HasStandardization with HasWeightCol with HasThreshold with
HasAggregationDepth
- with HasMaxBlockSizeInMB {
+ with HasMaxBlockSizeInMB with HasIntermediateStorageLevel {
import
org.apache.spark.ml.classification.LogisticRegression.supportedFamilyNames
@@ -451,6 +451,10 @@ class LogisticRegression @Since("1.2.0") (
@Since("3.1.0")
def setMaxBlockSizeInMB(value: Double): this.type = set(maxBlockSizeInMB,
value)
+ /** @group expertSetParam */
+ @Since("4.4.0")
+ def setIntermediateStorageLevel(value: String): this.type =
set(intermediateStorageLevel, value)
+
private def assertBoundConstrainedOptimizationParamsValid(
numCoefficientSets: Int,
numFeatures: Int): Unit = {
@@ -510,7 +514,7 @@ class LogisticRegression @Since("1.2.0") (
instr.logDataset(dataset)
instr.logParams(this, labelCol, weightCol, featuresCol, predictionCol,
rawPredictionCol,
probabilityCol, regParam, elasticNetParam, standardization, threshold,
thresholds, maxIter,
- tol, fitIntercept, maxBlockSizeInMB)
+ tol, fitIntercept, maxBlockSizeInMB, intermediateStorageLevel)
if (dataset.storageLevel != StorageLevel.NONE) {
instr.logWarning("Input instances will be standardized, blockified to
blocks, and " +
@@ -962,7 +966,7 @@ class LogisticRegression @Since("1.2.0") (
val maxMemUsage = (actualBlockSizeInMB * 1024L * 1024L).ceil.toLong
val blocks = InstanceBlock.blokifyWithMaxMemUsage(scaled, maxMemUsage)
- .persist(StorageLevel.MEMORY_AND_DISK)
+ .persist(StorageLevel.fromString($(intermediateStorageLevel)))
.setName(s"$uid: training blocks (blockSizeInMB=$actualBlockSizeInMB)")
val costFun = if (multinomial) {
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
index 7066beb39923..5614239bbc31 100644
---
a/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/ml/classification/FMClassifierSuite.scala
@@ -42,7 +42,11 @@ class FMClassifierSuite extends MLTest with
DefaultReadWriteTest {
}
test("params") {
- ParamsSuite.checkParams(new FMClassifier)
+ val classifier = new FMClassifier
+ ParamsSuite.checkParams(classifier)
+ assert(classifier.getIntermediateStorageLevel === "MEMORY_AND_DISK")
+ classifier.setIntermediateStorageLevel("DISK_ONLY")
+ assert(classifier.getIntermediateStorageLevel === "DISK_ONLY")
val model = new FMClassificationModel("fmc_test", 0.0, Vectors.dense(0.0),
new DenseMatrix(1, 8, new Array[Double](8)))
ParamsSuite.checkParams(model)
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
index 9dc03c147f6d..98908994bd7d 100644
---
a/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/ml/classification/LinearSVCSuite.scala
@@ -93,7 +93,11 @@ class LinearSVCSuite extends MLTest with
DefaultReadWriteTest {
}
test("params") {
- ParamsSuite.checkParams(new LinearSVC)
+ val classifier = new LinearSVC
+ ParamsSuite.checkParams(classifier)
+ assert(classifier.getIntermediateStorageLevel === "MEMORY_AND_DISK")
+ classifier.setIntermediateStorageLevel("DISK_ONLY")
+ assert(classifier.getIntermediateStorageLevel === "DISK_ONLY")
val model = new LinearSVCModel("linearSVC", Vectors.dense(0.0), 0.0)
ParamsSuite.checkParams(model)
}
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
index 124a08db3de6..7c0929123261 100644
---
a/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/ml/classification/LogisticRegressionSuite.scala
@@ -179,7 +179,11 @@ class LogisticRegressionSuite extends MLTest with
DefaultReadWriteTest {
}
test("params") {
- ParamsSuite.checkParams(new LogisticRegression)
+ val classifier = new LogisticRegression
+ ParamsSuite.checkParams(classifier)
+ assert(classifier.getIntermediateStorageLevel === "MEMORY_AND_DISK")
+ classifier.setIntermediateStorageLevel("DISK_ONLY")
+ assert(classifier.getIntermediateStorageLevel === "DISK_ONLY")
val model = new LogisticRegressionModel("logReg", Vectors.dense(0.0), 0.0)
ParamsSuite.checkParams(model)
}
diff --git a/python/pyspark/ml/classification.py
b/python/pyspark/ml/classification.py
index 0006cbc619ee..fdd81c312ade 100644
--- a/python/pyspark/ml/classification.py
+++ b/python/pyspark/ml/classification.py
@@ -48,6 +48,7 @@ from pyspark.ml.param.shared import (
HasBlockSize,
HasElasticNetParam,
HasFitIntercept,
+ HasIntermediateStorageLevel,
HasMaxBlockSizeInMB,
HasMaxIter,
HasParallelism,
@@ -607,6 +608,7 @@ class _LinearSVCParams(
HasAggregationDepth,
HasThreshold,
HasMaxBlockSizeInMB,
+ HasIntermediateStorageLevel,
):
"""
Params for :py:class:`LinearSVC` and :py:class:`LinearSVCModel`.
@@ -738,12 +740,14 @@ class LinearSVC(
weightCol: Optional[str] = None,
aggregationDepth: int = 2,
maxBlockSizeInMB: float = 0.0,
+ intermediateStorageLevel: str = "MEMORY_AND_DISK",
):
"""
__init__(self, \\*, featuresCol="features", labelCol="label",
predictionCol="prediction", \
maxIter=100, regParam=0.0, tol=1e-6,
rawPredictionCol="rawPrediction", \
fitIntercept=True, standardization=True, threshold=0.0,
weightCol=None, \
- aggregationDepth=2, maxBlockSizeInMB=0.0):
+ aggregationDepth=2, maxBlockSizeInMB=0.0, \
+ intermediateStorageLevel="MEMORY_AND_DISK"):
"""
super().__init__()
self._java_obj = self._new_java_obj(
@@ -770,12 +774,14 @@ class LinearSVC(
weightCol: Optional[str] = None,
aggregationDepth: int = 2,
maxBlockSizeInMB: float = 0.0,
+ intermediateStorageLevel: str = "MEMORY_AND_DISK",
) -> "LinearSVC":
"""
setParams(self, \\*, featuresCol="features", labelCol="label",
predictionCol="prediction", \
maxIter=100, regParam=0.0, tol=1e-6,
rawPredictionCol="rawPrediction", \
fitIntercept=True, standardization=True, threshold=0.0,
weightCol=None, \
- aggregationDepth=2, maxBlockSizeInMB=0.0):
+ aggregationDepth=2, maxBlockSizeInMB=0.0, \
+ intermediateStorageLevel="MEMORY_AND_DISK"):
Sets params for Linear SVM Classifier.
"""
kwargs = self._input_kwargs
@@ -847,6 +853,13 @@ class LinearSVC(
"""
return self._set(maxBlockSizeInMB=value)
+ @since("4.4.0")
+ def setIntermediateStorageLevel(self, value: str) -> "LinearSVC":
+ """
+ Sets the value of :py:attr:`intermediateStorageLevel`.
+ """
+ return self._set(intermediateStorageLevel=value)
+
class LinearSVCModel(
_JavaClassificationModel[Vector],
@@ -952,6 +965,7 @@ class _LogisticRegressionParams(
HasAggregationDepth,
HasThreshold,
HasMaxBlockSizeInMB,
+ HasIntermediateStorageLevel,
):
"""
Params for :py:class:`LogisticRegression` and
:py:class:`LogisticRegressionModel`.
@@ -1263,6 +1277,7 @@ class LogisticRegression(
lowerBoundsOnIntercepts: Optional[Vector] = ...,
upperBoundsOnIntercepts: Optional[Vector] = ...,
maxBlockSizeInMB: float = ...,
+ intermediateStorageLevel: str = ...,
): ...
@overload
@@ -1289,6 +1304,7 @@ class LogisticRegression(
lowerBoundsOnIntercepts: Optional[Vector] = ...,
upperBoundsOnIntercepts: Optional[Vector] = ...,
maxBlockSizeInMB: float = ...,
+ intermediateStorageLevel: str = ...,
): ...
@keyword_only
@@ -1316,6 +1332,7 @@ class LogisticRegression(
lowerBoundsOnIntercepts: Optional[Vector] = None,
upperBoundsOnIntercepts: Optional[Vector] = None,
maxBlockSizeInMB: float = 0.0,
+ intermediateStorageLevel: str = "MEMORY_AND_DISK",
):
"""
__init__(self, \\*, featuresCol="features", labelCol="label",
predictionCol="prediction", \
@@ -1325,7 +1342,8 @@ class LogisticRegression(
aggregationDepth=2, family="auto", \
lowerBoundsOnCoefficients=None,
upperBoundsOnCoefficients=None, \
lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, \
- maxBlockSizeInMB=0.0):
+ maxBlockSizeInMB=0.0, \
+ intermediateStorageLevel="MEMORY_AND_DISK"):
If the threshold and thresholds Params are both set, they must be
equivalent.
"""
super().__init__()
@@ -1360,6 +1378,7 @@ class LogisticRegression(
lowerBoundsOnIntercepts: Optional[Vector] = ...,
upperBoundsOnIntercepts: Optional[Vector] = ...,
maxBlockSizeInMB: float = ...,
+ intermediateStorageLevel: str = ...,
) -> "LogisticRegression": ...
@overload
@@ -1386,6 +1405,7 @@ class LogisticRegression(
lowerBoundsOnIntercepts: Optional[Vector] = ...,
upperBoundsOnIntercepts: Optional[Vector] = ...,
maxBlockSizeInMB: float = ...,
+ intermediateStorageLevel: str = ...,
) -> "LogisticRegression": ...
@keyword_only
@@ -1414,6 +1434,7 @@ class LogisticRegression(
lowerBoundsOnIntercepts: Optional[Vector] = None,
upperBoundsOnIntercepts: Optional[Vector] = None,
maxBlockSizeInMB: float = 0.0,
+ intermediateStorageLevel: str = "MEMORY_AND_DISK",
) -> "LogisticRegression":
"""
setParams(self, \\*, featuresCol="features", labelCol="label",
predictionCol="prediction", \
@@ -1423,7 +1444,8 @@ class LogisticRegression(
aggregationDepth=2, family="auto", \
lowerBoundsOnCoefficients=None,
upperBoundsOnCoefficients=None, \
lowerBoundsOnIntercepts=None, upperBoundsOnIntercepts=None, \
- maxBlockSizeInMB=0.0):
+ maxBlockSizeInMB=0.0, \
+ intermediateStorageLevel="MEMORY_AND_DISK"):
Sets params for logistic regression.
If the threshold and thresholds Params are both set, they must be
equivalent.
"""
@@ -1525,6 +1547,13 @@ class LogisticRegression(
"""
return self._set(maxBlockSizeInMB=value)
+ @since("4.4.0")
+ def setIntermediateStorageLevel(self, value: str) -> "LogisticRegression":
+ """
+ Sets the value of :py:attr:`intermediateStorageLevel`.
+ """
+ return self._set(intermediateStorageLevel=value)
+
class LogisticRegressionModel(
_JavaProbabilisticClassificationModel[Vector],
@@ -4042,6 +4071,7 @@ class OneVsRestModelWriter(MLWriter):
class FMClassifier(
_JavaProbabilisticClassifier["FMClassificationModel"],
_FactorizationMachinesParams,
+ HasIntermediateStorageLevel,
JavaMLWritable,
JavaMLReadable["FMClassifier"],
):
@@ -4129,13 +4159,15 @@ class FMClassifier(
solver: str = "adamW",
thresholds: Optional[List[float]] = None,
seed: Optional[int] = None,
+ intermediateStorageLevel: str = "MEMORY_AND_DISK",
):
"""
__init__(self, \\*, featuresCol="features", labelCol="label",
predictionCol="prediction", \
probabilityCol="probability",
rawPredictionCol="rawPrediction", \
factorSize=8, fitIntercept=True, fitLinear=True,
regParam=0.0, \
miniBatchFraction=1.0, initStd=0.01, maxIter=100,
stepSize=1.0, \
- tol=1e-6, solver="adamW", thresholds=None, seed=None)
+ tol=1e-6, solver="adamW", thresholds=None, seed=None, \
+ intermediateStorageLevel="MEMORY_AND_DISK")
"""
super().__init__()
self._java_obj = self._new_java_obj(
@@ -4166,13 +4198,15 @@ class FMClassifier(
solver: str = "adamW",
thresholds: Optional[List[float]] = None,
seed: Optional[int] = None,
+ intermediateStorageLevel: str = "MEMORY_AND_DISK",
) -> "FMClassifier":
"""
setParams(self, \\*, featuresCol="features", labelCol="label",
predictionCol="prediction", \
probabilityCol="probability",
rawPredictionCol="rawPrediction", \
factorSize=8, fitIntercept=True, fitLinear=True,
regParam=0.0, \
miniBatchFraction=1.0, initStd=0.01, maxIter=100,
stepSize=1.0, \
- tol=1e-6, solver="adamW", thresholds=None, seed=None)
+ tol=1e-6, solver="adamW", thresholds=None, seed=None, \
+ intermediateStorageLevel="MEMORY_AND_DISK")
Sets Params for FMClassifier.
"""
kwargs = self._input_kwargs
@@ -4258,10 +4292,18 @@ class FMClassifier(
"""
return self._set(regParam=value)
+ @since("4.4.0")
+ def setIntermediateStorageLevel(self, value: str) -> "FMClassifier":
+ """
+ Sets the value of :py:attr:`intermediateStorageLevel`.
+ """
+ return self._set(intermediateStorageLevel=value)
+
class FMClassificationModel(
_JavaProbabilisticClassificationModel[Vector],
_FactorizationMachinesParams,
+ HasIntermediateStorageLevel,
JavaMLWritable,
JavaMLReadable["FMClassificationModel"],
HasTrainingSummary,
diff --git a/python/pyspark/ml/tests/test_classification.py
b/python/pyspark/ml/tests/test_classification.py
index 761ec650fac1..ad3428bb4c87 100644
--- a/python/pyspark/ml/tests/test_classification.py
+++ b/python/pyspark/ml/tests/test_classification.py
@@ -58,6 +58,29 @@ from pyspark.testing.sqlutils import ReusedSQLTestCase
class ClassificationTestsMixin:
+ def test_intermediate_storage_level(self):
+ dataset = self.spark.createDataFrame(
+ [(0.0, Vectors.dense(0.0, 1.0)), (1.0, Vectors.dense(1.0, 0.0))],
+ ["label", "features"],
+ )
+ classifiers = [
+ LogisticRegression(maxIter=2,
intermediateStorageLevel="DISK_ONLY"),
+ LinearSVC(maxIter=2, intermediateStorageLevel="DISK_ONLY"),
+ FMClassifier(maxIter=2, seed=42,
intermediateStorageLevel="DISK_ONLY"),
+ ]
+ for classifier in classifiers:
+ with self.subTest(classifier=type(classifier).__name__):
+ self.assertEqual(
+ type(classifier)().getIntermediateStorageLevel(),
"MEMORY_AND_DISK"
+ )
+ self.assertEqual(classifier.getIntermediateStorageLevel(),
"DISK_ONLY")
+ classifier.setParams(intermediateStorageLevel="MEMORY_ONLY")
+ self.assertEqual(classifier.getIntermediateStorageLevel(),
"MEMORY_ONLY")
+ classifier.setIntermediateStorageLevel("DISK_ONLY")
+ self.assertEqual(classifier.getIntermediateStorageLevel(),
"DISK_ONLY")
+ model = classifier.fit(dataset)
+ self.assertEqual(model.getIntermediateStorageLevel(),
"DISK_ONLY")
+
def test_naive_bayes(self):
spark = self.spark
df = spark.createDataFrame(
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]