This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 289160d37f3e [SPARK-58911][ML][FOLLOWUP] Validate effective
CountVectorizer DF range
289160d37f3e is described below
commit 289160d37f3e24e16a1e5adf90482e0bf5f92517
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Fri Aug 28 09:59:12 2026 +0800
[SPARK-58911][ML][FOLLOWUP] Validate effective CountVectorizer DF range
### What changes were proposed in this pull request?
This follow-up to #57755 validates the effective `minDF` and `maxDF` values
after fractional
thresholds are converted using the input document count. If the effective
maximum is below the
minimum, the DataFrame expression raises an error. Regression coverage
includes both combinations
of fractional and absolute thresholds.
### Why are the changes needed?
The DataFrame rewrite retained validation when both thresholds use the same
units, but dropped the
post-conversion validation needed when one threshold is fractional and the
other is absolute. An
invalid effective range consequently produced an empty vocabulary instead
of reporting the invalid
parameters.
### Does this PR introduce _any_ user-facing change?
Yes. On the unreleased `master` branch, invalid mixed-unit DF ranges now
raise an error instead of
silently producing an empty vocabulary. This restores the validation
behavior from before #57755.
### How was this patch tested?
Added regression coverage to `CountVectorizerSuite` for both mixed-unit
invalid-range directions.
The test suite was not run locally. `git diff --check` passes.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #58327 from zhengruifeng/ml-countvectorizer-df-validation-dev-6.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../apache/spark/ml/feature/CountVectorizer.scala | 7 +++++--
.../spark/ml/feature/CountVectorizerSuite.scala | 20 ++++++++++++++++++++
2 files changed, 25 insertions(+), 2 deletions(-)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
b/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
index 2ef1bf1f43a3..bd1917bb9e34 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/CountVectorizer.scala
@@ -194,8 +194,11 @@ class CountVectorizer @Since("1.5.0") (@Since("1.5.0")
override val uid: String)
val vocSize = $(vocabSize)
val input = dataset.select($(inputCol))
val inputSizeCol = input.select(count(lit(0))).scalar()
- val minDfCol = if ($(minDF) >= 1.0) lit($(minDF)) else inputSizeCol *
lit($(minDF))
- val maxDfCol = if ($(maxDF) >= 1.0) lit($(maxDF)) else inputSizeCol *
lit($(maxDF))
+ val rawMinDfCol = if ($(minDF) >= 1.0) lit($(minDF)) else inputSizeCol *
lit($(minDF))
+ val rawMaxDfCol = if ($(maxDF) >= 1.0) lit($(maxDF)) else inputSizeCol *
lit($(maxDF))
+ val invalidDfRangeError = raise_error(lit("maxDF must be >= minDF."))
+ val minDfCol = when(rawMaxDfCol >= rawMinDfCol,
rawMinDfCol).otherwise(invalidDfRangeError)
+ val maxDfCol = when(rawMaxDfCol >= rawMinDfCol,
rawMaxDfCol).otherwise(invalidDfRangeError)
val filteringRequired = isSet(minDF) || isSet(maxDF)
val wordCounts = if (filteringRequired) {
input
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala
index c00733998050..40474ea816b2 100644
---
a/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala
+++
b/mllib/src/test/scala/org/apache/spark/ml/feature/CountVectorizerSuite.scala
@@ -16,6 +16,7 @@
*/
package org.apache.spark.ml.feature
+import org.apache.spark.SparkRuntimeException
import org.apache.spark.ml.linalg.{Vector, Vectors}
import org.apache.spark.ml.param.ParamsSuite
import org.apache.spark.ml.util.{DefaultReadWriteTest, MLTest, MLTestingUtils}
@@ -214,6 +215,25 @@ class CountVectorizerSuite extends MLTest with
DefaultReadWriteTest {
}
}
+ test("CountVectorizer validates mixed minDF and maxDF") {
+ val df = Seq(Seq("a"), Seq("b"), Seq("c"), Seq("d")).toDF("words")
+
+ Seq((0.75, 2.0), (3.0, 0.5)).foreach { case (minDf, maxDf) =>
+ val error = intercept[SparkRuntimeException] {
+ new CountVectorizer()
+ .setInputCol("words")
+ .setOutputCol("features")
+ .setMinDF(minDf)
+ .setMaxDF(maxDf)
+ .fit(df)
+ }
+ checkError(
+ exception = error,
+ condition = "USER_RAISED_EXCEPTION",
+ parameters = Map("errorMessage" -> "maxDF must be >= minDF."))
+ }
+ }
+
test("CountVectorizerModel with minTF count") {
val df = Seq(
(0, split("a a a b b c c c d "), Vectors.sparse(4, Seq((0, 3.0), (2,
3.0)))),
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
