This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 7a6c598a1b2c [SPARK-58535][ML] Create StringIndexer lookup maps during
transformation
7a6c598a1b2c is described below
commit 7a6c598a1b2c14e60de25318c96d27f9cbfa4c80
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Aug 4 14:56:00 2026 +0800
[SPARK-58535][ML] Create StringIndexer lookup maps during transformation
### What changes were proposed in this pull request?
This patch creates the StringIndexerModel label-to-index lookup maps inside
`transform` and passes them to invalid-value filtering. The maps are no longer
retained as derived model state.
### Why are the changes needed?
The lookup maps are only needed to transform input data. Creating them
during transformation avoids retaining them on models that are fit or loaded
but never transformed.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
`build/sbt 'mllib/testOnly
org.apache.spark.ml.feature.StringIndexerSuite'`\n\nAll 28 tests passed. No new
test was added because this is an internal lifecycle refactor; the existing
suite covers normal indexing and invalid-value handling.\n\n### Was this patch
authored or co-authored using generative AI tooling?\n\nGenerated-by: Codex
(GPT-5)
Closes #57735 from zhengruifeng/lazy-string-indexer-labels-to-index.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../apache/spark/ml/feature/StringIndexer.scala | 28 ++++++++++------------
1 file changed, 13 insertions(+), 15 deletions(-)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
index 8bb07d3f0260..c0f7e312dcbb 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StringIndexer.scala
@@ -316,20 +316,8 @@ class StringIndexerModel (
labelsArray(0)
}
- // Prepares the maps for string values to corresponding index values.
- private val labelsToIndexArray: Array[OpenHashMap[String, Double]] = {
- for (labels <- labelsArray) yield {
- val n = labels.length
- val map = new OpenHashMap[String, Double](n)
- labels.zipWithIndex.foreach { case (label, idx) =>
- map.update(label, idx)
- }
- map
- }
- }
-
private[spark] override def estimatedSize: Long =
- estimateMatadataSize + SizeEstimator.estimate((labelsArray,
labelsToIndexArray))
+ estimateMatadataSize + SizeEstimator.estimate(labelsArray)
/** @group setParam */
@Since("1.6.0")
@@ -353,7 +341,10 @@ class StringIndexerModel (
// This filters out any null values and also the input labels which are not
in
// the dataset used for fitting.
- private def filterInvalidData(dataset: Dataset[_], inputColNames:
Seq[String]): Dataset[_] = {
+ private def filterInvalidData(
+ dataset: Dataset[_],
+ inputColNames: Seq[String],
+ labelsToIndexArray: Array[OpenHashMap[String, Double]]): Dataset[_] = {
val conditions: Seq[Column] = inputColNames.indices.map { i =>
val inputColName = inputColNames(i)
val labelToIndex = labelsToIndexArray(i)
@@ -401,11 +392,18 @@ class StringIndexerModel (
transformSchema(dataset.schema, logging = true)
val (inputColNames, outputColNames) = getInOutCols()
+ val labelsToIndexArray = labelsArray.map { labels =>
+ val map = new OpenHashMap[String, Double](labels.length)
+ labels.zipWithIndex.foreach { case (label, idx) =>
+ map.update(label, idx)
+ }
+ map
+ }
val outputColumns = new Array[Column](outputColNames.length)
// Skips invalid rows if `handleInvalid` is set to
`StringIndexer.SKIP_INVALID`.
val filteredDataset = if (getHandleInvalid == StringIndexer.SKIP_INVALID) {
- filterInvalidData(dataset, inputColNames.toImmutableArraySeq)
+ filterInvalidData(dataset, inputColNames.toImmutableArraySeq,
labelsToIndexArray)
} else {
dataset
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
