This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new cddd9aefe2ab [SPARK-58732][ML] Avoid MLlib vector conversion in
Normalizer
cddd9aefe2ab is described below
commit cddd9aefe2ab342d62073a5babde9930015dd78d
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Thu Aug 13 07:49:32 2026 +0800
[SPARK-58732][ML] Avoid MLlib vector conversion in Normalizer
### What changes were proposed in this pull request?
Update ML Normalizer to calculate the norm and normalize values with ML
vectors directly. The normalization implementation follows the existing MLlib
Normalizer transform method, including cloning dense and sparse value arrays
and reusing sparse indices.
### Why are the changes needed?
ML Normalizer currently converts every input vector from ML to MLlib and
converts the normalized result back to ML. Avoiding those conversions reduces
allocation and transform overhead.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
The existing NormalizerSuite covers dense, sparse, zero-vector, and
parameterized normalization behavior.
build/sbt 'mllib/testOnly org.apache.spark.ml.feature.NormalizerSuite'
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57955 from zhengruifeng/SPARK-58732-normalizer-native-vector.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../org/apache/spark/ml/feature/Normalizer.scala | 43 +++++++++++++++++++---
1 file changed, 38 insertions(+), 5 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala
b/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala
index 83b6092edc2d..bdd749e7c485 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Normalizer.scala
@@ -20,11 +20,10 @@ package org.apache.spark.ml.feature
import org.apache.spark.annotation.Since
import org.apache.spark.ml.UnaryTransformer
import org.apache.spark.ml.attribute.AttributeGroup
-import org.apache.spark.ml.linalg.{SQLDataTypes, Vector, VectorUDT}
+import org.apache.spark.ml.linalg.{DenseVector, SparseVector, SQLDataTypes,
Vector, Vectors,
+ VectorUDT}
import org.apache.spark.ml.param.{DoubleParam, ParamValidators}
import org.apache.spark.ml.util._
-import org.apache.spark.mllib.feature
-import org.apache.spark.mllib.linalg.{Vectors => OldVectors}
import org.apache.spark.sql.types._
/**
@@ -56,8 +55,42 @@ class Normalizer @Since("1.4.0") (@Since("1.4.0") override
val uid: String)
def setP(value: Double): this.type = set(p, value)
override protected def createTransformFunc: Vector => Vector = {
- val normalizer = new feature.Normalizer($(p))
- vector => normalizer.transform(OldVectors.fromML(vector)).asML
+ val localP = $(p)
+ vector => {
+ val norm = Vectors.norm(vector, localP)
+ if (norm != 0.0) {
+ val scale = 1.0 / norm
+ // For dense vector, we've to allocate new memory for new output
vector.
+ // However, for sparse vector, the `index` array will not be changed,
+ // so we can re-use it to save memory.
+ vector match {
+ case DenseVector(vs) =>
+ val values = vs.clone()
+ val size = values.length
+ var i = 0
+ while (i < size) {
+ values(i) *= scale
+ i += 1
+ }
+ Vectors.dense(values)
+ case SparseVector(size, ids, vs) =>
+ val values = vs.clone()
+ val nnz = values.length
+ var i = 0
+ while (i < nnz) {
+ values(i) *= scale
+ i += 1
+ }
+ Vectors.sparse(size, ids, values)
+ case v => throw new IllegalArgumentException("Do not support vector
type " + v.getClass)
+ }
+ } else {
+ // Since the norm is zero, return the input vector object itself.
+ // Note that it's safe since we always assume that the data in RDD
+ // should be immutable.
+ vector
+ }
+ }
}
override protected def validateInputType(inputType: DataType): Unit = {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
