This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 6daabb86ceaa [SPARK-58253][ML][CONNECT] Add size estimates for
RobustScalerModel, MinMaxScalerModel, MaxAbsScalerModel, and StandardScalerModel
6daabb86ceaa is described below
commit 6daabb86ceaaf5a57157acdd8bd8bf2da978b765
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Thu Jul 23 10:21:33 2026 +0800
[SPARK-58253][ML][CONNECT] Add size estimates for RobustScalerModel,
MinMaxScalerModel, MaxAbsScalerModel, and StandardScalerModel
### What changes were proposed in this pull request?
Add explicit `estimatedSize` implementations for RobustScalerModel,
MinMaxScalerModel, MaxAbsScalerModel, and StandardScalerModel. Each directly
accounts for model metadata and its learned vectors via `getSizeInBytes`,
rather than using a reflection-based object-graph walk.
### Why are the changes needed?
SPARK-57521 already makes the default `Model.estimatedSize` estimate a
parentless copy, avoiding parent-estimator and SparkSession traversal. These
explicit implementations make size accounting cheaper and more precise for
scalar models, and align them with models that already provide specialized
estimates.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
- Added fitted-model size bounds to the corresponding feature algorithm
suites.
- `JAVA_HOME=/usr/lib/jvm/java-17-openjdk-amd64 build/sbt
'mllib/Test/compile'`\n- Targeted suites were not run.\n\n### Was this patch
authored or co-authored using generative AI tooling?\n\nGenerated-by: OpenAI
Codex (GPT-5)
Closes #57424 from zhengruifeng/scalar-model-size-estimates-master-dev3.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../scala/org/apache/spark/ml/feature/MaxAbsScaler.scala | 8 ++++++++
.../scala/org/apache/spark/ml/feature/MinMaxScaler.scala | 11 +++++++++++
.../scala/org/apache/spark/ml/feature/RobustScaler.scala | 11 +++++++++++
.../scala/org/apache/spark/ml/feature/StandardScaler.scala | 11 +++++++++++
.../org/apache/spark/ml/feature/MaxAbsScalerSuite.scala | 13 +++++++++++++
.../org/apache/spark/ml/feature/MinMaxScalerSuite.scala | 13 +++++++++++++
.../org/apache/spark/ml/feature/RobustScalerSuite.scala | 11 +++++++++++
.../org/apache/spark/ml/feature/StandardScalerSuite.scala | 11 +++++++++++
8 files changed, 89 insertions(+)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/feature/MaxAbsScaler.scala
b/mllib/src/main/scala/org/apache/spark/ml/feature/MaxAbsScaler.scala
index 6d2e39d65302..a962ce3784c9 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/MaxAbsScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/MaxAbsScaler.scala
@@ -112,6 +112,14 @@ class MaxAbsScalerModel private[ml] (
// For ml connect only
private[ml] def this() = this("", Vectors.empty)
+ private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
+ if (maxAbs != null) {
+ size += maxAbs.getSizeInBytes
+ }
+ size
+ }
+
/** @group setParam */
@Since("2.0.0")
def setInputCol(value: String): this.type = set(inputCol, value)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
b/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
index 36942bd08a3a..9bf13c48b22a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/MinMaxScaler.scala
@@ -158,6 +158,17 @@ class MinMaxScalerModel private[ml] (
// For ml connect only
private[ml] def this() = this("", Vectors.empty, Vectors.empty)
+ private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
+ if (originalMin != null) {
+ size += originalMin.getSizeInBytes
+ }
+ if (originalMax != null) {
+ size += originalMax.getSizeInBytes
+ }
+ size
+ }
+
/** @group setParam */
@Since("1.5.0")
def setInputCol(value: String): this.type = set(inputCol, value)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
b/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
index 098976495085..0b520542566c 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/RobustScaler.scala
@@ -234,6 +234,17 @@ class RobustScalerModel private[ml] (
// For ml connect only
private[ml] def this() = this("", Vectors.empty, Vectors.empty)
+ private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
+ if (range != null) {
+ size += range.getSizeInBytes
+ }
+ if (median != null) {
+ size += median.getSizeInBytes
+ }
+ size
+ }
+
/** @group setParam */
def setInputCol(value: String): this.type = set(inputCol, value)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
b/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
index 9cd2fba004a6..fd61753c25dd 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/StandardScaler.scala
@@ -151,6 +151,17 @@ class StandardScalerModel private[ml] (
// For ml connect only
private[ml] def this() = this("", Vectors.empty, Vectors.empty)
+ private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
+ if (std != null) {
+ size += std.getSizeInBytes
+ }
+ if (mean != null) {
+ size += mean.getSizeInBytes
+ }
+ size
+ }
+
/** @group setParam */
@Since("1.2.0")
def setInputCol(value: String): this.type = set(inputCol, value)
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/feature/MaxAbsScalerSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/feature/MaxAbsScalerSuite.scala
index 5de938fa40c4..103a0d3b5960 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/MaxAbsScalerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/MaxAbsScalerSuite.scala
@@ -25,6 +25,19 @@ class MaxAbsScalerSuite extends MLTest with
DefaultReadWriteTest {
import testImplicits._
+ test("MaxAbsScalerModel estimated size") {
+ val model = new MaxAbsScaler()
+ .setInputCol("features")
+ .setOutputCol("scaled")
+ .fit(Seq(
+ Tuple1(Vectors.dense(1.0, 0.0)),
+ Tuple1(Vectors.dense(0.0, 1.0))).toDF("features"))
+
+ val maxSize = 2 * 1024
+ assert(model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
test("MaxAbsScaler fit basic case") {
val data = Array(
Vectors.dense(1, 0, 100),
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/feature/MinMaxScalerSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/feature/MinMaxScalerSuite.scala
index 9b2b0c48f4f6..4fdb8ea8a99f 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/MinMaxScalerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/MinMaxScalerSuite.scala
@@ -25,6 +25,19 @@ class MinMaxScalerSuite extends MLTest with
DefaultReadWriteTest {
import testImplicits._
+ test("MinMaxScalerModel estimated size") {
+ val model = new MinMaxScaler()
+ .setInputCol("features")
+ .setOutputCol("scaled")
+ .fit(Seq(
+ Tuple1(Vectors.dense(1.0, 0.0)),
+ Tuple1(Vectors.dense(0.0, 1.0))).toDF("features"))
+
+ val maxSize = 2 * 1024
+ assert(model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
test("MinMaxScaler fit basic case") {
val data = Array(
Vectors.dense(1, 0, Long.MinValue),
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/feature/RobustScalerSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/feature/RobustScalerSuite.scala
index b3eb595b971a..bd99c91e47c3 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/RobustScalerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/RobustScalerSuite.scala
@@ -170,6 +170,17 @@ class RobustScalerSuite extends MLTest with
DefaultReadWriteTest {
Vectors.dense(1.0), Vectors.dense(2.0)))
}
+ test("RobustScalerModel estimated size") {
+ val model = new RobustScaler()
+ .setInputCol("features")
+ .setOutputCol("scaled")
+ .fit(data.map(Tuple1.apply).toSeq.toDF("features"))
+
+ val maxSize = 4 * 1024
+ assert(model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
test("Scaling with default parameter") {
val df0 = data.zip(resWithScaling).toSeq.toDF("features", "expected")
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/feature/StandardScalerSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/feature/StandardScalerSuite.scala
index 07645b36153c..436527330b9e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/StandardScalerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/StandardScalerSuite.scala
@@ -69,6 +69,17 @@ class StandardScalerSuite extends MLTest with
DefaultReadWriteTest {
Vectors.dense(1.0), Vectors.dense(2.0)))
}
+ test("StandardScalerModel estimated size") {
+ val model = new StandardScaler()
+ .setInputCol("features")
+ .setOutputCol("scaled")
+ .fit(data.map(Tuple1.apply).toSeq.toDF("features"))
+
+ val maxSize = 2 * 1024
+ assert(model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
test("Standardization with default parameter") {
val df0 = data.zip(resWithStd).toSeq.toDF("features", "expected")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
