This is an automated email from the ASF dual-hosted git repository.
uros-b pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b2918df83111 [SPARK-58397][ML][CONNECT] Add Bucketizer size estimate
b2918df83111 is described below
commit b2918df83111fe7683c4b7e6c9aac3a2b347231d
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Jul 28 10:14:33 2026 +0000
[SPARK-58397][ML][CONNECT] Add Bucketizer size estimate
### What changes were proposed in this pull request?
Adds a specialized `Bucketizer.estimatedSize` implementation that charges
its parameter metadata. Adds a bounded-size regression test.
### Why are the changes needed?
Bucketizer is a model whose state is entirely represented by its parameter
maps, so its cache charge can use the precise metadata estimate instead of a
generic object-graph walk.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Added `Bucketizer estimated size` coverage in `BucketizerSuite`. The suite
was not run locally.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57591 from zhengruifeng/bucketizer-model-size-metadata-dev3.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Uros Bojanic <[email protected]>
---
mllib/src/main/scala/org/apache/spark/ml/feature/Bucketizer.scala | 2 ++
.../test/scala/org/apache/spark/ml/feature/BucketizerSuite.scala | 8 ++++++++
2 files changed, 10 insertions(+)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/Bucketizer.scala
b/mllib/src/main/scala/org/apache/spark/ml/feature/Bucketizer.scala
index 967d46ff7cd5..dbb26c53d145 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/Bucketizer.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/Bucketizer.scala
@@ -217,6 +217,8 @@ final class Bucketizer @Since("1.4.0") (@Since("1.4.0")
override val uid: String
defaultCopy[Bucketizer](extra).setParent(parent)
}
+ private[spark] override def estimatedSize: Long = estimateMatadataSize
+
@Since("3.0.0")
override def toString: String = {
s"Bucketizer: uid=$uid" +
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/feature/BucketizerSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/feature/BucketizerSuite.scala
index 2634d6320048..702363dbcd7b 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/BucketizerSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/BucketizerSuite.scala
@@ -37,6 +37,14 @@ class BucketizerSuite extends MLTest with
DefaultReadWriteTest {
ParamsSuite.checkParams(new Bucketizer)
}
+ test("Bucketizer estimated size") {
+ val model = new Bucketizer().setSplits(Array(-1.0, 0.0, 1.0))
+ val maxSize = 1024 * 4
+ assert(
+ model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
+ }
+
test("Bucket continuous features, without -inf,inf") {
// Check a set of valid feature values.
val splits = Array(-0.5, 0.0, 0.5)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]