This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 8f4bfef5545b [SPARK-58182][ML][CONNECT] Include parameter metadata in
tree classifier size estimates
8f4bfef5545b is described below
commit 8f4bfef5545b42f281b2bae1441d1cf8b6843e74
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Mon Jul 20 13:48:37 2026 +0800
[SPARK-58182][ML][CONNECT] Include parameter metadata in tree classifier
size estimates
### What changes were proposed in this pull request?
This patch adds `estimateMatadataSize` to the custom size estimates for
`DecisionTreeClassificationModel`, `GBTClassificationModel`, and
`RandomForestClassificationModel`. Their existing estimates count only learned
tree nodes.
### Why are the changes needed?
Spark Connect uses `Model.estimatedSize` for server-side ML cache
accounting. These three custom overrides omitted the models' parameter maps and
UID, unlike the other custom model estimators. Including the existing metadata
estimate makes their cache accounting include both model configuration and
learned tree state.
The parameter values for these models are simple: strings, primitive
numeric/boolean values, and the `Array[Double]` classifier thresholds. They do
not include DataFrames, Spark sessions, nested stages, vectors, or matrices, so
this metadata estimate cannot traverse a session-backed object graph.
### Does this PR introduce _any_ user-facing change?
no — internal ML cache-size accounting only
### How was this patch tested?
`git diff --check` passed. I attempted to run:
```bash
build/sbt 'mllib/testOnly
org.apache.spark.ml.classification.DecisionTreeClassifierSuite
org.apache.spark.ml.classification.GBTClassifierSuite
org.apache.spark.ml.classification.RandomForestClassifierSuite'
```
The suites did not start because this environment uses Java 11; the build
requires Java 17.0.11 or later. No tests were added for this focused change.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57322 from zhengruifeng/SPARK-57521-tree-model-size-metadata-dev4.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../org/apache/spark/ml/classification/DecisionTreeClassifier.scala | 2 +-
.../main/scala/org/apache/spark/ml/classification/GBTClassifier.scala | 2 +-
.../org/apache/spark/ml/classification/RandomForestClassifier.scala | 2 +-
.../src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala | 4 ++--
4 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala
b/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala
index d5564f6a3fbd..3506c1a2502d 100644
---
a/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala
+++
b/mllib/src/main/scala/org/apache/spark/ml/classification/DecisionTreeClassifier.scala
@@ -201,7 +201,7 @@ class DecisionTreeClassificationModel private[ml] (
// For ml connect only
private[ml] def this() = this("", Node.dummyNode, -1, -1)
- private[spark] override def estimatedSize: Long = getEstimatedSize()
+ private[spark] override def estimatedSize: Long = estimateMatadataSize +
getEstimatedSize()
override def predict(features: Vector): Double = {
rootNode.predictImpl(features).prediction
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala
b/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala
index 2c5c7e7740a3..905a092eb889 100644
---
a/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala
+++
b/mllib/src/main/scala/org/apache/spark/ml/classification/GBTClassifier.scala
@@ -276,7 +276,7 @@ class GBTClassificationModel private[ml](
private[ml] def this() = this("",
Array(new DecisionTreeRegressionModel), Array(Double.NaN), -1, -1)
- private[spark] override def estimatedSize: Long = getEstimatedSize()
+ private[spark] override def estimatedSize: Long = estimateMatadataSize +
getEstimatedSize()
@Since("1.4.0")
override def trees: Array[DecisionTreeRegressionModel] = _trees
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
index 2c22ca5b4230..565663196a99 100644
---
a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
+++
b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
@@ -245,7 +245,7 @@ class RandomForestClassificationModel private[ml] (
// For ml connect only
private[ml] def this() = this("", Array(new
DecisionTreeClassificationModel), -1, -1)
- private[spark] override def estimatedSize: Long = getEstimatedSize()
+ private[spark] override def estimatedSize: Long = estimateMatadataSize +
getEstimatedSize()
@Since("1.4.0")
override def trees: Array[DecisionTreeClassificationModel] = _trees
diff --git
a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
index 6c9b1819021f..9ba5a499ba8f 100644
---
a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
+++
b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
@@ -358,7 +358,7 @@ class MLSuite extends MLHelper {
.set(Connect.CONNECT_SESSION_CONNECT_ML_CACHE_MEMORY_CONTROL_ENABLED.key,
"true")
for (estimator <- Seq(getDecisionTreeClassifier,
getRandomForestClassifier)) {
- for (maxModelSize <- Seq(20000, 50000)) {
+ for (maxModelSize <- Seq(25000, 50000)) {
sessionHolder.session.conf.set(
Connect.CONNECT_SESSION_CONNECT_ML_CACHE_MEMORY_CONTROL_MAX_MODEL_SIZE.key,
maxModelSize.toString)
@@ -375,7 +375,7 @@ class MLSuite extends MLHelper {
sessionHolder.session.conf
.set(Connect.CONNECT_SESSION_CONNECT_ML_CACHE_MEMORY_CONTROL_ENABLED.key,
"true")
- for (maxModelSize <- Seq(20000, 50000, 130000)) {
+ for (maxModelSize <- Seq(25000, 50000, 130000)) {
sessionHolder.session.conf.set(
Connect.CONNECT_SESSION_CONNECT_ML_CACHE_MEMORY_CONTROL_MAX_MODEL_SIZE.key,
maxModelSize.toString)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
