This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 4d486e342217 [SPARK-59047][ML] Avoid retaining random forest tree
weights
4d486e342217 is described below
commit 4d486e3422176a53668d2d559ebd7afadc1d79bd
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Fri Aug 28 13:54:22 2026 +0800
[SPARK-59047][ML] Avoid retaining random forest tree weights
### What changes were proposed in this pull request?
Generate unit tree weights on demand for random forest classification and
regression models instead
of retaining a lazily initialized array. Cache the generated weights
locally while saving an
ensemble model to avoid repeated allocation in the per-tree loop.
This also removes the stale comments about possibly supporting weighted
random forests later.
### Why are the changes needed?
Random forest tree weights are rarely accessed and are not used during
prediction. Generating the
unit weights when requested avoids retaining an otherwise unused array for
the lifetime of the
model.
### Does this PR introduce _any_ user-facing change?
No. Random forest tree weights continue to contain one unit weight for
every tree.
### How was this patch tested?
The existing classification and regression model persistence tests were run:
```
build/sbt 'mllib/testOnly *RandomForestClassifierSuite -- -z "read/write"'
build/sbt 'mllib/testOnly *RandomForestRegressorSuite -- -z "read/write"'
```
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-5)
Closes #58371 from zhengruifeng/ml-spark-59047-tree-weights-dev-5.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit 182f00c6480b75d303276cb062e474fa62f29d63)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../org/apache/spark/ml/classification/RandomForestClassifier.scala | 5 +----
.../scala/org/apache/spark/ml/regression/RandomForestRegressor.scala | 5 +----
mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala | 3 ++-
3 files changed, 4 insertions(+), 9 deletions(-)
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
index 565663196a99..41bcf3729e5a 100644
---
a/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
+++
b/mllib/src/main/scala/org/apache/spark/ml/classification/RandomForestClassifier.scala
@@ -250,11 +250,8 @@ class RandomForestClassificationModel private[ml] (
@Since("1.4.0")
override def trees: Array[DecisionTreeClassificationModel] = _trees
- // Note: We may add support for weights (based on tree performance) later on.
- private lazy val _treeWeights: Array[Double] =
Array.fill[Double](_trees.length)(1.0)
-
@Since("1.4.0")
- override def treeWeights: Array[Double] = _treeWeights
+ override def treeWeights: Array[Double] =
Array.fill[Double](_trees.length)(1.0)
/**
* Gets summary of model on training set. An exception is thrown
diff --git
a/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
b/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
index 9d42d601b1f1..4d9a039ecaca 100644
---
a/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
+++
b/mllib/src/main/scala/org/apache/spark/ml/regression/RandomForestRegressor.scala
@@ -220,11 +220,8 @@ class RandomForestRegressionModel private[ml] (
@Since("1.4.0")
override def trees: Array[DecisionTreeRegressionModel] = _trees
- // Note: We may add support for weights (based on tree performance) later on.
- private lazy val _treeWeights: Array[Double] =
Array.fill[Double](_trees.length)(1.0)
-
@Since("1.4.0")
- override def treeWeights: Array[Double] = _treeWeights
+ override def treeWeights: Array[Double] =
Array.fill[Double](_trees.length)(1.0)
@Since("1.4.0")
override def transformSchema(schema: StructType): StructType = {
diff --git a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
index 1cc64e58d980..2b275ce6fd64 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/tree/treeModels.scala
@@ -527,10 +527,11 @@ private[ml] object EnsembleModelReadWrite {
sparkSession: SparkSession,
extraMetadata: JObject): Unit = {
DefaultParamsWriter.saveMetadata(instance, path, sparkSession,
Some(extraMetadata))
+ val treeWeights = instance.treeWeights
val treesMetadataWeights = instance.trees.zipWithIndex.map { case (tree,
treeID) =>
(treeID,
DefaultParamsWriter.getMetadataToSave(tree.asInstanceOf[Params],
sparkSession),
- instance.treeWeights(treeID))
+ treeWeights(treeID))
}
val treesMetadataPath = new Path(path, "treesMetadata").toString
ReadWriteUtils.saveArray[(Int, String, Double)](
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
