This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 446f17f2f3d2 [SPARK-58367][ML][CONNECT] Include ALS metadata in size
estimates
446f17f2f3d2 is described below
commit 446f17f2f3d204a6eaf764237c02b862bdaaaf2a
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Jul 28 08:47:18 2026 +0800
[SPARK-58367][ML][CONNECT] Include ALS metadata in size estimates
### What changes were proposed in this pull request?
This PR updates ALS pre-fit and fitted-model size estimation to include
parameter metadata as well as the distributed user and item factor data.
### Why are the changes needed?
ALS has a specialized estimate for distributed factor data but previously
omitted the model's parameter metadata. Including it makes ALS cache accounting
consistent with other Spark ML implementations.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
Updated `ALSSuite` to verify the metadata contribution in both estimation
paths and assert that a fitted model remains below the expected upper bound.
Also ran `git diff --check`, ASCII, and source-line-length checks.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57554 from zhengruifeng/als-model-size-metadata-dev3.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
(cherry picked from commit 6f05a8309ab9d464ddf17d6ca34cc2e9a24c0090)
Signed-off-by: Ruifeng Zheng <[email protected]>
---
mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala | 8 ++++++--
.../test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala | 8 ++++++--
2 files changed, 12 insertions(+), 4 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
index 43fa2c0ed3d2..9a54b70f7777 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
@@ -526,9 +526,11 @@ class ALSModel private[ml] (
}
private[spark] override def estimatedSize: Long = {
+ var size = estimateMatadataSize
val userCount = userFactors.count()
val itemCount = itemFactors.count()
- (userCount + itemCount) * (rank + 1) * 4
+ size += (userCount + itemCount) * (rank + 1) * 4
+ size
}
}
@@ -805,10 +807,12 @@ class ALS(@Since("1.4.0") override val uid: String)
extends Estimator[ALSModel]
override def copy(extra: ParamMap): ALS = defaultCopy(extra)
override def estimateModelSize(dataset: Dataset[_]): Long = {
+ var size = estimateMatadataSize
val Row(userCount: Long, itemCount: Long) =
dataset.select(count_distinct(col(getUserCol)),
count_distinct(col(getItemCol))).head()
val rank = getRank
- (userCount + itemCount) * (rank + 1) * 4
+ size += (userCount + itemCount) * (rank + 1) * 4
+ size
}
}
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index e6e15bb1b348..a5bff057bfec 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -1132,10 +1132,14 @@ class ALSStorageSuite extends SparkFunSuite with
MLlibTestSparkContext with Defa
(111, 2, 1.0),
(111, 1, 0.1)
)).toDF("item", "user", "rating")
- assert(als.estimateModelSize(df) === estimatedDFSize)
+ assert(als.estimateModelSize(df) === estimatedDFSize +
als.estimateMatadataSize)
val model = als.fit(df)
- assert(model.estimatedSize == estimatedDFSize)
+ assert(model.estimatedSize === estimatedDFSize +
model.estimateMatadataSize)
+ val maxSize = 1024 * 16
+ assert(
+ model.estimatedSize < maxSize,
+ s"Estimation (${model.estimatedSize}) should be less than $maxSize")
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]