This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new 446f17f2f3d2 [SPARK-58367][ML][CONNECT] Include ALS metadata in size 
estimates
446f17f2f3d2 is described below

commit 446f17f2f3d204a6eaf764237c02b862bdaaaf2a
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Tue Jul 28 08:47:18 2026 +0800

    [SPARK-58367][ML][CONNECT] Include ALS metadata in size estimates
    
    ### What changes were proposed in this pull request?
    
    This PR updates ALS pre-fit and fitted-model size estimation to include 
parameter metadata as well as the distributed user and item factor data.
    
    ### Why are the changes needed?
    
    ALS has a specialized estimate for distributed factor data but previously 
omitted the model's parameter metadata. Including it makes ALS cache accounting 
consistent with other Spark ML implementations.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Updated `ALSSuite` to verify the metadata contribution in both estimation 
paths and assert that a fitted model remains below the expected upper bound.
    
    Also ran `git diff --check`, ASCII, and source-line-length checks.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #57554 from zhengruifeng/als-model-size-metadata-dev3.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
    (cherry picked from commit 6f05a8309ab9d464ddf17d6ca34cc2e9a24c0090)
    Signed-off-by: Ruifeng Zheng <[email protected]>
---
 mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala | 8 ++++++--
 .../test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala  | 8 ++++++--
 2 files changed, 12 insertions(+), 4 deletions(-)

diff --git a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala 
b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
index 43fa2c0ed3d2..9a54b70f7777 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/recommendation/ALS.scala
@@ -526,9 +526,11 @@ class ALSModel private[ml] (
   }
 
   private[spark] override def estimatedSize: Long = {
+    var size = estimateMatadataSize
     val userCount = userFactors.count()
     val itemCount = itemFactors.count()
-    (userCount + itemCount) * (rank + 1) * 4
+    size += (userCount + itemCount) * (rank + 1) * 4
+    size
   }
 }
 
@@ -805,10 +807,12 @@ class ALS(@Since("1.4.0") override val uid: String) 
extends Estimator[ALSModel]
   override def copy(extra: ParamMap): ALS = defaultCopy(extra)
 
   override def estimateModelSize(dataset: Dataset[_]): Long = {
+    var size = estimateMatadataSize
     val Row(userCount: Long, itemCount: Long) =
       dataset.select(count_distinct(col(getUserCol)), 
count_distinct(col(getItemCol))).head()
     val rank = getRank
-    (userCount + itemCount) * (rank + 1) * 4
+    size += (userCount + itemCount) * (rank + 1) * 4
+    size
   }
 }
 
diff --git 
a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala 
b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
index e6e15bb1b348..a5bff057bfec 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/recommendation/ALSSuite.scala
@@ -1132,10 +1132,14 @@ class ALSStorageSuite extends SparkFunSuite with 
MLlibTestSparkContext with Defa
       (111, 2, 1.0),
       (111, 1, 0.1)
     )).toDF("item", "user", "rating")
-    assert(als.estimateModelSize(df) === estimatedDFSize)
+    assert(als.estimateModelSize(df) === estimatedDFSize + 
als.estimateMatadataSize)
 
     val model = als.fit(df)
-    assert(model.estimatedSize == estimatedDFSize)
+    assert(model.estimatedSize === estimatedDFSize + 
model.estimateMatadataSize)
+    val maxSize = 1024 * 16
+    assert(
+      model.estimatedSize < maxSize,
+      s"Estimation (${model.estimatedSize}) should be less than $maxSize")
   }
 }
 


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to