This is an automated email from the ASF dual-hosted git repository.

zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 523596fdc549 [SPARK-60807][ML][CONNECT] Restore offloaded ML model 
summaries during fetch
523596fdc549 is described below

commit 523596fdc5495bb191a0af903622355999a62eaa
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Wed Nov 4 08:23:05 2026 +0800

    [SPARK-60807][ML][CONNECT] Restore offloaded ML model summaries during fetch
    
    ### What changes were proposed in this pull request?
    
    Save the dataset plan received by the existing CreateSummary command beside 
the offloaded model. Command and relation FETCH can use this plan to restore a 
missing summary after further evictions. The plan follows the model's existing 
disk cleanup lifecycle.
    
    Evaluate each FETCH against the same loaded model instance used to check or 
restore its summary. No protobuf or production Python client changes are needed.
    
    ### Why are the changes needed?
    
    The current recovery sends CreateSummary and retries FETCH once. If the 
model is offloaded between those RPCs, the retry reloads a model without its 
summary and fails with CONNECT_ML.MODEL_SUMMARY_LOST. Relation FETCH also 
performs another cache lookup after restoring the summary, which can retrieve a 
different model instance.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Training-summary attributes remain accessible after model offloading, 
including when the in-memory ML cache cannot retain the model.
    
    ### How was this patch tested?
    
    - `build/sbt 'connect/testOnly org.apache.spark.sql.connect.ml.MLSuite'` 
(15 tests passed).
    - `build/sbt -Phive package`, followed by `python/run-tests --testnames 
pyspark.ml.tests.connect.test_connect_model_offloading --python-executables 
python3`.
    - `dev/lint-python --ruff`.
    - `build/sbt 'connect/scalastyle' 'connect/Test/scalastyle'`.
    
    The Scala regression covers zero-size and nonzero caches, repeated eviction 
between CreateSummary and FETCH, both CreateSummary outcomes, deep plans, and 
relation FETCH with and without a dataset.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: OpenAI Codex (GPT-6)
    
    Closes #60022 from zhengruifeng/spark-60807-connect-ml-summary-fetch.
    
    Authored-by: Ruifeng Zheng <[email protected]>
    Signed-off-by: Ruifeng Zheng <[email protected]>
---
 .../apache/spark/sql/connect/ml/MLHandler.scala    | 89 +++++++++++++---------
 .../org/apache/spark/sql/connect/ml/MLSuite.scala  | 75 ++++++++++++++++++
 2 files changed, 129 insertions(+), 35 deletions(-)

diff --git 
a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
 
b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
index 7f9909fc01db..e325f3b3e218 100644
--- 
a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
+++ 
b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.connect.ml
 
 import java.io.{PrintWriter, StringWriter}
 import java.lang.ThreadLocal
+import java.nio.file.Files
 import java.util.concurrent.ConcurrentHashMap
 
 import scala.collection.mutable
@@ -33,7 +34,8 @@ import org.apache.spark.ml.param.{ParamMap, Params}
 import org.apache.spark.ml.tree.TreeConfig
 import org.apache.spark.ml.util.{HasTrainingSummary, MLWritable, Summary}
 import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.connect.common.LiteralValueProtoConverter
+import org.apache.spark.sql.connect.common.{LiteralValueProtoConverter, 
ProtoUtils}
+import org.apache.spark.sql.connect.config.Connect
 import org.apache.spark.sql.connect.ml.Serializer.deserializeMethodArguments
 import org.apache.spark.sql.connect.service.SessionHolder
 import org.apache.spark.util.Utils
@@ -58,9 +60,9 @@ private class AttributeHelper(
     obj
   }
   // Get the attribute by reflection
-  def getAttribute: Any = {
+  def getAttribute(instance: Object): Any = {
     assert(methods.length >= 1)
-    methods.foldLeft(instance()) { (obj, m) =>
+    methods.foldLeft(instance) { (obj, m) =>
       if (m.argValues.isEmpty) {
         MLUtils.invokeMethodAllowed(obj, m.name)
       } else {
@@ -258,16 +260,7 @@ private[connect] object MLHandler extends Logging {
           .build()
 
       case proto.MlCommand.CommandCase.FETCH =>
-        val objRefId = mlCommand.getFetch.getObjRef.getId
-        val methods = mlCommand.getFetch.getMethodsList.asScala.toArray
-        val obj = sessionHolder.mlCache.get(objRefId)
-        if (obj != null && obj.isInstanceOf[HasTrainingSummary[_]]
-          && methods(0).getMethod == "summary"
-          && !obj.asInstanceOf[HasTrainingSummary[_]].hasSummary) {
-          throw MLModelSummaryLostException(objRefId)
-        }
-        val helper = AttributeHelper(sessionHolder, objRefId, methods)
-        val attrResult = helper.getAttribute
+        val attrResult = fetchAttribute(sessionHolder, mlCommand.getFetch)
         attrResult match {
           case s: Summary =>
             val id = mlCache.register(s)
@@ -487,6 +480,42 @@ private[connect] object MLHandler extends Logging {
       .asInstanceOf[proto.MlCommandResult]
   }
 
+  private def fetchAttribute(
+      sessionHolder: SessionHolder,
+      fetch: proto.Fetch,
+      summaryDataset: Option[proto.Relation] = None): Any = {
+    val objRefId = fetch.getObjRef.getId
+    val methods = fetch.getMethodsList.asScala.toArray
+    val obj = sessionHolder.mlCache.get(objRefId)
+    if (obj == null) {
+      throw MLCacheInvalidException(objRefId, 
sessionHolder.mlCache.getOffloadingTimeoutMinute)
+    }
+    sessionHolder.mlCache.synchronized {
+      if (obj.isInstanceOf[HasTrainingSummary[_]] && methods(0).getMethod == 
"summary"
+        && !obj.asInstanceOf[HasTrainingSummary[_]].hasSummary) {
+        val modelPath = sessionHolder.mlCache.getModelOffloadingPath(objRefId)
+        val dataset = summaryDataset.getOrElse {
+          val datasetPath = modelPath.resolve("summary_dataset")
+          if (!Files.exists(datasetPath)) {
+            throw MLModelSummaryLostException(objRefId)
+          }
+          ProtoUtils.parseWithRecursionLimit(
+            Files.readAllBytes(datasetPath),
+            proto.Relation.parser(),
+            sessionHolder.session.sparkContext.conf
+              .get(Connect.CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT))
+        }
+        obj
+          .asInstanceOf[HasTrainingSummary[_]]
+          .loadSummary(
+            modelPath.resolve("summary").toString,
+            MLUtils.parseRelationProto(dataset, sessionHolder))
+      }
+    }
+    // Use the same model even if its cache entry is evicted while restoring 
the summary.
+    AttributeHelper(sessionHolder, objRefId, methods).getAttribute(obj)
+  }
+
   private def createModelSummary(
       sessionHolder: SessionHolder,
       createSummaryCmd: proto.MlCommand.CreateSummary): proto.MlCommandResult =
@@ -502,6 +531,13 @@ private[connect] object MLHandler extends Logging {
       } else {
         false
       }
+      if (sessionHolder.mlCache.getMemoryControlEnabled) {
+        // Keep the dataset plan on disk so recovery survives subsequent model 
evictions.
+        val datasetPath = sessionHolder.mlCache
+          .getModelOffloadingPath(refId)
+          .resolve("summary_dataset")
+        Files.write(datasetPath, createSummaryCmd.getDataset.toByteArray)
+      }
       proto.MlCommandResult
         .newBuilder()
         .setParam(LiteralValueProtoConverter.toLiteralProto(isCreated))
@@ -539,29 +575,12 @@ private[connect] object MLHandler extends Logging {
 
       // Get the attribute from a cached object which could be a model or 
summary
       case proto.MlRelation.MlTypeCase.FETCH =>
-        val objRefId = relation.getFetch.getObjRef.getId
-        val methods = relation.getFetch.getMethodsList.asScala.toArray
-        val obj = sessionHolder.mlCache.get(objRefId)
-        sessionHolder.mlCache.synchronized {
-          if (obj != null && obj.isInstanceOf[HasTrainingSummary[_]]
-            && methods(0).getMethod == "summary"
-            && !obj.asInstanceOf[HasTrainingSummary[_]].hasSummary) {
-
-            if (relation.hasModelSummaryDataset) {
-              val dataset =
-                MLUtils.parseRelationProto(relation.getModelSummaryDataset, 
sessionHolder)
-              val modelPath = 
sessionHolder.mlCache.getModelOffloadingPath(objRefId)
-              val summaryPath = modelPath.resolve("summary").toString
-              obj.asInstanceOf[HasTrainingSummary[_]].loadSummary(summaryPath, 
dataset)
-            } else {
-              // For old Spark client backward compatibility.
-              throw MLModelSummaryLostException(objRefId)
-            }
-          }
+        val summaryDataset = if (relation.hasModelSummaryDataset) {
+          Some(relation.getModelSummaryDataset)
+        } else {
+          None
         }
-
-        val helper = AttributeHelper(sessionHolder, objRefId, methods)
-        helper.getAttribute.asInstanceOf[DataFrame]
+        fetchAttribute(sessionHolder, relation.getFetch, 
summaryDataset).asInstanceOf[DataFrame]
 
       case other => throw MlUnsupportedException(s"$other not supported")
     }
diff --git 
a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
 
b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
index 76dca9c938a7..c30f1d3b95ab 100644
--- 
a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
+++ 
b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
@@ -459,6 +459,81 @@ class MLSuite extends MLHelper {
     }
   }
 
+  gridTest("fetch offloaded model summaries")(Seq(0, 16384)) { cacheSize =>
+    val sessionHolder = SparkConnectTestUtils.createDummySessionHolder(spark)
+    sessionHolder.session.conf
+      
.set(Connect.CONNECT_SESSION_CONNECT_ML_CACHE_MEMORY_CONTROL_ENABLED.key, 
"true")
+    // A zero limit also evicts models reloaded while restoring or fetching a 
summary.
+    sessionHolder.session.conf.set(
+      
Connect.CONNECT_SESSION_CONNECT_ML_CACHE_MEMORY_CONTROL_MAX_IN_MEMORY_SIZE.key,
+      cacheSize)
+    val modelId = trainLogisticRegressionModel(sessionHolder)
+    // Use a plan deeper than the default protobuf recursion limit.
+    val dataset = (0 until 60).foldLeft(createLocalRelationProto) { (input, _) 
=>
+      proto.Relation
+        .newBuilder()
+        .setLimit(proto.Limit.newBuilder().setInput(input).setLimit(4))
+        .build()
+    }
+    val summaryFetch = proto.Fetch
+      .newBuilder()
+      .setObjRef(proto.ObjectRef.newBuilder().setId(modelId))
+      .addMethods(proto.Fetch.Method.newBuilder().setMethod("summary"))
+      .addMethods(proto.Fetch.Method.newBuilder().setMethod("labels"))
+      .build()
+    val command = proto.MlCommand.newBuilder().setFetch(summaryFetch).build()
+
+    try {
+      if (cacheSize == 0) {
+        intercept[MLModelSummaryLostException] {
+          MLHandler.handleMlCommand(sessionHolder, command)
+        }
+      }
+      val createSummary = proto.MlCommand
+        .newBuilder()
+        .setCreateSummary(
+          proto.MlCommand.CreateSummary
+            .newBuilder()
+            .setModelRef(proto.ObjectRef.newBuilder().setId(modelId))
+            .setDataset(dataset))
+        .build()
+      // Persist the plan even when CreateSummary finds the model already has 
its summary.
+      assert(
+        MLHandler
+          .handleMlCommand(sessionHolder, createSummary)
+          .getParam
+          .getBoolean === (cacheSize == 0))
+      for (_ <- 0 until 2) {
+        sessionHolder.mlCache.remove(modelId, evictOnly = true)
+        val result = MLHandler.handleMlCommand(sessionHolder, command)
+        assert(
+          result.getParam.getArray.getElementsList.asScala.map(_.getDouble) 
=== Seq(0.0, 1.0))
+        if (cacheSize == 0) {
+          assert(sessionHolder.mlCache.cachedModel.isEmpty)
+        }
+      }
+
+      val predictions = summaryFetch.toBuilder
+        .setMethods(1, 
proto.Fetch.Method.newBuilder().setMethod("predictions"))
+      val relation = proto.MlRelation
+        .newBuilder()
+        .setFetch(predictions)
+        .setModelSummaryDataset(dataset)
+        .build()
+      sessionHolder.mlCache.remove(modelId, evictOnly = true)
+      assert(MLHandler.transformMLRelation(relation, sessionHolder).count() 
=== 4)
+      sessionHolder.mlCache.remove(modelId, evictOnly = true)
+      assert(MLHandler
+        
.transformMLRelation(relation.toBuilder.clearModelSummaryDataset().build(), 
sessionHolder)
+        .count() === 4)
+      if (cacheSize == 0) {
+        assert(sessionHolder.mlCache.cachedModel.isEmpty)
+      }
+    } finally {
+      sessionHolder.mlCache.clear()
+    }
+  }
+
   test("Model size limit") {
     val sessionHolder = SparkConnectTestUtils.createDummySessionHolder(spark)
     sessionHolder.session.conf


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to