This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 523596fdc549 [SPARK-60807][ML][CONNECT] Restore offloaded ML model
summaries during fetch
523596fdc549 is described below
commit 523596fdc5495bb191a0af903622355999a62eaa
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Wed Nov 4 08:23:05 2026 +0800
[SPARK-60807][ML][CONNECT] Restore offloaded ML model summaries during fetch
### What changes were proposed in this pull request?
Save the dataset plan received by the existing CreateSummary command beside
the offloaded model. Command and relation FETCH can use this plan to restore a
missing summary after further evictions. The plan follows the model's existing
disk cleanup lifecycle.
Evaluate each FETCH against the same loaded model instance used to check or
restore its summary. No protobuf or production Python client changes are needed.
### Why are the changes needed?
The current recovery sends CreateSummary and retries FETCH once. If the
model is offloaded between those RPCs, the retry reloads a model without its
summary and fails with CONNECT_ML.MODEL_SUMMARY_LOST. Relation FETCH also
performs another cache lookup after restoring the summary, which can retrieve a
different model instance.
### Does this PR introduce _any_ user-facing change?
Yes. Training-summary attributes remain accessible after model offloading,
including when the in-memory ML cache cannot retain the model.
### How was this patch tested?
- `build/sbt 'connect/testOnly org.apache.spark.sql.connect.ml.MLSuite'`
(15 tests passed).
- `build/sbt -Phive package`, followed by `python/run-tests --testnames
pyspark.ml.tests.connect.test_connect_model_offloading --python-executables
python3`.
- `dev/lint-python --ruff`.
- `build/sbt 'connect/scalastyle' 'connect/Test/scalastyle'`.
The Scala regression covers zero-size and nonzero caches, repeated eviction
between CreateSummary and FETCH, both CreateSummary outcomes, deep plans, and
relation FETCH with and without a dataset.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: OpenAI Codex (GPT-6)
Closes #60022 from zhengruifeng/spark-60807-connect-ml-summary-fetch.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../apache/spark/sql/connect/ml/MLHandler.scala | 89 +++++++++++++---------
.../org/apache/spark/sql/connect/ml/MLSuite.scala | 75 ++++++++++++++++++
2 files changed, 129 insertions(+), 35 deletions(-)
diff --git
a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
index 7f9909fc01db..e325f3b3e218 100644
---
a/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
+++
b/sql/connect/server/src/main/scala/org/apache/spark/sql/connect/ml/MLHandler.scala
@@ -19,6 +19,7 @@ package org.apache.spark.sql.connect.ml
import java.io.{PrintWriter, StringWriter}
import java.lang.ThreadLocal
+import java.nio.file.Files
import java.util.concurrent.ConcurrentHashMap
import scala.collection.mutable
@@ -33,7 +34,8 @@ import org.apache.spark.ml.param.{ParamMap, Params}
import org.apache.spark.ml.tree.TreeConfig
import org.apache.spark.ml.util.{HasTrainingSummary, MLWritable, Summary}
import org.apache.spark.sql.DataFrame
-import org.apache.spark.sql.connect.common.LiteralValueProtoConverter
+import org.apache.spark.sql.connect.common.{LiteralValueProtoConverter,
ProtoUtils}
+import org.apache.spark.sql.connect.config.Connect
import org.apache.spark.sql.connect.ml.Serializer.deserializeMethodArguments
import org.apache.spark.sql.connect.service.SessionHolder
import org.apache.spark.util.Utils
@@ -58,9 +60,9 @@ private class AttributeHelper(
obj
}
// Get the attribute by reflection
- def getAttribute: Any = {
+ def getAttribute(instance: Object): Any = {
assert(methods.length >= 1)
- methods.foldLeft(instance()) { (obj, m) =>
+ methods.foldLeft(instance) { (obj, m) =>
if (m.argValues.isEmpty) {
MLUtils.invokeMethodAllowed(obj, m.name)
} else {
@@ -258,16 +260,7 @@ private[connect] object MLHandler extends Logging {
.build()
case proto.MlCommand.CommandCase.FETCH =>
- val objRefId = mlCommand.getFetch.getObjRef.getId
- val methods = mlCommand.getFetch.getMethodsList.asScala.toArray
- val obj = sessionHolder.mlCache.get(objRefId)
- if (obj != null && obj.isInstanceOf[HasTrainingSummary[_]]
- && methods(0).getMethod == "summary"
- && !obj.asInstanceOf[HasTrainingSummary[_]].hasSummary) {
- throw MLModelSummaryLostException(objRefId)
- }
- val helper = AttributeHelper(sessionHolder, objRefId, methods)
- val attrResult = helper.getAttribute
+ val attrResult = fetchAttribute(sessionHolder, mlCommand.getFetch)
attrResult match {
case s: Summary =>
val id = mlCache.register(s)
@@ -487,6 +480,42 @@ private[connect] object MLHandler extends Logging {
.asInstanceOf[proto.MlCommandResult]
}
+ private def fetchAttribute(
+ sessionHolder: SessionHolder,
+ fetch: proto.Fetch,
+ summaryDataset: Option[proto.Relation] = None): Any = {
+ val objRefId = fetch.getObjRef.getId
+ val methods = fetch.getMethodsList.asScala.toArray
+ val obj = sessionHolder.mlCache.get(objRefId)
+ if (obj == null) {
+ throw MLCacheInvalidException(objRefId,
sessionHolder.mlCache.getOffloadingTimeoutMinute)
+ }
+ sessionHolder.mlCache.synchronized {
+ if (obj.isInstanceOf[HasTrainingSummary[_]] && methods(0).getMethod ==
"summary"
+ && !obj.asInstanceOf[HasTrainingSummary[_]].hasSummary) {
+ val modelPath = sessionHolder.mlCache.getModelOffloadingPath(objRefId)
+ val dataset = summaryDataset.getOrElse {
+ val datasetPath = modelPath.resolve("summary_dataset")
+ if (!Files.exists(datasetPath)) {
+ throw MLModelSummaryLostException(objRefId)
+ }
+ ProtoUtils.parseWithRecursionLimit(
+ Files.readAllBytes(datasetPath),
+ proto.Relation.parser(),
+ sessionHolder.session.sparkContext.conf
+ .get(Connect.CONNECT_GRPC_MARSHALLER_RECURSION_LIMIT))
+ }
+ obj
+ .asInstanceOf[HasTrainingSummary[_]]
+ .loadSummary(
+ modelPath.resolve("summary").toString,
+ MLUtils.parseRelationProto(dataset, sessionHolder))
+ }
+ }
+ // Use the same model even if its cache entry is evicted while restoring
the summary.
+ AttributeHelper(sessionHolder, objRefId, methods).getAttribute(obj)
+ }
+
private def createModelSummary(
sessionHolder: SessionHolder,
createSummaryCmd: proto.MlCommand.CreateSummary): proto.MlCommandResult =
@@ -502,6 +531,13 @@ private[connect] object MLHandler extends Logging {
} else {
false
}
+ if (sessionHolder.mlCache.getMemoryControlEnabled) {
+ // Keep the dataset plan on disk so recovery survives subsequent model
evictions.
+ val datasetPath = sessionHolder.mlCache
+ .getModelOffloadingPath(refId)
+ .resolve("summary_dataset")
+ Files.write(datasetPath, createSummaryCmd.getDataset.toByteArray)
+ }
proto.MlCommandResult
.newBuilder()
.setParam(LiteralValueProtoConverter.toLiteralProto(isCreated))
@@ -539,29 +575,12 @@ private[connect] object MLHandler extends Logging {
// Get the attribute from a cached object which could be a model or
summary
case proto.MlRelation.MlTypeCase.FETCH =>
- val objRefId = relation.getFetch.getObjRef.getId
- val methods = relation.getFetch.getMethodsList.asScala.toArray
- val obj = sessionHolder.mlCache.get(objRefId)
- sessionHolder.mlCache.synchronized {
- if (obj != null && obj.isInstanceOf[HasTrainingSummary[_]]
- && methods(0).getMethod == "summary"
- && !obj.asInstanceOf[HasTrainingSummary[_]].hasSummary) {
-
- if (relation.hasModelSummaryDataset) {
- val dataset =
- MLUtils.parseRelationProto(relation.getModelSummaryDataset,
sessionHolder)
- val modelPath =
sessionHolder.mlCache.getModelOffloadingPath(objRefId)
- val summaryPath = modelPath.resolve("summary").toString
- obj.asInstanceOf[HasTrainingSummary[_]].loadSummary(summaryPath,
dataset)
- } else {
- // For old Spark client backward compatibility.
- throw MLModelSummaryLostException(objRefId)
- }
- }
+ val summaryDataset = if (relation.hasModelSummaryDataset) {
+ Some(relation.getModelSummaryDataset)
+ } else {
+ None
}
-
- val helper = AttributeHelper(sessionHolder, objRefId, methods)
- helper.getAttribute.asInstanceOf[DataFrame]
+ fetchAttribute(sessionHolder, relation.getFetch,
summaryDataset).asInstanceOf[DataFrame]
case other => throw MlUnsupportedException(s"$other not supported")
}
diff --git
a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
index 76dca9c938a7..c30f1d3b95ab 100644
---
a/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
+++
b/sql/connect/server/src/test/scala/org/apache/spark/sql/connect/ml/MLSuite.scala
@@ -459,6 +459,81 @@ class MLSuite extends MLHelper {
}
}
+ gridTest("fetch offloaded model summaries")(Seq(0, 16384)) { cacheSize =>
+ val sessionHolder = SparkConnectTestUtils.createDummySessionHolder(spark)
+ sessionHolder.session.conf
+
.set(Connect.CONNECT_SESSION_CONNECT_ML_CACHE_MEMORY_CONTROL_ENABLED.key,
"true")
+ // A zero limit also evicts models reloaded while restoring or fetching a
summary.
+ sessionHolder.session.conf.set(
+
Connect.CONNECT_SESSION_CONNECT_ML_CACHE_MEMORY_CONTROL_MAX_IN_MEMORY_SIZE.key,
+ cacheSize)
+ val modelId = trainLogisticRegressionModel(sessionHolder)
+ // Use a plan deeper than the default protobuf recursion limit.
+ val dataset = (0 until 60).foldLeft(createLocalRelationProto) { (input, _)
=>
+ proto.Relation
+ .newBuilder()
+ .setLimit(proto.Limit.newBuilder().setInput(input).setLimit(4))
+ .build()
+ }
+ val summaryFetch = proto.Fetch
+ .newBuilder()
+ .setObjRef(proto.ObjectRef.newBuilder().setId(modelId))
+ .addMethods(proto.Fetch.Method.newBuilder().setMethod("summary"))
+ .addMethods(proto.Fetch.Method.newBuilder().setMethod("labels"))
+ .build()
+ val command = proto.MlCommand.newBuilder().setFetch(summaryFetch).build()
+
+ try {
+ if (cacheSize == 0) {
+ intercept[MLModelSummaryLostException] {
+ MLHandler.handleMlCommand(sessionHolder, command)
+ }
+ }
+ val createSummary = proto.MlCommand
+ .newBuilder()
+ .setCreateSummary(
+ proto.MlCommand.CreateSummary
+ .newBuilder()
+ .setModelRef(proto.ObjectRef.newBuilder().setId(modelId))
+ .setDataset(dataset))
+ .build()
+ // Persist the plan even when CreateSummary finds the model already has
its summary.
+ assert(
+ MLHandler
+ .handleMlCommand(sessionHolder, createSummary)
+ .getParam
+ .getBoolean === (cacheSize == 0))
+ for (_ <- 0 until 2) {
+ sessionHolder.mlCache.remove(modelId, evictOnly = true)
+ val result = MLHandler.handleMlCommand(sessionHolder, command)
+ assert(
+ result.getParam.getArray.getElementsList.asScala.map(_.getDouble)
=== Seq(0.0, 1.0))
+ if (cacheSize == 0) {
+ assert(sessionHolder.mlCache.cachedModel.isEmpty)
+ }
+ }
+
+ val predictions = summaryFetch.toBuilder
+ .setMethods(1,
proto.Fetch.Method.newBuilder().setMethod("predictions"))
+ val relation = proto.MlRelation
+ .newBuilder()
+ .setFetch(predictions)
+ .setModelSummaryDataset(dataset)
+ .build()
+ sessionHolder.mlCache.remove(modelId, evictOnly = true)
+ assert(MLHandler.transformMLRelation(relation, sessionHolder).count()
=== 4)
+ sessionHolder.mlCache.remove(modelId, evictOnly = true)
+ assert(MLHandler
+
.transformMLRelation(relation.toBuilder.clearModelSummaryDataset().build(),
sessionHolder)
+ .count() === 4)
+ if (cacheSize == 0) {
+ assert(sessionHolder.mlCache.cachedModel.isEmpty)
+ }
+ } finally {
+ sessionHolder.mlCache.clear()
+ }
+ }
+
test("Model size limit") {
val sessionHolder = SparkConnectTestUtils.createDummySessionHolder(spark)
sessionHolder.session.conf
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]