This is an automated email from the ASF dual-hosted git repository.

dbtsai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 3395e2ce5058 [SPARK-58175][SQL][PROTOBUF] Cache descriptor trees built 
from a binary FileDescriptorSet
3395e2ce5058 is described below

commit 3395e2ce50587218962135f828c4689296e95052
Author: bhollis-dbx <[email protected]>
AuthorDate: Thu Jul 30 13:05:59 2026 -0700

    [SPARK-58175][SQL][PROTOBUF] Cache descriptor trees built from a binary 
FileDescriptorSet
    
    ### What changes were proposed in this pull request?
    
    `from_protobuf` / `to_protobuf` used with a binary `FileDescriptorSet` hold 
their descriptor behind a `transient lazy val`, and building it parses the 
bytes into a large `FileDescriptor` object graph -- the tree that holds the 
(potentially millions of) `FieldDescriptor` instances. There is no caching, so 
each task instance in an executor JVM parses the set independently. Under high 
fan-out (many partitions -> many concurrent tasks in one JVM) with large 
descriptor sets (hundreds of KB [...]
    
    This PR adds a process-wide bounded cache at the `parseFileDescriptorSet` 
layer -- the layer that materializes the `FieldDescriptor` graph -- so 
concurrent task instances share one parsed graph per `FileDescriptorSet` 
instead of each rebuilding it. Because both `buildDescriptorFromFDS` and the 
`convertAnyFieldsToJson` `buildTypeRegistry(bytes)` path
      go through this layer, a given set is parsed at most once per JVM 
regardless of entry point. Building a `DescriptorWithExtensions` on top of the 
cached graph is cheap and happens once per task-instance init, so it is not 
itself cached; the extension-support flag is read per call, so a shared parse 
still yields the flag-appropriate result.
    
    The cache is keyed on a content hash of the descriptor bytes (which keeps 
the key small and avoids pinning the byte array) and does not cache failed 
parses. It uses `NonFateSharingCache` (SPARK-43300) so that a task cancelled 
while populating an entry does not cause spurious failures in other tasks 
blocked on the same key. Its size is bounded by a new internal config, 
`spark.sql.protobuf.descriptorCacheSize` (default 8; setting it to 0 disables 
the cache as a kill-switch).
    
    ### Why are the changes needed?
    
    To eliminate redundant, concurrently-retained copies of large parsed 
descriptor graphs and the resulting executor GC pressure for descriptor-heavy 
`from_protobuf` / `to_protobuf` workloads.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No. A new internal config is added, but behavior is unchanged.
    
    ### How was this patch tested?
    
    New unit tests in `ProtobufFunctionsSuite` covering: repeated builds on the 
same bytes sharing one cached parse, distinct bytes cached separately, the 
`buildTypeRegistry(bytes)` path sharing the parse, the extension-support flag 
being honored per call over a shared parse, the disabled (size 0) kill-switch, 
an unknown message name surfacing the domain exception, and a failed parse not 
being cached (and surfacing the domain exception rather than a Guava wrapper). 
Existing `from_protobuf [...]
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Yes, drafted with assistance from generative AI tooling.
    
    Closes #57321 from bhollis-dbx/protobuf-descriptor-cache.
    
    Authored-by: bhollis-dbx <[email protected]>
    Signed-off-by: DB Tsai <[email protected]>
---
 .../spark/sql/protobuf/utils/ProtobufUtils.scala   | 49 ++++++++++++-
 .../sql/protobuf/ProtobufFunctionsSuite.scala      | 81 ++++++++++++++++++++++
 .../org/apache/spark/sql/internal/SQLConf.scala    | 15 ++++
 3 files changed, 142 insertions(+), 3 deletions(-)

diff --git 
a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
 
b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
index 47898f73d016..533311e311ea 100644
--- 
a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
+++ 
b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
@@ -18,11 +18,14 @@
 package org.apache.spark.sql.protobuf.utils
 
 import java.util.Locale
+import java.util.concurrent.{ExecutionException, TimeUnit}
 
 import scala.collection.mutable
 import scala.collection.mutable.ArrayBuffer
 import scala.jdk.CollectionConverters._
 
+import com.google.common.hash.Hashing
+import com.google.common.util.concurrent.{ExecutionError, 
UncheckedExecutionException}
 import com.google.protobuf.{DescriptorProtos, Descriptors, DynamicMessage, 
ExtensionRegistry, InvalidProtocolBufferException, Message}
 import com.google.protobuf.DescriptorProtos.{FileDescriptorProto, 
FileDescriptorSet}
 import com.google.protobuf.Descriptors.{Descriptor, FieldDescriptor}
@@ -33,7 +36,7 @@ import org.apache.spark.internal.Logging
 import org.apache.spark.sql.errors.QueryCompilationErrors
 import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.types._
-import org.apache.spark.util.Utils
+import org.apache.spark.util.{NonFateSharingCache, Utils}
 
 private[sql] object ProtobufUtils extends Logging {
 
@@ -238,10 +241,37 @@ private[sql] object ProtobufUtils extends Logging {
     DescriptorWithExtensions(descriptor, ExtensionRegistry.getEmptyRegistry, 
Map.empty)
   }
 
+  // Parsing a FileDescriptorSet materializes a potentially-large 
FileDescriptor graph, and each
+  // task instance would otherwise parse the same bytes independently.
+
+  // Read fresh each call so size 0 is an immediate kill-switch; the built 
cache's bound is fixed.
+  private def isDescriptorCacheEnabled: Boolean =
+    SQLConf.get.getConf(SQLConf.PROTOBUF_DESCRIPTOR_CACHE_SIZE) > 0
+
+  private def descriptorCacheMaxSize: Long =
+    SQLConf.get.getConf(SQLConf.PROTOBUF_DESCRIPTOR_CACHE_SIZE).toLong
+
+  private type FileDescriptors = List[Descriptors.FileDescriptor]
+
+  // Keyed on a content hash rather than the bytes, to avoid pinning the 
descriptor arrays.
+  // NonFateSharingCache (SPARK-43300) prevents a cancelled task's failed load 
from failing the
+  // other tasks blocked on the same key. The loader is passed per get() 
rather than as a
+  // CacheLoader because the key is the hash, not the bytes needed to parse; 
this overload also
+  // keeps the shaded Guava Cache type out of the signature (SPARK-44064).
+  private lazy val fileDescriptorCache: NonFateSharingCache[String, 
FileDescriptors] =
+    NonFateSharingCache(descriptorCacheMaxSize, 0, TimeUnit.SECONDS)
+
+  private[protobuf] def clearDescriptorCacheForTesting(): Unit =
+    fileDescriptorCache.invalidateAll()
+
+  private[protobuf] def fileDescriptorCacheSizeForTesting(): Long = 
fileDescriptorCache.size()
+
+  private def contentHash(bytes: Array[Byte]): String =
+    Hashing.sha256().hashBytes(bytes).toString
+
   def buildDescriptorFromFDS(
       messageName: String,
       binaryFileDescriptorSet: Array[Byte]): DescriptorWithExtensions = {
-    // Find the first message descriptor that matches the name.
     val fileDescriptors = parseFileDescriptorSet(binaryFileDescriptorSet)
     val descriptor = fileDescriptors
       .flatMap { fileDesc =>
@@ -262,7 +292,20 @@ private[sql] object ProtobufUtils extends Logging {
     }
   }
 
-  private def parseFileDescriptorSet(bytes: Array[Byte]): 
List[Descriptors.FileDescriptor] = {
+  private def parseFileDescriptorSet(bytes: Array[Byte]): FileDescriptors = {
+    if (!isDescriptorCacheEnabled) {
+      return doParseFileDescriptorSet(bytes)
+    }
+    try {
+      fileDescriptorCache.get(contentHash(bytes), () => 
doParseFileDescriptorSet(bytes))
+    } catch {
+      case e: UncheckedExecutionException if e.getCause != null => throw 
e.getCause
+      case e: ExecutionException if e.getCause != null => throw e.getCause
+      case e: ExecutionError if e.getCause != null => throw e.getCause
+    }
+  }
+
+  private def doParseFileDescriptorSet(bytes: Array[Byte]): FileDescriptors = {
     var fileDescriptorSet: DescriptorProtos.FileDescriptorSet = null
     try {
       fileDescriptorSet = DescriptorProtos.FileDescriptorSet.parseFrom(bytes)
diff --git 
a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
 
b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
index 5c875989e714..c25b4b264c65 100644
--- 
a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
+++ 
b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
@@ -27,6 +27,7 @@ import org.json4s.jackson.JsonMethods
 import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Row}
 import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.functions.{array, lit, map, struct, typedLit}
+import org.apache.spark.sql.internal.SQLConf
 import org.apache.spark.sql.protobuf.protos.Proto2Messages.Proto2AllTypes
 import org.apache.spark.sql.protobuf.protos.SimpleMessageProtos._
 import 
org.apache.spark.sql.protobuf.protos.SimpleMessageProtos.SimpleMessageRepeated.NestedEnum
@@ -2368,4 +2369,84 @@ class ProtobufFunctionsSuite extends SharedSparkSession 
with ProtobufTestBase
     assert(expectedDf.schema === fromProtoDf.schema)
     checkAnswer(fromProtoDf, expectedDf)
   }
+
+  test("descriptor cache: repeated builds on the same bytes share the cached 
parse") {
+    ProtobufUtils.clearDescriptorCacheForTesting()
+    val first = ProtobufUtils.buildDescriptor("BasicMessage", 
Some(testFileDesc))
+    val second = ProtobufUtils.buildDescriptor("BasicMessage", 
Some(testFileDesc))
+    // Same descriptor instance (from the shared parse), not a rebuilt copy.
+    assert(first.descriptor eq second.descriptor)
+    val repeated = ProtobufUtils.buildDescriptor("RepeatedMessage", 
Some(testFileDesc))
+    assert(repeated.descriptor ne first.descriptor)
+    // Distinct message names on the same bytes still share one parse.
+    assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 1)
+  }
+
+  test("descriptor cache: distinct bytes are parsed and cached separately") {
+    ProtobufUtils.clearDescriptorCacheForTesting()
+    // Derive a second, genuinely distinct descriptor set from the same source 
rather than relying
+    // on testFileDesc vs proto2FileDesc being different bytes: under SBT both 
point at one combined
+    // descriptor file, so they hash to the same cache key. 
proto2_messages.proto has no imports, so
+    // its single-file set parses standalone and is distinct from the full 
combined set either way.
+    val proto2Standalone = descriptorSetWithoutImports(proto2FileDesc, 
"FoobarWithRequiredFieldBar")
+    val basic = ProtobufUtils.buildDescriptor("BasicMessage", 
Some(testFileDesc))
+    val proto2 = ProtobufUtils.buildDescriptor("FoobarWithRequiredFieldBar", 
Some(proto2Standalone))
+    assert(proto2.descriptor ne basic.descriptor)
+    assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 2)
+  }
+
+  test("descriptor cache: buildTypeRegistry shares the parse with 
buildDescriptor") {
+    ProtobufUtils.clearDescriptorCacheForTesting()
+    ProtobufUtils.buildDescriptor("BasicMessage", Some(testFileDesc))
+    assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 1)
+    // The other entry point reuses the same parse rather than adding an entry.
+    ProtobufUtils.buildTypeRegistry(testFileDesc)
+    assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 1)
+  }
+
+  test("descriptor cache: the extensions-enabled flag is honored per call") {
+    ProtobufUtils.clearDescriptorCacheForTesting()
+    val extConf = SQLConf.PROTOBUF_EXTENSIONS_SUPPORT_ENABLED
+    // The flag is read per build, so a shared cached parse still yields the 
flag-correct result.
+    withSQLConf(extConf.key -> "false") {
+      val disabled = ProtobufUtils.buildDescriptor("BasicMessage", 
Some(testFileDesc))
+      assert(disabled.extensionRegistry eq 
com.google.protobuf.ExtensionRegistry.getEmptyRegistry)
+    }
+    withSQLConf(extConf.key -> "true") {
+      val enabled = ProtobufUtils.buildDescriptor("BasicMessage", 
Some(testFileDesc))
+      assert(enabled.extensionRegistry ne 
com.google.protobuf.ExtensionRegistry.getEmptyRegistry)
+    }
+    assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 1)
+  }
+
+  test("descriptor cache: disabled (size 0) reparses every time and caches 
nothing") {
+    ProtobufUtils.clearDescriptorCacheForTesting()
+    withSQLConf(SQLConf.PROTOBUF_DESCRIPTOR_CACHE_SIZE.key -> "0") {
+      val first = ProtobufUtils.buildDescriptor("BasicMessage", 
Some(testFileDesc))
+      val second = ProtobufUtils.buildDescriptor("BasicMessage", 
Some(testFileDesc))
+      // Each call reparses, so the descriptors come from different graphs.
+      assert(first.descriptor ne second.descriptor)
+      assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 0)
+    }
+  }
+
+  test("descriptor cache: an unknown message name surfaces the domain 
exception") {
+    ProtobufUtils.clearDescriptorCacheForTesting()
+    // The parse succeeds; only the message lookup fails, so the parse stays 
cached below.
+    intercept[AnalysisException] {
+      ProtobufUtils.buildDescriptor("NoSuchMessage", Some(testFileDesc))
+    }
+    val good = ProtobufUtils.buildDescriptor("BasicMessage", 
Some(testFileDesc))
+    assert(good.descriptor != null)
+    assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 1)
+  }
+
+  test("descriptor cache: a failed parse is not cached") {
+    ProtobufUtils.clearDescriptorCacheForTesting()
+    intercept[AnalysisException] {
+      ProtobufUtils.buildTypeRegistry(Array[Byte](1, 2, 3, 4))
+    }
+    // A failed parse leaves the cache empty.
+    assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 0)
+  }
 }
diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 97594802e258..ee9d06683360 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -8036,6 +8036,21 @@ object SQLConf {
       .booleanConf
       .createWithDefault(false)
 
+  val PROTOBUF_DESCRIPTOR_CACHE_SIZE =
+    buildConf("spark.sql.protobuf.descriptorCacheSize")
+      .internal()
+      .doc("Maximum number of entries in the per-JVM cache of parsed Protobuf 
FileDescriptor " +
+        "graphs built from a binary FileDescriptorSet, used by 
from_protobuf/to_protobuf. This " +
+        "knob only bounds memory (each graph can be hundreds of KB). Setting 
it to 0 disables " +
+        "the cache entirely and acts as an immediate kill-switch; a non-zero 
size bound is fixed " +
+        "when the cache is first built for a JVM, so changing it has no effect 
on an " +
+        "already-built cache.")
+      .version("4.3.0")
+      .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+      .intConf
+      .checkValue(_ >= 0, "descriptorCacheSize must be non-negative; 0 
disables the cache")
+      .createWithDefault(8)
+
   val LISTAGG_ALLOW_DISTINCT_CAST_WITH_ORDER =
     buildConf("spark.sql.listagg.allowDistinctCastWithOrder.enabled")
       .internal()


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to