This is an automated email from the ASF dual-hosted git repository.
dbtsai pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 4e75ef4b1b4b [SPARK-58175][SQL][PROTOBUF] Cache descriptor trees built
from a binary FileDescriptorSet
4e75ef4b1b4b is described below
commit 4e75ef4b1b4bdebbe6f925ac58f0b5b2421cd052
Author: bhollis-dbx <[email protected]>
AuthorDate: Thu Jul 30 13:05:59 2026 -0700
[SPARK-58175][SQL][PROTOBUF] Cache descriptor trees built from a binary
FileDescriptorSet
### What changes were proposed in this pull request?
`from_protobuf` / `to_protobuf` used with a binary `FileDescriptorSet` hold
their descriptor behind a `transient lazy val`, and building it parses the
bytes into a large `FileDescriptor` object graph -- the tree that holds the
(potentially millions of) `FieldDescriptor` instances. There is no caching, so
each task instance in an executor JVM parses the set independently. Under high
fan-out (many partitions -> many concurrent tasks in one JVM) with large
descriptor sets (hundreds of KB [...]
This PR adds a process-wide bounded cache at the `parseFileDescriptorSet`
layer -- the layer that materializes the `FieldDescriptor` graph -- so
concurrent task instances share one parsed graph per `FileDescriptorSet`
instead of each rebuilding it. Because both `buildDescriptorFromFDS` and the
`convertAnyFieldsToJson` `buildTypeRegistry(bytes)` path
go through this layer, a given set is parsed at most once per JVM
regardless of entry point. Building a `DescriptorWithExtensions` on top of the
cached graph is cheap and happens once per task-instance init, so it is not
itself cached; the extension-support flag is read per call, so a shared parse
still yields the flag-appropriate result.
The cache is keyed on a content hash of the descriptor bytes (which keeps
the key small and avoids pinning the byte array) and does not cache failed
parses. It uses `NonFateSharingCache` (SPARK-43300) so that a task cancelled
while populating an entry does not cause spurious failures in other tasks
blocked on the same key. Its size is bounded by a new internal config,
`spark.sql.protobuf.descriptorCacheSize` (default 8; setting it to 0 disables
the cache as a kill-switch).
### Why are the changes needed?
To eliminate redundant, concurrently-retained copies of large parsed
descriptor graphs and the resulting executor GC pressure for descriptor-heavy
`from_protobuf` / `to_protobuf` workloads.
### Does this PR introduce _any_ user-facing change?
No. A new internal config is added, but behavior is unchanged.
### How was this patch tested?
New unit tests in `ProtobufFunctionsSuite` covering: repeated builds on the
same bytes sharing one cached parse, distinct bytes cached separately, the
`buildTypeRegistry(bytes)` path sharing the parse, the extension-support flag
being honored per call over a shared parse, the disabled (size 0) kill-switch,
an unknown message name surfacing the domain exception, and a failed parse not
being cached (and surfacing the domain exception rather than a Guava wrapper).
Existing `from_protobuf [...]
### Was this patch authored or co-authored using generative AI tooling?
Yes, drafted with assistance from generative AI tooling.
Closes #57321 from bhollis-dbx/protobuf-descriptor-cache.
Authored-by: bhollis-dbx <[email protected]>
Signed-off-by: DB Tsai <[email protected]>
(cherry picked from commit 3395e2ce50587218962135f828c4689296e95052)
Signed-off-by: DB Tsai <[email protected]>
---
.../spark/sql/protobuf/utils/ProtobufUtils.scala | 49 ++++++++++++-
.../sql/protobuf/ProtobufFunctionsSuite.scala | 81 ++++++++++++++++++++++
.../org/apache/spark/sql/internal/SQLConf.scala | 15 ++++
3 files changed, 142 insertions(+), 3 deletions(-)
diff --git
a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
index 47898f73d016..533311e311ea 100644
---
a/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
+++
b/connector/protobuf/src/main/scala/org/apache/spark/sql/protobuf/utils/ProtobufUtils.scala
@@ -18,11 +18,14 @@
package org.apache.spark.sql.protobuf.utils
import java.util.Locale
+import java.util.concurrent.{ExecutionException, TimeUnit}
import scala.collection.mutable
import scala.collection.mutable.ArrayBuffer
import scala.jdk.CollectionConverters._
+import com.google.common.hash.Hashing
+import com.google.common.util.concurrent.{ExecutionError,
UncheckedExecutionException}
import com.google.protobuf.{DescriptorProtos, Descriptors, DynamicMessage,
ExtensionRegistry, InvalidProtocolBufferException, Message}
import com.google.protobuf.DescriptorProtos.{FileDescriptorProto,
FileDescriptorSet}
import com.google.protobuf.Descriptors.{Descriptor, FieldDescriptor}
@@ -33,7 +36,7 @@ import org.apache.spark.internal.Logging
import org.apache.spark.sql.errors.QueryCompilationErrors
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import org.apache.spark.util.Utils
+import org.apache.spark.util.{NonFateSharingCache, Utils}
private[sql] object ProtobufUtils extends Logging {
@@ -238,10 +241,37 @@ private[sql] object ProtobufUtils extends Logging {
DescriptorWithExtensions(descriptor, ExtensionRegistry.getEmptyRegistry,
Map.empty)
}
+ // Parsing a FileDescriptorSet materializes a potentially-large
FileDescriptor graph, and each
+ // task instance would otherwise parse the same bytes independently.
+
+ // Read fresh each call so size 0 is an immediate kill-switch; the built
cache's bound is fixed.
+ private def isDescriptorCacheEnabled: Boolean =
+ SQLConf.get.getConf(SQLConf.PROTOBUF_DESCRIPTOR_CACHE_SIZE) > 0
+
+ private def descriptorCacheMaxSize: Long =
+ SQLConf.get.getConf(SQLConf.PROTOBUF_DESCRIPTOR_CACHE_SIZE).toLong
+
+ private type FileDescriptors = List[Descriptors.FileDescriptor]
+
+ // Keyed on a content hash rather than the bytes, to avoid pinning the
descriptor arrays.
+ // NonFateSharingCache (SPARK-43300) prevents a cancelled task's failed load
from failing the
+ // other tasks blocked on the same key. The loader is passed per get()
rather than as a
+ // CacheLoader because the key is the hash, not the bytes needed to parse;
this overload also
+ // keeps the shaded Guava Cache type out of the signature (SPARK-44064).
+ private lazy val fileDescriptorCache: NonFateSharingCache[String,
FileDescriptors] =
+ NonFateSharingCache(descriptorCacheMaxSize, 0, TimeUnit.SECONDS)
+
+ private[protobuf] def clearDescriptorCacheForTesting(): Unit =
+ fileDescriptorCache.invalidateAll()
+
+ private[protobuf] def fileDescriptorCacheSizeForTesting(): Long =
fileDescriptorCache.size()
+
+ private def contentHash(bytes: Array[Byte]): String =
+ Hashing.sha256().hashBytes(bytes).toString
+
def buildDescriptorFromFDS(
messageName: String,
binaryFileDescriptorSet: Array[Byte]): DescriptorWithExtensions = {
- // Find the first message descriptor that matches the name.
val fileDescriptors = parseFileDescriptorSet(binaryFileDescriptorSet)
val descriptor = fileDescriptors
.flatMap { fileDesc =>
@@ -262,7 +292,20 @@ private[sql] object ProtobufUtils extends Logging {
}
}
- private def parseFileDescriptorSet(bytes: Array[Byte]):
List[Descriptors.FileDescriptor] = {
+ private def parseFileDescriptorSet(bytes: Array[Byte]): FileDescriptors = {
+ if (!isDescriptorCacheEnabled) {
+ return doParseFileDescriptorSet(bytes)
+ }
+ try {
+ fileDescriptorCache.get(contentHash(bytes), () =>
doParseFileDescriptorSet(bytes))
+ } catch {
+ case e: UncheckedExecutionException if e.getCause != null => throw
e.getCause
+ case e: ExecutionException if e.getCause != null => throw e.getCause
+ case e: ExecutionError if e.getCause != null => throw e.getCause
+ }
+ }
+
+ private def doParseFileDescriptorSet(bytes: Array[Byte]): FileDescriptors = {
var fileDescriptorSet: DescriptorProtos.FileDescriptorSet = null
try {
fileDescriptorSet = DescriptorProtos.FileDescriptorSet.parseFrom(bytes)
diff --git
a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
index 5c875989e714..c25b4b264c65 100644
---
a/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
+++
b/connector/protobuf/src/test/scala/org/apache/spark/sql/protobuf/ProtobufFunctionsSuite.scala
@@ -27,6 +27,7 @@ import org.json4s.jackson.JsonMethods
import org.apache.spark.sql.{AnalysisException, Column, DataFrame, Row}
import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.functions.{array, lit, map, struct, typedLit}
+import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.protobuf.protos.Proto2Messages.Proto2AllTypes
import org.apache.spark.sql.protobuf.protos.SimpleMessageProtos._
import
org.apache.spark.sql.protobuf.protos.SimpleMessageProtos.SimpleMessageRepeated.NestedEnum
@@ -2368,4 +2369,84 @@ class ProtobufFunctionsSuite extends SharedSparkSession
with ProtobufTestBase
assert(expectedDf.schema === fromProtoDf.schema)
checkAnswer(fromProtoDf, expectedDf)
}
+
+ test("descriptor cache: repeated builds on the same bytes share the cached
parse") {
+ ProtobufUtils.clearDescriptorCacheForTesting()
+ val first = ProtobufUtils.buildDescriptor("BasicMessage",
Some(testFileDesc))
+ val second = ProtobufUtils.buildDescriptor("BasicMessage",
Some(testFileDesc))
+ // Same descriptor instance (from the shared parse), not a rebuilt copy.
+ assert(first.descriptor eq second.descriptor)
+ val repeated = ProtobufUtils.buildDescriptor("RepeatedMessage",
Some(testFileDesc))
+ assert(repeated.descriptor ne first.descriptor)
+ // Distinct message names on the same bytes still share one parse.
+ assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 1)
+ }
+
+ test("descriptor cache: distinct bytes are parsed and cached separately") {
+ ProtobufUtils.clearDescriptorCacheForTesting()
+ // Derive a second, genuinely distinct descriptor set from the same source
rather than relying
+ // on testFileDesc vs proto2FileDesc being different bytes: under SBT both
point at one combined
+ // descriptor file, so they hash to the same cache key.
proto2_messages.proto has no imports, so
+ // its single-file set parses standalone and is distinct from the full
combined set either way.
+ val proto2Standalone = descriptorSetWithoutImports(proto2FileDesc,
"FoobarWithRequiredFieldBar")
+ val basic = ProtobufUtils.buildDescriptor("BasicMessage",
Some(testFileDesc))
+ val proto2 = ProtobufUtils.buildDescriptor("FoobarWithRequiredFieldBar",
Some(proto2Standalone))
+ assert(proto2.descriptor ne basic.descriptor)
+ assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 2)
+ }
+
+ test("descriptor cache: buildTypeRegistry shares the parse with
buildDescriptor") {
+ ProtobufUtils.clearDescriptorCacheForTesting()
+ ProtobufUtils.buildDescriptor("BasicMessage", Some(testFileDesc))
+ assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 1)
+ // The other entry point reuses the same parse rather than adding an entry.
+ ProtobufUtils.buildTypeRegistry(testFileDesc)
+ assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 1)
+ }
+
+ test("descriptor cache: the extensions-enabled flag is honored per call") {
+ ProtobufUtils.clearDescriptorCacheForTesting()
+ val extConf = SQLConf.PROTOBUF_EXTENSIONS_SUPPORT_ENABLED
+ // The flag is read per build, so a shared cached parse still yields the
flag-correct result.
+ withSQLConf(extConf.key -> "false") {
+ val disabled = ProtobufUtils.buildDescriptor("BasicMessage",
Some(testFileDesc))
+ assert(disabled.extensionRegistry eq
com.google.protobuf.ExtensionRegistry.getEmptyRegistry)
+ }
+ withSQLConf(extConf.key -> "true") {
+ val enabled = ProtobufUtils.buildDescriptor("BasicMessage",
Some(testFileDesc))
+ assert(enabled.extensionRegistry ne
com.google.protobuf.ExtensionRegistry.getEmptyRegistry)
+ }
+ assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 1)
+ }
+
+ test("descriptor cache: disabled (size 0) reparses every time and caches
nothing") {
+ ProtobufUtils.clearDescriptorCacheForTesting()
+ withSQLConf(SQLConf.PROTOBUF_DESCRIPTOR_CACHE_SIZE.key -> "0") {
+ val first = ProtobufUtils.buildDescriptor("BasicMessage",
Some(testFileDesc))
+ val second = ProtobufUtils.buildDescriptor("BasicMessage",
Some(testFileDesc))
+ // Each call reparses, so the descriptors come from different graphs.
+ assert(first.descriptor ne second.descriptor)
+ assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 0)
+ }
+ }
+
+ test("descriptor cache: an unknown message name surfaces the domain
exception") {
+ ProtobufUtils.clearDescriptorCacheForTesting()
+ // The parse succeeds; only the message lookup fails, so the parse stays
cached below.
+ intercept[AnalysisException] {
+ ProtobufUtils.buildDescriptor("NoSuchMessage", Some(testFileDesc))
+ }
+ val good = ProtobufUtils.buildDescriptor("BasicMessage",
Some(testFileDesc))
+ assert(good.descriptor != null)
+ assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 1)
+ }
+
+ test("descriptor cache: a failed parse is not cached") {
+ ProtobufUtils.clearDescriptorCacheForTesting()
+ intercept[AnalysisException] {
+ ProtobufUtils.buildTypeRegistry(Array[Byte](1, 2, 3, 4))
+ }
+ // A failed parse leaves the cache empty.
+ assert(ProtobufUtils.fileDescriptorCacheSizeForTesting() == 0)
+ }
}
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
index 97594802e258..ee9d06683360 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/internal/SQLConf.scala
@@ -8036,6 +8036,21 @@ object SQLConf {
.booleanConf
.createWithDefault(false)
+ val PROTOBUF_DESCRIPTOR_CACHE_SIZE =
+ buildConf("spark.sql.protobuf.descriptorCacheSize")
+ .internal()
+ .doc("Maximum number of entries in the per-JVM cache of parsed Protobuf
FileDescriptor " +
+ "graphs built from a binary FileDescriptorSet, used by
from_protobuf/to_protobuf. This " +
+ "knob only bounds memory (each graph can be hundreds of KB). Setting
it to 0 disables " +
+ "the cache entirely and acts as an immediate kill-switch; a non-zero
size bound is fixed " +
+ "when the cache is first built for a JVM, so changing it has no effect
on an " +
+ "already-built cache.")
+ .version("4.3.0")
+ .withBindingPolicy(ConfigBindingPolicy.NOT_APPLICABLE)
+ .intConf
+ .checkValue(_ >= 0, "descriptorCacheSize must be non-negative; 0
disables the cache")
+ .createWithDefault(8)
+
val LISTAGG_ALLOW_DISTINCT_CAST_WITH_ORDER =
buildConf("spark.sql.listagg.allowDistinctCastWithOrder.enabled")
.internal()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]