This is an automated email from the ASF dual-hosted git repository.

cloud-fan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 18ca27f34cc1 [SPARK-58069][SQL][FOLLOWUP] Handle empty approx_top_k 
combine buffers
18ca27f34cc1 is described below

commit 18ca27f34cc1fe72acbc6971e0cb89ea823b7bac
Author: Wenchen Fan <[email protected]>
AuthorDate: Mon Jul 27 13:38:48 2026 +0800

    [SPARK-58069][SQL][FOLLOWUP] Handle empty approx_top_k combine buffers
    
    ### What changes were proposed in this pull request?
    
    Followup to https://github.com/apache/spark/pull/57177.
    
    This change makes `CombineInternal` serialize and deserialize the 
placeholder buffer created for
    an empty partition. It encodes a missing item type as a zero-length type 
section and uses a default
    string serde for the necessarily empty sketch.
    
    ### Why are the changes needed?
    
    An untouched `approx_top_k_combine` buffer has no item type. It can 
nevertheless be serialized
    between aggregation stages, where selecting a type-specific serde or 
serializing the type currently
    throws. Preserving the empty placeholder allows the later merge to 
initialize it from real input.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. `approx_top_k_combine` no longer fails when an empty partition 
contributes an untouched
    placeholder buffer between aggregation stages.
    
    ### How was this patch tested?
    
    Added a regression test to `ApproxTopKSuite` that round-trips an empty 
combine buffer and verifies
    that its missing item type, sketch, and configured size are preserved. The 
staged workflow will run
    the required fail-before and pass-after suite cycle before opening the pull 
request.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex (GPT-5)
    
    Closes #57375 from cloud-fan/SPARK-58069-followup-238032.
    
    Authored-by: Wenchen Fan <[email protected]>
    Signed-off-by: Wenchen Fan <[email protected]>
---
 .../aggregate/ApproxTopKAggregates.scala           | 31 ++++++++++++----
 .../org/apache/spark/sql/ApproxTopKSuite.scala     | 41 ++++++++++++++++++++++
 2 files changed, 66 insertions(+), 6 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala
index b646f6e6e59d..186056ab7e6a 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala
@@ -723,9 +723,20 @@ class CombineInternal[T](
    * boundaries (SPARK-58069).
    */
   def serialize(): Array[Byte] = {
-    val sketchWithNullCountBytes = sketchWithNullCount.serialize(
-      
ApproxTopK.genSketchSerDe(itemDataType).asInstanceOf[ArrayOfItemsSerDe[T]])
-    val typeBytes: Array[Byte] = 
itemDataType.json.getBytes(StandardCharsets.UTF_8)
+    // An empty partition has a placeholder buffer whose itemDataType has not 
been initialized.
+    // It can still be serialized between aggregation stages, so use a default 
serde for its empty
+    // sketch and encode the missing type as a zero-length section.
+    val serDe: ArrayOfItemsSerDe[T] = if (itemDataType == null) {
+      new ArrayOfStringsSerDe().asInstanceOf[ArrayOfItemsSerDe[T]]
+    } else {
+      
ApproxTopK.genSketchSerDe(itemDataType).asInstanceOf[ArrayOfItemsSerDe[T]]
+    }
+    val sketchWithNullCountBytes = sketchWithNullCount.serialize(serDe)
+    val typeBytes: Array[Byte] = if (itemDataType == null) {
+      Array.emptyByteArray
+    } else {
+      itemDataType.json.getBytes(StandardCharsets.UTF_8)
+    }
     val byteArray = new Array[Byte](
       sketchWithNullCountBytes.length + Integer.BYTES + Integer.BYTES + 
typeBytes.length)
 
@@ -755,12 +766,20 @@ object CombineInternal {
     val typeLength = byteBuffer.getInt
     val typeBytes = new Array[Byte](typeLength)
     byteBuffer.get(typeBytes)
-    val itemDataType = DataType.fromJson(new String(typeBytes, 
StandardCharsets.UTF_8))
+    val itemDataType = if (typeLength == 0) {
+      null
+    } else {
+      DataType.fromJson(new String(typeBytes, StandardCharsets.UTF_8))
+    }
     // read sketchBytes
     val sketchBytes = new Array[Byte](buffer.length - Integer.BYTES - 
Integer.BYTES - typeLength)
     byteBuffer.get(sketchBytes)
-    val sketchWithNullCount = ApproxTopKAggregateBuffer.deserialize(
-      sketchBytes, ApproxTopK.genSketchSerDe(itemDataType))
+    val serDe = if (itemDataType == null) {
+      new ArrayOfStringsSerDe().asInstanceOf[ArrayOfItemsSerDe[Any]]
+    } else {
+      ApproxTopK.genSketchSerDe(itemDataType)
+    }
+    val sketchWithNullCount = 
ApproxTopKAggregateBuffer.deserialize(sketchBytes, serDe)
     new CombineInternal[Any](sketchWithNullCount, itemDataType, 
maxItemsTracked)
   }
 }
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ApproxTopKSuite.scala 
b/sql/core/src/test/scala/org/apache/spark/sql/ApproxTopKSuite.scala
index 0832262e64ff..1d462c75f20f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ApproxTopKSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ApproxTopKSuite.scala
@@ -20,8 +20,12 @@ package org.apache.spark.sql
 import java.sql.{Date, Timestamp}
 import java.time.{LocalDateTime, LocalTime}
 
+import org.apache.datasketches.frequencies.ItemsSketch
+
 import org.apache.spark.{SparkArithmeticException, SparkRuntimeException}
 import org.apache.spark.sql.catalyst.ExtendedAnalysisException
+import org.apache.spark.sql.catalyst.expressions.aggregate.{ApproxTopK, 
ApproxTopKAggregateBuffer,
+  CombineInternal}
 import org.apache.spark.sql.errors.DataTypeErrors.toSQLType
 import org.apache.spark.sql.test.SharedSparkSession
 import org.apache.spark.sql.types.{BooleanType, ByteType, DataType, DateType, 
DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType, 
StringType, TimestampNTZType, TimestampType, TimeType}
@@ -575,6 +579,43 @@ class ApproxTopKSuite extends SharedSparkSession {
         "CAST('13:00:00.123' AS TIME(3))"))
   )
 
+  test("SPARK-58069: serialize an empty approx_top_k_combine buffer") {
+    Seq(100, ApproxTopK.VOID_MAX_ITEMS_TRACKED).foreach { maxItemsTracked =>
+      val buffer = new CombineInternal[Any](
+        new ApproxTopKAggregateBuffer[Any](new ItemsSketch[Any](128), 0L),
+        null,
+        maxItemsTracked)
+
+      val restored = CombineInternal.deserialize(buffer.serialize())
+
+      assert(restored.getItemDataType == null)
+      assert(restored.getMaxItemsTracked == maxItemsTracked)
+      assert(restored.getSketchWithNullCount.sketch.isEmpty)
+      assert(restored.serialize().sameElements(buffer.serialize()))
+    }
+  }
+
+  test("SPARK-58069: merge an empty approx_top_k_combine buffer") {
+    val initializedBuffer = new CombineInternal[Any](
+      new ApproxTopKAggregateBuffer[Any](new ItemsSketch[Any](128), 0L),
+      StringType,
+      100)
+    initializedBuffer.updateMaxItemsTracked(
+      combineSizeSpecified = false, ApproxTopK.VOID_MAX_ITEMS_TRACKED)
+    assert(initializedBuffer.getMaxItemsTracked == 100)
+  }
+
+  test("SPARK-58069: combine sketches with empty shuffle partitions") {
+    val sketches = sql(
+      "SELECT approx_top_k_accumulate(id) AS sketch FROM range(10) GROUP BY 
id")
+      .repartition(20)
+    val combined = sketches.selectExpr("approx_top_k_combine(sketch) AS 
sketch")
+
+    checkAnswer(
+      combined.selectExpr("inline(approx_top_k_estimate(sketch, 10))"),
+      (0L until 10L).map(Row(_, 1L)))
+  }
+
   // positive tests for approx_top_k_combine on every types
   gridTest("SPARK-52798: same type, same size, specified combine size - 
success")(itemsWithTopK) {
     case (input, expected) =>


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to