This is an automated email from the ASF dual-hosted git repository.
cloud-fan pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 3069a5404d07 [SPARK-58069][SQL][FOLLOWUP] Handle empty approx_top_k
combine buffers
3069a5404d07 is described below
commit 3069a5404d075e9028a404733c5431b033565430
Author: Wenchen Fan <[email protected]>
AuthorDate: Mon Jul 27 13:38:48 2026 +0800
[SPARK-58069][SQL][FOLLOWUP] Handle empty approx_top_k combine buffers
### What changes were proposed in this pull request?
Followup to https://github.com/apache/spark/pull/57177.
This change makes `CombineInternal` serialize and deserialize the
placeholder buffer created for
an empty partition. It encodes a missing item type as a zero-length type
section and uses a default
string serde for the necessarily empty sketch.
### Why are the changes needed?
An untouched `approx_top_k_combine` buffer has no item type. It can
nevertheless be serialized
between aggregation stages, where selecting a type-specific serde or
serializing the type currently
throws. Preserving the empty placeholder allows the later merge to
initialize it from real input.
### Does this PR introduce _any_ user-facing change?
Yes. `approx_top_k_combine` no longer fails when an empty partition
contributes an untouched
placeholder buffer between aggregation stages.
### How was this patch tested?
Added a regression test to `ApproxTopKSuite` that round-trips an empty
combine buffer and verifies
that its missing item type, sketch, and configured size are preserved. The
staged workflow will run
the required fail-before and pass-after suite cycle before opening the pull
request.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex (GPT-5)
Closes #57375 from cloud-fan/SPARK-58069-followup-238032.
Authored-by: Wenchen Fan <[email protected]>
Signed-off-by: Wenchen Fan <[email protected]>
(cherry picked from commit 18ca27f34cc1fe72acbc6971e0cb89ea823b7bac)
Signed-off-by: Wenchen Fan <[email protected]>
---
.../aggregate/ApproxTopKAggregates.scala | 31 ++++++++++++----
.../org/apache/spark/sql/ApproxTopKSuite.scala | 41 ++++++++++++++++++++++
2 files changed, 66 insertions(+), 6 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala
index b646f6e6e59d..186056ab7e6a 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/aggregate/ApproxTopKAggregates.scala
@@ -723,9 +723,20 @@ class CombineInternal[T](
* boundaries (SPARK-58069).
*/
def serialize(): Array[Byte] = {
- val sketchWithNullCountBytes = sketchWithNullCount.serialize(
-
ApproxTopK.genSketchSerDe(itemDataType).asInstanceOf[ArrayOfItemsSerDe[T]])
- val typeBytes: Array[Byte] =
itemDataType.json.getBytes(StandardCharsets.UTF_8)
+ // An empty partition has a placeholder buffer whose itemDataType has not
been initialized.
+ // It can still be serialized between aggregation stages, so use a default
serde for its empty
+ // sketch and encode the missing type as a zero-length section.
+ val serDe: ArrayOfItemsSerDe[T] = if (itemDataType == null) {
+ new ArrayOfStringsSerDe().asInstanceOf[ArrayOfItemsSerDe[T]]
+ } else {
+
ApproxTopK.genSketchSerDe(itemDataType).asInstanceOf[ArrayOfItemsSerDe[T]]
+ }
+ val sketchWithNullCountBytes = sketchWithNullCount.serialize(serDe)
+ val typeBytes: Array[Byte] = if (itemDataType == null) {
+ Array.emptyByteArray
+ } else {
+ itemDataType.json.getBytes(StandardCharsets.UTF_8)
+ }
val byteArray = new Array[Byte](
sketchWithNullCountBytes.length + Integer.BYTES + Integer.BYTES +
typeBytes.length)
@@ -755,12 +766,20 @@ object CombineInternal {
val typeLength = byteBuffer.getInt
val typeBytes = new Array[Byte](typeLength)
byteBuffer.get(typeBytes)
- val itemDataType = DataType.fromJson(new String(typeBytes,
StandardCharsets.UTF_8))
+ val itemDataType = if (typeLength == 0) {
+ null
+ } else {
+ DataType.fromJson(new String(typeBytes, StandardCharsets.UTF_8))
+ }
// read sketchBytes
val sketchBytes = new Array[Byte](buffer.length - Integer.BYTES -
Integer.BYTES - typeLength)
byteBuffer.get(sketchBytes)
- val sketchWithNullCount = ApproxTopKAggregateBuffer.deserialize(
- sketchBytes, ApproxTopK.genSketchSerDe(itemDataType))
+ val serDe = if (itemDataType == null) {
+ new ArrayOfStringsSerDe().asInstanceOf[ArrayOfItemsSerDe[Any]]
+ } else {
+ ApproxTopK.genSketchSerDe(itemDataType)
+ }
+ val sketchWithNullCount =
ApproxTopKAggregateBuffer.deserialize(sketchBytes, serDe)
new CombineInternal[Any](sketchWithNullCount, itemDataType,
maxItemsTracked)
}
}
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/ApproxTopKSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/ApproxTopKSuite.scala
index 0832262e64ff..1d462c75f20f 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/ApproxTopKSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/ApproxTopKSuite.scala
@@ -20,8 +20,12 @@ package org.apache.spark.sql
import java.sql.{Date, Timestamp}
import java.time.{LocalDateTime, LocalTime}
+import org.apache.datasketches.frequencies.ItemsSketch
+
import org.apache.spark.{SparkArithmeticException, SparkRuntimeException}
import org.apache.spark.sql.catalyst.ExtendedAnalysisException
+import org.apache.spark.sql.catalyst.expressions.aggregate.{ApproxTopK,
ApproxTopKAggregateBuffer,
+ CombineInternal}
import org.apache.spark.sql.errors.DataTypeErrors.toSQLType
import org.apache.spark.sql.test.SharedSparkSession
import org.apache.spark.sql.types.{BooleanType, ByteType, DataType, DateType,
DecimalType, DoubleType, FloatType, IntegerType, LongType, ShortType,
StringType, TimestampNTZType, TimestampType, TimeType}
@@ -575,6 +579,43 @@ class ApproxTopKSuite extends SharedSparkSession {
"CAST('13:00:00.123' AS TIME(3))"))
)
+ test("SPARK-58069: serialize an empty approx_top_k_combine buffer") {
+ Seq(100, ApproxTopK.VOID_MAX_ITEMS_TRACKED).foreach { maxItemsTracked =>
+ val buffer = new CombineInternal[Any](
+ new ApproxTopKAggregateBuffer[Any](new ItemsSketch[Any](128), 0L),
+ null,
+ maxItemsTracked)
+
+ val restored = CombineInternal.deserialize(buffer.serialize())
+
+ assert(restored.getItemDataType == null)
+ assert(restored.getMaxItemsTracked == maxItemsTracked)
+ assert(restored.getSketchWithNullCount.sketch.isEmpty)
+ assert(restored.serialize().sameElements(buffer.serialize()))
+ }
+ }
+
+ test("SPARK-58069: merge an empty approx_top_k_combine buffer") {
+ val initializedBuffer = new CombineInternal[Any](
+ new ApproxTopKAggregateBuffer[Any](new ItemsSketch[Any](128), 0L),
+ StringType,
+ 100)
+ initializedBuffer.updateMaxItemsTracked(
+ combineSizeSpecified = false, ApproxTopK.VOID_MAX_ITEMS_TRACKED)
+ assert(initializedBuffer.getMaxItemsTracked == 100)
+ }
+
+ test("SPARK-58069: combine sketches with empty shuffle partitions") {
+ val sketches = sql(
+ "SELECT approx_top_k_accumulate(id) AS sketch FROM range(10) GROUP BY
id")
+ .repartition(20)
+ val combined = sketches.selectExpr("approx_top_k_combine(sketch) AS
sketch")
+
+ checkAnswer(
+ combined.selectExpr("inline(approx_top_k_estimate(sketch, 10))"),
+ (0L until 10L).map(Row(_, 1L)))
+ }
+
// positive tests for approx_top_k_combine on every types
gridTest("SPARK-52798: same type, same size, specified combine size -
success")(itemsWithTopK) {
case (input, expected) =>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]