This is an automated email from the ASF dual-hosted git repository.
MaxGekk pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-4.x by this push:
new 3d1b2d3c452e [SPARK-56893][SQL][TEST] Add
ParquetDictionaryDecodeBenchmark for dictionary ID decode throughput
3d1b2d3c452e is described below
commit 3d1b2d3c452e8838c7d4b3ead2478b05ccc38731
Author: Ismaël Mejía <[email protected]>
AuthorDate: Mon Jun 22 22:01:29 2026 +0200
[SPARK-56893][SQL][TEST] Add ParquetDictionaryDecodeBenchmark for
dictionary ID decode throughput
### What changes were proposed in this pull request?
Add a micro-benchmark for `ParquetVectorUpdater.decodeDictionaryIds` -- the
second pass of dictionary-encoded Parquet reads that translates dictionary IDs
into decoded values.
Coverage:
- **Core primitive Updaters:** Integer, Long, Float, Double.
- **Type-converting Updaters:** IntegerToLong, FloatToDouble.
Each group is tested with three null fractions (0%, 10%, 50%) to exercise
the no-null fast path and the per-element null-check path.
The benchmark includes a global pre-warm phase that interleaves both
`hasNull()` branches for every updater class to ensure C2 compiles with
balanced profiles, avoiding uncommon-trap demotion bias.
### Why are the changes needed?
This establishes a committed baseline for #55920 (SPARK-56893), which
optimizes the dictionary decode path with a `hasNull()` fast path and per-class
updater overrides. Landing the benchmark first allows the optimization PR to
regenerate the results, so the `.txt` diff directly shows the throughput gain
-- matching the pattern used in #55922 and #55924.
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
The benchmark compiles and runs successfully:
```
build/sbt "sql/Test/runMain
org.apache.spark.sql.execution.datasources.parquet.ParquetDictionaryDecodeBenchmark"
```
Benchmark results will be generated via the GitHub Actions benchmark
workflow.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: GitHub Copilot (claude-opus-4.6)
Closes #56670 from iemejia/SPARK-56893-parquet-dict-decode-benchmark.
Authored-by: Ismaël Mejía <[email protected]>
Signed-off-by: Max Gekk <[email protected]>
(cherry picked from commit ba8e7fe6f0443da07dba88118de448ef982962b0)
Signed-off-by: Max Gekk <[email protected]>
---
...quetDictionaryDecodeBenchmark-jdk21-results.txt | 48 +++++
...quetDictionaryDecodeBenchmark-jdk25-results.txt | 48 +++++
.../ParquetDictionaryDecodeBenchmark-results.txt | 48 +++++
.../parquet/ParquetDictionaryDecodeBenchmark.scala | 229 +++++++++++++++++++++
4 files changed, 373 insertions(+)
diff --git
a/sql/core/benchmarks/ParquetDictionaryDecodeBenchmark-jdk21-results.txt
b/sql/core/benchmarks/ParquetDictionaryDecodeBenchmark-jdk21-results.txt
new file mode 100644
index 000000000000..041372e7f0af
--- /dev/null
+++ b/sql/core/benchmarks/ParquetDictionaryDecodeBenchmark-jdk21-results.txt
@@ -0,0 +1,48 @@
+================================================================================================
+Dictionary Decode (no nulls)
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.11+10-LTS on Linux 6.17.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Dictionary Decode (no nulls): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+IntegerUpdater 6 6
0 177.6 5.6 1.0X
+LongUpdater 6 6
0 176.4 5.7 1.0X
+FloatUpdater 6 6
0 178.3 5.6 1.0X
+DoubleUpdater 6 6
0 177.2 5.6 1.0X
+IntegerToLongUpdater (INT32 -> Long) 6 6
0 175.8 5.7 1.0X
+FloatToDoubleUpdater (FLOAT -> Double) 6 6
0 172.2 5.8 1.0X
+
+
+================================================================================================
+Dictionary Decode (10% nulls)
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.11+10-LTS on Linux 6.17.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Dictionary Decode (10% nulls): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+IntegerUpdater 7 7
0 159.9 6.3 1.0X
+LongUpdater 7 7
0 158.4 6.3 1.0X
+FloatUpdater 7 7
0 159.9 6.3 1.0X
+DoubleUpdater 7 7
0 159.8 6.3 1.0X
+IntegerToLongUpdater (INT32 -> Long) 7 7
0 160.4 6.2 1.0X
+FloatToDoubleUpdater (FLOAT -> Double) 7 7
0 157.0 6.4 1.0X
+
+
+================================================================================================
+Dictionary Decode (50% nulls)
+================================================================================================
+
+OpenJDK 64-Bit Server VM 21.0.11+10-LTS on Linux 6.17.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Dictionary Decode (50% nulls): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+IntegerUpdater 9 9
0 113.2 8.8 1.0X
+LongUpdater 9 9
0 117.3 8.5 1.0X
+FloatUpdater 9 9
0 117.7 8.5 1.0X
+DoubleUpdater 9 9
0 117.4 8.5 1.0X
+IntegerToLongUpdater (INT32 -> Long) 9 9
0 113.5 8.8 1.0X
+FloatToDoubleUpdater (FLOAT -> Double) 9 9
0 117.6 8.5 1.0X
+
+
diff --git
a/sql/core/benchmarks/ParquetDictionaryDecodeBenchmark-jdk25-results.txt
b/sql/core/benchmarks/ParquetDictionaryDecodeBenchmark-jdk25-results.txt
new file mode 100644
index 000000000000..538cc8c3445b
--- /dev/null
+++ b/sql/core/benchmarks/ParquetDictionaryDecodeBenchmark-jdk25-results.txt
@@ -0,0 +1,48 @@
+================================================================================================
+Dictionary Decode (no nulls)
+================================================================================================
+
+OpenJDK 64-Bit Server VM 25.0.3+9-LTS on Linux 6.17.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Dictionary Decode (no nulls): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+IntegerUpdater 6 6
0 179.1 5.6 1.0X
+LongUpdater 6 6
0 178.6 5.6 1.0X
+FloatUpdater 6 6
0 178.1 5.6 1.0X
+DoubleUpdater 6 6
0 177.5 5.6 1.0X
+IntegerToLongUpdater (INT32 -> Long) 6 6
0 175.1 5.7 1.0X
+FloatToDoubleUpdater (FLOAT -> Double) 6 6
0 178.8 5.6 1.0X
+
+
+================================================================================================
+Dictionary Decode (10% nulls)
+================================================================================================
+
+OpenJDK 64-Bit Server VM 25.0.3+9-LTS on Linux 6.17.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Dictionary Decode (10% nulls): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+IntegerUpdater 7 7
0 159.0 6.3 1.0X
+LongUpdater 7 7
0 159.3 6.3 1.0X
+FloatUpdater 7 7
0 158.6 6.3 1.0X
+DoubleUpdater 7 7
0 158.2 6.3 1.0X
+IntegerToLongUpdater (INT32 -> Long) 7 7
0 158.6 6.3 1.0X
+FloatToDoubleUpdater (FLOAT -> Double) 7 7
0 159.2 6.3 1.0X
+
+
+================================================================================================
+Dictionary Decode (50% nulls)
+================================================================================================
+
+OpenJDK 64-Bit Server VM 25.0.3+9-LTS on Linux 6.17.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Dictionary Decode (50% nulls): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+IntegerUpdater 9 9
0 116.4 8.6 1.0X
+LongUpdater 9 9
0 111.4 9.0 1.0X
+FloatUpdater 9 9
0 116.4 8.6 1.0X
+DoubleUpdater 9 9
0 116.6 8.6 1.0X
+IntegerToLongUpdater (INT32 -> Long) 9 9
0 114.0 8.8 1.0X
+FloatToDoubleUpdater (FLOAT -> Double) 9 9
0 116.2 8.6 1.0X
+
+
diff --git a/sql/core/benchmarks/ParquetDictionaryDecodeBenchmark-results.txt
b/sql/core/benchmarks/ParquetDictionaryDecodeBenchmark-results.txt
new file mode 100644
index 000000000000..c43127858a13
--- /dev/null
+++ b/sql/core/benchmarks/ParquetDictionaryDecodeBenchmark-results.txt
@@ -0,0 +1,48 @@
+================================================================================================
+Dictionary Decode (no nulls)
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.19+10-LTS on Linux 6.17.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Dictionary Decode (no nulls): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+IntegerUpdater 6 6
0 186.7 5.4 1.0X
+LongUpdater 6 6
0 185.1 5.4 1.0X
+FloatUpdater 6 6
0 188.6 5.3 1.0X
+DoubleUpdater 6 6
0 187.6 5.3 1.0X
+IntegerToLongUpdater (INT32 -> Long) 6 6
0 184.4 5.4 1.0X
+FloatToDoubleUpdater (FLOAT -> Double) 6 6
0 187.3 5.3 1.0X
+
+
+================================================================================================
+Dictionary Decode (10% nulls)
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.19+10-LTS on Linux 6.17.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Dictionary Decode (10% nulls): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+IntegerUpdater 6 6
0 164.8 6.1 1.0X
+LongUpdater 6 6
0 165.7 6.0 1.0X
+FloatUpdater 6 6
0 166.1 6.0 1.0X
+DoubleUpdater 6 6
0 165.7 6.0 1.0X
+IntegerToLongUpdater (INT32 -> Long) 6 6
0 165.7 6.0 1.0X
+FloatToDoubleUpdater (FLOAT -> Double) 6 6
0 165.7 6.0 1.0X
+
+
+================================================================================================
+Dictionary Decode (50% nulls)
+================================================================================================
+
+OpenJDK 64-Bit Server VM 17.0.19+10-LTS on Linux 6.17.0-1018-azure
+AMD EPYC 9V74 80-Core Processor
+Dictionary Decode (50% nulls): Best Time(ms) Avg Time(ms)
Stdev(ms) Rate(M/s) Per Row(ns) Relative
+------------------------------------------------------------------------------------------------------------------------
+IntegerUpdater 9 9
0 117.7 8.5 1.0X
+LongUpdater 9 9
0 116.2 8.6 1.0X
+FloatUpdater 9 9
0 117.7 8.5 1.0X
+DoubleUpdater 9 9
0 117.6 8.5 1.0X
+IntegerToLongUpdater (INT32 -> Long) 9 9
0 118.6 8.4 1.0X
+FloatToDoubleUpdater (FLOAT -> Double) 9 9
0 117.4 8.5 1.0X
+
+
diff --git
a/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetDictionaryDecodeBenchmark.scala
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetDictionaryDecodeBenchmark.scala
new file mode 100644
index 000000000000..e7e2e6881fb1
--- /dev/null
+++
b/sql/core/src/test/scala/org/apache/spark/sql/execution/datasources/parquet/ParquetDictionaryDecodeBenchmark.scala
@@ -0,0 +1,229 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.execution.datasources.parquet
+
+import java.time.ZoneOffset
+
+import org.apache.parquet.column.{ColumnDescriptor, Encoding}
+import org.apache.parquet.schema.{LogicalTypeAnnotation, Types}
+import org.apache.parquet.schema.PrimitiveType.PrimitiveTypeName
+import org.apache.parquet.schema.Type.Repetition
+
+import org.apache.spark.benchmark.{Benchmark, BenchmarkBase}
+import org.apache.spark.sql.execution.vectorized.{OnHeapColumnVector,
WritableColumnVector}
+import org.apache.spark.sql.types._
+
+/**
+ * Benchmark for `ParquetVectorUpdater.decodeDictionaryIds` -- the second pass
of
+ * dictionary-encoded Parquet reads. After
`VectorizedRleValuesReader.readIntegers`
+ * populates dictionary IDs and null markers, `decodeDictionaryIds` translates
the IDs
+ * into decoded values.
+ *
+ * Coverage:
+ * A. Core primitive Updaters: Integer, Long, Float, Double.
+ * B. Type-converting Updaters: IntegerToLong, FloatToDouble.
+ *
+ * Each group is tested with three null fractions (0%, 10%, 50%) to exercise
the
+ * no-null fast path and the per-element null-check path.
+ *
+ * The dictionary is an anonymous `org.apache.parquet.column.Dictionary`
backed by
+ * pre-populated arrays (100 entries), matching the production decode-to-xxx
methods.
+ * Dictionary IDs are uniform-random in [0, 100).
+ *
+ * JIT note: `decodeDictionaryIds` has two branches (no-null vs has-null).
Running one
+ * branch extensively can bias the JIT against the other via uncommon-trap
demotion.
+ * A global pre-warm phase interleaves both branches for every updater class
before any
+ * measurement to ensure C2 compiles with balanced profiles.
+ *
+ * To run this benchmark:
+ * {{{
+ * 1. build/sbt "sql/Test/runMain <this class>"
+ * 2. generate result:
+ * SPARK_GENERATE_BENCHMARK_FILES=1 build/sbt "sql/Test/runMain <this
class>"
+ * Results in "benchmarks/ParquetDictionaryDecodeBenchmark-results.txt".
+ * }}}
+ */
+object ParquetDictionaryDecodeBenchmark extends BenchmarkBase {
+
+ private val NUM_ROWS = 1024 * 1024
+ private val NUM_ITERS = 5
+ private val DICT_SIZE = 100
+
+ // --------------- Helpers ---------------
+
+ private def descriptor(
+ name: PrimitiveTypeName,
+ logical: LogicalTypeAnnotation = null): ColumnDescriptor = {
+ var builder = Types.primitive(name, Repetition.OPTIONAL)
+ if (logical != null) builder = builder.as(logical)
+ new ColumnDescriptor(Array("col"), builder.named("col"), 0, 1)
+ }
+
+ private def factory(desc: ColumnDescriptor): ParquetVectorUpdaterFactory =
+ ParquetTestAccess.newFactory(
+ desc.getPrimitiveType.getLogicalTypeAnnotation,
+ ZoneOffset.UTC, "CORRECTED", "UTC", "CORRECTED", "UTC")
+
+ /**
+ * Creates a parquet-mr Dictionary backed by pre-populated arrays.
+ * Supports decodeToInt, decodeToLong, decodeToFloat, decodeToDouble.
+ */
+ private def createDictionary(size: Int):
org.apache.parquet.column.Dictionary = {
+ val intVals = Array.tabulate(size)(i => i * 7)
+ val longVals = Array.tabulate(size)(i => i.toLong * 13)
+ val floatVals = Array.tabulate(size)(i => i * 0.1f)
+ val doubleVals = Array.tabulate(size)(i => i * 0.01)
+
+ new org.apache.parquet.column.Dictionary(Encoding.PLAIN) {
+ override def getMaxId: Int = size - 1
+ override def decodeToInt(id: Int): Int = intVals(id)
+ override def decodeToLong(id: Int): Long = longVals(id)
+ override def decodeToFloat(id: Int): Float = floatVals(id)
+ override def decodeToDouble(id: Int): Double = doubleVals(id)
+ }
+ }
+
+ /** Populates a column vector with random dictionary IDs in [0, dictSize). */
+ private def populateDictIds(
+ dictIds: WritableColumnVector, count: Int, dictSize: Int): Unit = {
+ val rng = new java.util.Random(42)
+ var i = 0
+ while (i < count) {
+ dictIds.putInt(i, rng.nextInt(dictSize))
+ i += 1
+ }
+ }
+
+ /** Sets null markers on a column vector using the given null fraction. */
+ private def setNulls(
+ values: WritableColumnVector, count: Int, nullFraction: Double): Unit = {
+ val rng = new java.util.Random(123)
+ var i = 0
+ while (i < count) {
+ if (rng.nextDouble() < nullFraction) values.putNull(i)
+ i += 1
+ }
+ }
+
+ /** Updater configurations: (sparkType, descriptor). */
+ private val updaterConfigs: Seq[(DataType, ColumnDescriptor)] = Seq(
+ (IntegerType, descriptor(PrimitiveTypeName.INT32)),
+ (LongType, descriptor(PrimitiveTypeName.INT64)),
+ (FloatType, descriptor(PrimitiveTypeName.FLOAT)),
+ (DoubleType, descriptor(PrimitiveTypeName.DOUBLE)),
+ (LongType, descriptor(PrimitiveTypeName.INT32)), // IntegerToLong
+ (DoubleType, descriptor(PrimitiveTypeName.FLOAT)) // FloatToDouble
+ )
+
+ /**
+ * Pre-warms all updater classes by interleaving no-null and has-null calls.
+ * This trains C2 to compile both `hasNull()` branches as hot paths, avoiding
+ * the uncommon-trap demotion that occurs when one branch dominates
profiling.
+ */
+ private def globalPreWarm(dict: org.apache.parquet.column.Dictionary): Unit
= {
+ val warmIters = 50
+ for ((sparkType, desc) <- updaterConfigs) {
+ val updater = factory(desc).getUpdater(desc, sparkType)
+
+ val noNullVec = new OnHeapColumnVector(NUM_ROWS, sparkType)
+ val nullVec = new OnHeapColumnVector(NUM_ROWS, sparkType)
+ val dictIds = new OnHeapColumnVector(NUM_ROWS, IntegerType)
+ populateDictIds(dictIds, NUM_ROWS, DICT_SIZE)
+ setNulls(nullVec, NUM_ROWS, 0.5)
+
+ var iter = 0
+ while (iter < warmIters) {
+ updater.decodeDictionaryIds(NUM_ROWS, 0, noNullVec, dictIds, dict)
+ updater.decodeDictionaryIds(NUM_ROWS, 0, nullVec, dictIds, dict)
+ iter += 1
+ }
+ }
+ }
+
+ // --------------- Per-case runner ---------------
+
+ private val updaterLabels: Seq[String] = Seq(
+ "IntegerUpdater",
+ "LongUpdater",
+ "FloatUpdater",
+ "DoubleUpdater",
+ "IntegerToLongUpdater (INT32 -> Long)",
+ "FloatToDoubleUpdater (FLOAT -> Double)"
+ )
+
+ /**
+ * Registers a benchmark case that decodes `NUM_ROWS` dictionary IDs via
+ * `updater.decodeDictionaryIds`. The values vector has null markers pre-set
+ * (for the given null fraction) and is NOT reset between iterations -- the
+ * decoder reads nulls and overwrites non-null slots, so the null state is
+ * stable across iterations.
+ */
+ private def addDictDecodeCase(
+ benchmark: Benchmark,
+ label: String,
+ sparkType: DataType,
+ desc: ColumnDescriptor,
+ dict: org.apache.parquet.column.Dictionary,
+ nullFraction: Double): Unit = {
+ val updater = factory(desc).getUpdater(desc, sparkType)
+ val values = new OnHeapColumnVector(NUM_ROWS, sparkType)
+ val dictIds = new OnHeapColumnVector(NUM_ROWS, IntegerType)
+
+ populateDictIds(dictIds, NUM_ROWS, DICT_SIZE)
+ if (nullFraction > 0.0) setNulls(values, NUM_ROWS, nullFraction)
+
+ // Per-case pre-warm (supplements globalPreWarm)
+ updater.decodeDictionaryIds(NUM_ROWS, 0, values, dictIds, dict)
+
+ benchmark.addCase(label) { _ =>
+ updater.decodeDictionaryIds(NUM_ROWS, 0, values, dictIds, dict)
+ }
+ }
+
+ // --------------- Benchmark groups ---------------
+
+ private def runDictionaryDecodeBenchmark(
+ nullFraction: Double,
+ dict: org.apache.parquet.column.Dictionary): Unit = {
+ val label = if (nullFraction == 0.0) "no nulls"
+ else s"${(nullFraction * 100).toInt}% nulls"
+ val benchmark = new Benchmark(
+ s"Dictionary Decode ($label)", NUM_ROWS.toLong, NUM_ITERS, output =
output)
+
+ updaterConfigs.zip(updaterLabels).foreach { case ((sparkType, desc), name)
=>
+ addDictDecodeCase(benchmark, name, sparkType, desc, dict, nullFraction)
+ }
+
+ benchmark.run()
+ }
+
+ override def runBenchmarkSuite(mainArgs: Array[String]): Unit = {
+ val dict = createDictionary(DICT_SIZE)
+ globalPreWarm(dict)
+
+ runBenchmark("Dictionary Decode (no nulls)") {
+ runDictionaryDecodeBenchmark(0.0, dict)
+ }
+ runBenchmark("Dictionary Decode (10% nulls)") {
+ runDictionaryDecodeBenchmark(0.1, dict)
+ }
+ runBenchmark("Dictionary Decode (50% nulls)") {
+ runDictionaryDecodeBenchmark(0.5, dict)
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
