This is an automated email from the ASF dual-hosted git repository.
HyukjinKwon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new fb151ff46ebe [SPARK-60636][SQL] Use SQL signed-zero equality in
constant-map hash lookups
fb151ff46ebe is described below
commit fb151ff46ebe84ad86bfb5867c6f4c13c9454d38
Author: Linhong Liu <[email protected]>
AuthorDate: Thu Nov 5 06:51:04 2026 +0900
[SPARK-60636][SQL] Use SQL signed-zero equality in constant-map hash lookups
### What changes were proposed in this pull request?
Normalize signed zero when building and probing interpreted hash indexes
for constant maps with FLOAT or DOUBLE keys. Keep driver and generated hash
calculations consistent, and preserve the first matching physical entry when a
map contains both zero signs.
### Why are the changes needed?
SQL equality treats `-0.0` and `0.0` as equal, but interpreted hash lookups
use boxed floating-point keys that distinguish them. Such a lookup can return
NULL where the linear scan finds a value. Generated evaluation already finds
either zero sign; normalizing its full hash keeps the calculations consistent
without changing its buckets or results.
### Does this PR introduce _any_ user-facing change?
Yes. Interpreted hash lookups now find a stored key with either zero sign,
matching the linear scan and generated evaluation. Generated results, ordinary
keys, NaN, missing keys, and NULL lookups retain their existing behavior.
### How was this patch tested?
Eight regression cases cover FLOAT and DOUBLE, either stored zero sign,
both duplicate orders, forced hash and linear strategies, `GetMapValue` and
`ElementAt`, interpreted and generated evaluation, and
ordinary/NaN/missing/NULL controls.
`ComplexTypeSuite` passed: 33 tests, zero failures. The new regression
cases failed against the unmodified production code.
```bash
build/sbt 'catalyst/testOnly *ComplexTypeSuite'
```
Scala style checks passed for the Catalyst and SQL source and test code.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Codex CLI 0.164.0 (GPT-6)
Closes #59883 from linhongliu-db/fix-constant-map-signed-zero.
Authored-by: Linhong Liu <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
---
.../expressions/complexTypeExtractors.scala | 24 +++++++++++++-----
.../catalyst/expressions/ComplexTypeSuite.scala | 29 ++++++++++++++++++++++
2 files changed, 47 insertions(+), 6 deletions(-)
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index 074bcc63ce87..4d6acef7f58f 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -555,12 +555,20 @@ trait GetMapValueUtil extends BinaryExpression with
ImplicitCastInputTypes {
while (i < len) {
// putIfAbsent preserves first-match semantics for maps with duplicate
keys (allowed at
// the physical level by [[ArrayBasedMapData]]), matching the linear
scan path.
- hm.putIfAbsent(keys.get(i, keyType), i)
+ hm.putIfAbsent(normalizeFloatingPointKey(keys.get(i, keyType)), i)
i += 1
}
hm
}
+ // SQL equality treats both signs of zero as equal; boxed Float/Double
equality does not.
+ // Normalize stored and lookup keys so hashing agrees with the linear SQL
comparison.
+ private def normalizeFloatingPointKey(key: Any): Any = key match {
+ case value: Float if value == 0.0f => 0.0f
+ case value: Double if value == 0.0d => 0.0d
+ case _ => key
+ }
+
/**
* Builds a power-of-two open-addressed bucket table mapping `hash(key) &
hashMask` to the
* key's index in `map.keyArray()`. Used by the codegen hash path so
primitive-keyed lookups
@@ -598,22 +606,26 @@ trait GetMapValueUtil extends BinaryExpression with
ImplicitCastInputTypes {
case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType
| _: TimeType =>
val l = v.asInstanceOf[Long]
(l ^ (l >>> 32)).toInt
- case FloatType => java.lang.Float.floatToIntBits(v.asInstanceOf[Float])
+ case FloatType =>
+
java.lang.Float.floatToIntBits(normalizeFloatingPointKey(v).asInstanceOf[Float])
case DoubleType =>
- val l = java.lang.Double.doubleToLongBits(v.asInstanceOf[Double])
+ val l =
java.lang.Double.doubleToLongBits(normalizeFloatingPointKey(v).asInstanceOf[Double])
(l ^ (l >>> 32)).toInt
case _ => v.hashCode()
}
+ // Signed-zero normalization must match hashKeyOnDriver so generated probes
start in the
+ // same bucket as SQL-equal keys, regardless of their zero sign.
/** Java-side hash expression over the primitive/object `v`. Mirrors
[[hashKeyOnDriver]]. */
private def genHash(v: String, keyType: DataType): String = keyType match {
case BooleanType => s"($v ? 1 : 0)"
case ByteType | ShortType | IntegerType | DateType | _:
YearMonthIntervalType => s"$v"
case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType
| _: TimeType =>
s"(int)($v ^ ($v >>> 32))"
- case FloatType => s"Float.floatToIntBits($v)"
+ case FloatType => s"($v == 0.0f ? 0 : Float.floatToIntBits($v))"
case DoubleType =>
- s"(int)(Double.doubleToLongBits($v) ^ (Double.doubleToLongBits($v) >>>
32))"
+ s"($v == 0.0d ? 0 : " +
+ s"(int)(Double.doubleToLongBits($v) ^ (Double.doubleToLongBits($v) >>>
32)))"
case _ => s"$v.hashCode()"
}
@@ -743,7 +755,7 @@ trait GetMapValueUtil extends BinaryExpression with
ImplicitCastInputTypes {
ordinal: Any,
keyType: DataType,
ordering: Ordering[Any]): Any = {
- val idx = index.getOrDefault(ordinal, -1)
+ val idx = index.getOrDefault(normalizeFloatingPointKey(ordinal), -1)
if (idx == -1 || values.isNullAt(idx)) null else values.get(idx,
dataType)
}
diff --git
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
index 66c3be203f5d..90a73efe2fcd 100644
---
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
+++
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.{SPARK_DOC_ROOT, SparkFunSuite, SparkRuntimeException}
import org.apache.spark.sql.AnalysisException
import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.InternalRow
import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult,
UnresolvedExtractValue}
import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
import org.apache.spark.sql.catalyst.dsl.expressions._
@@ -397,6 +398,34 @@ class ComplexTypeSuite extends SparkFunSuite with
ExpressionEvalHelper {
}
}
+ for (keyType <- Seq(FloatType, DoubleType);
+ zeros <- Seq(Seq(-0.0d), Seq(0.0d), Seq(-0.0d, 0.0d), Seq(0.0d,
-0.0d))) {
+ test(s"map lookup uses SQL signed-zero equality: $keyType, $zeros") {
+ def key(value: Double): Any =
+ if (keyType == FloatType) (value.toFloat: Any) else value
+ val keys = (zeros ++ Seq(1.0d, Double.NaN)).map(key).toArray
+ val values = keys.indices.map(_ + 10).toArray
+ val map = Literal.create(new ArrayBasedMapData(
+ new GenericArrayData(keys), new GenericArrayData(values)),
+ MapType(keyType, IntegerType, valueContainsNull = false))
+ for (threshold <- Seq(0, Int.MaxValue)) {
+ withSQLConf(SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key ->
threshold.toString) {
+ val lookup = BoundReference(0, keyType, nullable = true)
+ val get = GetMapValue(map, lookup)
+ assert(get.usesFoldableHashLookup == (threshold == 0))
+ for (expression <- Seq(get, ElementAt(map, lookup, failOnError =
false))) {
+ checkEvaluation(expression, 10, InternalRow(key(-0.0d)))
+ checkEvaluation(expression, 10, InternalRow(key(0.0d)))
+ checkEvaluation(expression, zeros.size + 10,
InternalRow(key(1.0d)))
+ checkEvaluation(expression, zeros.size + 11,
InternalRow(key(Double.NaN)))
+ checkEvaluation(expression, null, InternalRow(key(42.0d)))
+ checkEvaluation(expression, null, InternalRow(null))
+ }
+ }
+ }
+ }
+ }
+
test("GetMapValue - non-foldable map always uses linear scan") {
// With threshold=0, a foldable map would take the hash path. A
non-foldable map (backed by
// a row column) must still fall back to linear scan, because its hash
index cannot be
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]