This is an automated email from the ASF dual-hosted git repository.

HyukjinKwon pushed a commit to branch branch-4.x
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/branch-4.x by this push:
     new c9c5f776254f [SPARK-60636][SQL] Use SQL signed-zero equality in 
constant-map hash lookups
c9c5f776254f is described below

commit c9c5f776254fa2953b43a21e3b5adc85df5f53d5
Author: Linhong Liu <[email protected]>
AuthorDate: Thu Nov 5 06:51:04 2026 +0900

    [SPARK-60636][SQL] Use SQL signed-zero equality in constant-map hash lookups
    
    ### What changes were proposed in this pull request?
    
    Normalize signed zero when building and probing interpreted hash indexes 
for constant maps with FLOAT or DOUBLE keys. Keep driver and generated hash 
calculations consistent, and preserve the first matching physical entry when a 
map contains both zero signs.
    
    ### Why are the changes needed?
    
    SQL equality treats `-0.0` and `0.0` as equal, but interpreted hash lookups 
use boxed floating-point keys that distinguish them. Such a lookup can return 
NULL where the linear scan finds a value. Generated evaluation already finds 
either zero sign; normalizing its full hash keeps the calculations consistent 
without changing its buckets or results.
    
    ### Does this PR introduce _any_ user-facing change?
    
    Yes. Interpreted hash lookups now find a stored key with either zero sign, 
matching the linear scan and generated evaluation. Generated results, ordinary 
keys, NaN, missing keys, and NULL lookups retain their existing behavior.
    
    ### How was this patch tested?
    
    Eight regression cases cover FLOAT and DOUBLE, either stored zero sign, 
both duplicate orders, forced hash and linear strategies, `GetMapValue` and 
`ElementAt`, interpreted and generated evaluation, and 
ordinary/NaN/missing/NULL controls.
    
    `ComplexTypeSuite` passed: 33 tests, zero failures. The new regression 
cases failed against the unmodified production code.
    
    ```bash
    build/sbt 'catalyst/testOnly *ComplexTypeSuite'
    ```
    
    Scala style checks passed for the Catalyst and SQL source and test code.
    
    ### Was this patch authored or co-authored using generative AI tooling?
    
    Generated-by: Codex CLI 0.164.0 (GPT-6)
    
    Closes #59883 from linhongliu-db/fix-constant-map-signed-zero.
    
    Authored-by: Linhong Liu <[email protected]>
    Signed-off-by: Hyukjin Kwon <[email protected]>
    (cherry picked from commit fb151ff46ebe84ad86bfb5867c6f4c13c9454d38)
    Signed-off-by: Hyukjin Kwon <[email protected]>
---
 .../expressions/complexTypeExtractors.scala        | 24 +++++++++++++-----
 .../catalyst/expressions/ComplexTypeSuite.scala    | 29 ++++++++++++++++++++++
 2 files changed, 47 insertions(+), 6 deletions(-)

diff --git 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
index 074bcc63ce87..4d6acef7f58f 100644
--- 
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
+++ 
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/complexTypeExtractors.scala
@@ -555,12 +555,20 @@ trait GetMapValueUtil extends BinaryExpression with 
ImplicitCastInputTypes {
     while (i < len) {
       // putIfAbsent preserves first-match semantics for maps with duplicate 
keys (allowed at
       // the physical level by [[ArrayBasedMapData]]), matching the linear 
scan path.
-      hm.putIfAbsent(keys.get(i, keyType), i)
+      hm.putIfAbsent(normalizeFloatingPointKey(keys.get(i, keyType)), i)
       i += 1
     }
     hm
   }
 
+  // SQL equality treats both signs of zero as equal; boxed Float/Double 
equality does not.
+  // Normalize stored and lookup keys so hashing agrees with the linear SQL 
comparison.
+  private def normalizeFloatingPointKey(key: Any): Any = key match {
+    case value: Float if value == 0.0f => 0.0f
+    case value: Double if value == 0.0d => 0.0d
+    case _ => key
+  }
+
   /**
    * Builds a power-of-two open-addressed bucket table mapping `hash(key) & 
hashMask` to the
    * key's index in `map.keyArray()`. Used by the codegen hash path so 
primitive-keyed lookups
@@ -598,22 +606,26 @@ trait GetMapValueUtil extends BinaryExpression with 
ImplicitCastInputTypes {
     case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType 
| _: TimeType =>
       val l = v.asInstanceOf[Long]
       (l ^ (l >>> 32)).toInt
-    case FloatType => java.lang.Float.floatToIntBits(v.asInstanceOf[Float])
+    case FloatType =>
+      
java.lang.Float.floatToIntBits(normalizeFloatingPointKey(v).asInstanceOf[Float])
     case DoubleType =>
-      val l = java.lang.Double.doubleToLongBits(v.asInstanceOf[Double])
+      val l = 
java.lang.Double.doubleToLongBits(normalizeFloatingPointKey(v).asInstanceOf[Double])
       (l ^ (l >>> 32)).toInt
     case _ => v.hashCode()
   }
 
+  // Signed-zero normalization must match hashKeyOnDriver so generated probes 
start in the
+  // same bucket as SQL-equal keys, regardless of their zero sign.
   /** Java-side hash expression over the primitive/object `v`. Mirrors 
[[hashKeyOnDriver]]. */
   private def genHash(v: String, keyType: DataType): String = keyType match {
     case BooleanType => s"($v ? 1 : 0)"
     case ByteType | ShortType | IntegerType | DateType | _: 
YearMonthIntervalType => s"$v"
     case LongType | TimestampType | TimestampNTZType | _: DayTimeIntervalType 
| _: TimeType =>
       s"(int)($v ^ ($v >>> 32))"
-    case FloatType => s"Float.floatToIntBits($v)"
+    case FloatType => s"($v == 0.0f ? 0 : Float.floatToIntBits($v))"
     case DoubleType =>
-      s"(int)(Double.doubleToLongBits($v) ^ (Double.doubleToLongBits($v) >>> 
32))"
+      s"($v == 0.0d ? 0 : " +
+        s"(int)(Double.doubleToLongBits($v) ^ (Double.doubleToLongBits($v) >>> 
32)))"
     case _ => s"$v.hashCode()"
   }
 
@@ -743,7 +755,7 @@ trait GetMapValueUtil extends BinaryExpression with 
ImplicitCastInputTypes {
         ordinal: Any,
         keyType: DataType,
         ordering: Ordering[Any]): Any = {
-      val idx = index.getOrDefault(ordinal, -1)
+      val idx = index.getOrDefault(normalizeFloatingPointKey(ordinal), -1)
       if (idx == -1 || values.isNullAt(idx)) null else values.get(idx, 
dataType)
     }
 
diff --git 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
index 66c3be203f5d..90a73efe2fcd 100644
--- 
a/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
+++ 
b/sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/ComplexTypeSuite.scala
@@ -20,6 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
 import org.apache.spark.{SPARK_DOC_ROOT, SparkFunSuite, SparkRuntimeException}
 import org.apache.spark.sql.AnalysisException
 import org.apache.spark.sql.Row
+import org.apache.spark.sql.catalyst.InternalRow
 import org.apache.spark.sql.catalyst.analysis.{TypeCheckResult, 
UnresolvedExtractValue}
 import org.apache.spark.sql.catalyst.analysis.TypeCheckResult.DataTypeMismatch
 import org.apache.spark.sql.catalyst.dsl.expressions._
@@ -397,6 +398,34 @@ class ComplexTypeSuite extends SparkFunSuite with 
ExpressionEvalHelper {
     }
   }
 
+  for (keyType <- Seq(FloatType, DoubleType);
+       zeros <- Seq(Seq(-0.0d), Seq(0.0d), Seq(-0.0d, 0.0d), Seq(0.0d, 
-0.0d))) {
+    test(s"map lookup uses SQL signed-zero equality: $keyType, $zeros") {
+      def key(value: Double): Any =
+        if (keyType == FloatType) (value.toFloat: Any) else value
+      val keys = (zeros ++ Seq(1.0d, Double.NaN)).map(key).toArray
+      val values = keys.indices.map(_ + 10).toArray
+      val map = Literal.create(new ArrayBasedMapData(
+        new GenericArrayData(keys), new GenericArrayData(values)),
+        MapType(keyType, IntegerType, valueContainsNull = false))
+      for (threshold <- Seq(0, Int.MaxValue)) {
+        withSQLConf(SQLConf.MAP_LOOKUP_HASH_THRESHOLD.key -> 
threshold.toString) {
+          val lookup = BoundReference(0, keyType, nullable = true)
+          val get = GetMapValue(map, lookup)
+          assert(get.usesFoldableHashLookup == (threshold == 0))
+          for (expression <- Seq(get, ElementAt(map, lookup, failOnError = 
false))) {
+            checkEvaluation(expression, 10, InternalRow(key(-0.0d)))
+            checkEvaluation(expression, 10, InternalRow(key(0.0d)))
+            checkEvaluation(expression, zeros.size + 10, 
InternalRow(key(1.0d)))
+            checkEvaluation(expression, zeros.size + 11, 
InternalRow(key(Double.NaN)))
+            checkEvaluation(expression, null, InternalRow(key(42.0d)))
+            checkEvaluation(expression, null, InternalRow(null))
+          }
+        }
+      }
+    }
+  }
+
   test("GetMapValue - non-foldable map always uses linear scan") {
     // With threshold=0, a foldable map would take the hash path. A 
non-foldable map (backed by
     // a row column) must still fall back to linear scan, because its hash 
index cannot be


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to