comphead commented on code in PR #68:
URL: https://github.com/apache/datafusion-spark/pull/68#discussion_r4566026784
##########
crates/spark/src/function/hash/murmur3.rs:
##########
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int32Array, types::ArrowDictionaryKeyType};
+use arrow::buffer::{Buffer, ScalarBuffer};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::{Result, ScalarValue, internal_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature,
+ Volatility,
+};
+
+use super::utils::hash_dictionary_values;
+use crate::create_hashes_internal;
+
+const DEFAULT_SEED: u32 = 42;
+
+/// Spark-compatible `hash` function (32-bit Murmur3, seed 42).
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#hash>
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkHash {
+ signature: Signature,
+}
+
+impl Default for SparkHash {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkHash {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::variadic_any(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkHash {
+ fn name(&self) -> &str {
+ "hash"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, args: ReturnFieldArgs) ->
Result<FieldRef> {
+ if args.arg_fields.is_empty() {
+ return plan_err!("hash requires at least one argument");
+ }
+ // Spark's HashExpression overrides nullable to false: NULL inputs are
+ // skipped and the seed is used, so the result is never null.
+ Ok(Arc::new(Field::new(self.name(), DataType::Int32, false)))
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let num_rows = args.number_rows;
+ let mut hashes: Vec<u32> = vec![DEFAULT_SEED; num_rows];
+
+ let arrays = ColumnarValue::values_to_arrays(&args.args)?;
+ create_murmur3_hashes(&arrays, &mut hashes)?;
+
+ if num_rows == 1 {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(
+ hashes[0] as i32,
+ ))))
+ } else {
+ // Reinterpret Vec<u32> as ScalarBuffer<i32> without copying.
+ let buffer = ScalarBuffer::<i32>::from(Buffer::from_vec(hashes));
+ Ok(ColumnarValue::Array(Arc::new(Int32Array::new(
+ buffer, None,
+ ))))
+ }
+ }
+}
+
+#[inline]
+fn mix_k1(k1: u32) -> u32 {
+ k1.wrapping_mul(0xcc9e2d51)
+ .rotate_left(15)
+ .wrapping_mul(0x1b873593)
+}
+
+#[inline]
+fn mix_h1(h1: u32, k1: u32) -> u32 {
+ (h1 ^ k1)
+ .rotate_left(13)
+ .wrapping_mul(5)
+ .wrapping_add(0xe6546b64)
+}
+
+#[inline]
+fn fmix(mut h1: u32, len: u32) -> u32 {
+ h1 ^= len;
+ h1 ^= h1 >> 16;
+ h1 = h1.wrapping_mul(0x85ebca6b);
+ h1 ^= h1 >> 13;
+ h1 = h1.wrapping_mul(0xc2b2ae35);
+ h1 ^= h1 >> 16;
+ h1
+}
+
+/// Spark's `Murmur3_x86_32.hashUnsafeBytes`: whole 4-byte little-endian words
+/// first, then each remaining tail byte is mixed separately as a sign-extended
+/// word (this differs from the canonical MurmurHash3 tail handling).
+#[inline]
+fn spark_compatible_murmur3_hash<T: AsRef<[u8]>>(data: T, seed: u32) -> u32 {
+ let data = data.as_ref();
+ let mut chunks = data.chunks_exact(4);
+ let mut h1 = seed;
+ for chunk in &mut chunks {
+ let word = u32::from_le_bytes([chunk[0], chunk[1], chunk[2],
chunk[3]]);
+ h1 = mix_h1(h1, mix_k1(word));
+ }
+ for &byte in chunks.remainder() {
+ h1 = mix_h1(h1, mix_k1(byte as i8 as i32 as u32));
+ }
+ fmix(h1, data.len() as u32)
+}
+
+/// Hash the values in a dictionary array using murmur3.
+fn create_murmur3_hashes_dictionary<K: ArrowDictionaryKeyType>(
+ array: &ArrayRef,
+ hashes_buffer: &mut [u32],
+ seeds_are_pristine: bool,
+) -> Result<()> {
+ hash_dictionary_values::<K, u32>(
+ array,
+ hashes_buffer,
+ seeds_are_pristine,
+ create_murmur3_hashes,
+ )
+}
+
+/// Create murmur3 hash values for every row, based on the values in the
columns.
+///
+/// The number of rows to hash is determined by `hashes_buffer.len()`.
+/// `hashes_buffer` should be pre-sized appropriately and seeded with the
+/// initial hash value (Spark uses `42`).
+fn create_murmur3_hashes(arrays: &[ArrayRef], hashes_buffer: &mut [u32]) ->
Result<()> {
+ create_hashes_internal!(
+ arrays,
+ hashes_buffer,
+ spark_compatible_murmur3_hash,
Review Comment:
For `Decimal128` with precision above 18, Spark hashes
`d.toJavaBigDecimal.unscaledValue().toByteArray()` (`hash.scala`,
`InterpretedHashFunction.hash`). Those are the minimal big-endian two's
complement bytes. The shared `hash_array_decimal!` in `utils.rs` feeds the
16-byte little-endian `i128` instead, so `hash(CAST(1 AS DECIMAL(20, 0)))` will
not match Spark. `xxhash64` has the same bug on main. Comet fixed it in
apache/datafusion-comet#6548 with a `hash_decimal!` macro that drops the
redundant sign bytes:
```rust
let sign_bits = value.leading_zeros().max(value.leading_ones());
let start = (sign_bits.saturating_sub(1) / 8) as usize;
hash_method(&value.to_be_bytes()[start..], seed)
```
Could we port that, including the list element arms, and add a wide decimal
case with a value from Spark? The precision <= 18 path is correct.
##########
crates/spark/src/function/hash/murmur3.rs:
##########
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int32Array, types::ArrowDictionaryKeyType};
+use arrow::buffer::{Buffer, ScalarBuffer};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::{Result, ScalarValue, internal_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature,
+ Volatility,
+};
+
+use super::utils::hash_dictionary_values;
+use crate::create_hashes_internal;
+
+const DEFAULT_SEED: u32 = 42;
+
+/// Spark-compatible `hash` function (32-bit Murmur3, seed 42).
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#hash>
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkHash {
+ signature: Signature,
+}
+
+impl Default for SparkHash {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkHash {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::variadic_any(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkHash {
+ fn name(&self) -> &str {
+ "hash"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, args: ReturnFieldArgs) ->
Result<FieldRef> {
+ if args.arg_fields.is_empty() {
+ return plan_err!("hash requires at least one argument");
+ }
+ // Spark's HashExpression overrides nullable to false: NULL inputs are
+ // skipped and the seed is used, so the result is never null.
+ Ok(Arc::new(Field::new(self.name(), DataType::Int32, false)))
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let num_rows = args.number_rows;
+ let mut hashes: Vec<u32> = vec![DEFAULT_SEED; num_rows];
+
+ let arrays = ColumnarValue::values_to_arrays(&args.args)?;
Review Comment:
`hashes` has `number_rows` entries, but `values_to_arrays` returns 1-row
arrays when every argument is a scalar. `ScalarFunctionExpr::evaluate` passes
`batch.num_rows()` as `number_rows`, so an all-literal `hash(...)` that was not
constant-folded (for example a physical expression built directly, as Comet
does) would index past the end of those arrays. `xxhash64` has the same
pattern. Using `a.to_array(num_rows)` for each argument, or hashing one row and
returning a scalar when all inputs are scalars, avoids it. I have not run this
case.
##########
crates/sqllogictest/test_files/hash/hash.slt:
##########
@@ -0,0 +1,207 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under
one or more contributor license agreements
+
+# Tests for Spark-compatible hash (32-bit Murmur3, seed 42)
+# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#hash
+# Expected values come from Comet's Spark-verified murmur3 tests.
+
+# Int32
+
+query I
+SELECT hash(arrow_cast(1, 'Int32'));
+----
+-559580957
+
+query I
+SELECT hash(arrow_cast(0, 'Int32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-1, 'Int32'));
+----
+-1604776387
+
+query I
+SELECT hash(arrow_cast(2147483647, 'Int32'));
+----
+133916647
+
+query I
+SELECT hash(arrow_cast(-2147483648, 'Int32'));
+----
+723455942
+
+# Int64
+
+query I
+SELECT hash(arrow_cast(1, 'Int64'));
+----
+-1712319331
+
+query I
+SELECT hash(arrow_cast(0, 'Int64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-1, 'Int64'));
+----
+-939490007
+
+query I
+SELECT hash(arrow_cast(9223372036854775807, 'Int64'));
+----
+-1604625029
+
+query I
+SELECT hash(arrow_cast(-9223372036854775808, 'Int64'));
+----
+-853646085
+
+# Float64: -0.0 hashes like 0.0
+
+query I
+SELECT hash(arrow_cast(1.0, 'Float64'));
+----
+-460888942
+
+query I
+SELECT hash(arrow_cast(0.0, 'Float64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-0.0, 'Float64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-1.0, 'Float64'));
+----
+332927831
+
+# Float32: -0.0 hashes like 0.0
+
+query I
+SELECT hash(arrow_cast(1.0, 'Float32'));
+----
+-466301895
+
+query I
+SELECT hash(arrow_cast(0.0, 'Float32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-0.0, 'Float32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-1.0, 'Float32'));
+----
+-603084565
+
+# Strings: every tail length, multi-byte UTF-8
+
+query I
+SELECT hash('hello');
+----
+-1008564952
+
+query I
+SELECT hash('bar');
+----
+-1808790533
+
+query I
+SELECT hash('');
+----
+142593372
+
+query I
+SELECT hash('a');
+----
+1485273170
+
+query I
+SELECT hash('ab');
+----
+-97053317
+
+query I
+SELECT hash('abc');
+----
+1322437556
+
+query I
+SELECT hash('abcd');
+----
+-396302900
+
+query I
+SELECT hash('abcde');
+----
+814637928
+
+query I
+SELECT hash('😁');
+----
+885025535
+
+query I
+SELECT hash('天地');
+----
+-1899966402
+
+# NULL leaves the seed (42) unchanged; the result is never NULL
+query I
+SELECT hash(NULL);
+----
+42
+
+query I
+SELECT hash(arrow_cast(NULL, 'Int32'));
+----
+42
+
+# NULL arguments are skipped when hashing several columns
+query B
+SELECT hash(1, NULL) = hash(1);
+----
+true
+
+query B
+SELECT hash(NULL, 1) = hash(1);
+----
+true
+
+# Arguments chain: the hash of the previous argument seeds the next
+query B
+SELECT hash(1, 2) = hash(2, 1);
+----
+false
+
+# Table data (VALUES integers are Int64, which hash as 8 bytes)
+statement ok
+CREATE TABLE hash_t AS VALUES (1, 'hello'), (NULL, 'bar'), (0, NULL);
+
+query I
+SELECT hash(column1) FROM hash_t;
+----
+-1712319331
+42
+-1670924195
+
+query I
+SELECT hash(column2) FROM hash_t;
+----
+-1008564952
+-1808790533
+42
+
+# Spark requires at least one argument
+statement error
Review Comment:
`Signature::variadic_any` already makes the planner reject `hash()` with
"'hash' does not support zero arguments". So the `plan_err!` in
`return_field_from_args` and `test_hash_requires_an_argument` cover a path that
SQL cannot reach. Could we drop both and pin the message here, as in `statement
error does not support zero arguments`?
##########
crates/sqllogictest/test_files/hash/hash.slt:
##########
@@ -0,0 +1,207 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under
one or more contributor license agreements
+
+# Tests for Spark-compatible hash (32-bit Murmur3, seed 42)
+# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#hash
+# Expected values come from Comet's Spark-verified murmur3 tests.
+
+# Int32
+
+query I
+SELECT hash(arrow_cast(1, 'Int32'));
+----
+-559580957
+
+query I
+SELECT hash(arrow_cast(0, 'Int32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-1, 'Int32'));
+----
+-1604776387
+
+query I
+SELECT hash(arrow_cast(2147483647, 'Int32'));
+----
+133916647
+
+query I
+SELECT hash(arrow_cast(-2147483648, 'Int32'));
+----
+723455942
+
+# Int64
+
+query I
+SELECT hash(arrow_cast(1, 'Int64'));
+----
+-1712319331
+
+query I
+SELECT hash(arrow_cast(0, 'Int64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-1, 'Int64'));
+----
+-939490007
+
+query I
+SELECT hash(arrow_cast(9223372036854775807, 'Int64'));
+----
+-1604625029
+
+query I
+SELECT hash(arrow_cast(-9223372036854775808, 'Int64'));
+----
+-853646085
+
+# Float64: -0.0 hashes like 0.0
+
+query I
+SELECT hash(arrow_cast(1.0, 'Float64'));
+----
+-460888942
+
+query I
+SELECT hash(arrow_cast(0.0, 'Float64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-0.0, 'Float64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-1.0, 'Float64'));
+----
+332927831
+
+# Float32: -0.0 hashes like 0.0
+
+query I
+SELECT hash(arrow_cast(1.0, 'Float32'));
+----
+-466301895
+
+query I
+SELECT hash(arrow_cast(0.0, 'Float32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-0.0, 'Float32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-1.0, 'Float32'));
+----
+-603084565
+
+# Strings: every tail length, multi-byte UTF-8
+
+query I
+SELECT hash('hello');
+----
+-1008564952
+
+query I
+SELECT hash('bar');
+----
+-1808790533
+
+query I
+SELECT hash('');
+----
+142593372
+
+query I
+SELECT hash('a');
+----
+1485273170
+
+query I
+SELECT hash('ab');
+----
+-97053317
+
+query I
+SELECT hash('abc');
+----
+1322437556
+
+query I
+SELECT hash('abcd');
+----
+-396302900
+
+query I
+SELECT hash('abcde');
+----
+814637928
+
+query I
+SELECT hash('😁');
+----
+885025535
+
+query I
+SELECT hash('天地');
+----
+-1899966402
+
+# NULL leaves the seed (42) unchanged; the result is never NULL
+query I
+SELECT hash(NULL);
+----
+42
+
+query I
+SELECT hash(arrow_cast(NULL, 'Int32'));
+----
+42
+
+# NULL arguments are skipped when hashing several columns
+query B
+SELECT hash(1, NULL) = hash(1);
+----
+true
+
+query B
+SELECT hash(NULL, 1) = hash(1);
+----
+true
+
+# Arguments chain: the hash of the previous argument seeds the next
+query B
+SELECT hash(1, 2) = hash(2, 1);
Review Comment:
This only shows that argument order matters. It does not check the seed
chaining against Spark. Spark's own example pins it: `SELECT hash('Spark',
array(123), 2)` returns `-1321691492` (`Murmur3Hash` `ExpressionDescription`).
It needs `INT` casts here because DataFusion types `2` and `123` as BIGINT.
There is also no `.slt` case for decimals, lists, structs, maps or
dictionaries. CI runs only `cargo test -p datafusion-spark-sqllogictest`, not
`cargo test -p datafusion-spark`, so the Rust unit tests here do not run in CI.
So `test_hash_i32`, `test_hash_str`, `test_hash_negative_zero` and
`test_hash_null_keeps_seed` could move here as `.slt` cases.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]