andygrove commented on code in PR #68:
URL: https://github.com/apache/datafusion-spark/pull/68#discussion_r4566762195
##########
crates/spark/src/function/hash/murmur3.rs:
##########
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int32Array, types::ArrowDictionaryKeyType};
+use arrow::buffer::{Buffer, ScalarBuffer};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::{Result, ScalarValue, internal_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature,
+ Volatility,
+};
+
+use super::utils::hash_dictionary_values;
+use crate::create_hashes_internal;
+
+const DEFAULT_SEED: u32 = 42;
+
+/// Spark-compatible `hash` function (32-bit Murmur3, seed 42).
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#hash>
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkHash {
+ signature: Signature,
+}
+
+impl Default for SparkHash {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkHash {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::variadic_any(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkHash {
+ fn name(&self) -> &str {
+ "hash"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, args: ReturnFieldArgs) ->
Result<FieldRef> {
+ if args.arg_fields.is_empty() {
+ return plan_err!("hash requires at least one argument");
+ }
+ // Spark's HashExpression overrides nullable to false: NULL inputs are
+ // skipped and the seed is used, so the result is never null.
+ Ok(Arc::new(Field::new(self.name(), DataType::Int32, false)))
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let num_rows = args.number_rows;
+ let mut hashes: Vec<u32> = vec![DEFAULT_SEED; num_rows];
+
+ let arrays = ColumnarValue::values_to_arrays(&args.args)?;
+ create_murmur3_hashes(&arrays, &mut hashes)?;
+
+ if num_rows == 1 {
+ Ok(ColumnarValue::Scalar(ScalarValue::Int32(Some(
+ hashes[0] as i32,
+ ))))
+ } else {
+ // Reinterpret Vec<u32> as ScalarBuffer<i32> without copying.
+ let buffer = ScalarBuffer::<i32>::from(Buffer::from_vec(hashes));
+ Ok(ColumnarValue::Array(Arc::new(Int32Array::new(
+ buffer, None,
+ ))))
+ }
+ }
+}
+
+#[inline]
+fn mix_k1(k1: u32) -> u32 {
+ k1.wrapping_mul(0xcc9e2d51)
+ .rotate_left(15)
+ .wrapping_mul(0x1b873593)
+}
+
+#[inline]
+fn mix_h1(h1: u32, k1: u32) -> u32 {
+ (h1 ^ k1)
+ .rotate_left(13)
+ .wrapping_mul(5)
+ .wrapping_add(0xe6546b64)
+}
+
+#[inline]
+fn fmix(mut h1: u32, len: u32) -> u32 {
+ h1 ^= len;
+ h1 ^= h1 >> 16;
+ h1 = h1.wrapping_mul(0x85ebca6b);
+ h1 ^= h1 >> 13;
+ h1 = h1.wrapping_mul(0xc2b2ae35);
+ h1 ^= h1 >> 16;
+ h1
+}
+
+/// Spark's `Murmur3_x86_32.hashUnsafeBytes`: whole 4-byte little-endian words
+/// first, then each remaining tail byte is mixed separately as a sign-extended
+/// word (this differs from the canonical MurmurHash3 tail handling).
+#[inline]
+fn spark_compatible_murmur3_hash<T: AsRef<[u8]>>(data: T, seed: u32) -> u32 {
+ let data = data.as_ref();
+ let mut chunks = data.chunks_exact(4);
+ let mut h1 = seed;
+ for chunk in &mut chunks {
+ let word = u32::from_le_bytes([chunk[0], chunk[1], chunk[2],
chunk[3]]);
+ h1 = mix_h1(h1, mix_k1(word));
+ }
+ for &byte in chunks.remainder() {
+ h1 = mix_h1(h1, mix_k1(byte as i8 as i32 as u32));
+ }
+ fmix(h1, data.len() as u32)
+}
+
+/// Hash the values in a dictionary array using murmur3.
+fn create_murmur3_hashes_dictionary<K: ArrowDictionaryKeyType>(
+ array: &ArrayRef,
+ hashes_buffer: &mut [u32],
+ seeds_are_pristine: bool,
+) -> Result<()> {
+ hash_dictionary_values::<K, u32>(
+ array,
+ hashes_buffer,
+ seeds_are_pristine,
+ create_murmur3_hashes,
+ )
+}
+
+/// Create murmur3 hash values for every row, based on the values in the
columns.
+///
+/// The number of rows to hash is determined by `hashes_buffer.len()`.
+/// `hashes_buffer` should be pre-sized appropriately and seeded with the
+/// initial hash value (Spark uses `42`).
+fn create_murmur3_hashes(arrays: &[ArrayRef], hashes_buffer: &mut [u32]) ->
Result<()> {
+ create_hashes_internal!(
+ arrays,
+ hashes_buffer,
+ spark_compatible_murmur3_hash,
Review Comment:
Thanks, ported in 4e49569. The fix is in the shared code in `utils.rs`, so
both `hash` and `xxhash64` now hash the minimal big-endian bytes. Decimal list
elements go through the generic recursive path, so they pick it up too. Added
Spark-verified `.slt` cases to both files for `DECIMAL(20, 0)` around the
sign-byte boundaries (0, ±1, 127, 128, -128, -129), `DECIMAL(38, x)` at ±max,
and wide decimals inside a list and a struct.
##########
crates/spark/src/function/hash/murmur3.rs:
##########
@@ -0,0 +1,268 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+use std::sync::Arc;
+
+use arrow::array::{ArrayRef, Int32Array, types::ArrowDictionaryKeyType};
+use arrow::buffer::{Buffer, ScalarBuffer};
+use arrow::datatypes::{DataType, Field, FieldRef};
+use datafusion_common::{Result, ScalarValue, internal_err, plan_err};
+use datafusion_expr::{
+ ColumnarValue, ReturnFieldArgs, ScalarFunctionArgs, ScalarUDFImpl,
Signature,
+ Volatility,
+};
+
+use super::utils::hash_dictionary_values;
+use crate::create_hashes_internal;
+
+const DEFAULT_SEED: u32 = 42;
+
+/// Spark-compatible `hash` function (32-bit Murmur3, seed 42).
+/// <https://spark.apache.org/docs/latest/api/sql/index.html#hash>
+#[derive(Debug, PartialEq, Eq, Hash)]
+pub struct SparkHash {
+ signature: Signature,
+}
+
+impl Default for SparkHash {
+ fn default() -> Self {
+ Self::new()
+ }
+}
+
+impl SparkHash {
+ pub fn new() -> Self {
+ Self {
+ signature: Signature::variadic_any(Volatility::Immutable),
+ }
+ }
+}
+
+impl ScalarUDFImpl for SparkHash {
+ fn name(&self) -> &str {
+ "hash"
+ }
+
+ fn signature(&self) -> &Signature {
+ &self.signature
+ }
+
+ fn return_type(&self, _arg_types: &[DataType]) -> Result<DataType> {
+ internal_err!("return_field_from_args should be used instead")
+ }
+
+ fn return_field_from_args(&self, args: ReturnFieldArgs) ->
Result<FieldRef> {
+ if args.arg_fields.is_empty() {
+ return plan_err!("hash requires at least one argument");
+ }
+ // Spark's HashExpression overrides nullable to false: NULL inputs are
+ // skipped and the seed is used, so the result is never null.
+ Ok(Arc::new(Field::new(self.name(), DataType::Int32, false)))
+ }
+
+ fn invoke_with_args(&self, args: ScalarFunctionArgs) ->
Result<ColumnarValue> {
+ let num_rows = args.number_rows;
+ let mut hashes: Vec<u32> = vec![DEFAULT_SEED; num_rows];
+
+ let arrays = ColumnarValue::values_to_arrays(&args.args)?;
Review Comment:
Fixed for both `hash` and `xxhash64` in 4e49569. When every argument is a
scalar, the function hashes one row and returns a scalar; otherwise each
argument is expanded with `to_array(number_rows)`. Added Rust tests that call
`invoke_with_args` with all-scalar arguments and `number_rows = 3`, since SQL
constant-folds this case.
##########
crates/sqllogictest/test_files/hash/hash.slt:
##########
@@ -0,0 +1,207 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under
one or more contributor license agreements
+
+# Tests for Spark-compatible hash (32-bit Murmur3, seed 42)
+# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#hash
+# Expected values come from Comet's Spark-verified murmur3 tests.
+
+# Int32
+
+query I
+SELECT hash(arrow_cast(1, 'Int32'));
+----
+-559580957
+
+query I
+SELECT hash(arrow_cast(0, 'Int32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-1, 'Int32'));
+----
+-1604776387
+
+query I
+SELECT hash(arrow_cast(2147483647, 'Int32'));
+----
+133916647
+
+query I
+SELECT hash(arrow_cast(-2147483648, 'Int32'));
+----
+723455942
+
+# Int64
+
+query I
+SELECT hash(arrow_cast(1, 'Int64'));
+----
+-1712319331
+
+query I
+SELECT hash(arrow_cast(0, 'Int64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-1, 'Int64'));
+----
+-939490007
+
+query I
+SELECT hash(arrow_cast(9223372036854775807, 'Int64'));
+----
+-1604625029
+
+query I
+SELECT hash(arrow_cast(-9223372036854775808, 'Int64'));
+----
+-853646085
+
+# Float64: -0.0 hashes like 0.0
+
+query I
+SELECT hash(arrow_cast(1.0, 'Float64'));
+----
+-460888942
+
+query I
+SELECT hash(arrow_cast(0.0, 'Float64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-0.0, 'Float64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-1.0, 'Float64'));
+----
+332927831
+
+# Float32: -0.0 hashes like 0.0
+
+query I
+SELECT hash(arrow_cast(1.0, 'Float32'));
+----
+-466301895
+
+query I
+SELECT hash(arrow_cast(0.0, 'Float32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-0.0, 'Float32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-1.0, 'Float32'));
+----
+-603084565
+
+# Strings: every tail length, multi-byte UTF-8
+
+query I
+SELECT hash('hello');
+----
+-1008564952
+
+query I
+SELECT hash('bar');
+----
+-1808790533
+
+query I
+SELECT hash('');
+----
+142593372
+
+query I
+SELECT hash('a');
+----
+1485273170
+
+query I
+SELECT hash('ab');
+----
+-97053317
+
+query I
+SELECT hash('abc');
+----
+1322437556
+
+query I
+SELECT hash('abcd');
+----
+-396302900
+
+query I
+SELECT hash('abcde');
+----
+814637928
+
+query I
+SELECT hash('😁');
+----
+885025535
+
+query I
+SELECT hash('天地');
+----
+-1899966402
+
+# NULL leaves the seed (42) unchanged; the result is never NULL
+query I
+SELECT hash(NULL);
+----
+42
+
+query I
+SELECT hash(arrow_cast(NULL, 'Int32'));
+----
+42
+
+# NULL arguments are skipped when hashing several columns
+query B
+SELECT hash(1, NULL) = hash(1);
+----
+true
+
+query B
+SELECT hash(NULL, 1) = hash(1);
+----
+true
+
+# Arguments chain: the hash of the previous argument seeds the next
+query B
+SELECT hash(1, 2) = hash(2, 1);
Review Comment:
Added `hash('Spark', array(123), 2)` = -1321691492 with INT casts (and the
same query for `xxhash64`), plus `.slt` cases for decimals, lists, structs,
maps and dictionaries. Spark 4.2.0 rejects `hash` on maps by default, so the
map values were taken with `spark.sql.legacy.allowHashOnMapType=true`, as a
comment in the file notes. The four Rust tests are removed because their values
were already in `hash.slt`. Done in 4e49569.
##########
crates/sqllogictest/test_files/hash/hash.slt:
##########
@@ -0,0 +1,207 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under
one or more contributor license agreements
+
+# Tests for Spark-compatible hash (32-bit Murmur3, seed 42)
+# Reference: https://spark.apache.org/docs/latest/api/sql/index.html#hash
+# Expected values come from Comet's Spark-verified murmur3 tests.
+
+# Int32
+
+query I
+SELECT hash(arrow_cast(1, 'Int32'));
+----
+-559580957
+
+query I
+SELECT hash(arrow_cast(0, 'Int32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-1, 'Int32'));
+----
+-1604776387
+
+query I
+SELECT hash(arrow_cast(2147483647, 'Int32'));
+----
+133916647
+
+query I
+SELECT hash(arrow_cast(-2147483648, 'Int32'));
+----
+723455942
+
+# Int64
+
+query I
+SELECT hash(arrow_cast(1, 'Int64'));
+----
+-1712319331
+
+query I
+SELECT hash(arrow_cast(0, 'Int64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-1, 'Int64'));
+----
+-939490007
+
+query I
+SELECT hash(arrow_cast(9223372036854775807, 'Int64'));
+----
+-1604625029
+
+query I
+SELECT hash(arrow_cast(-9223372036854775808, 'Int64'));
+----
+-853646085
+
+# Float64: -0.0 hashes like 0.0
+
+query I
+SELECT hash(arrow_cast(1.0, 'Float64'));
+----
+-460888942
+
+query I
+SELECT hash(arrow_cast(0.0, 'Float64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-0.0, 'Float64'));
+----
+-1670924195
+
+query I
+SELECT hash(arrow_cast(-1.0, 'Float64'));
+----
+332927831
+
+# Float32: -0.0 hashes like 0.0
+
+query I
+SELECT hash(arrow_cast(1.0, 'Float32'));
+----
+-466301895
+
+query I
+SELECT hash(arrow_cast(0.0, 'Float32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-0.0, 'Float32'));
+----
+933211791
+
+query I
+SELECT hash(arrow_cast(-1.0, 'Float32'));
+----
+-603084565
+
+# Strings: every tail length, multi-byte UTF-8
+
+query I
+SELECT hash('hello');
+----
+-1008564952
+
+query I
+SELECT hash('bar');
+----
+-1808790533
+
+query I
+SELECT hash('');
+----
+142593372
+
+query I
+SELECT hash('a');
+----
+1485273170
+
+query I
+SELECT hash('ab');
+----
+-97053317
+
+query I
+SELECT hash('abc');
+----
+1322437556
+
+query I
+SELECT hash('abcd');
+----
+-396302900
+
+query I
+SELECT hash('abcde');
+----
+814637928
+
+query I
+SELECT hash('😁');
+----
+885025535
+
+query I
+SELECT hash('天地');
+----
+-1899966402
+
+# NULL leaves the seed (42) unchanged; the result is never NULL
+query I
+SELECT hash(NULL);
+----
+42
+
+query I
+SELECT hash(arrow_cast(NULL, 'Int32'));
+----
+42
+
+# NULL arguments are skipped when hashing several columns
+query B
+SELECT hash(1, NULL) = hash(1);
+----
+true
+
+query B
+SELECT hash(NULL, 1) = hash(1);
+----
+true
+
+# Arguments chain: the hash of the previous argument seeds the next
+query B
+SELECT hash(1, 2) = hash(2, 1);
+----
+false
+
+# Table data (VALUES integers are Int64, which hash as 8 bytes)
+statement ok
+CREATE TABLE hash_t AS VALUES (1, 'hello'), (NULL, 'bar'), (0, NULL);
+
+query I
+SELECT hash(column1) FROM hash_t;
+----
+-1712319331
+42
+-1670924195
+
+query I
+SELECT hash(column2) FROM hash_t;
+----
+-1008564952
+-1808790533
+42
+
+# Spark requires at least one argument
+statement error
Review Comment:
Done in 4e49569. Removed the `plan_err!` and its test, and pinned the
planner message in `.slt`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]