This is an automated email from the ASF dual-hosted git repository.
marong pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 8420d75d5 [VL] Fix wrong result caused by missing metadata in hash
registration (#5355)
8420d75d5 is described below
commit 8420d75d52f28809ff3a8c7fa610fc94e8010d4b
Author: Rong Ma <[email protected]>
AuthorDate: Wed Apr 10 20:13:21 2024 +0800
[VL] Fix wrong result caused by missing metadata in hash registration
(#5355)
---
.../scala/org/apache/gluten/execution/TestOperator.scala | 14 ++++++++++++++
cpp/velox/operators/functions/RegistrationAllFunctions.cc | 10 ----------
cpp/velox/substrait/SubstraitParser.cc | 1 +
3 files changed, 15 insertions(+), 10 deletions(-)
diff --git
a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
index 08cf032b6..c3e14ef19 100644
---
a/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
+++
b/backends-velox/src/test/scala/org/apache/gluten/execution/TestOperator.scala
@@ -372,6 +372,20 @@ class TestOperator extends VeloxWholeStageTransformerSuite
{
}
}
+ test("hash") {
+ withTempView("t") {
+ Seq[(Integer, String)]((1, "a"), (2, null), (null, "b"))
+ .toDF("a", "b")
+ .createOrReplaceTempView("t")
+ runQueryAndCompare("select hash(a, b) from t") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ runQueryAndCompare("select xxhash64(a, b) from t") {
+ checkGlutenOperatorMatch[ProjectExecTransformer]
+ }
+ }
+ }
+
test("decimal abs") {
runQueryAndCompare("""
|select abs(cast (l_quantity * (-1.0) as decimal(12,
2))),
diff --git a/cpp/velox/operators/functions/RegistrationAllFunctions.cc
b/cpp/velox/operators/functions/RegistrationAllFunctions.cc
index 8f8b82d8d..2d2e820f1 100644
--- a/cpp/velox/operators/functions/RegistrationAllFunctions.cc
+++ b/cpp/velox/operators/functions/RegistrationAllFunctions.cc
@@ -36,16 +36,6 @@ using namespace facebook;
namespace gluten {
namespace {
void registerFunctionOverwrite() {
- velox::exec::registerStatefulVectorFunction(
- "murmur3hash",
- velox::functions::sparksql::hashWithSeedSignatures(),
- velox::functions::sparksql::makeHashWithSeed);
-
- velox::exec::registerStatefulVectorFunction(
- "xxhash64",
- velox::functions::sparksql::xxhash64WithSeedSignatures(),
- velox::functions::sparksql::makeXxHash64WithSeed);
-
velox::functions::registerUnaryNumeric<RoundFunction>({"round"});
velox::registerFunction<RoundFunction, int8_t, int8_t, int32_t>({"round"});
velox::registerFunction<RoundFunction, int16_t, int16_t, int32_t>({"round"});
diff --git a/cpp/velox/substrait/SubstraitParser.cc
b/cpp/velox/substrait/SubstraitParser.cc
index 9dc45a5fb..75f4c246b 100644
--- a/cpp/velox/substrait/SubstraitParser.cc
+++ b/cpp/velox/substrait/SubstraitParser.cc
@@ -399,6 +399,7 @@ std::unordered_map<std::string, std::string>
SubstraitParser::substraitVeloxFunc
{"bitwise_or", "spark_bitwise_or"},
{"bitwise_xor", "spark_bitwise_xor"},
{"murmur3hash", "hash_with_seed"},
+ {"xxhash64", "xxhash64_with_seed"},
{"modulus", "remainder"},
{"date_format", "format_datetime"},
{"collect_set", "set_agg"}};
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
