This is an automated email from the ASF dual-hosted git repository.
zhangzc pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 4ab76a149 [CH] Fix array distinct core dump (#6256)
4ab76a149 is described below
commit 4ab76a1493a97ecbc86fb25bacd9f88179ffed1e
Author: LiuNeng <[email protected]>
AuthorDate: Fri Jun 28 10:13:52 2024 +0800
[CH] Fix array distinct core dump (#6256)
[CH] Fix array distinct core dump
Co-authored-by: liuneng1994 <[email protected]>
---
.../execution/GlutenClickhouseFunctionSuite.scala | 21 +++++++++++++++++++++
.../Functions/SparkFunctionArrayDistinct.cpp | 5 ++++-
2 files changed, 25 insertions(+), 1 deletion(-)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala
index 7e46a6989..26e997281 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/GlutenClickhouseFunctionSuite.scala
@@ -190,4 +190,25 @@ class GlutenClickhouseFunctionSuite extends
GlutenClickHouseTPCHAbstractSuite {
)(df => checkFallbackOperators(df, 0))
spark.sql("drop table json_t1")
}
+
+ test("Fix arrayDistinct(Array(Nullable(Decimal))) core dump") {
+ val create_sql =
+ """
+ |create table if not exists test(
+ | dec array<decimal(10, 2)>
+ |) using parquet
+ |""".stripMargin
+ val fill_sql =
+ """
+ |insert into test values(array(1, 2, null)), (array(null, 2,3, 5))
+ |""".stripMargin
+ val query_sql =
+ """
+ |select array_distinct(dec) from test;
+ |""".stripMargin
+ spark.sql(create_sql)
+ spark.sql(fill_sql)
+ compareResultsAgainstVanillaSpark(query_sql, true, { _ => })
+ spark.sql("drop table test")
+ }
}
diff --git a/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp
b/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp
index 89598ff7a..0779346b6 100644
--- a/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp
+++ b/cpp-ch/local-engine/Functions/SparkFunctionArrayDistinct.cpp
@@ -314,7 +314,10 @@ void FunctionArrayDistinctSpark::executeHashed(
if (!set.find(hash))
{
set.insert(hash);
- res_data_col.insertFrom(src_data, j);
+ if (nullable_col)
+ res_data_col.insertFrom(*nullable_col, j);
+ else
+ res_data_col.insertFrom(src_data, j);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
