This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new 1a560c61 fix: requested character too large for encoding in chr
function (#552)
1a560c61 is described below
commit 1a560c615ff148b19cdec2eed3f01ad246a4aec6
Author: Vipul Vaibhaw <[email protected]>
AuthorDate: Fri Jun 14 20:47:37 2024 +0530
fix: requested character too large for encoding in chr function (#552)
* adding support for negative integers and modulo checks for int above 256,
also added test case
* fix test case to avoid cast integer overflow error
* Update core/src/execution/datafusion/expressions/scalar_funcs/chr.rs
Co-authored-by: Andy Grove <[email protected]>
* returning negative ints as empty string earlier
* adding scalar test and removing unreachable code
* Update core/src/execution/datafusion/expressions/scalar_funcs/chr.rs
Co-authored-by: Andy Grove <[email protected]>
* Update spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
Co-authored-by: Andy Grove <[email protected]>
* add a test case where integer & 0xFF == 0
---------
Co-authored-by: Andy Grove <[email protected]>
---
.../datafusion/expressions/scalar_funcs/chr.rs | 32 ++++++++++++++--------
.../org/apache/comet/CometExpressionSuite.scala | 24 ++++++++++++++++
2 files changed, 44 insertions(+), 12 deletions(-)
diff --git a/core/src/execution/datafusion/expressions/scalar_funcs/chr.rs
b/core/src/execution/datafusion/expressions/scalar_funcs/chr.rs
index 3d62d324..0421b491 100644
--- a/core/src/execution/datafusion/expressions/scalar_funcs/chr.rs
+++ b/core/src/execution/datafusion/expressions/scalar_funcs/chr.rs
@@ -44,10 +44,15 @@ pub fn chr(args: &[ArrayRef]) -> Result<ArrayRef> {
.iter()
.map(|integer: Option<i64>| {
integer
- .map(|integer| match core::char::from_u32(integer as u32) {
- Some(integer) => Ok(integer.to_string()),
- None => {
- exec_err!("requested character too large for
encoding.")
+ .map(|integer| {
+ if integer < 0 {
+ return Ok("".to_string()); // Return empty string for
negative integers
+ }
+ match core::char::from_u32((integer % 256) as u32) {
+ Some(ch) => Ok(ch.to_string()),
+ None => {
+ exec_err!("requested character not compatible for
encoding.")
+ }
}
})
.transpose()
@@ -106,16 +111,19 @@ fn handle_chr_fn(args: &[ColumnarValue]) ->
Result<ColumnarValue> {
Ok(ColumnarValue::Array(array))
}
ColumnarValue::Scalar(ScalarValue::Int64(Some(value))) => {
- match core::char::from_u32(value as u32) {
- Some(ch) => Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(
- ch.to_string(),
- )))),
- None => exec_err!("requested character too large for
encoding."),
+ if value < 0 {
+ Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(
+ "".to_string(),
+ ))))
+ } else {
+ match core::char::from_u32((value % 256) as u32) {
+ Some(ch) =>
Ok(ColumnarValue::Scalar(ScalarValue::Utf8(Some(
+ ch.to_string(),
+ )))),
+ None => exec_err!("requested character was incompatible
for encoding."),
+ }
}
}
- ColumnarValue::Scalar(ScalarValue::Int64(None)) => {
- Ok(ColumnarValue::Scalar(ScalarValue::Utf8(None)))
- }
_ => exec_err!("The argument must be an Int64 array or scalar."),
}
}
diff --git a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
index 2457f70a..ec45c984 100644
--- a/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
+++ b/spark/src/test/scala/org/apache/comet/CometExpressionSuite.scala
@@ -1054,6 +1054,30 @@ class CometExpressionSuite extends CometTestBase with
AdaptiveSparkPlanHelper {
}
}
+ test("Chr with negative and large value") {
+ Seq(false, true).foreach { dictionary =>
+ withSQLConf("parquet.enable.dictionary" -> dictionary.toString) {
+ val table = "test0"
+ withTable(table) {
+ sql(s"create table $table(c9 int, c4 int) using parquet")
+ sql(
+ s"insert into $table values(0, 0), (61231, -61231), (-1700, 1700),
(0, -4000), (-40, 40), (256, 512)")
+ val query = s"SELECT chr(c9), chr(c4) FROM $table"
+ checkSparkAnswerAndOperator(query)
+ }
+ }
+ }
+
+ withParquetTable((0 until 5).map(i => (i % 5, i % 3)), "tbl") {
+ withSQLConf(
+ "spark.sql.optimizer.excludedRules" ->
"org.apache.spark.sql.catalyst.optimizer.ConstantFolding") {
+ for (n <- Seq("0", "-0", "0.5", "-0.5", "555", "-555", "null")) {
+ checkSparkAnswerAndOperator(s"select chr(cast(${n} as int)) FROM
tbl")
+ }
+ }
+ }
+ }
+
test("InitCap") {
Seq(false, true).foreach { dictionary =>
withSQLConf("parquet.enable.dictionary" -> dictionary.toString) {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
