This is an automated email from the ASF dual-hosted git repository.
exmy pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-gluten.git
The following commit(s) were added to refs/heads/main by this push:
new 3097d7ab60 [GLUTEN-9049][CH] Fix diff for cast complex type to string
(#9072)
3097d7ab60 is described below
commit 3097d7ab6013869e9a786bc9dc4f687186ff5e0e
Author: exmy <[email protected]>
AuthorDate: Mon Mar 24 17:57:29 2025 +0800
[GLUTEN-9049][CH] Fix diff for cast complex type to string (#9072)
---
.../GlutenClickhouseFunctionSuite.scala | 70 ++++
...tring.cpp => SparkCastComplexTypesToString.cpp} | 8 +-
.../Functions/SparkCastComplexTypesToString.h | 431 +++++++++++++++++++++
.../Functions/SparkFunctionArrayToString.h | 165 --------
.../Functions/SparkFunctionMapToString.cpp | 28 --
.../Functions/SparkFunctionMapToString.h | 177 ---------
cpp-ch/local-engine/Parser/ExpressionParser.cpp | 23 +-
cpp-ch/local-engine/Parser/ExpressionParser.h | 6 +-
8 files changed, 515 insertions(+), 393 deletions(-)
diff --git
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
index bf84c46c71..907a5323a3 100644
---
a/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
+++
b/backends-clickhouse/src/test/scala/org/apache/gluten/execution/compatibility/GlutenClickhouseFunctionSuite.scala
@@ -20,7 +20,12 @@ import org.apache.gluten.config.GlutenConfig
import org.apache.gluten.execution.{GlutenClickHouseTPCHAbstractSuite,
ProjectExecTransformer}
import org.apache.spark.SparkConf
+import org.apache.spark.sql.catalyst.optimizer.{ConstantFolding,
NullPropagation}
import
org.apache.spark.sql.execution.datasources.v2.clickhouse.ClickHouseConfig
+import org.apache.spark.sql.internal.SQLConf
+
+// Some sqls' line length exceeds 100
+// scalastyle:off line.size.limit
class GlutenClickhouseFunctionSuite extends GlutenClickHouseTPCHAbstractSuite {
override protected val needCopyParquetToTablePath = true
@@ -442,6 +447,71 @@ class GlutenClickhouseFunctionSuite extends
GlutenClickHouseTPCHAbstractSuite {
}
}
+ test("GLUTEN-9049: cast complex type to string") {
+ withTable("test_9049") {
+ sql("""
+ |CREATE TABLE test_9049 (
+ | id INT,
+ | v1 array<string>,
+ | v2 array<array<string>>,
+ | v3 array<struct<s1:string, s2:int>>,
+ | v4 array<map<string, int>>,
+ | v5 map<string, array<string>>,
+ | v6 map<array<string>, map<string, struct<s1:string, s2:int>>>,
+ | v7 struct<s1:array<string>, s2:string, s3:map<string, int>,
s4:struct<ss1:string, ss2:int>>
+ |) using orc;
+ |""".stripMargin)
+ sql("""
+ |insert overwrite table test_9049 values
+ |(1,
+ |array('123', '\'456\'', null),
+ |array(array('abc', '\'edf\'', null), null),
+ |array(struct("\'abc\'", 100), struct("\'edf\'", 200), null,
struct("\'123\'", 300)),
+ |array(map('k1', 1), map('\'k2\'', 2), map('k3', null), null),
+ |map('k1', array('v1', 'v2', null), "'k2'", null),
+ |map(array('a1', 'a2', null), map('aa1', struct('s1', 123))),
+ |struct(array('sa1', null, "'sa2'"), null, map('sm1', null),
struct("ss1", 123))
+ |),
+ |(2,
+ |array('345', null, '\'678\''),
+ |array(array('abc', '\'edf\'', null), null),
+ |array(struct("\'abc\'", 400), struct("\'edf\'", 500), null,
struct("\'123\'", 600)),
+ |array(map('k1', 1), map('\'k2\'', 2), map('k3', null), null),
+ |map('k1', array('v1', 'v2', null), "'k2'", null),
+ |map(array('a1', 'a2', null), map('aa1', struct('s1', 234))),
+ |struct(array('sa1', null, "'sa2'"), null, map('sm1', null),
struct("ss1", 345))
+ |),
+ |(3, null, null, null, null, null, null, null);
+ |""".stripMargin)
+ val checkSql =
+ """
+ |select id,
+ |cast(v1 as string), cast(v2 as string),
+ |cast(v3 as string), cast(v4 as string),
+ |cast(v5 as string), cast(v6 as string),
+ |cast(v7 as string) from test_9049;
+ |""".stripMargin
+ compareResultsAgainstVanillaSpark(checkSql, true, { _ => })
+
+ withSQLConf(
+ SQLConf.OPTIMIZER_EXCLUDED_RULES.key -> (ConstantFolding.ruleName +
"," + NullPropagation.ruleName)) {
+ runQueryAndCompare(
+ """
+ |select
+ |cast(array("'123'", null) as string),
+ |cast(array(array('123\'')) as string),
+ |cast(array(struct(1, '123\'')) as string),
+ |cast(array(null) as string),
+ |cast(map(array(1), map("aa", "123\'")) as string),
+ |cast(named_struct("a", "test\'", "b", 1) as string),
+ |cast(named_struct("a", "test\'", "b", 1, "c", struct("\'test"),
"d", array('123\'')) as string)
+ |""".stripMargin,
+ noFallBack = false
+ )(checkGlutenOperatorMatch[ProjectExecTransformer])
+ }
+ }
+ }
+
test("GLUTEN-8921: Type mismatch at checkDecimalOverflowSparkOrNull") {
compareResultsAgainstVanillaSpark(
"""
diff --git a/cpp-ch/local-engine/Functions/SparkFunctionArrayToString.cpp
b/cpp-ch/local-engine/Functions/SparkCastComplexTypesToString.cpp
similarity index 82%
rename from cpp-ch/local-engine/Functions/SparkFunctionArrayToString.cpp
rename to cpp-ch/local-engine/Functions/SparkCastComplexTypesToString.cpp
index 1914c47c04..20903743b3 100644
--- a/cpp-ch/local-engine/Functions/SparkFunctionArrayToString.cpp
+++ b/cpp-ch/local-engine/Functions/SparkCastComplexTypesToString.cpp
@@ -15,14 +15,14 @@
* limitations under the License.
*/
-#include <Functions/SparkFunctionArrayToString.h>
+#include <Functions/SparkCastComplexTypesToString.h>
namespace local_engine
{
-REGISTER_FUNCTION(SparkFunctionArrayToString)
+REGISTER_FUNCTION(SparkCastComplexTypesToString)
{
- factory.registerFunction<local_eingine::SparkFunctionArrayToString>();
+ factory.registerFunction<local_engine::SparkCastComplexTypesToString>();
}
-}
+}
\ No newline at end of file
diff --git a/cpp-ch/local-engine/Functions/SparkCastComplexTypesToString.h
b/cpp-ch/local-engine/Functions/SparkCastComplexTypesToString.h
new file mode 100644
index 0000000000..e26dfcd65b
--- /dev/null
+++ b/cpp-ch/local-engine/Functions/SparkCastComplexTypesToString.h
@@ -0,0 +1,431 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+#include <Columns/ColumnNullable.h>
+#include <Columns/ColumnTuple.h>
+#include <Columns/ColumnMap.h>
+#include <Columns/ColumnArray.h>
+#include <Columns/ColumnStringHelpers.h>
+#include <DataTypes/DataTypeNullable.h>
+#include <DataTypes/DataTypeString.h>
+#include <DataTypes/DataTypeTuple.h>
+#include <DataTypes/DataTypeMap.h>
+#include <DataTypes/DataTypeArray.h>
+#include <Formats/FormatFactory.h>
+#include <Functions/FunctionFactory.h>
+#include <Functions/FunctionHelpers.h>
+#include <IO/WriteHelpers.h>
+
+namespace DB
+{
+namespace ErrorCodes
+{
+ extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
+ extern const int ILLEGAL_TYPE_OF_ARGUMENT;
+}
+}
+
+namespace local_engine
+{
+class SparkCastComplexTypesToString : public DB::IFunction
+{
+public:
+ static constexpr auto name = "sparkCastComplexTypesToString";
+
+ static DB::FunctionPtr create(DB::ContextPtr context) { return
std::make_shared<SparkCastComplexTypesToString>(context); }
+
+ explicit SparkCastComplexTypesToString(DB::ContextPtr context_) :
context(context_) {}
+
+ ~SparkCastComplexTypesToString() override = default;
+
+ String getName() const override { return name; }
+
+ size_t getNumberOfArguments() const override { return 1; }
+
+ bool isSuitableForShortCircuitArgumentsExecution(const
DB::DataTypesWithConstInfo & /*arguments*/) const override { return true; }
+
+ bool useDefaultImplementationForNulls() const override { return false; }
+
+ bool useDefaultImplementationForLowCardinalityColumns() const override {
return false; }
+
+ DB::DataTypePtr getReturnTypeImpl(const DB::ColumnsWithTypeAndName &
arguments) const override
+ {
+ if (arguments.size() != 1)
+ throw
DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}
argument size must be 1", name);
+
+ auto arg_type =
DB::WhichDataType(DB::removeNullable(arguments[0].type));
+
+ if (!arg_type.isTuple() && !arg_type.isMap() && !arg_type.isArray())
+ throw DB::Exception(
+ DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of
argument[0] of function {}, should be struct, map or array",
+ arguments[0].type->getName(), getName());
+
+ if (arguments[0].type->isNullable())
+ return DB::makeNullable(std::make_shared<DB::DataTypeString>());
+ else
+ return std::make_shared<DB::DataTypeString>();
+ }
+
+ DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments,
const DB::DataTypePtr & result_type, size_t input_rows_count) const override
+ {
+ DB::ColumnUInt8::MutablePtr null_map = nullptr;
+ if (const auto * col_nullable =
DB::checkAndGetColumn<DB::ColumnNullable>(arguments[0].column.get()))
+ {
+ null_map = DB::ColumnUInt8::create();
+ null_map->insertRangeFrom(col_nullable->getNullMapColumn(), 0,
col_nullable->size());
+ }
+
+ const auto & nested_col_with_type_and_name =
columnGetNested(arguments[0]);
+
+ if (const auto * col_const =
DB::checkAndGetColumn<DB::ColumnConst>(nested_col_with_type_and_name.column.get()))
+ {
+ DB::ColumnsWithTypeAndName new_arguments {1};
+ new_arguments[0] = {col_const->getDataColumnPtr(),
nested_col_with_type_and_name.type, nested_col_with_type_and_name.name};
+ auto col = executeImpl(new_arguments, result_type, 1);
+ return DB::ColumnConst::create(std::move(col), input_rows_count);
+ }
+
+ DB::FormatSettings format_settings = context ?
DB::getFormatSettings(context) : DB::FormatSettings{};
+ format_settings.pretty.charset =
DB::FormatSettings::Pretty::Charset::ASCII; /// Use ASCII for pretty output.
+
+ auto res_col = removeNullable(result_type)->createColumn();
+ DB::ColumnStringHelpers::WriteHelper
write_helper(assert_cast<DB::ColumnString &>(*res_col), input_rows_count);
+ auto & write_buffer = write_helper.getWriteBuffer();
+
+ // TODO: respect spark.sql.legacy.castComplexTypesToString.enabled
+ if (const auto * tuple_col =
DB::checkAndGetColumn<DB::ColumnTuple>(nested_col_with_type_and_name.column.get()))
+ {
+ const auto * tuple_type =
DB::checkAndGetDataType<DB::DataTypeTuple>(nested_col_with_type_and_name.type.get());
+ for (size_t row = 0; row < input_rows_count; ++row)
+ {
+ serializeTuple(*tuple_col, row, tuple_type->getElements(),
write_buffer, format_settings);
+ write_helper.rowWritten();
+ }
+ write_helper.finalize();
+ }
+ else if (const auto * map_col =
DB::checkAndGetColumn<DB::ColumnMap>(nested_col_with_type_and_name.column.get()))
+ {
+ const auto * map_type =
DB::checkAndGetDataType<DB::DataTypeMap>(nested_col_with_type_and_name.type.get());
+ const auto & key_type = map_type->getKeyType();
+ const auto & value_type = map_type->getValueType();
+ for (size_t row = 0; row < input_rows_count; ++row)
+ {
+ serializeMap(*map_col, row, key_type, value_type,
write_buffer, format_settings);
+ write_helper.rowWritten();
+ }
+ write_helper.finalize();
+ }
+ else if (const auto * array_col =
DB::checkAndGetColumn<DB::ColumnArray>(nested_col_with_type_and_name.column.get()))
+ {
+ const auto * array_type =
DB::checkAndGetDataType<DB::DataTypeArray>(nested_col_with_type_and_name.type.get());
+ for (size_t row = 0; row < input_rows_count; ++row)
+ {
+ serializeArray(*array_col, row, array_type->getNestedType(),
write_buffer, format_settings);
+ write_helper.rowWritten();
+ }
+ write_helper.finalize();
+ }
+ else
+ throw DB::Exception(
+ DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of
argument[0] of function {}, should be struct, map or array",
+ arguments[0].type->getName(), getName());
+
+ if (result_type->isNullable() && null_map)
+ return DB::ColumnNullable::create(std::move(res_col),
std::move(null_map));
+ return res_col;
+ }
+
+private:
+ DB::ContextPtr context;
+
+ void serializeTuple(
+ const DB::ColumnTuple & tuple_col,
+ size_t row_num,
+ const DB::DataTypes & elems_type,
+ DB::WriteBuffer & ostr,
+ const DB::FormatSettings & settings) const
+ {
+ writeChar('{', ostr);
+ for (size_t i = 0; i < tuple_col.tupleSize(); ++i)
+ {
+ if (i != 0)
+ writeString(", ", ostr);
+
+ const auto & elem_type = elems_type[i];
+ const auto & elem_column = tuple_col.getColumn(i);
+
+ if (DB::WhichDataType(elem_type).isNullable())
+ {
+ serializeNullable(
+ assert_cast<const DB::ColumnNullable &>(elem_column),
+ row_num,
+ DB::removeNullable(elem_type),
+ ostr,
+ settings);
+ }
+ else if (isTuple(elem_type))
+ {
+ serializeTuple(
+ assert_cast<const DB::ColumnTuple &>(elem_column),
+ row_num,
+ assert_cast<const DB::DataTypeTuple
&>(*elem_type).getElements(),
+ ostr,
+ settings);
+ }
+ else if (isMap(elem_type))
+ {
+ serializeMap(
+ assert_cast<const DB::ColumnMap &>(elem_column),
+ row_num,
+ assert_cast<const DB::DataTypeMap
&>(*elem_type).getKeyType(),
+ assert_cast<const DB::DataTypeMap
&>(*elem_type).getValueType(),
+ ostr,
+ settings);
+ }
+ else if (isArray(elem_type))
+ {
+ const auto & elem_array_type = assert_cast<const
DB::DataTypeArray &>(*elem_type);
+ serializeArray(
+ assert_cast<const DB::ColumnArray &>(elem_column),
+ row_num,
+ elem_array_type.getNestedType(),
+ ostr,
+ settings);
+ }
+ else
+
elem_type->getDefaultSerialization()->serializeText(elem_column, row_num, ostr,
settings);
+ }
+ writeChar('}', ostr);
+ }
+
+ void serializeMap(
+ const DB::ColumnMap & map_col,
+ size_t row_num,
+ const DB::DataTypePtr & key_type,
+ const DB::DataTypePtr & value_type,
+ DB::WriteBuffer & ostr,
+ const DB::FormatSettings & settings) const
+ {
+ const auto & nested_array = map_col.getNestedColumn();
+ const auto & nested_tuple = map_col.getNestedData();
+ const auto & offsets = nested_array.getOffsets();
+
+ const auto & key_column = nested_tuple.getColumn(0);
+ const auto & value_column = nested_tuple.getColumn(1);
+
+ const auto & key_serializer = key_type->getDefaultSerialization();
+ const auto & value_serializer = value_type->getDefaultSerialization();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ writeChar('{', ostr);
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ writeCString(", ", ostr);
+
+ if (DB::WhichDataType(key_type).isNullable())
+ {
+ serializeNullable(
+ assert_cast<const DB::ColumnNullable &>(key_column),
+ i,
+ DB::removeNullable(key_type),
+ ostr,
+ settings);
+ }
+ // The key of map cannot be/contain map.
+ else if (isArray(key_type))
+ {
+ serializeArray(
+ assert_cast<const DB::ColumnArray &>(key_column),
+ i,
+ assert_cast<const DB::DataTypeArray
&>(*key_type).getNestedType(),
+ ostr,
+ settings);
+ }
+ else if (isTuple(key_type))
+ {
+ serializeTuple(
+ assert_cast<const DB::ColumnTuple &>(key_column),
+ i,
+ assert_cast<const DB::DataTypeTuple
&>(*key_type).getElements(),
+ ostr,
+ settings);
+ }
+ else
+ key_serializer->serializeText(key_column, i, ostr, settings);
+
+ writeCString(" -> ", ostr);
+
+ if (DB::WhichDataType(value_type).isNullable())
+ {
+ serializeNullable(
+ assert_cast<const DB::ColumnNullable &>(value_column),
+ i,
+ DB::removeNullable(value_type),
+ ostr,
+ settings);
+ }
+ else if (isArray(value_type))
+ {
+ serializeArray(
+ assert_cast<const DB::ColumnArray &>(value_column),
+ i,
+ assert_cast<const DB::DataTypeArray
&>(*value_type).getNestedType(),
+ ostr,
+ settings);
+ }
+ else if (isMap(value_type))
+ {
+ serializeMap(
+ assert_cast<const DB::ColumnMap &>(value_column),
+ i,
+ assert_cast<const DB::DataTypeMap
&>(*value_type).getKeyType(),
+ assert_cast<const DB::DataTypeMap
&>(*value_type).getValueType(),
+ ostr,
+ settings);
+ }
+ else if (isTuple(value_type))
+ {
+ serializeTuple(
+ assert_cast<const DB::ColumnTuple &>(value_column),
+ i,
+ assert_cast<const DB::DataTypeTuple
&>(*value_type).getElements(),
+ ostr,
+ settings);
+ }
+ else
+ value_serializer->serializeText(value_column, i, ostr,
settings);
+ }
+ writeChar('}', ostr);
+ }
+
+ void serializeArray(
+ const DB::ColumnArray & array_col,
+ size_t row_num,
+ const DB::DataTypePtr & value_type,
+ DB::WriteBuffer & ostr,
+ const DB::FormatSettings & settings) const
+ {
+ const DB::ColumnArray::Offsets & offsets = array_col.getOffsets();
+ const auto & nested_column = array_col.getDataPtr();
+
+ size_t offset = offsets[row_num - 1];
+ size_t next_offset = offsets[row_num];
+
+ const auto & value_serializer = value_type->getDefaultSerialization();
+
+ writeChar('[', ostr);
+ for (size_t i = offset; i < next_offset; ++i)
+ {
+ if (i != offset)
+ writeCString(", ", ostr);
+
+ if (DB::WhichDataType(value_type).isNullable())
+ {
+ serializeNullable(
+ assert_cast<const DB::ColumnNullable &>(*nested_column),
+ i,
+ DB::removeNullable(value_type),
+ ostr,
+ settings);
+ }
+ else if (isArray(value_type))
+ {
+ serializeArray(
+ assert_cast<const DB::ColumnArray &>(*nested_column),
+ i,
+ assert_cast<const DB::DataTypeArray
&>(*value_type).getNestedType(),
+ ostr,
+ settings);
+ }
+ else if (isMap(value_type))
+ {
+ serializeMap(
+ assert_cast<const DB::ColumnMap &>(*nested_column),
+ i,
+ assert_cast<const DB::DataTypeMap
&>(*value_type).getKeyType(),
+ assert_cast<const DB::DataTypeMap
&>(*value_type).getValueType(),
+ ostr,
+ settings);
+ }
+ else if (isTuple(value_type))
+ {
+ serializeTuple(
+ assert_cast<const DB::ColumnTuple &>(*nested_column),
+ i,
+ assert_cast<const DB::DataTypeTuple
&>(*value_type).getElements(),
+ ostr,
+ settings);
+ }
+ else
+ value_serializer->serializeText(*nested_column, i, ostr,
settings);
+ }
+ writeChar(']', ostr);
+ }
+
+ void serializeNullable(
+ const DB::ColumnNullable & column,
+ size_t row_num,
+ const DB::DataTypePtr & nested_type,
+ DB::WriteBuffer & ostr,
+ const DB::FormatSettings & settings) const
+ {
+ if (column.isNullAt(row_num))
+ {
+ writeCString("null", ostr);
+ return;
+ }
+
+ const auto & nested_col = column.getNestedColumn();
+
+ if (isArray(nested_type))
+ {
+ serializeArray(
+ assert_cast<const DB::ColumnArray &>(nested_col),
+ row_num,
+ assert_cast<const DB::DataTypeArray
&>(*nested_type).getNestedType(),
+ ostr,
+ settings);
+ }
+ else if (isMap(nested_type))
+ {
+ serializeMap(
+ assert_cast<const DB::ColumnMap &>(nested_col),
+ row_num,
+ assert_cast<const DB::DataTypeMap
&>(*nested_type).getKeyType(),
+ assert_cast<const DB::DataTypeMap
&>(*nested_type).getValueType(),
+ ostr,
+ settings);
+ }
+ else if (isTuple(nested_type))
+ {
+ serializeTuple(
+ assert_cast<const DB::ColumnTuple &>(nested_col),
+ row_num,
+ assert_cast<const DB::DataTypeTuple
&>(*nested_type).getElements(),
+ ostr,
+ settings);
+ }
+ else
+ nested_type->getDefaultSerialization()->serializeText(nested_col,
row_num, ostr, settings);
+ }
+};
+
+}
diff --git a/cpp-ch/local-engine/Functions/SparkFunctionArrayToString.h
b/cpp-ch/local-engine/Functions/SparkFunctionArrayToString.h
deleted file mode 100644
index fce339d294..0000000000
--- a/cpp-ch/local-engine/Functions/SparkFunctionArrayToString.h
+++ /dev/null
@@ -1,165 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#pragma once
-#include <memory>
-#include <Columns/ColumnArray.h>
-#include <Columns/ColumnNullable.h>
-#include <Columns/ColumnStringHelpers.h>
-#include <DataTypes/DataTypeNullable.h>
-#include <DataTypes/DataTypeString.h>
-#include <DataTypes/DataTypeArray.h>
-#include <Formats/FormatFactory.h>
-#include <Functions/FunctionFactory.h>
-#include <Functions/FunctionHelpers.h>
-#include <IO/WriteHelpers.h>
-
-namespace DB
-{
-namespace ErrorCodes
-{
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
- extern const int ILLEGAL_TYPE_OF_ARGUMENT;
-}
-}
-
-namespace local_eingine
-{
-class SparkFunctionArrayToString : public DB::IFunction
-{
-public:
- static constexpr auto name = "sparkCastArrayToString";
-
- static DB::FunctionPtr create(DB::ContextPtr context) { return
std::make_shared<SparkFunctionArrayToString>(context); }
-
- explicit SparkFunctionArrayToString(DB::ContextPtr context_) :
context(context_) {}
-
- ~SparkFunctionArrayToString() override = default;
-
- String getName() const override { return name; }
-
- size_t getNumberOfArguments() const override { return 1; }
-
- bool isSuitableForShortCircuitArgumentsExecution(const
DB::DataTypesWithConstInfo & /*arguments*/) const override { return true; }
-
- bool useDefaultImplementationForNulls() const override { return false; }
-
- bool useDefaultImplementationForLowCardinalityColumns() const override {
return false; }
-
- DB::DataTypePtr getReturnTypeImpl(const DB::ColumnsWithTypeAndName &
arguments) const override
- {
- if (arguments.size() != 1)
- throw
DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}
argument size must be 1", name);
-
- auto arg_type = DB::removeNullable(arguments[0].type);
- if (!DB::WhichDataType(arg_type).isArray())
- throw DB::Exception(
- DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT, "Illegal type {} of
argument[0] of function {}",
- arguments[0].type->getName(), getName());
-
- if (arguments[0].type->isNullable())
- return makeNullable(std::make_shared<DB::DataTypeString>());
- else
- return std::make_shared<DB::DataTypeString>();
- }
-
- DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments,
const DB::DataTypePtr & result_type, size_t input_rows_count) const override
- {
- DB::ColumnUInt8::MutablePtr null_map = nullptr;
- if (const auto * col_nullable =
checkAndGetColumn<DB::ColumnNullable>(arguments[0].column.get()))
- {
- null_map = DB::ColumnUInt8::create();
- null_map->insertRangeFrom(col_nullable->getNullMapColumn(), 0,
col_nullable->size());
- }
-
- const auto & nested_col_with_type_and_name =
columnGetNested(arguments[0]);
-
- if (const auto * col_const = typeid_cast<const DB::ColumnConst
*>(nested_col_with_type_and_name.column.get()))
- {
- DB::ColumnsWithTypeAndName new_arguments {1};
- new_arguments[0] = {col_const->getDataColumnPtr(),
nested_col_with_type_and_name.type, nested_col_with_type_and_name.name};
- auto col = executeImpl(new_arguments, result_type, 1);
- return DB::ColumnConst::create(std::move(col), input_rows_count);
- }
-
- const DB::IColumn & col_from = *nested_col_with_type_and_name.column;
- size_t size = col_from.size();
- auto col_to = removeNullable(result_type)->createColumn();
-
- DB::FormatSettings format_settings = context ?
DB::getFormatSettings(context) : DB::FormatSettings{};
- format_settings.pretty.charset =
DB::FormatSettings::Pretty::Charset::ASCII; /// Use ASCII for pretty output.
-
- DB::ColumnStringHelpers::WriteHelper write_helper(
- assert_cast<DB::ColumnString &>(*col_to),
- size);
-
- auto & write_buffer = write_helper.getWriteBuffer();
-
- const auto * array_type =
checkAndGetDataType<DB::DataTypeArray>(nested_col_with_type_and_name.type.get());
- if (!array_type)
- throw DB::Exception(DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
"Argument #1 for function {} must be an array, not {}",
- name, arguments[0].type->getName());
-
- DB::DataTypePtr value_type = array_type->getNestedType();
- auto value_serializer = value_type->getDefaultSerialization();
-
- for (size_t row = 0; row < size; ++row)
- {
- serializeInSparkStyle(col_from,row,write_buffer,format_settings,
value_serializer);
- write_helper.rowWritten();
- }
-
- write_helper.finalize();
-
- if (result_type->isNullable() && null_map)
- return DB::ColumnNullable::create(std::move(col_to),
std::move(null_map));
- return col_to;
- }
-
-private:
- DB::ContextPtr context;
-
- void serializeInSparkStyle(
- const DB::IColumn & column,
- size_t row_num,
- DB::WriteBuffer & ostr,
- const DB::FormatSettings & settings,
- const DB::SerializationPtr & value_serializer) const
- {
- const auto & column_array = assert_cast<const DB::ColumnArray
&>(column);
-
- const auto & nested_column= column_array.getData();
- const DB::ColumnArray::Offsets & offsets = column_array.getOffsets();
-
- size_t offset = offsets[row_num - 1];
- size_t next_offset = offsets[row_num];
-
- writeChar('[', ostr);
- if (offset != next_offset)
- {
- value_serializer->serializeText(nested_column, offset, ostr,
settings);
- for (size_t i = offset + 1; i < next_offset; ++i)
- {
- writeString(std::string_view(", "), ostr);
- value_serializer->serializeText(nested_column, i, ostr,
settings);
- }
- }
- writeChar(']', ostr);
- }
-};
-
-}
diff --git a/cpp-ch/local-engine/Functions/SparkFunctionMapToString.cpp
b/cpp-ch/local-engine/Functions/SparkFunctionMapToString.cpp
deleted file mode 100644
index 499f64e48e..0000000000
--- a/cpp-ch/local-engine/Functions/SparkFunctionMapToString.cpp
+++ /dev/null
@@ -1,28 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <Functions/SparkFunctionMapToString.h>
-
-namespace local_engine
-{
-
-REGISTER_FUNCTION(SparkFunctionMapToString)
-{
- factory.registerFunction<local_eingine::SparkFunctionMapToString>();
-}
-
-}
\ No newline at end of file
diff --git a/cpp-ch/local-engine/Functions/SparkFunctionMapToString.h
b/cpp-ch/local-engine/Functions/SparkFunctionMapToString.h
deleted file mode 100644
index 34fb4a7de6..0000000000
--- a/cpp-ch/local-engine/Functions/SparkFunctionMapToString.h
+++ /dev/null
@@ -1,177 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-#pragma once
-
-#include <Columns/ColumnMap.h>
-#include <Columns/ColumnNullable.h>
-#include <Columns/ColumnStringHelpers.h>
-#include <Columns/ColumnTuple.h>
-#include <Columns/ColumnsNumber.h>
-#include <DataTypes/DataTypeNullable.h>
-#include <DataTypes/DataTypeString.h>
-#include <Formats/FormatFactory.h>
-#include <Functions/FunctionFactory.h>
-#include <Functions/FunctionHelpers.h>
-#include <IO/WriteHelpers.h>
-
-namespace DB
-{
-namespace ErrorCodes
-{
- extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH;
- extern const int ILLEGAL_TYPE_OF_ARGUMENT;
-}
-}
-
-namespace local_eingine
-{
-class SparkFunctionMapToString : public DB::IFunction
-{
-public:
- static constexpr auto name = "sparkCastMapToString";
- static DB::FunctionPtr create(DB::ContextPtr context) { return
std::make_shared<SparkFunctionMapToString>(context); }
- explicit SparkFunctionMapToString(DB::ContextPtr context_) :
context(context_) {}
- ~SparkFunctionMapToString() override = default;
- String getName() const override { return name; }
- size_t getNumberOfArguments() const override { return 3; }
- bool isSuitableForShortCircuitArgumentsExecution(const
DB::DataTypesWithConstInfo & /*arguments*/) const override { return true; }
- bool useDefaultImplementationForNulls() const override { return false; }
- bool useDefaultImplementationForLowCardinalityColumns() const override {
return false; }
-
- DB::DataTypePtr getReturnTypeImpl(const DB::ColumnsWithTypeAndName &
arguments) const override
- {
- if (arguments.size() != 3)
- throw
DB::Exception(DB::ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, "Function {}
argument size must be 3", name);
-
- auto arg_type = DB::removeNullable(arguments[0].type);
- if (!DB::WhichDataType(arg_type).isMap())
- {
- throw DB::Exception(
- DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
- "Illegal type {} of argument[1] of function {}",
- arguments[0].type->getName(),
- getName());
- }
-
- auto key_type = DB::WhichDataType(removeNullable(arguments[1].type));
- auto value_type = DB::WhichDataType(removeNullable(arguments[2].type));
- // Not support complex types in key or value
- if (!key_type.isString() && !key_type.isNumber() &&
!value_type.isString() && !value_type.isNumber())
- {
- throw DB::Exception(
- DB::ErrorCodes::ILLEGAL_TYPE_OF_ARGUMENT,
- "Cast MapToString not support {}, {} as key value",
- arguments[1].type->getName(),
- arguments[2].type->getName());
- }
-
- if (arguments[0].type->isNullable())
- return makeNullable(std::make_shared<DB::DataTypeString>());
- else
- return std::make_shared<DB::DataTypeString>();
- }
-
- DB::ColumnPtr executeImpl(const DB::ColumnsWithTypeAndName & arguments,
const DB::DataTypePtr & result_type, size_t /*input_rows*/) const override
- {
- DB::ColumnUInt8::MutablePtr null_map = nullptr;
- if (const auto * col_nullable =
checkAndGetColumn<DB::ColumnNullable>(arguments[0].column.get()))
- {
- null_map = DB::ColumnUInt8::create();
- null_map->insertRangeFrom(col_nullable->getNullMapColumn(), 0,
col_nullable->size());
- }
-
- const auto & col_with_type_and_name = columnGetNested(arguments[0]);
- const DB::IColumn & column = *col_with_type_and_name.column;
- const DB::IColumn & col_from = column.isConst() ?
reinterpret_cast<const DB::ColumnConst &>(column).getDataColumn() : column;
-
- size_t size = col_from.size();
- auto col_to = removeNullable(result_type)->createColumn();
-
- {
- DB::FormatSettings format_settings = context ?
DB::getFormatSettings(context) : DB::FormatSettings{};
- DB::ColumnStringHelpers::WriteHelper write_helper(
- assert_cast<DB::ColumnString &>(*col_to),
- size);
-
- auto & write_buffer = write_helper.getWriteBuffer();
- auto key_serializer = arguments[1].type->getDefaultSerialization();
- auto value_serializer =
arguments[2].type->getDefaultSerialization();
-
- for (size_t row = 0; row < size; ++row)
- {
- serializeInSparkStyle(
- col_from,
- row,
- write_buffer,
- format_settings,
- key_serializer,
- value_serializer);
- write_helper.rowWritten();
- }
-
- write_helper.finalize();
- }
-
- if (result_type->isNullable() && null_map)
- return DB::ColumnNullable::create(std::move(col_to),
std::move(null_map));
- return col_to;
- }
-
-private:
- DB::ContextPtr context;
-
- void serializeInSparkStyle(
- const DB::IColumn & column,
- size_t row_num,
- DB::WriteBuffer & ostr,
- const DB::FormatSettings & settings,
- const DB::SerializationPtr & key_serializer,
- const DB::SerializationPtr & value_serializer) const
- {
-
- const auto & column_map = assert_cast<const DB::ColumnMap &>(column);
-
- const auto & nested_array = column_map.getNestedColumn();
- const auto & nested_tuple = column_map.getNestedData();
- const auto & offsets = nested_array.getOffsets();
- const auto& key_column = nested_tuple.getColumn(0);
- const auto& value_column = nested_tuple.getColumn(1);
-
- size_t offset = offsets[row_num - 1];
- size_t next_offset = offsets[row_num];
-
- writeChar('{', ostr);
- for (size_t i = offset; i < next_offset; ++i)
- {
- if (i != offset)
- {
- writeChar(',', ostr);
- writeChar(' ', ostr);
- }
-
- key_serializer->serializeText(key_column, i, ostr, settings);
- writeChar(' ', ostr);
- writeChar('-', ostr);
- writeChar('>', ostr);
- writeChar(' ', ostr);
- value_serializer->serializeText(value_column, i, ostr, settings);
- }
- writeChar('}', ostr);
- }
-};
-
-}
diff --git a/cpp-ch/local-engine/Parser/ExpressionParser.cpp
b/cpp-ch/local-engine/Parser/ExpressionParser.cpp
index 4261241f25..53dec15464 100644
--- a/cpp-ch/local-engine/Parser/ExpressionParser.cpp
+++ b/cpp-ch/local-engine/Parser/ExpressionParser.cpp
@@ -266,7 +266,7 @@ bool ExpressionParser::reuseCSE() const
}
ExpressionParser::NodeRawConstPtr
-ExpressionParser::addConstColumn(DB::ActionsDAG & actions_dag, const
DB::DataTypePtr type, const DB::Field & field) const
+ExpressionParser::addConstColumn(DB::ActionsDAG & actions_dag, const
DB::DataTypePtr & type, const DB::Field & field) const
{
String name = toString(field).substr(0, 10);
name = getUniqueName(name);
@@ -350,19 +350,10 @@ ExpressionParser::NodeRawConstPtr
ExpressionParser::parseExpression(ActionsDAG &
result_node = toFunctionNode(actions_dag,
"checkDecimalOverflowSparkOrNull", args);
}
}
- else if (isMap(denull_input_type) && isString(denull_output_type))
+ else if ((isMap(denull_input_type) || isArray(denull_input_type)
|| isTuple(denull_input_type)) && isString(denull_output_type))
{
- // ISSUE-7389: spark cast(map to string) has different
behavior with CH cast(map to string)
- auto map_input_type = std::static_pointer_cast<const
DataTypeMap>(denull_input_type);
- args.emplace_back(addConstColumn(actions_dag,
map_input_type->getKeyType(), map_input_type->getKeyType()->getDefault()));
- args.emplace_back(
- addConstColumn(actions_dag,
map_input_type->getValueType(), map_input_type->getValueType()->getDefault()));
- result_node = toFunctionNode(actions_dag,
"sparkCastMapToString", args);
- }
- else if (isArray(denull_input_type) &&
isString(denull_output_type))
- {
- // ISSUE-7602: spark cast(array to string) has different
result with CH cast(array to string)
- result_node = toFunctionNode(actions_dag,
"sparkCastArrayToString", args);
+ /// https://github.com/apache/incubator-gluten/issues/9049
+ result_node = toFunctionNode(actions_dag,
"sparkCastComplexTypesToString", args);
}
else if (isString(denull_input_type) && substrait_type.has_bool_())
{
@@ -374,7 +365,7 @@ ExpressionParser::NodeRawConstPtr
ExpressionParser::parseExpression(ActionsDAG &
{
/// Spark cast(x as INT) if x is String -> CH cast(trim(x) as
INT)
/// Refer to
https://github.com/apache/incubator-gluten/issues/4956 and
https://github.com/apache/incubator-gluten/issues/8598
- auto trim_str_arg = addConstColumn(actions_dag,
std::make_shared<DataTypeString>(), " \t\n\r\f");
+ const auto * trim_str_arg = addConstColumn(actions_dag,
std::make_shared<DataTypeString>(), " \t\n\r\f");
args[0] = toFunctionNode(actions_dag, "trimBothSpark",
{args[0], trim_str_arg});
args.emplace_back(addConstColumn(actions_dag,
std::make_shared<DataTypeString>(), output_type->getName()));
result_node = toFunctionNode(actions_dag, "CAST", args);
@@ -821,7 +812,7 @@ ExpressionParser::parseArrayJoin(const
substrait::Expression_ScalarFunction & fu
const auto * key_node = add_tuple_element(item_node, 1);
/// value = arrayJoin(arg_not_null).2.2
- const auto val_node = add_tuple_element(item_node, 2);
+ const auto * val_node = add_tuple_element(item_node, 2);
actions_dag.addOrReplaceInOutputs(*pos_node);
actions_dag.addOrReplaceInOutputs(*key_node);
@@ -862,7 +853,7 @@ ExpressionParser::parseJsonTuple(const
substrait::Expression_ScalarFunction & fu
const auto * extract_expr_node = addConstColumn(actions_dag,
std::make_shared<DB::DataTypeString>(), write_buffer.str());
auto json_extract_builder =
DB::FunctionFactory::instance().get("JSONExtract", context->queryContext());
auto json_extract_result_name = "JSONExtract(" +
json_expr_node->result_name + ", " + extract_expr_node->result_name + ")";
- const auto json_extract_node
+ const auto * json_extract_node
= &actions_dag.addFunction(json_extract_builder, {json_expr_node,
extract_expr_node}, json_extract_result_name);
auto tuple_element_builder =
DB::FunctionFactory::instance().get("sparkTupleElement",
context->queryContext());
auto tuple_index_type = std::make_shared<DB::DataTypeUInt32>();
diff --git a/cpp-ch/local-engine/Parser/ExpressionParser.h
b/cpp-ch/local-engine/Parser/ExpressionParser.h
index 1e4a48282a..9e094ffea5 100644
--- a/cpp-ch/local-engine/Parser/ExpressionParser.h
+++ b/cpp-ch/local-engine/Parser/ExpressionParser.h
@@ -41,16 +41,16 @@ class ExpressionParser
{
public:
using NodeRawConstPtr = const DB::ActionsDAG::Node *;
- ExpressionParser(const std::shared_ptr<const ParserContext> & context_) :
context(context_) { }
+ explicit ExpressionParser(const std::shared_ptr<const ParserContext> &
context_) : context(context_) { }
~ExpressionParser() = default;
/// Append a counter-suffix to name
String getUniqueName(const String & name) const;
- NodeRawConstPtr addConstColumn(DB::ActionsDAG & actions_dag, const
DB::DataTypePtr type, const DB::Field & field) const;
+ NodeRawConstPtr addConstColumn(DB::ActionsDAG & actions_dag, const
DB::DataTypePtr & type, const DB::Field & field) const;
/// Parse expr and add an expression node in actions_dag
- NodeRawConstPtr parseExpression(DB::ActionsDAG & actions_dag, const
substrait::Expression & expr) const;
+ NodeRawConstPtr parseExpression(DB::ActionsDAG & actions_dag, const
substrait::Expression & rel) const;
/// Build an actions dag that contains expressions. header is used as
input columns for the actions dag.
DB::ActionsDAG expressionsToActionsDAG(const
std::vector<substrait::Expression> & expressions, const DB::Block & header)
const;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
