github-actions[bot] commented on code in PR #63528:
URL: https://github.com/apache/doris/pull/63528#discussion_r4324150491
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java:
##########
@@ -581,4 +600,321 @@ private Expression transformHadoopFunctionExpr(Table tbl,
String columnName, Exp
}
return originExpr;
}
+
+ /**
+ * The explicit value of a default_value or replace_value mapping on an
ARRAY/MAP/STRUCT column was
+ * validated by {@link NereidsDataDescription#validateMappingFunction} as
a SQL literal, but BE casts
+ * the mapping expression as a string. Replace it with the canonical text
so that both agree, e.g.
+ * {@code '[1e3]'} for an array of INT is written as {@code [1000]}
instead of a NULL element.
+ */
+ private static Expression canonicalizeComplexValue(Column column,
Expression value) {
+ if (!column.getOriginType().isComplexType() || !(value instanceof
StringLikeLiteral)) {
+ return value;
+ }
+ return new StringLiteral(ComplexTypeDefaultValue.canonicalize(
+ DataType.fromCatalogType(column.getOriginType()),
((StringLikeLiteral) value).getStringValue()));
+ }
+
+ /**
+ * Arrow reads a column with its declared type, so an ARRAY/MAP/STRUCT
column keeps the complex type
+ * in its scan slot instead of being read as VARCHAR like the text
formats. The mapping expressions
+ * have to work on that complex type, otherwise the IF branches mix
complex and string values.
+ */
+ private boolean readsNativeComplexType(Column column) {
+ return column.getOriginType().isComplexType()
+ &&
fileGroupInfo.getFileGroup().getFileFormatProperties().getFileFormatType()
+ == TFileFormatType.FORMAT_ARROW;
+ }
+
+ /**
+ * The mapping value of a complex column is written as canonical text.
When the scan slot keeps the
+ * native complex type, parse that text into the column type: a string
branch and a complex branch
+ * have no common type under the legacy type coercion behavior ({@code
+ * enable_new_type_coercion_behavior=false}).
+ */
+ private static Expression mappingValue(Column column, Expression value,
boolean nativeComplex) {
+ Expression canonical = canonicalizeComplexValue(column, value);
+ if (!nativeComplex) {
+ return canonical;
+ }
+ return new Cast(canonical,
DataType.fromCatalogType(column.getOriginType()));
+ }
+
+ /**
+ * The NULL branch of a mapping IF. The typed VARCHAR NULL used for text
sources has no common type
+ * with a native complex branch under the legacy type coercion behavior,
so leave it untyped.
+ */
+ private static Expression nullMappingValue(boolean nativeComplex) {
+ return nativeComplex ? new NullLiteral() : new
NullLiteral(VarcharType.SYSTEM_DEFAULT);
+ }
+
+ /**
+ * A text source compares its raw value with the search value of {@code
replace_value}. A native
+ * complex slot needs a type-aware comparison instead of the display text,
whose nested strings are
+ * not escaped: a single element {@code a", "b} renders exactly like the
two element array
+ * {@code ["a", "b"]}. The search is a transient comparison value, so it
must not be canonicalized
+ * as a column default, whose format rejects quotes and backslashes that
the complex cast accepts.
+ */
+ private static Expression searchValueComparison(Expression slot, Column
column, Expression search,
+ boolean nativeComplex) throws AnalysisException {
+ if (!nativeComplex) {
+ return new EqualTo(slot, search);
+ }
+ DataType complexType =
DataType.fromCatalogType(column.getOriginType());
+ // Cast the search expression, literal or slot, to the column type so
that the comparison sees
+ // exactly the bytes the sink writes. The cast is strict: Stream Load
disables
+ // enable_strict_cast, so a malformed non-NULL search would otherwise
become SQL NULL, which
+ // the mapping IF treats as "not equal" and replaces every row,
including rows that do not
+ // match the search. A strict cast fails the load instead.
+ Expression typedSearch = new Cast(search, complexType, false, true);
+ return buildEquality(slot, typedSearch, complexType, true,
column.getOriginType().toSql());
+ }
+
+ /**
+ * Equality of two expressions of {@code type}, built from comparisons the
engine can evaluate.
+ * Primitive values and arrays of them are compared structurally ({@code
compare_at}), which keeps
+ * nested NULLs apart from values and treats two NULLs as equal like the
engine does inside a
+ * complex value; {@code <=>} expresses that for a nested value. MAP and
STRUCT are rejected by
+ * {@code ComparisonPredicate}, so their sub-values are compared
recursively: a MAP is identified
+ * by its deduplicated keys and values in order, and a STRUCT that cannot
be serialized losslessly
+ * enough for a whole JSON comparison both by its own NULL flag and by its
fields. A STRUCT whose
+ * JSON text is a lossless identity is compared as one value, and a STRUCT
whose leaves have no
+ * lossless JSON identity but are comparable recursively is compared by
one engine call, so the
+ * search is parsed a bounded number of times instead of once per field.
+ *
+ * <p>JSON is used only for a value the engine cannot compare
structurally. A MAP is never
+ * rendered as a JSON object: JSONB object keys are limited to 255 bytes
and lose the key NULL
+ * flag, so the empty string key and a NULL key would render the same.
Nested MAPs are rejected
+ * for the same reason.
+ *
+ * @param topLevel whether this is the whole search comparison. A NULL
search then keeps the SQL
+ * three-valued result of the text formats instead of
matching a NULL value.
+ */
+ private static Expression buildEquality(Expression left, Expression right,
DataType type,
+ boolean topLevel, String columnTypeSql) throws AnalysisException {
+ if (supportsStructuralComparison(type)) {
+ // A top level ARRAY is compared by EqualTo, which is NULL when
the search is NULL; a
+ // nested value uses <=> so that two NULLs match, like compare_at
inside a complex value.
+ return topLevel ? new EqualTo(left, right) : new
NullSafeEqual(left, right);
+ }
+ if (type.isMapType()) {
+ MapType mapType = (MapType) type;
+ DataType keyType = mapType.getKeyType();
+ DataType valueType = mapType.getValueType();
+ if (!supportsStructuralComparison(keyType) ||
!supportsMapValueComparison(valueType)) {
+ throw unsupportedComplexSearch(columnTypeSql, type);
+ }
+ // An Arrow MAP keeps every entry, so a repeated key makes its key
and value arrays
+ // longer than the ones of the search text, which the string cast
deduplicates with the
+ // last value winning. The sink applies the same last-wins rule
before it writes the
+ // column, so normalize the native source the same way; otherwise
a row that is written
+ // as the search value would not match it and would not be
replaced.
+ Expression leftMap = deduplicateMap(left);
+ // Only the top level search is the string cast, which already
deduplicated its keys. A
+ // MAP field of a STRUCT is parsed by the struct SerDe, which
leaves repeated keys in
+ // place, so it needs the same normalization as the source.
+ Expression rightMap = topLevel ? right : deduplicateMap(right);
Review Comment:
[P2] Normalize native MAP search slots before comparing
A `replace_value` search can be another native MAP slot, including the
source column itself. For Arrow `v MAP<INT,INT>` containing entries
`[(1,1),(1,2)]`, `v=replace_value(v,NULL)` deduplicates the left side to
`{1:2}` but leaves the right side with both entries here. The search's
same-type cast is a no-op, so its keys remain `[1,1]` instead of `[1]`; the
comparison is false and the load keeps `{1:2}` rather than replacing the column
with NULL. Normalize native search maps too instead of assuming every top-level
search passed through the string parser.
##########
be/src/exprs/function/comparison_equals.cpp:
##########
@@ -15,18 +15,86 @@
// specific language governing permissions and limitations
// under the License.
+#include "common/check.h"
#include "core/accurate_comparison.h"
#include "core/column/column.h"
+#include "core/column/column_nullable.h"
+#include "core/column/column_vector.h"
#include "core/data_type/data_type.h"
+#include "core/data_type/data_type_number.h"
#include "exprs/function/functions_comparison.h"
#include "exprs/function/simple_function_factory.h"
namespace doris {
using FunctionEquals = FunctionComparison<EqualsOp, NameEquals>;
using FunctionNotEquals = FunctionComparison<NotEqualsOp, NameNotEquals>;
+/// Recursive equality for a value the planner cannot compare as one
expression, for example a
+/// STRUCT with a float leaf whose JSON text is not a lossless identity. The
comparison is NULL
+/// safe: two NULL values (including a NULL nested field) are equal, and a
NULL and a non-NULL
+/// value are not.
+///
+/// The planner emits this function to keep the typed search in one place.
Expanding the fields in
+/// the expression tree instead embeds the search cast in every field
comparison, and each
+/// occurrence parses the whole search string again; an N field STRUCT then
converts N fields N+1
+/// times per row. Column::compare_at already applies the nested NULL rule and
compares every
+/// column type structurally, so the function only has to unwrap the two top
level null maps.
+class FunctionComplexEquals : public IFunction {
+public:
+ static constexpr auto name = "complex_equals";
+ static FunctionPtr create() { return
std::make_shared<FunctionComplexEquals>(); }
+
+ String get_name() const override { return name; }
+
+ size_t get_number_of_arguments() const override { return 2; }
+
+ bool use_default_implementation_for_nulls() const override { return false;
}
+
+ DataTypePtr get_return_type_impl(const DataTypes& /*arguments*/) const
override {
+ return std::make_shared<DataTypeBool>();
+ }
+
+ Status execute_impl(FunctionContext* /*context*/, Block& block, const
ColumnNumbers& arguments,
+ uint32_t result, size_t input_rows_count) const
override {
+ ColumnPtr left =
+
block.get_by_position(arguments[0]).column->convert_to_full_column_if_const();
+ ColumnPtr right =
+
block.get_by_position(arguments[1]).column->convert_to_full_column_if_const();
Review Comment:
[P2] Compare constant searches without expanding their payload
For an Arrow `replace_value` mapping on `STRUCT<a:ARRAY<INT>,d:DOUBLE>`, a
literal search reaches this function as a one-row `ColumnConst`, while the
source column is nonconstant. Expanding that search here copies its entire
nested array once per input row (`ColumnConst::convert_to_full_column` ->
`ColumnStruct::insert_many_from` -> `ColumnArray::insert_many_from`). A
100000-element search therefore adds about 2 GB of nullable INT payload for a
4096-row batch, even when every source array is empty and cannot match. Retain
the constant's data column and compare against row 0, as
`GenericComparisonImpl::vector_constant` already does.
##########
be/src/exprs/function/cast/cast_base.cpp:
##########
@@ -264,12 +265,20 @@ Status cast_from_string_to_complex_type(FunctionContext*
context, Block& block,
options.escape_char = '\\';
options.timezone = &context->state()->timezone_obj();
+ // The complex SerDes unescape nested strings in place, and get_data_at()
returns a view into
+ // the source column. That column can also be another load target, for
example
+ // `columns: k, s, s = replace_value(w, NULL), w`, so parsing the borrowed
text would rewrite
+ // the bytes the same row imports into w. Parse an owned copy instead;
reusing the buffer keeps
+ // the copy at one allocation per batch.
+ std::string owned_text;
for (size_t i = 0; i < input_rows_count; ++i) {
if (null_map && null_map[i]) {
nullable_col_to.insert_default();
} else {
auto str = col_from->get_data_at(i);
- Status st = to_serde->from_string(str, nested_column, options);
+ owned_text.assign(str.data, str.size);
Review Comment:
[P2] Charge the owned cast scratch buffer to the query tracker
Every string-to-complex cast now copies a whole input cell into
`std::string`, which bypasses Doris Allocator/MemTracker. For a 9 MiB STRING
containing non-array text, `CAST(s AS ARRAY<INT>)` allocates another roughly 9
MiB before rejecting the first character and returning only NULL; the tracked
source and output therefore omit the new live buffer. Concurrent evaluators
multiply this uncharged memory, and the string retains its largest capacity
until the batch ends. Keep the owned copy, but use reusable allocator-aware
storage in both wrappers (the strict twin is at line 323). This is separate
from the existing thread about persisted-default `Field` vectors.
##########
fe/fe-core/src/main/java/org/apache/doris/nereids/load/NereidsLoadScanProvider.java:
##########
@@ -581,4 +600,321 @@ private Expression transformHadoopFunctionExpr(Table tbl,
String columnName, Exp
}
return originExpr;
}
+
+ /**
+ * The explicit value of a default_value or replace_value mapping on an
ARRAY/MAP/STRUCT column was
+ * validated by {@link NereidsDataDescription#validateMappingFunction} as
a SQL literal, but BE casts
+ * the mapping expression as a string. Replace it with the canonical text
so that both agree, e.g.
+ * {@code '[1e3]'} for an array of INT is written as {@code [1000]}
instead of a NULL element.
+ */
+ private static Expression canonicalizeComplexValue(Column column,
Expression value) {
+ if (!column.getOriginType().isComplexType() || !(value instanceof
StringLikeLiteral)) {
+ return value;
+ }
+ return new StringLiteral(ComplexTypeDefaultValue.canonicalize(
+ DataType.fromCatalogType(column.getOriginType()),
((StringLikeLiteral) value).getStringValue()));
+ }
+
+ /**
+ * Arrow reads a column with its declared type, so an ARRAY/MAP/STRUCT
column keeps the complex type
+ * in its scan slot instead of being read as VARCHAR like the text
formats. The mapping expressions
+ * have to work on that complex type, otherwise the IF branches mix
complex and string values.
+ */
+ private boolean readsNativeComplexType(Column column) {
+ return column.getOriginType().isComplexType()
+ &&
fileGroupInfo.getFileGroup().getFileFormatProperties().getFileFormatType()
+ == TFileFormatType.FORMAT_ARROW;
+ }
+
+ /**
+ * The mapping value of a complex column is written as canonical text.
When the scan slot keeps the
+ * native complex type, parse that text into the column type: a string
branch and a complex branch
+ * have no common type under the legacy type coercion behavior ({@code
+ * enable_new_type_coercion_behavior=false}).
+ */
+ private static Expression mappingValue(Column column, Expression value,
boolean nativeComplex) {
+ Expression canonical = canonicalizeComplexValue(column, value);
+ if (!nativeComplex) {
+ return canonical;
+ }
+ return new Cast(canonical,
DataType.fromCatalogType(column.getOriginType()));
+ }
+
+ /**
+ * The NULL branch of a mapping IF. The typed VARCHAR NULL used for text
sources has no common type
+ * with a native complex branch under the legacy type coercion behavior,
so leave it untyped.
+ */
+ private static Expression nullMappingValue(boolean nativeComplex) {
+ return nativeComplex ? new NullLiteral() : new
NullLiteral(VarcharType.SYSTEM_DEFAULT);
+ }
+
+ /**
+ * A text source compares its raw value with the search value of {@code
replace_value}. A native
+ * complex slot needs a type-aware comparison instead of the display text,
whose nested strings are
+ * not escaped: a single element {@code a", "b} renders exactly like the
two element array
+ * {@code ["a", "b"]}. The search is a transient comparison value, so it
must not be canonicalized
+ * as a column default, whose format rejects quotes and backslashes that
the complex cast accepts.
+ */
+ private static Expression searchValueComparison(Expression slot, Column
column, Expression search,
+ boolean nativeComplex) throws AnalysisException {
+ if (!nativeComplex) {
+ return new EqualTo(slot, search);
+ }
+ DataType complexType =
DataType.fromCatalogType(column.getOriginType());
+ // Cast the search expression, literal or slot, to the column type so
that the comparison sees
+ // exactly the bytes the sink writes. The cast is strict: Stream Load
disables
+ // enable_strict_cast, so a malformed non-NULL search would otherwise
become SQL NULL, which
+ // the mapping IF treats as "not equal" and replaces every row,
including rows that do not
+ // match the search. A strict cast fails the load instead.
+ Expression typedSearch = new Cast(search, complexType, false, true);
+ return buildEquality(slot, typedSearch, complexType, true,
column.getOriginType().toSql());
+ }
+
+ /**
+ * Equality of two expressions of {@code type}, built from comparisons the
engine can evaluate.
+ * Primitive values and arrays of them are compared structurally ({@code
compare_at}), which keeps
+ * nested NULLs apart from values and treats two NULLs as equal like the
engine does inside a
+ * complex value; {@code <=>} expresses that for a nested value. MAP and
STRUCT are rejected by
+ * {@code ComparisonPredicate}, so their sub-values are compared
recursively: a MAP is identified
+ * by its deduplicated keys and values in order, and a STRUCT that cannot
be serialized losslessly
+ * enough for a whole JSON comparison both by its own NULL flag and by its
fields. A STRUCT whose
+ * JSON text is a lossless identity is compared as one value, and a STRUCT
whose leaves have no
+ * lossless JSON identity but are comparable recursively is compared by
one engine call, so the
+ * search is parsed a bounded number of times instead of once per field.
+ *
+ * <p>JSON is used only for a value the engine cannot compare
structurally. A MAP is never
+ * rendered as a JSON object: JSONB object keys are limited to 255 bytes
and lose the key NULL
+ * flag, so the empty string key and a NULL key would render the same.
Nested MAPs are rejected
+ * for the same reason.
+ *
+ * @param topLevel whether this is the whole search comparison. A NULL
search then keeps the SQL
+ * three-valued result of the text formats instead of
matching a NULL value.
+ */
+ private static Expression buildEquality(Expression left, Expression right,
DataType type,
+ boolean topLevel, String columnTypeSql) throws AnalysisException {
+ if (supportsStructuralComparison(type)) {
+ // A top level ARRAY is compared by EqualTo, which is NULL when
the search is NULL; a
+ // nested value uses <=> so that two NULLs match, like compare_at
inside a complex value.
+ return topLevel ? new EqualTo(left, right) : new
NullSafeEqual(left, right);
+ }
+ if (type.isMapType()) {
+ MapType mapType = (MapType) type;
+ DataType keyType = mapType.getKeyType();
+ DataType valueType = mapType.getValueType();
+ if (!supportsStructuralComparison(keyType) ||
!supportsMapValueComparison(valueType)) {
+ throw unsupportedComplexSearch(columnTypeSql, type);
+ }
+ // An Arrow MAP keeps every entry, so a repeated key makes its key
and value arrays
+ // longer than the ones of the search text, which the string cast
deduplicates with the
+ // last value winning. The sink applies the same last-wins rule
before it writes the
+ // column, so normalize the native source the same way; otherwise
a row that is written
+ // as the search value would not match it and would not be
replaced.
+ Expression leftMap = deduplicateMap(left);
+ // Only the top level search is the string cast, which already
deduplicated its keys. A
+ // MAP field of a STRUCT is parsed by the struct SerDe, which
leaves repeated keys in
+ // place, so it needs the same normalization as the source.
+ Expression rightMap = topLevel ? right : deduplicateMap(right);
+ List<Expression> equalities = Lists.newArrayList(
+ new NullSafeEqual(mapFunction("map_keys", leftMap),
mapFunction("map_keys", rightMap)));
+ if (supportsStructuralComparison(valueType)) {
+ equalities.add(new NullSafeEqual(mapFunction("map_values",
leftMap),
+ mapFunction("map_values", rightMap)));
+ } else {
+ // The value contains a STRUCT or an ARRAY of one, which the
engine cannot compare as
+ // an array element. Its JSON rendering is escaped, keeps NULL
apart from the "null"
+ // text and never uses the value as a JSON object key.
+ equalities.add(new
NullSafeEqual(jsonText(mapFunction("map_values", leftMap)),
+ jsonText(mapFunction("map_values", rightMap))));
+ }
+ Expression equality = new And(equalities);
+ return topLevel ? guardNullSearch(right, equality) : equality;
+ }
+ if (type.isStructType()) {
+ if (supportsJsonEncoding(type)) {
+ // The engine rejects a direct STRUCT comparison, so its JSON
text is the comparison
+ // key. Compare the whole value at once: expanding the fields
would embed the typed
+ // search in every field comparison and each occurrence parses
the whole search
+ // string, which makes a dynamic search over an N field STRUCT
convert N times the
+ // N fields for every row.
+ return topLevel
+ ? new EqualTo(jsonText(left), jsonText(right))
+ : new NullSafeEqual(jsonText(left), jsonText(right));
+ }
+ if (supportsDeepStructuralComparison(type)) {
+ // The JSON text is not a lossless identity for this STRUCT (a
float signed zero, a
+ // version one DATE/DATETIME, DECIMALV2, ...), so the fields
have to be compared
+ // one by one. Doing that in the expression tree embeds the
typed search in every
+ // field comparison, and each occurrence parses the whole
search string: an N field
+ // STRUCT converts N fields N+1 times per row. One engine call
compares both values
+ // recursively instead, so the search is parsed once.
+ Expression equality = complexEquals(left, right);
+ return topLevel ? guardNullSearch(right, equality) : equality;
+ }
+ List<StructField> fields = ((StructType) type).getFields();
+ List<Expression> equalities =
Lists.newArrayListWithCapacity(fields.size());
+ for (int i = 0; i < fields.size(); i++) {
Review Comment:
[P2] Reuse the typed search when a STRUCT contains MAP fields
For `v STRUCT<m1:MAP<INT,INT>,...,mN:MAP<INT,INT>>` and a dynamic STRING
source `w`, `v=replace_value(w,NULL)` falls through to this expansion because
both faster branches reject MAPs. Each field's key and value comparison embeds
another `CAST(w AS full_v_type STRICT)`, so a matching row parses all N fields
at least 2N times. Load plans are translated directly from the logical project
and do not run physical-project common-subexpression extraction, leaving
quadratic conversion work per row. Materialize the typed search once before
expanding the field comparisons, or compare a once-normalized value in one
engine call.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
