This is an automated email from the ASF dual-hosted git repository.
xuyang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4b743061b4 [feature](function) support type template in SQL function
(#17344)
4b743061b4 is described below
commit 4b743061b45dbf5a5bd86f368271c55c0a3baaea
Author: Kang <[email protected]>
AuthorDate: Wed Mar 8 10:51:31 2023 +0800
[feature](function) support type template in SQL function (#17344)
A new way just like c++ template is proposed in this PR. The previous
functions can be defined much simpler using template function.
# map element extract template function
[['element_at', '%element_extract%'], 'E', ['ARRAY<E>', 'BIGINT'],
'ALWAYS_NULLABLE', ['E']],
# map element extract template function
[['element_at', '%element_extract%'], 'V', ['MAP<K, V>', 'K'],
'ALWAYS_NULLABLE', ['K', 'V']],
BTW, the plain type function is not affected and the legacy ARRAY_X MAP_K_V
is still supported for compatability.
---
.../java/org/apache/doris/catalog/ArrayType.java | 22 ++++
.../java/org/apache/doris/catalog/MapType.java | 33 ++++++
.../org/apache/doris/catalog/PrimitiveType.java | 1 +
.../org/apache/doris/catalog/TemplateType.java | 132 +++++++++++++++++++++
.../main/java/org/apache/doris/catalog/Type.java | 17 +++
.../apache/doris/analysis/FunctionCallExpr.java | 59 +++++----
.../java/org/apache/doris/catalog/Function.java | 10 ++
.../java/org/apache/doris/catalog/FunctionSet.java | 67 +++++++++++
gensrc/script/doris_builtins_functions.py | 8 +-
gensrc/script/gen_builtins_functions.py | 41 +++++--
.../stream_load/test_map_load_and_function.out | 53 ++++++---
.../stream_load/test_map_load_and_function.groovy | 33 +++++-
12 files changed, 413 insertions(+), 63 deletions(-)
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
index 6cc9162dc6..a28f3c0302 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/ArrayType.java
@@ -27,6 +27,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
+import java.util.Map;
import java.util.Objects;
/**
@@ -92,6 +93,27 @@ public class ArrayType extends Type {
&& (((ArrayType) t).containsNull || !containsNull);
}
+ @Override
+ public boolean hasTemplateType() {
+ return itemType.hasTemplateType();
+ }
+
+ @Override
+ public Type specializeTemplateType(Type specificType, Map<String, Type>
specializedTypeMap,
+ boolean useSpecializedType) throws
TypeException {
+ if (!(specificType instanceof ArrayType)) {
+ throw new TypeException(specificType + " is not ArrayType");
+ }
+
+ ArrayType o = (ArrayType) specificType;
+ Type newItemType = itemType;
+ if (itemType.hasTemplateType()) {
+ newItemType = itemType.specializeTemplateType(o.itemType,
specializedTypeMap, useSpecializedType);
+ }
+
+ return new ArrayType(newItemType);
+ }
+
public static ArrayType create() {
return new ArrayType();
}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
index 6fd8da9c24..e72df77710 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/MapType.java
@@ -27,6 +27,7 @@ import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.gson.annotations.SerializedName;
+import java.util.Map;
import java.util.Objects;
/**
@@ -121,6 +122,38 @@ public class MapType extends Type {
&& (valueType.matchesType(((MapType) t).valueType));
}
+ @Override
+ public boolean hasTemplateType() {
+ return keyType.hasTemplateType() || valueType.hasTemplateType();
+ }
+
+ @Override
+ public Type specializeTemplateType(Type specificType, Map<String, Type>
specializedTypeMap,
+ boolean useSpecializedType) throws
TypeException {
+ if (!(specificType instanceof MapType)) {
+ throw new TypeException(specificType + " is not MapType");
+ }
+
+ MapType specificMapType = (MapType) specificType;
+ Type newKeyType = keyType;
+ if (keyType.hasTemplateType()) {
+ newKeyType = keyType.specializeTemplateType(
+ specificMapType.keyType, specializedTypeMap,
useSpecializedType);
+ }
+ Type newValueType = valueType;
+ if (valueType.hasTemplateType()) {
+ newValueType = valueType.specializeTemplateType(
+ specificMapType.valueType, specializedTypeMap,
useSpecializedType);
+ }
+
+ Type newMapType = new MapType(newKeyType, newValueType);
+ if (Type.canCastTo(specificType, newMapType)) {
+ return newMapType;
+ } else {
+ throw new TypeException(specificType + " can not cast to
specialize type " + newMapType);
+ }
+ }
+
@Override
public String toString() {
return toSql(0).toUpperCase();
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
index c4d1a4e8d8..fdcbfaedef 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/PrimitiveType.java
@@ -72,6 +72,7 @@ public enum PrimitiveType {
STRUCT("STRUCT", 16, TPrimitiveType.STRUCT),
STRING("STRING", 16, TPrimitiveType.STRING),
VARIANT("VARIANT", 24, TPrimitiveType.VARIANT),
+ TEMPLATE("TEMPLATE", -1, TPrimitiveType.INVALID_TYPE),
// Unsupported scalar types.
BINARY("BINARY", -1, TPrimitiveType.BINARY),
ALL("ALL", -1, TPrimitiveType.INVALID_TYPE);
diff --git
a/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java
new file mode 100644
index 0000000000..58f3fbe952
--- /dev/null
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/TemplateType.java
@@ -0,0 +1,132 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.thrift.TColumnType;
+import org.apache.doris.thrift.TTypeDesc;
+
+import com.google.common.base.Strings;
+import com.google.gson.annotations.SerializedName;
+
+import java.util.Map;
+
+/**
+ * Describes a TemplateType type, used for SQL function argument and return
type,
+ * NOT used for table column type.
+ */
+public class TemplateType extends Type {
+
+ @SerializedName(value = "name")
+ private final String name;
+
+ public TemplateType(String name) {
+ this.name = name;
+ }
+
+ @Override
+ public PrimitiveType getPrimitiveType() {
+ return PrimitiveType.TEMPLATE;
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (!(other instanceof TemplateType)) {
+ return false;
+ }
+ TemplateType o = (TemplateType) other;
+ return o.name.equals(name);
+ }
+
+ @Override
+ public boolean matchesType(Type t) {
+ // not matches any type
+ return false;
+ }
+
+ @Override
+ public boolean hasTemplateType() {
+ return true;
+ }
+
+ @Override
+ public Type specializeTemplateType(Type specificType, Map<String, Type>
specializedTypeMap,
+ boolean useSpecializedType) throws
TypeException {
+ if (specificType.hasTemplateType() && !specificType.isNull()) {
+ throw new TypeException(specificType + " should not
hasTemplateType");
+ }
+
+ Type specializedType = specializedTypeMap.get(name);
+ if (useSpecializedType) {
+ if (specializedType == null) {
+ throw new TypeException("template type " + name + " is not
specialized yet");
+ }
+ return specializedType;
+ }
+
+ if (specializedType != null
+ && !specificType.equals(specializedType)
+ && !specificType.matchesType(specializedType)
+ && !Type.isImplicitlyCastable(specificType, specializedType,
true)
+ && !Type.canCastTo(specificType, specializedType)) {
+ throw new TypeException(
+ String.format("can not specialize template type %s to %s since
it's already specialized as %s",
+ name, specificType, specializedType));
+ }
+
+ if (specializedType == null) {
+ specializedTypeMap.put(name, specificType);
+ }
+ return specializedTypeMap.get(name);
+ }
+
+ @Override
+ public String toSql(int depth) {
+ return name;
+ }
+
+ @Override
+ public String toString() {
+ return toSql(0).toUpperCase();
+ }
+
+ @Override
+ protected String prettyPrint(int lpad) {
+ String leftPadding = Strings.repeat(" ", lpad);
+ return leftPadding + toSql();
+ }
+
+ @Override
+ public boolean supportSubType(Type subType) {
+ throw new RuntimeException("supportSubType not implementd for
TemplateType");
+ }
+
+ @Override
+ public void toThrift(TTypeDesc container) {
+ throw new RuntimeException("can not call toThrift on TemplateType");
+ }
+
+ @Override
+ public TColumnType toColumnTypeThrift() {
+ throw new RuntimeException("can not call toColumnTypeThrift on
TemplateType");
+ }
+
+ @Override
+ public int hashCode() {
+ return name.hashCode();
+ }
+}
diff --git a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
index a4bd07d33a..4f7f5cb96e 100644
--- a/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
+++ b/fe/fe-common/src/main/java/org/apache/doris/catalog/Type.java
@@ -39,6 +39,7 @@ import java.time.LocalDate;
import java.time.LocalDateTime;
import java.util.ArrayList;
import java.util.List;
+import java.util.Map;
import java.util.Set;
/**
@@ -507,6 +508,21 @@ public abstract class Type {
return isScalarType(PrimitiveType.DATEV2);
}
+ public boolean hasTemplateType() {
+ return false;
+ }
+
+ // return a new type without template type, by specialize tempalte type in
this type
+ public Type specializeTemplateType(Type specificType, Map<String, Type>
specializedTypeMap,
+ boolean useSpecializedType) throws
TypeException {
+ if (hasTemplateType()) {
+ // throw exception by default, sub class should specialize
tempalte type properly
+ throw new TypeException("specializeTemplateType not implemented");
+ } else {
+ return this;
+ }
+ }
+
/**
* Returns true if Impala supports this type in the metdata. It does not
mean we
* can manipulate data of this type. For tables that contain columns with
these
@@ -1560,6 +1576,7 @@ public abstract class Type {
|| t1 == PrimitiveType.TIMEV2 || t2 ==
PrimitiveType.TIMEV2
|| t1 == PrimitiveType.MAP || t2 == PrimitiveType.MAP
|| t1 == PrimitiveType.STRUCT || t2 ==
PrimitiveType.STRUCT
+ || t1 == PrimitiveType.TEMPLATE || t2 ==
PrimitiveType.TEMPLATE
|| t1 == PrimitiveType.UNSUPPORTED || t2 ==
PrimitiveType.UNSUPPORTED
|| t1 == PrimitiveType.VARIANT || t2 ==
PrimitiveType.VARIANT) {
continue;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
index 9e101275a0..51b6b75cad 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/FunctionCallExpr.java
@@ -1369,37 +1369,34 @@ public class FunctionCallExpr extends Expr {
}
}
- if
(!fn.getFunctionName().getFunction().equals(ELEMENT_EXTRACT_FN_NAME)) {
- Type[] args = fn.getArgs();
- if (args.length > 0) {
- // Implicitly cast all the children to match the function if
necessary
- for (int i = 0; i < argTypes.length - orderByElements.size();
++i) {
- // For varargs, we must compare with the last type in
callArgs.argTypes.
- int ix = Math.min(args.length - 1, i);
- if (fnName.getFunction().equalsIgnoreCase("money_format")
- && children.get(0).getType().isDecimalV3() &&
args[ix].isDecimalV3()) {
- continue;
- } else if (fnName.getFunction().equalsIgnoreCase("array")
- && (children.get(0).getType().isDecimalV3() &&
args[ix].isDecimalV3()
- ||
children.get(0).getType().isDatetimeV2() && args[ix].isDatetimeV2())) {
- continue;
- } else if
((fnName.getFunction().equalsIgnoreCase("array_min") || fnName.getFunction()
- .equalsIgnoreCase("array_max") ||
fnName.getFunction().equalsIgnoreCase("element_at"))
- && ((
- children.get(0).getType().isDecimalV3() &&
((ArrayType) args[ix]).getItemType()
- .isDecimalV3())
- ||
(children.get(0).getType().isDatetimeV2()
- && ((ArrayType)
args[ix]).getItemType().isDatetimeV2())
- || (children.get(0).getType().isDecimalV2()
- && ((ArrayType)
args[ix]).getItemType().isDecimalV2()))) {
- continue;
- } else if (!argTypes[i].matchesType(args[ix])
- && !(argTypes[i].isDateOrDateTime() &&
args[ix].isDateOrDateTime())
- && (!fn.getReturnType().isDecimalV3()
- || (argTypes[i].isValid() &&
!argTypes[i].isDecimalV3()
- && args[ix].isDecimalV3()))) {
- uncheckedCastChild(args[ix], i);
- }
+ Type[] args = fn.getArgs();
+ if (args.length > 0) {
+ // Implicitly cast all the children to match the function if
necessary
+ for (int i = 0; i < argTypes.length - orderByElements.size(); ++i)
{
+ // For varargs, we must compare with the last type in
callArgs.argTypes.
+ int ix = Math.min(args.length - 1, i);
+ if (fnName.getFunction().equalsIgnoreCase("money_format")
+ && children.get(0).getType().isDecimalV3() &&
args[ix].isDecimalV3()) {
+ continue;
+ } else if (fnName.getFunction().equalsIgnoreCase("array")
+ && (children.get(0).getType().isDecimalV3() &&
args[ix].isDecimalV3()
+ || children.get(0).getType().isDatetimeV2() &&
args[ix].isDatetimeV2())) {
+ continue;
+ } else if ((fnName.getFunction().equalsIgnoreCase("array_min")
|| fnName.getFunction()
+ .equalsIgnoreCase("array_max") ||
fnName.getFunction().equalsIgnoreCase("element_at"))
+ && ((
+ children.get(0).getType().isDecimalV3() &&
((ArrayType) args[ix]).getItemType()
+ .isDecimalV3())
+ || (children.get(0).getType().isDatetimeV2()
+ && ((ArrayType) args[ix]).getItemType().isDatetimeV2())
+ || (children.get(0).getType().isDecimalV2()
+ && ((ArrayType)
args[ix]).getItemType().isDecimalV2()))) {
+ continue;
+ } else if (!argTypes[i].matchesType(args[ix]) && !(
+ argTypes[i].isDateOrDateTime() &&
args[ix].isDateOrDateTime())
+ && (!fn.getReturnType().isDecimalV3()
+ || (argTypes[i].isValid() &&
!argTypes[i].isDecimalV3() && args[ix].isDecimalV3()))) {
+ uncheckedCastChild(args[ix], i);
}
}
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java
index f1a608a9b8..8997b1bf32 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Function.java
@@ -814,4 +814,14 @@ public class Function implements Writable {
throw new UserException("failed to serialize function: " +
functionName(), t);
}
}
+
+ public boolean hasTemplateArg() {
+ for (Type t : getArgs()) {
+ if (t.hasTemplateType()) {
+ return true;
+ }
+ }
+
+ return false;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
index 0a4bed2789..cb501817b4 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionSet.java
@@ -1223,6 +1223,34 @@ public class FunctionSet<T> {
return null;
}
+ List<Function> normalFunctions = Lists.newArrayList();
+ List<Function> templateFunctions = Lists.newArrayList();
+ for (Function fn : fns) {
+ if (fn.hasTemplateArg()) {
+ templateFunctions.add(fn);
+ } else {
+ normalFunctions.add(fn);
+ }
+ }
+
+ // try normal functions first
+ Function fn = getFunction(desc, mode, normalFunctions);
+ if (fn != null) {
+ return fn;
+ }
+
+ // then specialize template functions and try them
+ List<Function> specializedTemplateFunctions = Lists.newArrayList();
+ for (Function f : templateFunctions) {
+ f = FunctionSet.specializeTemplateFunction(f, desc);
+ if (f != null) {
+ specializedTemplateFunctions.add(f);
+ }
+ }
+ return getFunction(desc, mode, specializedTemplateFunctions);
+ }
+
+ private Function getFunction(Function desc, Function.CompareMode mode,
List<Function> fns) {
// First check for identical
for (Function f : fns) {
if (f.compare(desc, Function.CompareMode.IS_IDENTICAL)) {
@@ -1262,6 +1290,45 @@ public class FunctionSet<T> {
return null;
}
+ public static Function specializeTemplateFunction(Function
templateFunction, Function requestFunction) {
+ try {
+ boolean hasTemplateType = false;
+ LOG.debug("templateFunction signature: " +
templateFunction.signatureString()
+ + " return: " + templateFunction.getReturnType());
+ LOG.debug("requestFunction signature: " +
requestFunction.signatureString()
+ + " return: " + requestFunction.getReturnType());
+ Function specializedFunction = templateFunction;
+ if (templateFunction instanceof ScalarFunction) {
+ ScalarFunction f = (ScalarFunction) templateFunction;
+ specializedFunction = new ScalarFunction(f.getFunctionName(),
Lists.newArrayList(f.getArgs()),
+ f.getReturnType(), f.hasVarArgs(),
f.getSymbolName(), f.getBinaryType(),
+ f.isUserVisible(),
f.isVectorized(), f.getNullableMode());
+ } else {
+ // TODO(xk)
+ }
+ Type[] args = specializedFunction.getArgs();
+ Map<String, Type> specializedTypeMap = Maps.newHashMap();
+ for (int i = 0; i < args.length; i++) {
+ if (args[i].hasTemplateType()) {
+ hasTemplateType = true;
+ args[i] =
args[i].specializeTemplateType(requestFunction.getArgs()[i],
specializedTypeMap, false);
+ }
+ }
+ if (specializedFunction.getReturnType().hasTemplateType()) {
+ hasTemplateType = true;
+ specializedFunction.setReturnType(
+
specializedFunction.getReturnType().specializeTemplateType(
+ requestFunction.getReturnType(), specializedTypeMap,
true));
+ }
+ LOG.debug("specializedFunction signature: " +
specializedFunction.signatureString()
+ + " return: " + specializedFunction.getReturnType());
+ return hasTemplateType ? specializedFunction : templateFunction;
+ } catch (TypeException e) {
+ LOG.warn("specializeTemplateFunction exception", e);
+ return null;
+ }
+ }
+
/**
* There are essential differences in the implementation of some functions
for different
* types params, which should be prohibited.
diff --git a/gensrc/script/doris_builtins_functions.py
b/gensrc/script/doris_builtins_functions.py
index f650e0a28a..b891b48513 100644
--- a/gensrc/script/doris_builtins_functions.py
+++ b/gensrc/script/doris_builtins_functions.py
@@ -25,7 +25,7 @@
# It contains all the meta data that describes the function.
# The format is:
-# [sql aliases], <return_type>, [<args>], <nullable mode>
+# [sql aliases], <return_type>, [<args>], <nullable mode>, [template_types]
#
# 'sql aliases' are the function names that can be used from sql. There must
be at least
# one per function.
@@ -33,6 +33,10 @@
# 'nullable mode' reflects whether the return value of the function is null.
See @Function.NullableMode
# for the specific mode and meaning.
#
+# 'template_types' is for template function just like C++. It is optional list.
+# eg. [['element_at', '%element_extract%'], 'V', ['MAP<K, V>', 'K'],
'ALWAYS_NULLABLE', ['K', 'V']],
+# 'K' and 'V' is type template and will be specialized at runtime in FE to
match specific args.
+#
visible_functions = [
# Bit and Byte functions
# For functions corresponding to builtin operators, we can reuse the
implementations
@@ -100,7 +104,7 @@ visible_functions = [
[['element_at', '%element_extract%'], 'STRING', ['ARRAY_STRING',
'BIGINT'], 'ALWAYS_NULLABLE'],
# map element
- [['element_at', '%element_extract%'], 'INT', ['MAP_STRING_INT', 'STRING'],
'ALWAYS_NULLABLE'],
+ [['element_at', '%element_extract%'], 'V', ['MAP<K, V>', 'K'],
'ALWAYS_NULLABLE', ['K', 'V']],
[['arrays_overlap'], 'BOOLEAN', ['ARRAY_BOOLEAN', 'ARRAY_BOOLEAN'],
'ALWAYS_NULLABLE'],
[['arrays_overlap'], 'BOOLEAN', ['ARRAY_TINYINT', 'ARRAY_TINYINT'],
'ALWAYS_NULLABLE'],
diff --git a/gensrc/script/gen_builtins_functions.py
b/gensrc/script/gen_builtins_functions.py
index 28b8812142..2aa6c74fcd 100755
--- a/gensrc/script/gen_builtins_functions.py
+++ b/gensrc/script/gen_builtins_functions.py
@@ -54,6 +54,7 @@ package org.apache.doris.builtins;\n\
\n\
import org.apache.doris.catalog.ArrayType;\n\
import org.apache.doris.catalog.MapType;\n\
+import org.apache.doris.catalog.TemplateType;\n\
import org.apache.doris.catalog.Type;\n\
import org.apache.doris.catalog.Function;\n\
import org.apache.doris.catalog.FunctionSet;\n\
@@ -73,14 +74,14 @@ print(FE_PATH)
# This contains all the metadata to describe all the builtins.
# Each meta data entry is itself a map to store all the meta data
-# - fn_name, ret_type, args, symbol, sql_names
+# - fn_name, ret_type, args, symbol, sql_names, template_types(optional)
meta_data_entries = []
# Read in the function and add it to the meta_data_entries map
def add_function(fn_meta_data, user_visible):
"""add function
"""
- assert len(fn_meta_data) == 4, \
+ assert len(fn_meta_data) >= 4, \
"Invalid function entry in doris_builtins_functions.py:\n\t" +
repr(fn_meta_data)
entry = {}
entry["sql_names"] = fn_meta_data[0]
@@ -91,6 +92,12 @@ def add_function(fn_meta_data, user_visible):
else:
entry['nullable_mode'] = 'DEPEND_ON_ARGUMENT'
+ # process template
+ if len(fn_meta_data) >= 5:
+ entry["template_types"] = fn_meta_data[4]
+ else:
+ entry["template_types"] = []
+
entry["user_visible"] = user_visible
meta_data_entries.append(entry)
@@ -103,15 +110,35 @@ for example:
in[ARRAY_INT] --> out[new ArrayType(Type.INT)]
in[MAP_STRING_INT] --> out[new MapType(Type.STRING,Type.INT)]
"""
-def generate_fe_datatype(str_type):
+def generate_fe_datatype(str_type, template_types):
+ # delete whitespace
+ str_type = str_type.replace(' ', '').replace('\t', '')
+
+ # process template
+ if str_type in template_types:
+ return 'new TemplateType("{}")'.format(str_type)
+
+ # process Array, Map, Struct template
+ template_start = str_type.find('<')
+ template_end = str_type.rfind('>')
+ if template_start >= 0 and template_end > 0:
+ # exclude <>
+ template = str_type[template_start + 1 : template_end]
+ if str_type.startswith("ARRAY<"):
+ return 'new ArrayType({})'.format(generate_fe_datatype(template,
template_types))
+ elif str_type.startswith("MAP<"):
+ types = template.split(',', 2)
+ return 'new MapType({}, {})'.format(generate_fe_datatype(types[0],
template_types), generate_fe_datatype(types[1], template_types))
+
+ # lagacy Array, Map syntax
if str_type.startswith("ARRAY_"):
vec_type = str_type.split('_', 1);
if len(vec_type) > 1 and vec_type[0] == "ARRAY":
- return "new ArrayType(" + generate_fe_datatype(vec_type[1]) + ")"
+ return "new ArrayType(" + generate_fe_datatype(vec_type[1],
template_types) + ")"
if str_type.startswith("MAP_"):
vec_type = str_type.split('_', 2)
if len(vec_type) > 2 and vec_type[0] == "MAP":
- return "new MapType(" + generate_fe_datatype(vec_type[1]) + "," +
generate_fe_datatype(vec_type[2])+")"
+ return "new MapType(" + generate_fe_datatype(vec_type[1],
template_types) + "," + generate_fe_datatype(vec_type[2], template_types)+")"
if str_type == "DECIMALV2":
return "Type.MAX_DECIMALV2_TYPE"
if str_type == "DECIMAL32":
@@ -136,7 +163,7 @@ def generate_fe_entry(entry, name):
else:
java_output += ", false"
java_output += ", Function.NullableMode." + entry["nullable_mode"]
- java_output += ", " + generate_fe_datatype(entry["ret_type"])
+ java_output += ", " + generate_fe_datatype(entry["ret_type"],
entry["template_types"])
# Check the last entry for varargs indicator.
if entry["args"] and entry["args"][-1] == "...":
@@ -145,7 +172,7 @@ def generate_fe_entry(entry, name):
else:
java_output += ", false"
for arg in entry["args"]:
- java_output += ", " + generate_fe_datatype(arg)
+ java_output += ", " + generate_fe_datatype(arg,
entry["template_types"])
return java_output
# Generates the FE builtins init file that registers all the builtins.
diff --git
a/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
b/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
index 1b7eb1f5a3..8e3b7905a4 100644
--- a/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
+++ b/regression-test/data/load_p0/stream_load/test_map_load_and_function.out
@@ -1,5 +1,5 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
--- !select --
+-- !select_all --
1 \N
2 {' 11amory ':23, 'beat':20, ' clever ':66}
3 {'k1':31, 'k2':300}
@@ -16,22 +16,37 @@
15 {'':2, 'k2':0}
16 {null:null}
--- !select --
-\N
-\N
-300
-\N
-\N
-400
-\N
-\N
-\N
-\N
-\N
-\N
-\N
-\N
-130
-0
-\N
+-- !select_m --
+1 \N
+2 \N
+3 300
+4 \N
+5 \N
+6 400
+7 \N
+8 \N
+9 \N
+10 \N
+11 \N
+12 \N
+13 \N
+15 0
+16 \N
+17 \N
+18 130
+
+-- !select_m1 --
+1 100 200 \N
+
+-- !select_m2 --
+1 k1 k2 \N
+
+-- !select_m3 --
+1 v1 v2 \N
+
+-- !select_m4 --
+1 10000 20000 \N
+
+-- !select_m5 --
+1 100 200 \N
diff --git
a/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy
b/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy
index d796c08b2e..38e4d295ca 100644
---
a/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy
+++
b/regression-test/suites/load_p0/stream_load/test_map_load_and_function.groovy
@@ -63,12 +63,37 @@ suite("test_map_load_and_function", "p0") {
}
// check result
- qt_select "SELECT * FROM ${testTable} ORDER BY id"
+ qt_select_all "SELECT * FROM ${testTable} ORDER BY id"
// insert into valid json rows
- sql """INSERT INTO ${testTable} VALUES(12, NULL)"""
- sql """INSERT INTO ${testTable} VALUES(13, {"k1":100, "k2": 130})"""
+ sql """INSERT INTO ${testTable} VALUES(17, NULL)"""
+ sql """INSERT INTO ${testTable} VALUES(18, {"k1":100, "k2": 130})"""
// map element_at
- qt_select "SELECT m['k2'] FROM ${testTable} ORDER BY id"
+ qt_select_m "SELECT id, m['k2'] FROM ${testTable} ORDER BY id"
+
+
+ testTable = "tbl_test_map2"
+ sql "DROP TABLE IF EXISTS ${testTable}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${testTable} (
+ id INT,
+ `m1` MAP<STRING, INT> NULL,
+ `m2` MAP<INT, STRING> NULL,
+ `m3` MAP<STRING, STRING> NULL,
+ `m4` MAP<INT, BIGINT> NULL,
+ `m5` MAP<BIGINT, INT> NULL
+ )
+ DUPLICATE KEY(id)
+ DISTRIBUTED BY HASH(id) BUCKETS 10
+ PROPERTIES("replication_num" = "1");
+ """
+ sql """INSERT INTO ${testTable} VALUES(1, {'k1':100, 'k2':200}, {100:'k1',
200:'k2'}, {'k1':'v1', 'k2':'v2'}, {100:10000, 200:20000}, {10000:100,
20000:200})"""
+
+ // map element_at
+ qt_select_m1 "SELECT id, m1['k1'], m1['k2'], m1['nokey'] FROM ${testTable}
ORDER BY id"
+ qt_select_m2 "SELECT id, m2[100], m2[200], m1[300] FROM ${testTable} ORDER
BY id"
+ qt_select_m3 "SELECT id, m3['k1'], m3['k2'], m3['nokey'] FROM ${testTable}
ORDER BY id"
+ qt_select_m4 "SELECT id, m4[100], m4[200], m4[300] FROM ${testTable} ORDER
BY id"
+ qt_select_m5 "SELECT id, m5[10000], m5[20000], m5[30000] FROM ${testTable}
ORDER BY id"
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
