This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 9c34a7f6b75 branch-3.0: [feat](udf) support "prefer_udf_over_builtin"
session variable #51195 (#51275)
9c34a7f6b75 is described below
commit 9c34a7f6b75f91811219ba46c971f7f098bd9a65
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Wed Jun 11 10:54:44 2025 +0800
branch-3.0: [feat](udf) support "prefer_udf_over_builtin" session variable
#51195 (#51275)
Cherry-picked from #51195
Co-authored-by: Mingyu Chen (Rayner) <[email protected]>
---
.../org/apache/doris/catalog/FunctionRegistry.java | 66 +++++++++++------
.../nereids/trees/expressions/functions/Udf.java | 9 +++
.../java/org/apache/doris/qe/SessionVariable.java | 9 +++
.../data/javaudf_p0/test_javaudf_override.out | Bin 0 -> 299 bytes
.../suites/javaudf_p0/test_javaudf_override.groovy | 82 +++++++++++++++++++++
5 files changed, 142 insertions(+), 24 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
index dc11b5d02be..0ff447811c0 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/FunctionRegistry.java
@@ -124,33 +124,28 @@ public class FunctionRegistry {
int arity = arguments.size();
String qualifiedName = StringUtils.isEmpty(dbName) ? name : dbName +
"." + name;
- if (StringUtils.isEmpty(dbName)) {
- // search internal function only if dbName is empty
- functionBuilders = name2BuiltinBuilders.get(name.toLowerCase());
- if (CollectionUtils.isEmpty(functionBuilders) &&
AggCombinerFunctionBuilder.isAggStateCombinator(name)) {
- String nestedName =
AggCombinerFunctionBuilder.getNestedName(name);
- String combinatorSuffix =
AggCombinerFunctionBuilder.getCombinatorSuffix(name);
- functionBuilders =
name2BuiltinBuilders.get(nestedName.toLowerCase());
- if (functionBuilders != null) {
- List<FunctionBuilder> candidateBuilders =
Lists.newArrayListWithCapacity(functionBuilders.size());
- for (FunctionBuilder functionBuilder : functionBuilders) {
- AggCombinerFunctionBuilder combinerBuilder
- = new
AggCombinerFunctionBuilder(combinatorSuffix, functionBuilder);
- if (combinerBuilder.canApply(arguments)) {
- candidateBuilders.add(combinerBuilder);
- }
- }
- functionBuilders = candidateBuilders;
- }
- }
- }
+ boolean preferUdfOverBuiltin = ConnectContext.get() == null ? false
+ :
ConnectContext.get().getSessionVariable().preferUdfOverBuiltin;
- // search udf
- if (CollectionUtils.isEmpty(functionBuilders)) {
+ if (preferUdfOverBuiltin) {
+ // find udf first, then find builtin function
functionBuilders = findUdfBuilder(dbName, name);
- if (functionBuilders == null || functionBuilders.isEmpty()) {
- throw new AnalysisException("Can not found function '" +
qualifiedName + "'");
+ if (CollectionUtils.isEmpty(functionBuilders) &&
StringUtils.isEmpty(dbName)) {
+ // if dbName is not empty, we should search builtin functions
first
+ functionBuilders = findBuiltinFunctionBuilder(name, arguments);
+ }
+ } else {
+ // find builtin function first, then find udf
+ if (StringUtils.isEmpty(dbName)) {
+ functionBuilders = findBuiltinFunctionBuilder(name, arguments);
}
+ if (CollectionUtils.isEmpty(functionBuilders)) {
+ functionBuilders = findUdfBuilder(dbName, name);
+ }
+ }
+
+ if (functionBuilders == null || functionBuilders.isEmpty()) {
+ throw new AnalysisException("Can not found function '" +
qualifiedName + "'");
}
// check the arity and type
@@ -205,6 +200,29 @@ public class FunctionRegistry {
return candidateBuilders.get(0);
}
+ private List<FunctionBuilder> findBuiltinFunctionBuilder(String name,
List<?> arguments) {
+ List<FunctionBuilder> functionBuilders;
+ // search internal function only if dbName is empty
+ functionBuilders = name2BuiltinBuilders.get(name.toLowerCase());
+ if (CollectionUtils.isEmpty(functionBuilders) &&
AggCombinerFunctionBuilder.isAggStateCombinator(name)) {
+ String nestedName = AggCombinerFunctionBuilder.getNestedName(name);
+ String combinatorSuffix =
AggCombinerFunctionBuilder.getCombinatorSuffix(name);
+ functionBuilders =
name2BuiltinBuilders.get(nestedName.toLowerCase());
+ if (functionBuilders != null) {
+ List<FunctionBuilder> candidateBuilders =
Lists.newArrayListWithCapacity(functionBuilders.size());
+ for (FunctionBuilder functionBuilder : functionBuilders) {
+ AggCombinerFunctionBuilder combinerBuilder
+ = new AggCombinerFunctionBuilder(combinatorSuffix,
functionBuilder);
+ if (combinerBuilder.canApply(arguments)) {
+ candidateBuilders.add(combinerBuilder);
+ }
+ }
+ functionBuilders = candidateBuilders;
+ }
+ }
+ return functionBuilders;
+ }
+
/**
* public for test.
*/
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java
index e1bdbf5add6..7b4229058c1 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/functions/Udf.java
@@ -46,4 +46,13 @@ public interface Udf extends ComputeNullable {
NullableMode getNullableMode();
List<Expression> children();
+
+ @Override
+ default boolean foldable() {
+ // Udf should not be folded in FE.
+ // When session variable "prefer_udf_fold" is set to true,
+ // we may find udf with same signature as builtin function.
+ // If return true, the FE will calculate the result of the udf with
builtin function's logic, which is wrong.
+ return false;
+ }
}
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index cfbdbdfd436..15affdff26b 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -716,6 +716,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String SQL_CONVERTOR_CONFIG = "sql_convertor_config";
+ public static final String PREFER_UDF_OVER_BUILTIN =
"prefer_udf_over_builtin";
+
/**
* If set false, user couldn't submit analyze SQL and FE won't allocate
any related resources.
*/
@@ -2450,6 +2452,13 @@ public class SessionVariable implements Serializable,
Writable {
})
public String sqlConvertorConfig = "{}";
+ @VariableMgr.VarAttr(name = PREFER_UDF_OVER_BUILTIN, needForward = true,
+ description = {
+ "是否优先查找 UDF 而不是内置函数",
+ "Whether to prefer UDF over builtin functions"
+ })
+ public boolean preferUdfOverBuiltin = false;
+
public void setEnableEsParallelScroll(boolean enableESParallelScroll) {
this.enableESParallelScroll = enableESParallelScroll;
}
diff --git a/regression-test/data/javaudf_p0/test_javaudf_override.out
b/regression-test/data/javaudf_p0/test_javaudf_override.out
new file mode 100644
index 00000000000..d000890723d
Binary files /dev/null and
b/regression-test/data/javaudf_p0/test_javaudf_override.out differ
diff --git a/regression-test/suites/javaudf_p0/test_javaudf_override.groovy
b/regression-test/suites/javaudf_p0/test_javaudf_override.groovy
new file mode 100644
index 00000000000..e95bcf989cc
--- /dev/null
+++ b/regression-test/suites/javaudf_p0/test_javaudf_override.groovy
@@ -0,0 +1,82 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+import org.codehaus.groovy.runtime.IOGroovyMethods
+
+import java.nio.charset.StandardCharsets
+import java.nio.file.Files
+import java.nio.file.Paths
+
+suite("test_javaudf_override") {
+ def tableName = "test_javaudf_override"
+ def jarPath =
"""${context.file.parent}/jars/java-udf-case-jar-with-dependencies.jar"""
+ scp_udf_file_to_all_be(jarPath)
+
+ log.info("Jar path: ${jarPath}".toString())
+ sql "drop database if exists test_javaudf_override_db"
+ sql "create database test_javaudf_override_db"
+ try {
+ sql """ use test_javaudf_override_db"""
+ sql """ DROP TABLE IF EXISTS ${tableName} """
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tableName} (
+ int_col int
+ )
+ DISTRIBUTED BY HASH(int_col) PROPERTIES("replication_num" = "1");
+ """
+ sql """insert into ${tableName} values(-100),(100),(0)"""
+
+ File path = new File(jarPath)
+ if (!path.exists()) {
+ throw new IllegalStateException("""${jarPath} doesn't exist! """)
+ }
+
+ // 1. create global udf "abs", same name as builtin function "abs"
+
+ sql """DROP GLOBAL FUNCTION IF EXISTS abs(int);"""
+ sql """CREATE GLOBAL FUNCTION abs(int) RETURNS int PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.Echo\$EchoInt",
+ "type"="JAVA_UDF"
+ );"""
+
+ // default, will use builtin function first
+ sql "set prefer_udf_over_builtin=false"
+ qt_sql01 """select abs(-1), abs(1), abs(int_col) from ${tableName}"""
+ // set prefer_udf_over_builtin, will use udf first
+ sql "set prefer_udf_over_builtin=true"
+ qt_sql02 """select abs(-1), abs(1), abs(int_col) from ${tableName}"""
+
+ // 2. create database udf "abs", same name as builtin function "abs"
+ def curdb = "test_javaudf_override_db"
+ sql """DROP GLOBAL FUNCTION IF EXISTS abs(int);"""
+ sql "use ${curdb}"
+ sql """CREATE FUNCTION abs(int) RETURNS int PROPERTIES (
+ "file"="file://${jarPath}",
+ "symbol"="org.apache.doris.udf.Echo\$EchoInt",
+ "type"="JAVA_UDF"
+ );"""
+
+
+ sql "set prefer_udf_over_builtin=false"
+ qt_sql03 """select abs(-1), abs(1), ${curdb}.abs(-1), ${curdb}.abs(1),
abs(int_col), ${curdb}.abs(int_col) from ${tableName}"""
+ sql "set prefer_udf_over_builtin=true"
+ qt_sql04 """select abs(-1), abs(1), ${curdb}.abs(-1), ${curdb}.abs(1),
abs(int_col), ${curdb}.abs(int_col) from ${tableName}"""
+
+ } finally {
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
