This is an automated email from the ASF dual-hosted git repository.
morrysnow pushed a commit to branch branch-3.1
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.1 by this push:
new 7d372b11a62 branch-3.1: [feat][fix](jdbc) support custom function
rules in catalog properties #51471 #51787 (#52467)
7d372b11a62 is described below
commit 7d372b11a62dcbd6a546a2b6a71eb16df0495e08
Author: Mingyu Chen (Rayner) <[email protected]>
AuthorDate: Mon Jun 30 10:42:07 2025 +0800
branch-3.1: [feat][fix](jdbc) support custom function rules in catalog
properties #51471 #51787 (#52467)
bp #51471 #51787
---------
Co-authored-by: zy-kkk <[email protected]>
---
.../org/apache/doris/catalog/JdbcResource.java | 1 +
.../java/org/apache/doris/catalog/JdbcTable.java | 20 +
.../apache/doris/datasource/ExternalCatalog.java | 104 +++--
.../apache/doris/datasource/ExternalDatabase.java | 74 ++--
.../doris/datasource/ExternalFunctionRules.java | 287 +++++++++++++
.../org/apache/doris/datasource/ExternalTable.java | 34 +-
.../doris/datasource/hive/HMSExternalCatalog.java | 2 +-
.../doris/datasource/jdbc/JdbcExternalCatalog.java | 20 +-
.../doris/datasource/jdbc/JdbcExternalTable.java | 1 +
.../jdbc/source/JdbcFunctionPushDownRule.java | 84 ++--
.../doris/datasource/jdbc/source/JdbcScanNode.java | 10 +-
.../ExternalFunctionPushDownRulesTest.java | 431 ++++++++++++++++++++
.../ExternalFunctionRewriteRulesTest.java | 443 +++++++++++++++++++++
.../datasource/iceberg/CreateIcebergTableTest.java | 4 +-
.../paimon/PaimonExternalCatalogTest.java | 2 +-
.../apache/doris/planner/HiveTableSinkTest.java | 2 +-
.../jdbc/test_clickhouse_jdbc_catalog.groovy | 71 +++-
.../jdbc/test_jdbc_query_mysql.groovy | 134 ++++++-
.../jdbc/test_mysql_jdbc_catalog.groovy | 76 +++-
.../jdbc/test_oracle_jdbc_catalog.groovy | 72 +++-
20 files changed, 1719 insertions(+), 153 deletions(-)
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
index bbd3e6df802..c3c8b4d49a6 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcResource.java
@@ -109,6 +109,7 @@ public class JdbcResource extends Resource {
public static final String CHECK_SUM = "checksum";
public static final String CREATE_TIME = "create_time";
public static final String TEST_CONNECTION = "test_connection";
+ public static final String FUNCTION_RULES = "function_rules";
private static final ImmutableList<String> ALL_PROPERTIES = new
ImmutableList.Builder<String>().add(
JDBC_URL,
diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
index 6dce40a2684..adf013576b3 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/JdbcTable.java
@@ -22,6 +22,7 @@ import org.apache.doris.common.DdlException;
import org.apache.doris.common.FeConstants;
import org.apache.doris.common.io.DeepCopy;
import org.apache.doris.common.io.Text;
+import org.apache.doris.datasource.ExternalFunctionRules;
import org.apache.doris.thrift.TJdbcTable;
import org.apache.doris.thrift.TOdbcTableType;
import org.apache.doris.thrift.TTableDescriptor;
@@ -103,6 +104,8 @@ public class JdbcTable extends Table {
private int connectionPoolMaxLifeTime;
private boolean connectionPoolKeepAlive;
+ private ExternalFunctionRules functionRules;
+
static {
Map<String, TOdbcTableType> tempMap = new CaseInsensitiveMap();
tempMap.put("mysql", TOdbcTableType.MYSQL);
@@ -128,6 +131,8 @@ public class JdbcTable extends Table {
throws DdlException {
super(id, name, TableType.JDBC, schema);
validate(properties);
+ // check and set external function rules
+ checkAndSetExternalFunctionRules(properties);
}
public JdbcTable(long id, String name, List<Column> schema, TableType
type) {
@@ -412,6 +417,12 @@ public class JdbcTable extends Table {
}
}
+ private void checkAndSetExternalFunctionRules(Map<String, String>
properties) throws DdlException {
+
ExternalFunctionRules.check(properties.getOrDefault(JdbcResource.FUNCTION_RULES,
""));
+ this.functionRules = ExternalFunctionRules.create(jdbcTypeName,
+ properties.getOrDefault(JdbcResource.FUNCTION_RULES, ""));
+ }
+
/**
* Formats the provided name (for example, a database, table, or schema
name) according to the specified parameters.
*
@@ -509,4 +520,13 @@ public class JdbcTable extends Table {
public static String formatNameWithRemoteName(String remoteName, String
wrapStart, String wrapEnd) {
return wrapStart + remoteName + wrapEnd;
}
+
+ // This is used when converting JdbcExternalTable to JdbcTable.
+ public void setExternalFunctionRules(ExternalFunctionRules functionRules) {
+ this.functionRules = functionRules;
+ }
+
+ public ExternalFunctionRules getExternalFunctionRules() {
+ return functionRules;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
index 439254bf9eb..25b74bb6eff 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java
@@ -78,7 +78,6 @@ import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
-import lombok.Data;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.math.NumberUtils;
@@ -106,7 +105,6 @@ import java.util.stream.Collectors;
/**
* The abstract class for all types of external catalogs.
*/
-@Data
public abstract class ExternalCatalog
implements CatalogIf<ExternalDatabase<? extends ExternalTable>>,
Writable, GsonPostProcessable {
private static final Logger LOG =
LogManager.getLogger(ExternalCatalog.class);
@@ -180,6 +178,8 @@ public abstract class ExternalCatalog
private volatile Configuration cachedConf = null;
private byte[] confLock = new byte[0];
+ private volatile boolean isInitializing = false;
+
public ExternalCatalog() {
}
@@ -302,38 +302,46 @@ public abstract class ExternalCatalog
* So you have to make sure the client of third system is initialized
before any method was called.
*/
public final synchronized void makeSureInitialized() {
- initLocalObjects();
- if (!initialized) {
- if (useMetaCache.get()) {
- if (metaCache == null) {
- metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
- name,
- OptionalLong.of(86400L),
-
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
- Config.max_meta_object_cache_num,
- ignored -> getFilteredDatabaseNames(),
- localDbName -> Optional.ofNullable(
- buildDbForInit(null, localDbName,
Util.genIdByName(name, localDbName), logType,
- true)),
- (key, value, cause) -> value.ifPresent(v ->
v.setUnInitialized(invalidCacheInInit)));
- }
- setLastUpdateTime(System.currentTimeMillis());
- } else {
- if (!Env.getCurrentEnv().isMaster()) {
- // Forward to master and wait the journal to replay.
- int waitTimeOut = ConnectContext.get() == null ? 300 :
ConnectContext.get().getExecTimeout();
- MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor(waitTimeOut * 1000);
- try {
- remoteExecutor.forward(id, -1);
- } catch (Exception e) {
- Util.logAndThrowRuntimeException(LOG,
- String.format("failed to forward init catalog
%s operation to master.", name), e);
+ if (isInitializing) {
+ return;
+ }
+ isInitializing = true;
+ try {
+ initLocalObjects();
+ if (!initialized) {
+ if (useMetaCache.get()) {
+ if (metaCache == null) {
+ metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
+ name,
+ OptionalLong.of(86400L),
+
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
+ Config.max_meta_object_cache_num,
+ ignored -> getFilteredDatabaseNames(),
+ localDbName -> Optional.ofNullable(
+ buildDbForInit(null, localDbName,
Util.genIdByName(name, localDbName), logType,
+ true)),
+ (key, value, cause) -> value.ifPresent(v ->
v.setUnInitialized(invalidCacheInInit)));
}
- return;
+ setLastUpdateTime(System.currentTimeMillis());
+ } else {
+ if (!Env.getCurrentEnv().isMaster()) {
+ // Forward to master and wait the journal to replay.
+ int waitTimeOut = ConnectContext.get() == null ? 300 :
ConnectContext.get().getExecTimeout();
+ MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor(waitTimeOut * 1000);
+ try {
+ remoteExecutor.forward(id, -1);
+ } catch (Exception e) {
+ Util.logAndThrowRuntimeException(LOG,
+ String.format("failed to forward init
catalog %s operation to master.", name), e);
+ }
+ return;
+ }
+ init();
}
- init();
+ initialized = true;
}
- initialized = true;
+ } finally {
+ isInitializing = false;
}
}
@@ -351,7 +359,7 @@ public abstract class ExternalCatalog
// check if all required properties are set when creating catalog
public void checkProperties() throws DdlException {
// check refresh parameter of catalog
- Map<String, String> properties = getCatalogProperty().getProperties();
+ Map<String, String> properties = catalogProperty.getProperties();
if (properties.containsKey(CatalogMgr.METADATA_REFRESH_INTERVAL_SEC)) {
try {
int metadataRefreshIntervalSec = Integer.parseInt(
@@ -384,7 +392,7 @@ public abstract class ExternalCatalog
* isDryRun: if true, it will try to create the custom access controller,
but will not add it to the access manager.
*/
public void initAccessController(boolean isDryRun) {
- Map<String, String> properties = getCatalogProperty().getProperties();
+ Map<String, String> properties = catalogProperty.getProperties();
// 1. get access controller class name
String className =
properties.getOrDefault(CatalogMgr.ACCESS_CONTROLLER_CLASS_PROP, "");
if (Strings.isNullOrEmpty(className)) {
@@ -544,7 +552,7 @@ public abstract class ExternalCatalog
* @param invalidCache if {@code true}, the catalog cache will be
invalidated
* and reloaded during the refresh process.
*/
- public void resetToUninitialized(boolean invalidCache) {
+ public synchronized void resetToUninitialized(boolean invalidCache) {
this.objectCreated = false;
this.initialized = false;
synchronized (this.propLock) {
@@ -997,6 +1005,14 @@ public abstract class ExternalCatalog
dbNameToId.put(ClusterNamespace.getNameFromFullName(db.getFullName()),
db.getId());
}
+ /**
+ * Set the initialized status for testing purposes only.
+ * This method should only be used in test cases.
+ */
+ public void setInitializedForTest(boolean initialized) {
+ this.initialized = initialized;
+ }
+
@Override
public void createDb(CreateDbStmt stmt) throws DdlException {
makeSureInitialized();
@@ -1244,4 +1260,24 @@ public abstract class ExternalCatalog
Env.getCurrentEnv().getExtMetaCacheMgr().invalidSchemaCache(id);
}
}
+
+ public CatalogProperty getCatalogProperty() {
+ return catalogProperty;
+ }
+
+ public Optional<Boolean> getUseMetaCache() {
+ return useMetaCache;
+ }
+
+ public Map<Pair<String, String>, String> getTableAutoAnalyzePolicy() {
+ return tableAutoAnalyzePolicy;
+ }
+
+ public TransactionManager getTransactionManager() {
+ return transactionManager;
+ }
+
+ public ThreadPoolExecutor getThreadPoolWithPreAuth() {
+ return threadPoolWithPreAuth;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
index 12facde4944..7dea087c5ee 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalDatabase.java
@@ -101,6 +101,8 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
private MetaCache<T> metaCache;
+ private volatile boolean isInitializing = false;
+
/**
* Create external database.
*
@@ -132,7 +134,7 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
}
}
- public void setUnInitialized(boolean invalidCache) {
+ public synchronized void setUnInitialized(boolean invalidCache) {
this.initialized = false;
this.invalidCacheInInit = invalidCache;
if (extCatalog.getUseMetaCache().isPresent()) {
@@ -154,39 +156,49 @@ public abstract class ExternalDatabase<T extends
ExternalTable>
}
public final synchronized void makeSureInitialized() {
- extCatalog.makeSureInitialized();
- if (!initialized) {
- if (extCatalog.getUseMetaCache().get()) {
- if (metaCache == null) {
- metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
- name,
- OptionalLong.of(86400L),
-
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
- Config.max_meta_object_cache_num,
- ignored -> listTableNames(),
- localTableName -> Optional.ofNullable(
- buildTableForInit(null, localTableName,
-
Util.genIdByName(extCatalog.getName(), name, localTableName), extCatalog,
- this, true)),
- (key, value, cause) ->
value.ifPresent(ExternalTable::unsetObjectCreated));
- }
- setLastUpdateTime(System.currentTimeMillis());
- } else {
- if (!Env.getCurrentEnv().isMaster()) {
- // Forward to master and wait the journal to replay.
- int waitTimeOut = ConnectContext.get() == null ? 300 :
ConnectContext.get().getExecTimeout();
- MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor(waitTimeOut * 1000);
- try {
- remoteExecutor.forward(extCatalog.getId(), id);
- } catch (Exception e) {
- Util.logAndThrowRuntimeException(LOG,
- String.format("failed to forward init external
db %s operation to master", name), e);
+ if (isInitializing) {
+ return;
+ }
+ isInitializing = true;
+ try {
+ extCatalog.makeSureInitialized();
+ if (!initialized) {
+ if (extCatalog.getUseMetaCache().get()) {
+ if (metaCache == null) {
+ metaCache =
Env.getCurrentEnv().getExtMetaCacheMgr().buildMetaCache(
+ name,
+ OptionalLong.of(86400L),
+
OptionalLong.of(Config.external_cache_expire_time_minutes_after_access * 60L),
+ Config.max_meta_object_cache_num,
+ ignored -> listTableNames(),
+ localTableName -> Optional.ofNullable(
+ buildTableForInit(null, localTableName,
+
Util.genIdByName(extCatalog.getName(), name, localTableName),
+ extCatalog,
+ this, true)),
+ (key, value, cause) ->
value.ifPresent(ExternalTable::unsetObjectCreated));
+ }
+ setLastUpdateTime(System.currentTimeMillis());
+ } else {
+ if (!Env.getCurrentEnv().isMaster()) {
+ // Forward to master and wait the journal to replay.
+ int waitTimeOut = ConnectContext.get() == null ? 300 :
ConnectContext.get().getExecTimeout();
+ MasterCatalogExecutor remoteExecutor = new
MasterCatalogExecutor(waitTimeOut * 1000);
+ try {
+ remoteExecutor.forward(extCatalog.getId(), id);
+ } catch (Exception e) {
+ Util.logAndThrowRuntimeException(LOG,
+ String.format("failed to forward init
external db %s operation to master", name),
+ e);
+ }
+ return;
}
- return;
+ init();
}
- init();
+ initialized = true;
}
- initialized = true;
+ } finally {
+ isInitializing = false;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalFunctionRules.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalFunctionRules.java
new file mode 100644
index 00000000000..c88eae9a686
--- /dev/null
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalFunctionRules.java
@@ -0,0 +1,287 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource;
+
+import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.jdbc.source.JdbcFunctionPushDownRule;
+
+import com.google.common.base.Strings;
+import com.google.common.collect.Maps;
+import com.google.common.collect.Sets;
+import com.google.gson.Gson;
+import lombok.Data;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+/**
+ * External push down rules for functions.
+ * This class provides a way to define which functions can be pushed down to
external data sources.
+ * It supports both supported and unsupported functions in a JSON format.
+ */
+public class ExternalFunctionRules {
+ private static final Logger LOG =
LogManager.getLogger(ExternalFunctionRules.class);
+
+ private FunctionPushDownRule functionPushDownRule;
+ private FunctionRewriteRules functionRewriteRules;
+
+ public static ExternalFunctionRules create(String datasource, String
jsonRules) {
+ ExternalFunctionRules rules = new ExternalFunctionRules();
+ rules.functionPushDownRule = FunctionPushDownRule.create(datasource,
jsonRules);
+ rules.functionRewriteRules = FunctionRewriteRules.create(datasource,
jsonRules);
+ return rules;
+ }
+
+ public static void check(String jsonRules) throws DdlException {
+ if (Strings.isNullOrEmpty(jsonRules)) {
+ return;
+ }
+ FunctionPushDownRule.check(jsonRules);
+ FunctionRewriteRules.check(jsonRules);
+ }
+
+ public FunctionPushDownRule getFunctionPushDownRule() {
+ return functionPushDownRule;
+ }
+
+ public FunctionRewriteRules getFunctionRewriteRule() {
+ return functionRewriteRules;
+ }
+
+ /**
+ * FunctionPushDownRule is used to determine if a function can be pushed
down
+ */
+ public static class FunctionPushDownRule {
+ private final Set<String> supportedFunctions = Sets.newHashSet();
+ private final Set<String> unsupportedFunctions = Sets.newHashSet();
+
+ public static FunctionPushDownRule create(String datasource, String
jsonRules) {
+ FunctionPushDownRule funcRule = new FunctionPushDownRule();
+ try {
+ // Add default push down rules
+ switch (datasource.toLowerCase()) {
+ case "mysql":
+
funcRule.unsupportedFunctions.addAll(JdbcFunctionPushDownRule.MYSQL_UNSUPPORTED_FUNCTIONS);
+ break;
+ case "clickhouse":
+
funcRule.supportedFunctions.addAll(JdbcFunctionPushDownRule.CLICKHOUSE_SUPPORTED_FUNCTIONS);
+ break;
+ case "oracle":
+
funcRule.supportedFunctions.addAll(JdbcFunctionPushDownRule.ORACLE_SUPPORTED_FUNCTIONS);
+ break;
+ default:
+ break;
+ }
+ if (!Strings.isNullOrEmpty(jsonRules)) {
+ // set custom rules
+ Gson gson = new Gson();
+ PushDownRules rules = gson.fromJson(jsonRules,
PushDownRules.class);
+ funcRule.setCustomRules(rules);
+ }
+ return funcRule;
+ } catch (Exception e) {
+ LOG.warn("should not happen", e);
+ return funcRule;
+ }
+ }
+
+ public static void check(String jsonRules) throws DdlException {
+ try {
+ Gson gson = new Gson();
+ PushDownRules rules = gson.fromJson(jsonRules,
PushDownRules.class);
+ if (rules == null) {
+ throw new DdlException("Push down rules cannot be null");
+ }
+ rules.check();
+ } catch (Exception e) {
+ throw new DdlException("Failed to parse push down rules: " +
jsonRules, e);
+ }
+ }
+
+ private void setCustomRules(PushDownRules rules) {
+ if (rules != null && rules.getPushdown() != null) {
+ if (rules.getPushdown().getSupported() != null) {
+ rules.getPushdown().getSupported().stream()
+ .map(String::toLowerCase)
+ .forEach(supportedFunctions::add);
+ }
+ if (rules.getPushdown().getUnsupported() != null) {
+ rules.getPushdown().getUnsupported().stream()
+ .map(String::toLowerCase)
+ .forEach(unsupportedFunctions::add);
+ }
+ }
+ }
+
+ /**
+ * Checks if the function can be pushed down.
+ *
+ * @param functionName the name of the function to check
+ * @return true if the function can be pushed down, false otherwise
+ */
+ public boolean canPushDown(String functionName) {
+ if (supportedFunctions.isEmpty() &&
unsupportedFunctions.isEmpty()) {
+ return false;
+ }
+
+ // If supportedFunctions is not empty, only functions in
supportedFunctions can return true
+ if (!supportedFunctions.isEmpty()) {
+ return supportedFunctions.contains(functionName.toLowerCase());
+ }
+
+ // For functions contained in unsupportedFunctions, return false
+ if (unsupportedFunctions.contains(functionName.toLowerCase())) {
+ return false;
+ }
+
+ // In other cases, return true
+ return true;
+ }
+ }
+
+ /**
+ * FunctionRewriteRule is used to rewrite function names based on provided
rules.
+ * It allows for mapping one function name to another.
+ */
+ public static class FunctionRewriteRules {
+ private final Map<String, String> rewriteMap = Maps.newHashMap();
+
+ public static FunctionRewriteRules create(String datasource, String
jsonRules) {
+ FunctionRewriteRules rewriteRule = new FunctionRewriteRules();
+ try {
+ // Add default rewrite rules
+ switch (datasource.toLowerCase()) {
+ case "mysql":
+
rewriteRule.rewriteMap.putAll(JdbcFunctionPushDownRule.REPLACE_MYSQL_FUNCTIONS);
+ break;
+ case "clickhouse":
+
rewriteRule.rewriteMap.putAll(JdbcFunctionPushDownRule.REPLACE_CLICKHOUSE_FUNCTIONS);
+ break;
+ case "oracle":
+
rewriteRule.rewriteMap.putAll(JdbcFunctionPushDownRule.REPLACE_ORACLE_FUNCTIONS);
+ break;
+ default:
+ break;
+ }
+ if (!Strings.isNullOrEmpty(jsonRules)) {
+ // set custom rules
+ Gson gson = new Gson();
+ RewriteRules rules = gson.fromJson(jsonRules,
RewriteRules.class);
+ rewriteRule.setCustomRules(rules);
+ }
+ return rewriteRule;
+ } catch (Exception e) {
+ LOG.warn("should not happen", e);
+ return rewriteRule;
+ }
+ }
+
+ private void setCustomRules(RewriteRules rules) {
+ if (rules != null && rules.getRewrite() != null) {
+ this.rewriteMap.putAll(rules.getRewrite());
+ }
+ }
+
+ public String rewriteFunction(String origFuncName) {
+ return rewriteMap.getOrDefault(origFuncName, origFuncName);
+ }
+
+ public static void check(String jsonRules) throws DdlException {
+ try {
+ Gson gson = new Gson();
+ RewriteRules rules = gson.fromJson(jsonRules,
RewriteRules.class);
+ if (rules == null) {
+ throw new DdlException("Rewrite rules cannot be null");
+ }
+ rules.check();
+ } catch (Exception e) {
+ throw new DdlException("Failed to parse rewrite rules: " +
jsonRules, e);
+ }
+ }
+ }
+
+ /**
+ * push down rules in json format.
+ * eg:
+ * {
+ * "pushdown": {
+ * "supported": ["function1", "function2"],
+ * "unsupported": ["function3", "function4"]
+ * }
+ * }
+ */
+ @Data
+ public static class PushDownRules {
+ private PushDown pushdown;
+
+ @Data
+ public static class PushDown {
+ private List<String> supported;
+ private List<String> unsupported;
+ }
+
+ public void check() {
+ if (pushdown != null) {
+ if (pushdown.getSupported() != null) {
+ for (String func : pushdown.getSupported()) {
+ if (Strings.isNullOrEmpty(func)) {
+ throw new IllegalArgumentException("Supported
function name cannot be empty");
+ }
+ }
+ }
+ if (pushdown.getUnsupported() != null) {
+ for (String func : pushdown.getUnsupported()) {
+ if (Strings.isNullOrEmpty(func)) {
+ throw new IllegalArgumentException("Unsupported
function name cannot be empty");
+ }
+ }
+ }
+ }
+ }
+ }
+
+ /**
+ * push down rules in json format.
+ * eg:
+ * {
+ * "rewrite": {
+ * "func1": "func2",
+ * "func3": "func4"
+ * }
+ * }
+ */
+ @Data
+ public static class RewriteRules {
+ private Map<String, String> rewrite;
+
+ public void check() {
+ if (rewrite != null) {
+ for (Map.Entry<String, String> entry : rewrite.entrySet()) {
+ String origFunc = entry.getKey();
+ String newFunc = entry.getValue();
+ if (Strings.isNullOrEmpty(origFunc) ||
Strings.isNullOrEmpty(newFunc)) {
+ throw new IllegalArgumentException("Function names in
rewrite rules cannot be empty");
+ }
+ }
+ }
+ }
+ }
+}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
index 60a1f172978..8e999df2aef 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
@@ -45,7 +45,6 @@ import org.apache.doris.thrift.TTableDescriptor;
import com.google.common.base.Objects;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
-import lombok.Getter;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.logging.log4j.LogManager;
@@ -64,7 +63,6 @@ import java.util.Set;
* External table represent tables that are not self-managed by Doris.
* Such as tables from hive, iceberg, es, etc.
*/
-@Getter
public class ExternalTable implements TableIf, Writable, GsonPostProcessable {
private static final Logger LOG =
LogManager.getLogger(ExternalTable.class);
@@ -466,4 +464,36 @@ public class ExternalTable implements TableIf, Writable,
GsonPostProcessable {
public int hashCode() {
return Objects.hashCode(name, db);
}
+
+ public long getSchemaUpdateTime() {
+ return schemaUpdateTime;
+ }
+
+ public long getDbId() {
+ return dbId;
+ }
+
+ public boolean isObjectCreated() {
+ return objectCreated;
+ }
+
+ public ExternalCatalog getCatalog() {
+ return catalog;
+ }
+
+ public ExternalDatabase getDb() {
+ return db;
+ }
+
+ public long getTimestamp() {
+ return timestamp;
+ }
+
+ public String getDbName() {
+ return dbName;
+ }
+
+ public TableAttributes getTableAttributes() {
+ return tableAttributes;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
index ac010abd071..f232c8a1a76 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java
@@ -190,7 +190,7 @@ public class HMSExternalCatalog extends ExternalCatalog {
}
@Override
- public void resetToUninitialized(boolean invalidCache) {
+ public synchronized void resetToUninitialized(boolean invalidCache) {
super.resetToUninitialized(invalidCache);
if (metadataOps != null) {
metadataOps.close();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
index a8561a6ff88..85e3b29e1a3 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalCatalog.java
@@ -28,6 +28,7 @@ import org.apache.doris.common.FeConstants;
import org.apache.doris.datasource.CatalogProperty;
import org.apache.doris.datasource.ExternalCatalog;
import org.apache.doris.datasource.ExternalDatabase;
+import org.apache.doris.datasource.ExternalFunctionRules;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.InitCatalogLog;
import org.apache.doris.datasource.SessionContext;
@@ -50,7 +51,6 @@ import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.protobuf.ByteString;
-import lombok.Getter;
import org.apache.commons.lang3.StringUtils;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -63,7 +63,6 @@ import java.util.Map;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.Future;
-@Getter
public class JdbcExternalCatalog extends ExternalCatalog {
private static final Logger LOG =
LogManager.getLogger(JdbcExternalCatalog.class);
@@ -77,6 +76,7 @@ public class JdbcExternalCatalog extends ExternalCatalog {
// or Gson will throw exception with HikariCP
private transient JdbcClient jdbcClient;
private IdentifierMapping identifierMapping;
+ private ExternalFunctionRules functionRules;
public JdbcExternalCatalog(long catalogId, String name, String resource,
Map<String, String> props,
String comment)
@@ -107,6 +107,9 @@ public class JdbcExternalCatalog extends ExternalCatalog {
getExcludeDatabaseMap());
JdbcResource.checkConnectionPoolProperties(getConnectionPoolMinSize(),
getConnectionPoolMaxSize(),
getConnectionPoolMaxWaitTime(),
getConnectionPoolMaxLifeTime());
+
+ // check function rules
+
ExternalFunctionRules.check(catalogProperty.getProperties().getOrDefault(JdbcResource.FUNCTION_RULES,
""));
}
@Override
@@ -125,7 +128,7 @@ public class JdbcExternalCatalog extends ExternalCatalog {
}
@Override
- public void resetToUninitialized(boolean invalidCache) {
+ public synchronized void resetToUninitialized(boolean invalidCache) {
super.resetToUninitialized(invalidCache);
this.identifierMapping = new JdbcIdentifierMapping(
(Env.isTableNamesCaseInsensitive() ||
Env.isStoredTableNamesLowerCase()),
@@ -157,6 +160,7 @@ public class JdbcExternalCatalog extends ExternalCatalog {
}
public String getDatabaseTypeName() {
+ makeSureInitialized();
return jdbcClient.getDbType();
}
@@ -222,6 +226,8 @@ public class JdbcExternalCatalog extends ExternalCatalog {
@Override
protected void initLocalObjectsImpl() {
jdbcClient = createJdbcClient();
+ this.functionRules =
ExternalFunctionRules.create(jdbcClient.getDbType(),
+ catalogProperty.getOrDefault(JdbcResource.FUNCTION_RULES, ""));
}
private JdbcClient createJdbcClient() {
@@ -436,4 +442,12 @@ public class JdbcExternalCatalog extends ExternalCatalog {
return testTable;
}
+
+ public ExternalFunctionRules getFunctionRules() {
+ return functionRules;
+ }
+
+ public IdentifierMapping getIdentifierMapping() {
+ return identifierMapping;
+ }
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
index b3ff728bb7d..2bcd2277251 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/JdbcExternalTable.java
@@ -187,6 +187,7 @@ public class JdbcExternalTable extends ExternalTable {
remoteColumnNames.put(column.getName(), remoteColumnName);
}
jdbcTable.setRemoteColumnNames(remoteColumnNames);
+ jdbcTable.setExternalFunctionRules(jdbcCatalog.getFunctionRules());
return jdbcTable;
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcFunctionPushDownRule.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcFunctionPushDownRule.java
index a765681c402..48aac798cf0 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcFunctionPushDownRule.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcFunctionPushDownRule.java
@@ -23,6 +23,8 @@ import org.apache.doris.analysis.FunctionName;
import org.apache.doris.analysis.TimestampArithmeticExpr;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.Config;
+import org.apache.doris.datasource.ExternalFunctionRules;
+import org.apache.doris.datasource.ExternalFunctionRules.FunctionRewriteRules;
import org.apache.doris.thrift.TOdbcTableType;
import com.google.common.base.Preconditions;
@@ -34,11 +36,10 @@ import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.TreeSet;
-import java.util.function.Predicate;
public class JdbcFunctionPushDownRule {
private static final Logger LOG =
LogManager.getLogger(JdbcFunctionPushDownRule.class);
- private static final TreeSet<String> MYSQL_UNSUPPORTED_FUNCTIONS = new
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ public static final TreeSet<String> MYSQL_UNSUPPORTED_FUNCTIONS = new
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
static {
MYSQL_UNSUPPORTED_FUNCTIONS.add("date_trunc");
@@ -47,14 +48,14 @@ public class JdbcFunctionPushDownRule {
MYSQL_UNSUPPORTED_FUNCTIONS.addAll(Arrays.asList(Config.jdbc_mysql_unsupported_pushdown_functions));
}
- private static final TreeSet<String> CLICKHOUSE_SUPPORTED_FUNCTIONS = new
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ public static final TreeSet<String> CLICKHOUSE_SUPPORTED_FUNCTIONS = new
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
static {
CLICKHOUSE_SUPPORTED_FUNCTIONS.add("from_unixtime");
CLICKHOUSE_SUPPORTED_FUNCTIONS.add("unix_timestamp");
}
- private static final TreeSet<String> ORACLE_SUPPORTED_FUNCTIONS = new
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
+ public static final TreeSet<String> ORACLE_SUPPORTED_FUNCTIONS = new
TreeSet<>(String.CASE_INSENSITIVE_ORDER);
static {
ORACLE_SUPPORTED_FUNCTIONS.add("nvl");
@@ -73,21 +74,21 @@ public class JdbcFunctionPushDownRule {
return
!ORACLE_SUPPORTED_FUNCTIONS.contains(functionName.toLowerCase());
}
- private static final Map<String, String> REPLACE_MYSQL_FUNCTIONS =
Maps.newHashMap();
+ public static final Map<String, String> REPLACE_MYSQL_FUNCTIONS =
Maps.newHashMap();
static {
REPLACE_MYSQL_FUNCTIONS.put("nvl", "ifnull");
REPLACE_MYSQL_FUNCTIONS.put("to_date", "date");
}
- private static final Map<String, String> REPLACE_CLICKHOUSE_FUNCTIONS =
Maps.newHashMap();
+ public static final Map<String, String> REPLACE_CLICKHOUSE_FUNCTIONS =
Maps.newHashMap();
static {
REPLACE_CLICKHOUSE_FUNCTIONS.put("from_unixtime", "FROM_UNIXTIME");
REPLACE_CLICKHOUSE_FUNCTIONS.put("unix_timestamp", "toUnixTimestamp");
}
- private static final Map<String, String> REPLACE_ORACLE_FUNCTIONS =
Maps.newHashMap();
+ public static final Map<String, String> REPLACE_ORACLE_FUNCTIONS =
Maps.newHashMap();
static {
REPLACE_ORACLE_FUNCTIONS.put("ifnull", "nvl");
@@ -105,77 +106,54 @@ public class JdbcFunctionPushDownRule {
return
REPLACE_ORACLE_FUNCTIONS.containsKey(functionName.toLowerCase());
}
- public static Expr processFunctions(TOdbcTableType tableType, Expr expr,
List<String> errors) {
- if (tableType == null || expr == null) {
+ public static Expr processFunctions(TOdbcTableType tableType, Expr expr,
List<String> errors,
+ ExternalFunctionRules functionRules) {
+ if (tableType == null || expr == null || functionRules == null) {
return expr;
}
- Predicate<String> checkFunction;
- Predicate<String> replaceFunction;
-
- if (TOdbcTableType.MYSQL.equals(tableType)) {
- replaceFunction =
JdbcFunctionPushDownRule::isReplaceMysqlFunctions;
- checkFunction =
JdbcFunctionPushDownRule::isMySQLFunctionUnsupported;
- } else if (TOdbcTableType.CLICKHOUSE.equals(tableType)) {
- replaceFunction =
JdbcFunctionPushDownRule::isReplaceClickHouseFunctions;
- checkFunction =
JdbcFunctionPushDownRule::isClickHouseFunctionUnsupported;
- } else if (TOdbcTableType.ORACLE.equals(tableType)) {
- replaceFunction =
JdbcFunctionPushDownRule::isReplaceOracleFunctions;
- checkFunction =
JdbcFunctionPushDownRule::isOracleFunctionUnsupported;
- } else {
- return expr;
- }
-
- return processFunctionsRecursively(expr, checkFunction,
replaceFunction, errors, tableType);
+ return processFunctionsRecursively(expr, functionRules, errors,
tableType);
}
- private static Expr processFunctionsRecursively(Expr expr,
Predicate<String> checkFunction,
- Predicate<String> replaceFunction, List<String> errors,
TOdbcTableType tableType) {
+ private static Expr processFunctionsRecursively(Expr expr,
ExternalFunctionRules functionRules, List<String> errors,
+ TOdbcTableType tableType) {
if (expr instanceof FunctionCallExpr) {
FunctionCallExpr functionCallExpr = (FunctionCallExpr) expr;
String func = functionCallExpr.getFnName().getFunction();
Preconditions.checkArgument(!func.isEmpty(), "function can not be
empty");
- if (checkFunction.test(func)) {
- String errMsg = "Unsupported function: " + func + " in expr: "
+ expr.toExternalSql(
- TableType.JDBC_EXTERNAL_TABLE, null)
- + " in JDBC Table Type: " + tableType;
- LOG.warn(errMsg);
- errors.add(errMsg);
+ // 1. check can push down
+ if (!functionRules.getFunctionPushDownRule().canPushDown(func)) {
+ if (LOG.isDebugEnabled()) {
+ String errMsg = "Unsupported function: " + func + " in
expr: " + expr.toExternalSql(
+ TableType.JDBC_EXTERNAL_TABLE, null)
+ + " in JDBC Table Type: " + tableType;
+ LOG.debug(errMsg);
+ }
+ errors.add("has error");
}
- replaceFunctionNameIfNecessary(func, replaceFunction,
functionCallExpr, tableType);
-
+ // 2. replace function
+ replaceFunctionNameIfNecessary(func,
functionRules.getFunctionRewriteRule(), functionCallExpr);
expr = replaceGenericFunctionExpr(functionCallExpr, func);
}
List<Expr> children = expr.getChildren();
for (int i = 0; i < children.size(); i++) {
Expr child = children.get(i);
- Expr newChild = processFunctionsRecursively(child, checkFunction,
replaceFunction, errors, tableType);
+ Expr newChild = processFunctionsRecursively(child, functionRules,
errors, tableType);
expr.setChild(i, newChild);
}
return expr;
}
- private static void replaceFunctionNameIfNecessary(String func,
Predicate<String> replaceFunction,
- FunctionCallExpr functionCallExpr, TOdbcTableType tableType) {
- if (replaceFunction.test(func)) {
- String newFunc;
- if (TOdbcTableType.MYSQL.equals(tableType)) {
- newFunc = REPLACE_MYSQL_FUNCTIONS.get(func.toLowerCase());
- } else if (TOdbcTableType.CLICKHOUSE.equals(tableType)) {
- newFunc = REPLACE_CLICKHOUSE_FUNCTIONS.get(func);
- } else if (TOdbcTableType.ORACLE.equals(tableType)) {
- newFunc = REPLACE_ORACLE_FUNCTIONS.get(func);
- } else {
- newFunc = null;
- }
- if (newFunc != null) {
-
functionCallExpr.setFnName(FunctionName.createBuiltinName(newFunc));
- }
+ private static void replaceFunctionNameIfNecessary(String func,
FunctionRewriteRules rewriteRule,
+ FunctionCallExpr functionCallExpr) {
+ String newFuncName = rewriteRule.rewriteFunction(func);
+ if (!newFuncName.equals(func)) {
+
functionCallExpr.setFnName(FunctionName.createBuiltinName(newFuncName));
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java
index 31026e6b877..019cceca6a7 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/datasource/jdbc/source/JdbcScanNode.java
@@ -39,6 +39,7 @@ import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.TableIf.TableType;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.UserException;
+import org.apache.doris.datasource.ExternalFunctionRules;
import org.apache.doris.datasource.ExternalScanNode;
import org.apache.doris.datasource.jdbc.JdbcExternalTable;
import org.apache.doris.planner.PlanNodeId;
@@ -131,7 +132,8 @@ public class JdbcScanNode extends ExternalScanNode {
ArrayList<Expr> conjunctsList = Expr.cloneList(conjuncts, sMap);
List<String> errors = Lists.newArrayList();
- List<Expr> pushDownConjuncts =
collectConjunctsToPushDown(conjunctsList, errors);
+ List<Expr> pushDownConjuncts =
collectConjunctsToPushDown(conjunctsList, errors,
+ tbl.getExternalFunctionRules());
for (Expr individualConjunct : pushDownConjuncts) {
String filter = conjunctExprToString(jdbcType, individualConjunct,
tbl);
@@ -140,13 +142,15 @@ public class JdbcScanNode extends ExternalScanNode {
}
}
- private List<Expr> collectConjunctsToPushDown(List<Expr> conjunctsList,
List<String> errors) {
+ private List<Expr> collectConjunctsToPushDown(List<Expr> conjunctsList,
List<String> errors,
+ ExternalFunctionRules functionRules) {
List<Expr> pushDownConjuncts = new ArrayList<>();
for (Expr p : conjunctsList) {
if (shouldPushDownConjunct(jdbcType, p)) {
List<Expr> individualConjuncts = p.getConjuncts();
for (Expr individualConjunct : individualConjuncts) {
- Expr newp =
JdbcFunctionPushDownRule.processFunctions(jdbcType, individualConjunct, errors);
+ Expr newp =
JdbcFunctionPushDownRule.processFunctions(jdbcType, individualConjunct, errors,
+ functionRules);
if (!errors.isEmpty()) {
errors.clear();
continue;
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionPushDownRulesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionPushDownRulesTest.java
new file mode 100644
index 00000000000..5f8591ed1d4
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionPushDownRulesTest.java
@@ -0,0 +1,431 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource;
+
+import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.ExternalFunctionRules.FunctionPushDownRule;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class ExternalFunctionPushDownRulesTest {
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithMysqlDataSource() {
+ // Test MySQL datasource with default rules
+ FunctionPushDownRule rule = FunctionPushDownRule.create("mysql", null);
+
+ // MySQL has unsupported functions by default, supported functions is
empty
+ Assertions.assertFalse(rule.canPushDown("date_trunc"));
+ Assertions.assertFalse(rule.canPushDown("money_format"));
+ Assertions.assertFalse(rule.canPushDown("negative"));
+
+ // Test case insensitivity
+ Assertions.assertFalse(rule.canPushDown("DATE_TRUNC"));
+ Assertions.assertFalse(rule.canPushDown("Money_Format"));
+
+ // Functions not in unsupported list should be allowed (since
supportedFunctions is empty)
+ Assertions.assertTrue(rule.canPushDown("sum"));
+ Assertions.assertTrue(rule.canPushDown("count"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithClickHouseDataSource() {
+ // Test ClickHouse datasource with default rules
+ FunctionPushDownRule rule = FunctionPushDownRule.create("clickhouse",
null);
+
+ // ClickHouse has supported functions by default, so only supported
functions return true
+ Assertions.assertTrue(rule.canPushDown("from_unixtime"));
+ Assertions.assertTrue(rule.canPushDown("unix_timestamp"));
+
+ // Test case insensitivity
+ Assertions.assertTrue(rule.canPushDown("FROM_UNIXTIME"));
+ Assertions.assertTrue(rule.canPushDown("Unix_Timestamp"));
+
+ // Functions not in supported list should be denied (since
supportedFunctions is not empty)
+ Assertions.assertFalse(rule.canPushDown("unknown_function"));
+ Assertions.assertFalse(rule.canPushDown("custom_func"));
+ Assertions.assertFalse(rule.canPushDown("sum"));
+ Assertions.assertFalse(rule.canPushDown("count"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithOracleDataSource() {
+ // Test Oracle datasource with default rules
+ FunctionPushDownRule rule = FunctionPushDownRule.create("oracle",
null);
+
+ // Oracle has supported functions by default, so only supported
functions return true
+ Assertions.assertTrue(rule.canPushDown("nvl"));
+ Assertions.assertTrue(rule.canPushDown("ifnull"));
+
+ // Test case insensitivity
+ Assertions.assertTrue(rule.canPushDown("NVL"));
+ Assertions.assertTrue(rule.canPushDown("IfNull"));
+
+ // Functions not in supported list should be denied (since
supportedFunctions is not empty)
+ Assertions.assertFalse(rule.canPushDown("unknown_function"));
+ Assertions.assertFalse(rule.canPushDown("custom_func"));
+ Assertions.assertFalse(rule.canPushDown("sum"));
+ Assertions.assertFalse(rule.canPushDown("count"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithUnknownDataSource() {
+ // Test unknown datasource should have no default rules
+ FunctionPushDownRule rule = FunctionPushDownRule.create("unknown",
null);
+
+ // With no rules, all functions should be denied
+ Assertions.assertFalse(rule.canPushDown("any_function"));
+ Assertions.assertFalse(rule.canPushDown("sum"));
+ Assertions.assertFalse(rule.canPushDown("count"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithValidCustomRules() {
+ // Test custom rules with supported functions
+ String jsonRules = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"supported\": [\"custom_func1\", \"Custom_Func2\"],\n"
+ + " \"unsupported\": [\"blocked_func1\",
\"Blocked_Func2\"]\n"
+ + " }\n"
+ + "}";
+
+ FunctionPushDownRule rule = FunctionPushDownRule.create("mysql",
jsonRules);
+
+ // Since supportedFunctions is not empty, only supported functions
return true
+ Assertions.assertTrue(rule.canPushDown("custom_func1"));
+ Assertions.assertTrue(rule.canPushDown("custom_func2")); // case
insensitive
+ Assertions.assertTrue(rule.canPushDown("CUSTOM_FUNC1"));
+
+ // Functions not in supported list should be denied (even if not in
unsupported list)
+ Assertions.assertFalse(rule.canPushDown("other_function"));
+ Assertions.assertFalse(rule.canPushDown("sum"));
+ Assertions.assertFalse(rule.canPushDown("count"));
+
+ // Functions in unsupported list should still be denied
+ Assertions.assertFalse(rule.canPushDown("blocked_func1"));
+ Assertions.assertFalse(rule.canPushDown("blocked_func2")); // case
insensitive
+ Assertions.assertFalse(rule.canPushDown("BLOCKED_FUNC1"));
+
+ // Default MySQL unsupported functions should be denied
+ Assertions.assertFalse(rule.canPushDown("date_trunc"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithOnlySupportedCustomRules() {
+ // Test custom rules with only supported functions
+ String jsonRules = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"supported\": [\"allowed_func1\", \"allowed_func2\"]\n"
+ + " }\n"
+ + "}";
+
+ FunctionPushDownRule rule = FunctionPushDownRule.create("unknown",
jsonRules);
+
+ // Since supportedFunctions is not empty, only supported functions
return true
+ Assertions.assertTrue(rule.canPushDown("allowed_func1"));
+ Assertions.assertTrue(rule.canPushDown("allowed_func2"));
+
+ // Other functions should be denied (since supportedFunctions is not
empty)
+ Assertions.assertFalse(rule.canPushDown("other_function"));
+ Assertions.assertFalse(rule.canPushDown("sum"));
+ Assertions.assertFalse(rule.canPushDown("count"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithOnlyUnsupportedCustomRules()
{
+ // Test custom rules with only unsupported functions
+ String jsonRules = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"unsupported\": [\"blocked_func1\",
\"blocked_func2\"]\n"
+ + " }\n"
+ + "}";
+
+ FunctionPushDownRule rule = FunctionPushDownRule.create("unknown",
jsonRules);
+
+ // Custom unsupported functions should be denied
+ Assertions.assertFalse(rule.canPushDown("blocked_func1"));
+ Assertions.assertFalse(rule.canPushDown("blocked_func2"));
+
+ // Other functions should be allowed (default behavior)
+ Assertions.assertTrue(rule.canPushDown("other_function"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithEmptyCustomRules() {
+ // Test empty custom rules
+ String jsonRules = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"supported\": [],\n"
+ + " \"unsupported\": []\n"
+ + " }\n"
+ + "}";
+
+ FunctionPushDownRule rule = FunctionPushDownRule.create("unknown",
jsonRules);
+
+ // With empty rules, all functions should be denied
+ Assertions.assertFalse(rule.canPushDown("any_function"));
+ Assertions.assertFalse(rule.canPushDown("sum"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithNullCustomRules() {
+ // Test null pushdown section
+ String jsonRules = "{\n"
+ + " \"pushdown\": null\n"
+ + "}";
+
+ FunctionPushDownRule rule = FunctionPushDownRule.create("unknown",
jsonRules);
+
+ // With null rules, all functions should be denied
+ Assertions.assertFalse(rule.canPushDown("any_function"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithInvalidJson() {
+ // Test invalid JSON should not throw exception but return default rule
+ String invalidJson = "{ invalid json }";
+
+ FunctionPushDownRule rule = FunctionPushDownRule.create("unknown",
invalidJson);
+
+ // Should return a rule with no functions configured
+ Assertions.assertFalse(rule.canPushDown("any_function"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateWithEmptyJsonRules() {
+ // Test empty string rules
+ FunctionPushDownRule rule = FunctionPushDownRule.create("mysql", "");
+
+ // Should only have default MySQL rules
+ Assertions.assertFalse(rule.canPushDown("date_trunc"));
+ Assertions.assertTrue(rule.canPushDown("other_function"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCreateCaseInsensitiveDataSource() {
+ // Test case insensitivity for datasource names
+ FunctionPushDownRule mysqlRule = FunctionPushDownRule.create("MYSQL",
null);
+ FunctionPushDownRule clickhouseRule =
FunctionPushDownRule.create("ClickHouse", null);
+ FunctionPushDownRule oracleRule =
FunctionPushDownRule.create("Oracle", null);
+
+ // Should apply correct default rules regardless of case
+ Assertions.assertFalse(mysqlRule.canPushDown("date_trunc"));
+ Assertions.assertTrue(clickhouseRule.canPushDown("from_unixtime"));
+ Assertions.assertTrue(oracleRule.canPushDown("nvl"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCanPushDownLogic() {
+ // Test the canPushDown logic with different scenarios
+
+ // 1. Both supported and unsupported are empty -> return false
+ FunctionPushDownRule emptyRule =
FunctionPushDownRule.create("unknown", null);
+ Assertions.assertFalse(emptyRule.canPushDown("any_function"));
+
+ // 2. Function in supported list -> return true (only when
supportedFunctions is not empty)
+ String supportedJson = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"supported\": [\"func1\"]\n"
+ + " }\n"
+ + "}";
+ FunctionPushDownRule supportedRule =
FunctionPushDownRule.create("unknown", supportedJson);
+ Assertions.assertTrue(supportedRule.canPushDown("func1"));
+
+ // 3. Function not in supported list when supportedFunctions is not
empty -> return false
+ Assertions.assertFalse(supportedRule.canPushDown("other_func"));
+
+ // 4. Function in unsupported list -> return false
+ String unsupportedJson = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"unsupported\": [\"func1\"]\n"
+ + " }\n"
+ + "}";
+ FunctionPushDownRule unsupportedRule =
FunctionPushDownRule.create("unknown", unsupportedJson);
+ Assertions.assertFalse(unsupportedRule.canPushDown("func1"));
+
+ // 5. Function not in unsupported list when supportedFunctions is
empty -> return true
+ Assertions.assertTrue(unsupportedRule.canPushDown("other_func"));
+
+ // 6. Priority test: when supportedFunctions is not empty, only
supported functions return true
+ String bothJson = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"supported\": [\"func1\"],\n"
+ + " \"unsupported\": [\"func2\"]\n"
+ + " }\n"
+ + "}";
+ FunctionPushDownRule bothRule = FunctionPushDownRule.create("unknown",
bothJson);
+ Assertions.assertTrue(bothRule.canPushDown("func1")); // in supported
list
+ Assertions.assertFalse(bothRule.canPushDown("func2")); // not in
supported list (even though in unsupported)
+ Assertions.assertFalse(bothRule.canPushDown("func3")); // not in
supported list
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCheck() throws DdlException {
+ // Test valid JSON rules
+ String validJson = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"supported\": [\"func1\", \"func2\"],\n"
+ + " \"unsupported\": [\"func3\", \"func4\"]\n"
+ + " }\n"
+ + "}";
+
+ // Should not throw exception
+ Assertions.assertDoesNotThrow(() -> {
+ FunctionPushDownRule.check(validJson);
+ });
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCheckWithInvalidJson() {
+ // Test invalid JSON rules
+ String invalidJson = "{ invalid json }";
+
+ // Should throw DdlException
+ DdlException exception = Assertions.assertThrows(DdlException.class,
() -> {
+ FunctionPushDownRule.check(invalidJson);
+ });
+
+ Assertions.assertTrue(exception.getMessage().contains("Failed to parse
push down rules"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCheckWithEmptyJson() throws
DdlException {
+ // Test empty JSON
+ String emptyJson = "{}";
+
+ // Should not throw exception
+ Assertions.assertDoesNotThrow(() -> {
+ FunctionPushDownRule.check(emptyJson);
+ });
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCheckWithNullJson() throws
DdlException {
+ // Test null JSON
+ String nullJson = null;
+
+ DdlException exception = Assertions.assertThrows(DdlException.class,
() -> {
+ FunctionPushDownRule.check(nullJson);
+ });
+ Assertions.assertTrue(exception.getMessage().contains("Failed to parse
push down rules"));
+ }
+
+ @Test
+ public void testFunctionPushDownRuleCheckWithMalformedJson() {
+ // Test various malformed JSON strings
+ String[] malformedJsons = {
+ "{ \"pushdown\": }", // Missing value
+ "{ \"pushdown\": { \"supported\": } }", // Missing array
+ "{ \"pushdown\" { \"supported\": [] } }", // Missing colon
+ "{ \"pushdown\": { \"supported\": [\"func1\",] } }", //
Trailing comma
+ "{ \"pushdown\": { \"supported\": [\"func1\" \"func2\"] } }"
// Missing comma
+ };
+
+ for (String malformedJson : malformedJsons) {
+ DdlException exception =
Assertions.assertThrows(DdlException.class, () -> {
+ FunctionPushDownRule.check(malformedJson);
+ });
+
+ Assertions.assertTrue(exception.getMessage().contains("Failed to
parse push down rules"));
+ }
+ }
+
+ @Test
+ public void testFunctionPushDownRuleWithComplexCustomRules() {
+ // Test complex custom rules that override and extend default rules
+ String complexJson = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"supported\": [\"date_trunc\", \"custom_func\"],\n"
+ + " \"unsupported\": [\"from_unixtime\",
\"another_func\"]\n"
+ + " }\n"
+ + "}";
+
+ // Test with MySQL (has default unsupported functions)
+ FunctionPushDownRule mysqlRule = FunctionPushDownRule.create("mysql",
complexJson);
+
+ // Since supportedFunctions is not empty, only supported functions
return true
+ Assertions.assertTrue(mysqlRule.canPushDown("date_trunc")); // in
supported list
+ Assertions.assertTrue(mysqlRule.canPushDown("custom_func")); // in
supported list
+
+ // Functions not in supported list should be denied
+ Assertions.assertFalse(mysqlRule.canPushDown("from_unixtime")); // not
in supported list
+ Assertions.assertFalse(mysqlRule.canPushDown("another_func")); // not
in supported list
+ Assertions.assertFalse(mysqlRule.canPushDown("money_format")); // not
in supported list
+ Assertions.assertFalse(mysqlRule.canPushDown("sum")); // not in
supported list
+ Assertions.assertFalse(mysqlRule.canPushDown("count")); // not in
supported list
+ }
+
+ @Test
+ public void testFunctionPushDownRuleNewLogicCases() {
+ // Additional test cases for the new logic
+
+ // Test case 1: Only unsupported functions defined (supportedFunctions
is empty)
+ String onlyUnsupportedJson = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"unsupported\": [\"blocked_func\"]\n"
+ + " }\n"
+ + "}";
+ FunctionPushDownRule onlyUnsupportedRule =
FunctionPushDownRule.create("unknown", onlyUnsupportedJson);
+
+ // Functions in unsupported list should be denied
+
Assertions.assertFalse(onlyUnsupportedRule.canPushDown("blocked_func"));
+
+ // Other functions should be allowed (since supportedFunctions is
empty)
+ Assertions.assertTrue(onlyUnsupportedRule.canPushDown("allowed_func"));
+ Assertions.assertTrue(onlyUnsupportedRule.canPushDown("sum"));
+
+ // Test case 2: Both supported and unsupported functions defined
+ String bothListsJson = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"supported\": [\"func1\", \"func2\"],\n"
+ + " \"unsupported\": [\"func3\", \"func4\"]\n"
+ + " }\n"
+ + "}";
+ FunctionPushDownRule bothListsRule =
FunctionPushDownRule.create("unknown", bothListsJson);
+
+ // Only supported functions return true
+ Assertions.assertTrue(bothListsRule.canPushDown("func1"));
+ Assertions.assertTrue(bothListsRule.canPushDown("func2"));
+
+ // Functions in unsupported list should be denied (not in supported
list)
+ Assertions.assertFalse(bothListsRule.canPushDown("func3"));
+ Assertions.assertFalse(bothListsRule.canPushDown("func4"));
+
+ // Other functions should be denied (not in supported list)
+ Assertions.assertFalse(bothListsRule.canPushDown("func5"));
+ Assertions.assertFalse(bothListsRule.canPushDown("other_func"));
+
+ // Test case 3: MySQL with custom supported functions
+ String mysqlSupportedJson = "{\n"
+ + " \"pushdown\": {\n"
+ + " \"supported\": [\"date_trunc\", \"money_format\"]\n"
+ + " }\n"
+ + "}";
+ FunctionPushDownRule mysqlSupportedRule =
FunctionPushDownRule.create("mysql", mysqlSupportedJson);
+
+ // Only supported functions return true (overrides default MySQL
unsupported functions)
+ Assertions.assertTrue(mysqlSupportedRule.canPushDown("date_trunc"));
+ Assertions.assertTrue(mysqlSupportedRule.canPushDown("money_format"));
+
+ // Other functions should be denied
+ Assertions.assertFalse(mysqlSupportedRule.canPushDown("negative"));
+ Assertions.assertFalse(mysqlSupportedRule.canPushDown("sum"));
+ Assertions.assertFalse(mysqlSupportedRule.canPushDown("count"));
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionRewriteRulesTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionRewriteRulesTest.java
new file mode 100644
index 00000000000..3bca159575a
--- /dev/null
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalFunctionRewriteRulesTest.java
@@ -0,0 +1,443 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.datasource;
+
+import org.apache.doris.common.DdlException;
+import org.apache.doris.datasource.ExternalFunctionRules.FunctionRewriteRules;
+
+import org.junit.jupiter.api.Assertions;
+import org.junit.jupiter.api.Test;
+
+public class ExternalFunctionRewriteRulesTest {
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithMysqlDataSource() {
+ // Test MySQL datasource with default rewrite rules
+ FunctionRewriteRules rule = FunctionRewriteRules.create("mysql", null);
+
+ // MySQL has default rewrite rules
+ Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl"));
+ Assertions.assertEquals("date", rule.rewriteFunction("to_date"));
+
+ // Test case sensitivity - original function names should be used as-is
+ Assertions.assertEquals("NVL", rule.rewriteFunction("NVL"));
+ Assertions.assertEquals("To_Date", rule.rewriteFunction("To_Date"));
+
+ // Functions not in rewrite map should return original name
+ Assertions.assertEquals("sum", rule.rewriteFunction("sum"));
+ Assertions.assertEquals("count", rule.rewriteFunction("count"));
+ Assertions.assertEquals("unknown_func",
rule.rewriteFunction("unknown_func"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithClickHouseDataSource() {
+ // Test ClickHouse datasource with default rewrite rules
+ FunctionRewriteRules rule = FunctionRewriteRules.create("clickhouse",
null);
+
+ // ClickHouse has default rewrite rules
+ Assertions.assertEquals("FROM_UNIXTIME",
rule.rewriteFunction("from_unixtime"));
+ Assertions.assertEquals("toUnixTimestamp",
rule.rewriteFunction("unix_timestamp"));
+
+ // Test case sensitivity
+ Assertions.assertEquals("FROM_UNIXTIME",
rule.rewriteFunction("FROM_UNIXTIME"));
+ Assertions.assertEquals("Unix_Timestamp",
rule.rewriteFunction("Unix_Timestamp"));
+
+ // Functions not in rewrite map should return original name
+ Assertions.assertEquals("sum", rule.rewriteFunction("sum"));
+ Assertions.assertEquals("count", rule.rewriteFunction("count"));
+ Assertions.assertEquals("unknown_func",
rule.rewriteFunction("unknown_func"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithOracleDataSource() {
+ // Test Oracle datasource with default rewrite rules
+ FunctionRewriteRules rule = FunctionRewriteRules.create("oracle",
null);
+
+ // Oracle has default rewrite rules
+ Assertions.assertEquals("nvl", rule.rewriteFunction("ifnull"));
+
+ // Test case sensitivity
+ Assertions.assertEquals("IFNULL", rule.rewriteFunction("IFNULL"));
+ Assertions.assertEquals("IfNull", rule.rewriteFunction("IfNull"));
+
+ // Functions not in rewrite map should return original name
+ Assertions.assertEquals("sum", rule.rewriteFunction("sum"));
+ Assertions.assertEquals("count", rule.rewriteFunction("count"));
+ Assertions.assertEquals("unknown_func",
rule.rewriteFunction("unknown_func"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithUnknownDataSource() {
+ // Test unknown datasource should have no default rewrite rules
+ FunctionRewriteRules rule = FunctionRewriteRules.create("unknown",
null);
+
+ // All functions should return original name (no rewrite rules)
+ Assertions.assertEquals("any_function",
rule.rewriteFunction("any_function"));
+ Assertions.assertEquals("sum", rule.rewriteFunction("sum"));
+ Assertions.assertEquals("count", rule.rewriteFunction("count"));
+ Assertions.assertEquals("nvl", rule.rewriteFunction("nvl"));
+ Assertions.assertEquals("ifnull", rule.rewriteFunction("ifnull"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithValidCustomRules() {
+ // Test custom rewrite rules
+ String jsonRules = "{\n"
+ + " \"rewrite\": {\n"
+ + " \"old_func1\": \"new_func1\",\n"
+ + " \"Old_Func2\": \"New_Func2\",\n"
+ + " \"CUSTOM_FUNC\": \"custom_replacement\"\n"
+ + " }\n"
+ + "}";
+
+ FunctionRewriteRules rule = FunctionRewriteRules.create("mysql",
jsonRules);
+
+ // Custom rewrite rules should work
+ Assertions.assertEquals("new_func1",
rule.rewriteFunction("old_func1"));
+ Assertions.assertEquals("New_Func2",
rule.rewriteFunction("Old_Func2"));
+ Assertions.assertEquals("custom_replacement",
rule.rewriteFunction("CUSTOM_FUNC"));
+
+ // Default MySQL rewrite rules should still work
+ Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl"));
+ Assertions.assertEquals("date", rule.rewriteFunction("to_date"));
+
+ // Functions not in any rewrite map should return original name
+ Assertions.assertEquals("other_function",
rule.rewriteFunction("other_function"));
+ Assertions.assertEquals("sum", rule.rewriteFunction("sum"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithEmptyCustomRules() {
+ // Test empty custom rewrite rules
+ String jsonRules = "{\n"
+ + " \"rewrite\": {}\n"
+ + "}";
+
+ FunctionRewriteRules rule = FunctionRewriteRules.create("mysql",
jsonRules);
+
+ // Default MySQL rewrite rules should still work
+ Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl"));
+ Assertions.assertEquals("date", rule.rewriteFunction("to_date"));
+
+ // Other functions should return original name
+ Assertions.assertEquals("other_function",
rule.rewriteFunction("other_function"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithNullCustomRules() {
+ // Test null rewrite section
+ String jsonRules = "{\n"
+ + " \"rewrite\": null\n"
+ + "}";
+
+ FunctionRewriteRules rule = FunctionRewriteRules.create("mysql",
jsonRules);
+
+ // Default MySQL rewrite rules should still work
+ Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl"));
+ Assertions.assertEquals("date", rule.rewriteFunction("to_date"));
+
+ // Other functions should return original name
+ Assertions.assertEquals("other_function",
rule.rewriteFunction("other_function"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithNullRewriteMap() {
+ // Test null rewrite map - this test is no longer relevant with the
new format
+ // Since rewrite is now directly the map, we just test with null
rewrite
+ String jsonRules = "{\n"
+ + " \"rewrite\": null\n"
+ + "}";
+
+ FunctionRewriteRules rule = FunctionRewriteRules.create("mysql",
jsonRules);
+
+ // Default MySQL rewrite rules should still work
+ Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl"));
+ Assertions.assertEquals("date", rule.rewriteFunction("to_date"));
+
+ // Other functions should return original name
+ Assertions.assertEquals("other_function",
rule.rewriteFunction("other_function"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithInvalidJson() {
+ // Test invalid JSON should not throw exception but return default rule
+ String invalidJson = "{ invalid json }";
+
+ FunctionRewriteRules rule = FunctionRewriteRules.create("mysql",
invalidJson);
+
+ // Should still have default MySQL rewrite rules
+ Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl"));
+ Assertions.assertEquals("date", rule.rewriteFunction("to_date"));
+
+ // Other functions should return original name
+ Assertions.assertEquals("other_function",
rule.rewriteFunction("other_function"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithEmptyJsonRules() {
+ // Test empty string rules
+ FunctionRewriteRules rule = FunctionRewriteRules.create("mysql", "");
+
+ // Should only have default MySQL rewrite rules
+ Assertions.assertEquals("ifnull", rule.rewriteFunction("nvl"));
+ Assertions.assertEquals("date", rule.rewriteFunction("to_date"));
+ Assertions.assertEquals("other_function",
rule.rewriteFunction("other_function"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateCaseInsensitiveDataSource() {
+ // Test case insensitivity for datasource names
+ FunctionRewriteRules mysqlRule = FunctionRewriteRules.create("MYSQL",
null);
+ FunctionRewriteRules clickhouseRule =
FunctionRewriteRules.create("ClickHouse", null);
+ FunctionRewriteRules oracleRule =
FunctionRewriteRules.create("Oracle", null);
+
+ // Should apply correct default rules regardless of case
+ Assertions.assertEquals("ifnull", mysqlRule.rewriteFunction("nvl"));
+ Assertions.assertEquals("FROM_UNIXTIME",
clickhouseRule.rewriteFunction("from_unixtime"));
+ Assertions.assertEquals("nvl", oracleRule.rewriteFunction("ifnull"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleRewriteFunction() {
+ // Test the rewriteFunction logic with different scenarios
+
+ // Test with custom rewrite rules
+ String jsonRules = "{\n"
+ + " \"rewrite\": {\n"
+ + " \"func1\": \"replacement1\",\n"
+ + " \"func2\": \"replacement2\"\n"
+ + " }\n"
+ + "}";
+
+ FunctionRewriteRules rule = FunctionRewriteRules.create("unknown",
jsonRules);
+
+ // Functions in rewrite map should be replaced
+ Assertions.assertEquals("replacement1", rule.rewriteFunction("func1"));
+ Assertions.assertEquals("replacement2", rule.rewriteFunction("func2"));
+
+ // Functions not in rewrite map should return original name
+ Assertions.assertEquals("func3", rule.rewriteFunction("func3"));
+ Assertions.assertEquals("unknown_func",
rule.rewriteFunction("unknown_func"));
+
+ // Test with null function name
+ Assertions.assertEquals(null, rule.rewriteFunction(null));
+
+ // Test with empty function name
+ Assertions.assertEquals("", rule.rewriteFunction(""));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCheck() throws DdlException {
+ // Test valid JSON rewrite rules
+ String validJson = "{\n"
+ + " \"rewrite\": {\n"
+ + " \"func1\": \"replacement1\",\n"
+ + " \"func2\": \"replacement2\"\n"
+ + " }\n"
+ + "}";
+
+ // Should not throw exception
+ Assertions.assertDoesNotThrow(() -> {
+ FunctionRewriteRules.check(validJson);
+ });
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCheckWithInvalidJson() {
+ // Test invalid JSON rules
+ String invalidJson = "{ invalid json }";
+
+ // Should throw DdlException
+ DdlException exception = Assertions.assertThrows(DdlException.class,
() -> {
+ FunctionRewriteRules.check(invalidJson);
+ });
+
+ Assertions.assertTrue(exception.getMessage().contains("Failed to parse
rewrite rules"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCheckWithEmptyJson() throws
DdlException {
+ // Test empty JSON
+ String emptyJson = "{}";
+
+ // Should not throw exception
+ Assertions.assertDoesNotThrow(() -> {
+ FunctionRewriteRules.check(emptyJson);
+ });
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCheckWithNullJson() {
+ // Test null JSON
+ String nullJson = null;
+
+ // Should throw DdlException due to new null check
+ DdlException exception = Assertions.assertThrows(DdlException.class,
() -> {
+ FunctionRewriteRules.check(nullJson);
+ });
+ Assertions.assertTrue(exception.getMessage().contains("Failed to parse
rewrite rules"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCheckWithMalformedJson() {
+ // Test various malformed JSON strings
+ String[] malformedJsons = {
+ "{ \"rewrite\": }", // Missing value
+ "{ \"rewrite\": { } }", // Missing object - this is actually
valid now
+ "{ \"rewrite\" { } }", // Missing colon
+ "{ \"rewrite\": {\"func1\": \"replacement1\",} }", // Trailing
comma
+ "{ \"rewrite\": {\"func1\" \"replacement1\"} }" // Missing
colon
+ };
+
+ for (String malformedJson : malformedJsons) {
+ // Skip the second case as it's now valid
+ if (malformedJson.equals("{ \"rewrite\": { } }")) {
+ continue;
+ }
+
+ DdlException exception =
Assertions.assertThrows(DdlException.class, () -> {
+ FunctionRewriteRules.check(malformedJson);
+ });
+
+ Assertions.assertTrue(exception.getMessage().contains("Failed to
parse rewrite rules"));
+ }
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCheckWithEmptyFunctionNames() {
+ // Test empty function names in rewrite rules should throw exception
+ String emptyKeyJson = "{\n"
+ + " \"rewrite\": {\n"
+ + " \"\": \"replacement1\",\n"
+ + " \"func2\": \"replacement2\"\n"
+ + " }\n"
+ + "}";
+
+ DdlException exception1 = Assertions.assertThrows(DdlException.class,
() -> {
+ FunctionRewriteRules.check(emptyKeyJson);
+ });
+ Assertions.assertTrue(exception1.getMessage().contains("Failed to
parse rewrite rules"));
+
+ String emptyValueJson = "{\n"
+ + " \"rewrite\": {\n"
+ + " \"func1\": \"\",\n"
+ + " \"func2\": \"replacement2\"\n"
+ + " }\n"
+ + "}";
+
+ DdlException exception2 = Assertions.assertThrows(DdlException.class,
() -> {
+ FunctionRewriteRules.check(emptyValueJson);
+ });
+ Assertions.assertTrue(exception2.getMessage().contains("Failed to
parse rewrite rules"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCheckWithNullFunctionNames() {
+ // Test null function names in rewrite rules should throw exception
+ // Note: JSON parsing will not allow null keys, but null values are
possible
+ String nullValueJson = "{\n"
+ + " \"rewrite\": {\n"
+ + " \"func1\": null,\n"
+ + " \"func2\": \"replacement2\"\n"
+ + " }\n"
+ + "}";
+
+ DdlException exception = Assertions.assertThrows(DdlException.class,
() -> {
+ FunctionRewriteRules.check(nullValueJson);
+ });
+ Assertions.assertTrue(exception.getMessage().contains("Failed to parse
rewrite rules"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleWithComplexCustomRules() {
+ // Test complex custom rewrite rules that override and extend default
rules
+ String complexJson = "{\n"
+ + " \"rewrite\": {\n"
+ + " \"nvl\": \"custom_nvl\",\n"
+ + " \"custom_func1\": \"transformed_func1\",\n"
+ + " \"old_function\": \"new_function\"\n"
+ + " }\n"
+ + "}";
+
+ // Test with MySQL (has default rewrite rules)
+ FunctionRewriteRules mysqlRule = FunctionRewriteRules.create("mysql",
complexJson);
+
+ // Custom rewrite rules should override default rules
+ Assertions.assertEquals("custom_nvl",
mysqlRule.rewriteFunction("nvl")); // overridden
+ Assertions.assertEquals("transformed_func1",
mysqlRule.rewriteFunction("custom_func1")); // custom
+ Assertions.assertEquals("new_function",
mysqlRule.rewriteFunction("old_function")); // custom
+
+ // Default MySQL rewrite rules that are not overridden should still
work
+ Assertions.assertEquals("date", mysqlRule.rewriteFunction("to_date"));
// default
+
+ // Functions not in any rewrite map should return original name
+ Assertions.assertEquals("sum", mysqlRule.rewriteFunction("sum"));
+ Assertions.assertEquals("count", mysqlRule.rewriteFunction("count"));
+ }
+
+ @Test
+ public void testFunctionRewriteRuleCreateWithMultipleDataSources() {
+ // Test that different datasources have different default rules
+ FunctionRewriteRules mysqlRule = FunctionRewriteRules.create("mysql",
null);
+ FunctionRewriteRules clickhouseRule =
FunctionRewriteRules.create("clickhouse", null);
+ FunctionRewriteRules oracleRule =
FunctionRewriteRules.create("oracle", null);
+
+ // Same function should be rewritten differently for different
datasources
+ Assertions.assertEquals("ifnull", mysqlRule.rewriteFunction("nvl"));
// MySQL: nvl -> ifnull
+ Assertions.assertEquals("nvl", clickhouseRule.rewriteFunction("nvl"));
// ClickHouse: no rewrite
+ Assertions.assertEquals("nvl", oracleRule.rewriteFunction("nvl")); //
Oracle: no rewrite
+
+ Assertions.assertEquals("ifnull",
mysqlRule.rewriteFunction("ifnull")); // MySQL: no rewrite
+ Assertions.assertEquals("ifnull",
clickhouseRule.rewriteFunction("ifnull")); // ClickHouse: no rewrite
+ Assertions.assertEquals("nvl", oracleRule.rewriteFunction("ifnull"));
// Oracle: ifnull -> nvl
+
+ Assertions.assertEquals("FROM_UNIXTIME",
+ clickhouseRule.rewriteFunction("from_unixtime")); //
ClickHouse specific
+ Assertions.assertEquals("from_unixtime",
mysqlRule.rewriteFunction("from_unixtime")); // No rewrite in MySQL
+ Assertions.assertEquals("from_unixtime",
oracleRule.rewriteFunction("from_unixtime")); // No rewrite in Oracle
+ }
+
+ @Test
+ public void testFunctionRewriteRuleRewriteFunctionEdgeCases() {
+ // Test edge cases for rewriteFunction method
+ String jsonRules = "{\n"
+ + " \"rewrite\": {\n"
+ + " \"normal_func\": \"replaced_func\",\n"
+ + " \"UPPER_CASE\": \"lower_case\",\n"
+ + " \"Mixed_Case\": \"another_case\"\n"
+ + " }\n"
+ + "}";
+
+ FunctionRewriteRules rule = FunctionRewriteRules.create("unknown",
jsonRules);
+
+ // Test exact matches
+ Assertions.assertEquals("replaced_func",
rule.rewriteFunction("normal_func"));
+ Assertions.assertEquals("lower_case",
rule.rewriteFunction("UPPER_CASE"));
+ Assertions.assertEquals("another_case",
rule.rewriteFunction("Mixed_Case"));
+
+ // Test case sensitivity - should not match different cases
+ Assertions.assertEquals("Normal_Func",
rule.rewriteFunction("Normal_Func")); // different case
+ Assertions.assertEquals("upper_case",
rule.rewriteFunction("upper_case")); // different case
+ Assertions.assertEquals("mixed_case",
rule.rewriteFunction("mixed_case")); // different case
+
+ // Test special characters
+ Assertions.assertEquals("func_with_underscore",
rule.rewriteFunction("func_with_underscore"));
+ Assertions.assertEquals("func123", rule.rewriteFunction("func123"));
+ Assertions.assertEquals("func-with-dash",
rule.rewriteFunction("func-with-dash"));
+ }
+}
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
index 2300ece6253..45fe7738783 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/CreateIcebergTableTest.java
@@ -72,7 +72,7 @@ public class CreateIcebergTableTest {
if (icebergCatalog.getUseMetaCache().get()) {
icebergCatalog.makeSureInitialized();
} else {
- icebergCatalog.setInitialized(true);
+ icebergCatalog.setInitializedForTest(true);
}
// create db
@@ -82,7 +82,7 @@ public class CreateIcebergTableTest {
if (icebergCatalog.getUseMetaCache().get()) {
icebergCatalog.makeSureInitialized();
} else {
- icebergCatalog.setInitialized(true);
+ icebergCatalog.setInitializedForTest(true);
}
IcebergExternalDatabase db = new
IcebergExternalDatabase(icebergCatalog, 1L, dbName, dbName);
icebergCatalog.addDatabaseForTest(db);
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java
index 35dd64515b6..da66b3c5ba9 100644
---
a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java
+++
b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalCatalogTest.java
@@ -31,7 +31,7 @@ public class PaimonExternalCatalogTest {
HashMap<String, String> props = new HashMap<>();
props.put("warehouse", "not_exist");
PaimonExternalCatalog catalog = new PaimonFileExternalCatalog(1,
"name", "resource", props, "comment");
- catalog.setInitialized(true);
+ catalog.setInitializedForTest(true);
try {
catalog.getPaimonTable("dbName", "tblName");
diff --git
a/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java
b/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java
index b57bbcb51a2..602b2c8a18a 100644
--- a/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java
+++ b/fe/fe-core/src/test/java/org/apache/doris/planner/HiveTableSinkTest.java
@@ -88,7 +88,7 @@ public class HiveTableSinkTest {
mockDifferLocationTable(location);
HMSExternalCatalog hmsExternalCatalog = new HMSExternalCatalog();
- hmsExternalCatalog.setInitialized(true);
+ hmsExternalCatalog.setInitializedForTest(true);
HMSExternalDatabase db = new
HMSExternalDatabase(hmsExternalCatalog, 10000, "hive_db1", "hive_db1");
HMSExternalTable tbl = new HMSExternalTable(10001, "hive_tbl1",
"hive_db1", hmsExternalCatalog, db);
HiveTableSink hiveTableSink = new HiveTableSink(tbl);
diff --git
a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
index 3e625596d99..9d4295fc633 100644
---
a/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
+++
b/regression-test/suites/external_table_p0/jdbc/test_clickhouse_jdbc_catalog.groovy
@@ -143,6 +143,75 @@ suite("test_clickhouse_jdbc_catalog",
"p0,external,clickhouse,external_docker,ex
order_qt_clickhouse_7_schema_tvf """ select * from query('catalog' =
'clickhouse_7_schema', 'query' = 'select * from doris_test.type;') order by 1;
"""
order_qt_clickhouse_7_schema_tvf_arr """ select * from query('catalog'
= 'clickhouse_7_schema', 'query' = 'select * from doris_test.arr;') order by 1;
"""
- sql """ drop catalog if exists clickhouse_7_schema """
+ // test function rules
+ // test push down
+ sql """ drop catalog if exists clickhouse_7_catalog """
+ // test invalid config
+ test {
+ sql """ create catalog if not exists clickhouse_7_catalog
properties(
+ "type"="jdbc",
+ "user"="default",
+ "password"="123456",
+ "jdbc_url" =
"jdbc:clickhouse://${externalEnvIp}:${clickhouse_port}/doris_test?databaseTerm=schema",
+ "driver_url" = "${driver_url_7}",
+ "driver_class" =
"com.clickhouse.jdbc.ClickHouseDriver",
+ "function_rules" = '{"pushdown" : {"supported" :
[null]}}'
+ );"""
+
+ exception """Failed to parse push down rules: {"pushdown" :
{"supported" : [null]}}"""
+ }
+
+ sql """ create catalog if not exists clickhouse_7_catalog properties(
+ "type"="jdbc",
+ "user"="default",
+ "password"="123456",
+ "jdbc_url" =
"jdbc:clickhouse://${externalEnvIp}:${clickhouse_port}/doris_test?databaseTerm=schema",
+ "driver_url" = "${driver_url_7}",
+ "driver_class" = "com.clickhouse.jdbc.ClickHouseDriver",
+ "function_rules" = '{"pushdown" : {"supported": ["abs"]}}'
+ );"""
+ sql "use clickhouse_7_catalog.doris_test"
+ explain {
+ sql("select k4 from type where abs(k4) > 0 and unix_timestamp(k4)
> 0")
+ contains """SELECT "k4" FROM "doris_test"."type" WHERE ((abs("k4")
> 0)) AND ((toUnixTimestamp("k4") > 0))"""
+ contains """PREDICATES: ((abs(CAST(k4[#3] AS double)) > 0) AND
(unix_timestamp(k4[#3]) > 0))"""
+ }
+ sql """alter catalog clickhouse_7_catalog set
properties("function_rules" = '');"""
+ explain {
+ sql("select k4 from type where abs(k4) > 0 and unix_timestamp(k4)
> 0")
+ contains """QUERY: SELECT "k4" FROM "doris_test"."type" WHERE
((toUnixTimestamp("k4") > 0))"""
+ contains """PREDICATES: ((abs(CAST(k4[#3] AS double)) > 0) AND
(unix_timestamp(k4[#3]) > 0))"""
+ }
+
+ sql """alter catalog clickhouse_7_catalog set
properties("function_rules" = '{"pushdown" : {"supported": ["abs"]}}')"""
+ explain {
+ sql("select k4 from type where abs(k4) > 0 and unix_timestamp(k4)
> 0")
+ contains """SELECT "k4" FROM "doris_test"."type" WHERE ((abs("k4")
> 0)) AND ((toUnixTimestamp("k4") > 0))"""
+ contains """PREDICATES: ((abs(CAST(k4[#3] AS double)) > 0) AND
(unix_timestamp(k4[#3]) > 0))"""
+ }
+
+ // test rewrite
+ sql """alter catalog clickhouse_7_catalog set
properties("function_rules" = '{"pushdown" : {"supported": ["abs"]}, "rewrite"
: {"unix_timestamp" : "rewrite_func"}}')"""
+ explain {
+ sql("select k4 from type where abs(k4) > 0 and unix_timestamp(k4)
> 0")
+ contains """QUERY: SELECT "k4" FROM "doris_test"."type" WHERE
((abs("k4") > 0)) AND ((rewrite_func("k4") > 0))"""
+ contains """((abs(CAST(k4[#3] AS double)) > 0) AND
(unix_timestamp(k4[#3]) > 0))"""
+ }
+
+ // reset function rules
+ sql """alter catalog clickhouse_7_catalog set
properties("function_rules" = '');"""
+ explain {
+ sql("select k4 from type where abs(k4) > 0 and unix_timestamp(k4)
> 0")
+ contains """QUERY: SELECT "k4" FROM "doris_test"."type" WHERE
((toUnixTimestamp("k4") > 0))"""
+ contains """PREDICATES: ((abs(CAST(k4[#3] AS double)) > 0) AND
(unix_timestamp(k4[#3]) > 0))"""
+ }
+
+ // test invalid config
+ test {
+ sql """alter catalog clickhouse_7_catalog set
properties("function_rules" = 'invalid_json')"""
+ exception """Failed to parse push down rules: invalid_json"""
+ }
+
+ // sql """ drop catalog if exists clickhouse_7_schema """
}
}
diff --git
a/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy
b/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy
index 4a32e2e206a..16c99826311 100644
--- a/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy
+++ b/regression-test/suites/external_table_p0/jdbc/test_jdbc_query_mysql.groovy
@@ -941,6 +941,121 @@ suite("test_jdbc_query_mysql",
"p0,external,mysql,external_docker,external_docke
order_qt_sql111 """ SELECT rank() OVER () FROM (SELECT k8 FROM
$jdbcMysql57Table1 LIMIT 10) as t LIMIT 3 """
order_qt_sql112 """ SELECT k7, count(DISTINCT k8) FROM
$jdbcMysql57Table1 WHERE k8 > 110 GROUP BY GROUPING SETS ((), (k7)) """
+ // test function rules
+ sql """ drop table if exists jdbc_table_function_rule """
+ test {
+ sql """
+ CREATE EXTERNAL TABLE `jdbc_table_function_rule` (
+ `products_id` int(11) NOT NULL,
+ `orders_id` int(11) NOT NULL,
+ `sales_add_time` datetime NOT NULL,
+ `sales_update_time` datetime NOT NULL,
+ `finance_admin` int(11) NOT NULL
+ ) ENGINE=JDBC
+ COMMENT "JDBC Mysql 外部表"
+ PROPERTIES (
+ "resource" = "$jdbcResourceMysql57",
+ "table" = "ex_tb4",
+ "table_type"="mysql",
+ "function_rules" = '{"pushdown" : {"supported" : [null]}}'
+ );
+ """
+
+ exception """Failed to parse push down rules: {"pushdown" :
{"supported" : [null]}}"""
+ }
+
+ sql """
+ CREATE EXTERNAL TABLE `jdbc_table_function_rule` (
+ `products_id` int(11) NOT NULL,
+ `orders_id` int(11) NOT NULL,
+ `sales_add_time` datetime NOT NULL,
+ `sales_update_time` datetime NOT NULL,
+ `finance_admin` int(11) NOT NULL
+ ) ENGINE=JDBC
+ COMMENT "JDBC Mysql 外部表"
+ PROPERTIES (
+ "resource" = "$jdbcResourceMysql57",
+ "table" = "ex_tb4",
+ "table_type"="mysql",
+ "function_rules" = '{"pushdown" : {"supported" :
["date_trunc"]}}'
+ );
+ """
+ explain {
+ sql """select products_id from jdbc_table_function_rule where
abs(products_id) > 0 and date_trunc(`sales_add_time`, "month") = "2013-10-01
00:00:00";"""
+ contains """QUERY: SELECT `products_id`, `sales_add_time` FROM
`ex_tb4` WHERE (date_trunc(`sales_add_time`, 'month') = '2013-10-01
00:00:00')"""
+ contains """PREDICATES: ((abs(products_id[#0]) > 0) AND
(date_trunc(sales_add_time[#2], 'month') = '2013-10-01 00:00:00'))"""
+ }
+
+ sql """drop table jdbc_table_function_rule"""
+ sql """
+ CREATE EXTERNAL TABLE `jdbc_table_function_rule` (
+ `products_id` int(11) NOT NULL,
+ `orders_id` int(11) NOT NULL,
+ `sales_add_time` datetime NOT NULL,
+ `sales_update_time` datetime NOT NULL,
+ `finance_admin` int(11) NOT NULL
+ ) ENGINE=JDBC
+ COMMENT "JDBC Mysql 外部表"
+ PROPERTIES (
+ "resource" = "$jdbcResourceMysql57",
+ "table" = "ex_tb4",
+ "table_type"="mysql",
+ "function_rules" = ''
+ );
+ """
+ explain {
+ sql """select products_id from jdbc_table_function_rule where
abs(products_id) > 0 and date_trunc(`sales_add_time`, "month") = "2013-10-01
00:00:00";"""
+ contains """QUERY: SELECT `products_id`, `sales_add_time` FROM
`ex_tb4` WHERE ((abs(`products_id`) > 0))"""
+ contains """PREDICATES: ((abs(products_id[#0]) > 0) AND
(date_trunc(sales_add_time[#2], 'month') = '2013-10-01 00:00:00'))"""
+ }
+
+ sql """drop table jdbc_table_function_rule"""
+ sql """
+ CREATE EXTERNAL TABLE `jdbc_table_function_rule` (
+ `products_id` int(11) NOT NULL,
+ `orders_id` int(11) NOT NULL,
+ `sales_add_time` datetime NOT NULL,
+ `sales_update_time` datetime NOT NULL,
+ `finance_admin` int(11) NOT NULL
+ ) ENGINE=JDBC
+ COMMENT "JDBC Mysql 外部表"
+ PROPERTIES (
+ "resource" = "$jdbcResourceMysql57",
+ "table" = "ex_tb4",
+ "table_type"="mysql",
+ "function_rules" = '{"pushdown" : {"supported":
["date_trunc"], "unsupported" : ["abs"]}}'
+ );
+ """
+ explain {
+ sql """select products_id from jdbc_table_function_rule where
abs(products_id) > 0 and date_trunc(`sales_add_time`, "month") = "2013-10-01
00:00:00";"""
+ contains """QUERY: SELECT `products_id`, `sales_add_time` FROM
`ex_tb4` WHERE (date_trunc(`sales_add_time`, 'month') = '2013-10-01
00:00:00')"""
+ contains """PREDICATES: ((abs(products_id[#0]) > 0) AND
(date_trunc(sales_add_time[#2], 'month') = '2013-10-01 00:00:00'))"""
+ }
+
+ // test rewrite
+ sql """drop table jdbc_table_function_rule"""
+ sql """
+ CREATE EXTERNAL TABLE `jdbc_table_function_rule` (
+ `products_id` int(11) NOT NULL,
+ `orders_id` int(11) NOT NULL,
+ `sales_add_time` datetime NOT NULL,
+ `sales_update_time` datetime NOT NULL,
+ `finance_admin` int(11) NOT NULL
+ ) ENGINE=JDBC
+ COMMENT "JDBC Mysql 外部表"
+ PROPERTIES (
+ "resource" = "$jdbcResourceMysql57",
+ "table" = "ex_tb4",
+ "table_type"="mysql",
+ "function_rules" = '{"pushdown" : {"supported": ["to_date"],
"unsupported" : ["abs"]}, "rewrite" : {"to_date" : "date2"}}'
+ );
+ """
+ explain {
+ sql """select products_id from jdbc_table_function_rule where
to_date(sales_add_time) = "2013-10-01" and abs(products_id) > 0 and
date_trunc(`sales_add_time`, "month") = "2013-10-01 00:00:00";"""
+ contains """QUERY: SELECT `products_id`, `sales_add_time` FROM
`ex_tb4` WHERE (date2(`sales_add_time`) = '2013-10-01')"""
+ contains """PREDICATES: (((to_date(sales_add_time[#2]) =
'2013-10-01') AND (abs(products_id[#0]) > 0)) AND
(date_trunc(sales_add_time[#2], 'month') = '2013-10-01 00:00:00'))"""
+ }
+
// TODO: check this, maybe caused by datasource in JDBC
// test alter resource
sql """alter resource $jdbcResourceMysql57 properties("password" =
"1234567")"""
@@ -950,25 +1065,6 @@ suite("test_jdbc_query_mysql",
"p0,external,mysql,external_docker,external_docke
}
sql """alter resource $jdbcResourceMysql57 properties("password" =
"123456")"""
-// // test for type check
-// sql """ drop table if exists ${exMysqlTypeTable} """
-// sql """
-// CREATE EXTERNAL TABLE ${exMysqlTypeTable} (
-// `id` bigint NOT NULL,
-// `count_value` varchar(100) NULL
-// ) ENGINE=JDBC
-// COMMENT "JDBC Mysql 外部表"
-// PROPERTIES (
-// "resource" = "$jdbcResourceMysql57",
-// "table" = "ex_tb2",
-// "table_type"="mysql"
-// );
-// """
-//
-// test {
-// sql """select * from ${exMysqlTypeTable} order by id"""
-// exception "Fail to convert jdbc type of java.lang.Integer to
doris type BIGINT on column: id"
-// }
}
}
diff --git
a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
index 46f2bb371e9..2e06703e115 100644
---
a/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
+++
b/regression-test/suites/external_table_p0/jdbc/test_mysql_jdbc_catalog.groovy
@@ -72,6 +72,7 @@ suite("test_mysql_jdbc_catalog",
"p0,external,mysql,external_docker,external_doc
String dt_null = "dt_null";
String test_zd = "test_zd"
+ sql """switch internal"""
try_sql("DROP USER ${user}")
sql """CREATE USER '${user}' IDENTIFIED BY '${pwd}'"""
@@ -96,7 +97,7 @@ suite("test_mysql_jdbc_catalog",
"p0,external,mysql,external_docker,external_doc
"driver_class" = "${driver_class}"
);"""
- sql """use ${internal_db_name}"""
+ sql """use internal.${internal_db_name}"""
sql """ drop table if exists ${internal_db_name}.${inDorisTable} """
sql """
CREATE TABLE ${internal_db_name}.${inDorisTable} (
@@ -653,6 +654,79 @@ suite("test_mysql_jdbc_catalog",
"p0,external,mysql,external_docker,external_doc
// so need to test both.
sql """drop catalog if exists mysql_conjuncts;"""
sql """set enable_nereids_planner=true"""
+
+
+ // test function rules
+ // test push down
+ sql """ drop catalog if exists mysql_function_rules"""
+ // test invalid config
+ test {
+ sql """create catalog if not exists mysql_function_rules
properties(
+ "type"="jdbc",
+ "user"="root",
+ "password"="123456",
+ "jdbc_url" =
"jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false&zeroDateTimeBehavior=convertToNull",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "${driver_class}",
+ "metadata_refresh_interval_sec" = "5",
+ "function_rules" = '{"pushdown" : {"supported" : [null]}}'
+ );"""
+
+ exception """Failed to parse push down rules: {"pushdown" :
{"supported" : [null]}}"""
+ }
+
+ sql """create catalog if not exists mysql_function_rules properties(
+ "type"="jdbc",
+ "user"="root",
+ "password"="123456",
+ "jdbc_url" =
"jdbc:mysql://${externalEnvIp}:${mysql_port}/doris_test?useSSL=false&zeroDateTimeBehavior=convertToNull",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "${driver_class}",
+ "metadata_refresh_interval_sec" = "5",
+ "function_rules" = '{"pushdown" : {"supported" : ["date_trunc"]}}'
+ );"""
+
+ sql "use mysql_function_rules.doris_test"
+ explain {
+ sql """select tinyint_u from all_types where abs(tinyint_u) > 0
and date_trunc(`datetime`, "month") = "2013-10-01 00:00:00";"""
+ contains """QUERY: SELECT `tinyint_u`, `datetime` FROM
`doris_test`.`all_types` WHERE (date_trunc(`datetime`, 'month') = '2013-10-01
00:00:00')"""
+ contains """PREDICATES: ((abs(tinyint_u[#0]) > 0) AND
(date_trunc(datetime[#17], 'month') = '2013-10-01 00:00:00'))"""
+ }
+ sql """alter catalog mysql_function_rules set
properties("function_rules" = '');"""
+ explain {
+ sql """select tinyint_u from all_types where abs(tinyint_u) > 0
and date_trunc(`datetime`, "month") = "2013-10-01 00:00:00";"""
+ contains """QUERY: SELECT `tinyint_u`, `datetime` FROM
`doris_test`.`all_types` WHERE ((abs(`tinyint_u`) > 0))"""
+ contains """PREDICATES: ((abs(tinyint_u[#0]) > 0) AND
(date_trunc(datetime[#17], 'month') = '2013-10-01 00:00:00'))"""
+ }
+
+ sql """alter catalog mysql_function_rules set
properties("function_rules" = '{"pushdown" : {"supported": ["date_trunc"],
"unsupported" : ["abs"]}}')"""
+ explain {
+ sql """select tinyint_u from all_types where abs(tinyint_u) > 0
and date_trunc(`datetime`, "month") = "2013-10-01 00:00:00";"""
+ contains """QUERY: SELECT `tinyint_u`, `datetime` FROM
`doris_test`.`all_types` WHERE (date_trunc(`datetime`, 'month') = '2013-10-01
00:00:00')"""
+ contains """PREDICATES: ((abs(tinyint_u[#0]) > 0) AND
(date_trunc(datetime[#17], 'month') = '2013-10-01 00:00:00'))"""
+ }
+
+ // test rewrite
+ sql """alter catalog mysql_function_rules set
properties("function_rules" = '{"pushdown" : {"supported": ["to_date"],
"unsupported" : ["abs"]}, "rewrite" : {"to_date" : "date2"}}');"""
+ explain {
+ sql """select tinyint_u from all_types where to_date(`datetime`) =
"2013-10-01" and abs(tinyint_u) > 0 and date_trunc(`datetime`, "month") =
"2013-10-01 00:00:00";"""
+ contains """QUERY: SELECT `tinyint_u`, `datetime` FROM
`doris_test`.`all_types` WHERE (date2(`datetime`) = '2013-10-01')"""
+ contains """PREDICATES: (((to_date(datetime[#17]) = '2013-10-01')
AND (abs(tinyint_u[#0]) > 0)) AND (date_trunc(datetime[#17], 'month') =
'2013-10-01 00:00:00'))"""
+ }
+
+ // reset function rules
+ sql """alter catalog mysql_function_rules set
properties("function_rules" = '');"""
+ explain {
+ sql """select tinyint_u from all_types where to_date(`datetime`) =
"2013-10-01" and abs(tinyint_u) > 0 and date_trunc(`datetime`, "month") =
"2013-10-01 00:00:00";"""
+ contains """QUERY: SELECT `tinyint_u`, `datetime` FROM
`doris_test`.`all_types` WHERE (date(`datetime`) = '2013-10-01') AND
((abs(`tinyint_u`) > 0))"""
+ contains """PREDICATES: (((to_date(datetime[#17]) = '2013-10-01')
AND (abs(tinyint_u[#0]) > 0)) AND (date_trunc(datetime[#17], 'month') =
'2013-10-01 00:00:00'))"""
+ }
+
+ // test invalid config
+ test {
+ sql """alter catalog mysql_function_rules set
properties("function_rules" = 'invalid_json')"""
+ exception """Failed to parse push down rules: invalid_json"""
+ }
}
}
diff --git
a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy
b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy
index 4dd2607d484..818de72bca7 100644
---
a/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy
+++
b/regression-test/suites/external_table_p0/jdbc/test_oracle_jdbc_catalog.groovy
@@ -387,7 +387,77 @@ suite("test_oracle_jdbc_catalog",
"p0,external,oracle,external_docker,external_d
order_qt_null_operator9 """ SELECT * FROM STUDENT WHERE (id IS NOT
NULL AND NULL); """
order_qt_null_operator10 """ SELECT * FROM STUDENT WHERE (name IS NULL
OR age IS NOT NULL); """
- sql """ drop catalog if exists oracle_null_operator; """
+ // test function rules
+ // test push down
+ sql """ drop catalog if exists oracle_function_rules"""
+ // test invalid config
+ test {
+ sql """create catalog if not exists oracle_function_rules
properties(
+ "type"="jdbc",
+ "user"="doris_test",
+ "password"="123456",
+ "jdbc_url" =
"jdbc:oracle:thin:@${externalEnvIp}:${oracle_port}:${SID}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "oracle.jdbc.driver.OracleDriver",
+ "function_rules" = '{"pushdown" : {"supported" : [null]}}'
+ );"""
+
+ exception """Failed to parse push down rules: {"pushdown" :
{"supported" : [null]}}"""
+ }
+
+ sql """create catalog if not exists oracle_function_rules properties(
+ "type"="jdbc",
+ "user"="doris_test",
+ "password"="123456",
+ "jdbc_url" =
"jdbc:oracle:thin:@${externalEnvIp}:${oracle_port}:${SID}",
+ "driver_url" = "${driver_url}",
+ "driver_class" = "oracle.jdbc.driver.OracleDriver",
+ "function_rules" = '{"pushdown" : {"supported" : ["abs"]}}'
+ );"""
+
+ sql "use oracle_function_rules.DORIS_TEST"
+ explain {
+ sql """select id from STUDENT where abs(id) > 0 and ifnull(id, 3)
= 3;"""
+ contains """QUERY: SELECT "ID" FROM "DORIS_TEST"."STUDENT" WHERE
((abs("ID") > 0)) AND ((nvl("ID", 3) = 3))"""
+ contains """PREDICATES: ((abs(ID[#0]) > 0) AND (ifnull(ID[#0], 3)
= 3))"""
+ }
+ sql """alter catalog oracle_function_rules set
properties("function_rules" = '');"""
+ explain {
+ sql """select id from STUDENT where abs(id) > 0 and ifnull(id, 3)
= 3;"""
+ contains """QUERY: SELECT "ID" FROM "DORIS_TEST"."STUDENT" WHERE
((nvl("ID", 3) = 3))"""
+ contains """PREDICATES: ((abs(ID[#0]) > 0) AND (ifnull(ID[#0], 3)
= 3))"""
+ }
+
+ sql """alter catalog oracle_function_rules set
properties("function_rules" = '{"pushdown" : {"supported": ["abs"],
"unsupported" : []}}')"""
+ explain {
+ sql """select id from STUDENT where abs(id) > 0 and ifnull(id, 3)
= 3;"""
+ contains """QUERY: SELECT "ID" FROM "DORIS_TEST"."STUDENT" WHERE
((abs("ID") > 0)) AND ((nvl("ID", 3) = 3))"""
+ contains """PREDICATES: ((abs(ID[#0]) > 0) AND (ifnull(ID[#0], 3)
= 3))"""
+ }
+
+ // test rewrite
+ sql """alter catalog oracle_function_rules set
properties("function_rules" = '{"pushdown" : {"supported": ["abs"]}, "rewrite"
: {"abs" : "abs2"}}');"""
+ explain {
+ sql """select id from STUDENT where abs(id) > 0 and ifnull(id, 3)
= 3;"""
+ contains """QUERY: SELECT "ID" FROM "DORIS_TEST"."STUDENT" WHERE
((abs2("ID") > 0)) AND ((nvl("ID", 3) = 3))"""
+ contains """PREDICATES: ((abs(ID[#0]) > 0) AND (ifnull(ID[#0], 3)
= 3))"""
+ }
+
+ // reset function rules
+ sql """alter catalog oracle_function_rules set
properties("function_rules" = '');"""
+ explain {
+ sql """select id from STUDENT where abs(id) > 0 and ifnull(id, 3)
= 3;"""
+ contains """QUERY: SELECT "ID" FROM "DORIS_TEST"."STUDENT" WHERE
((nvl("ID", 3) = 3))"""
+ contains """PREDICATES: ((abs(ID[#0]) > 0) AND (ifnull(ID[#0], 3)
= 3))"""
+ }
+
+ // test invalid config
+ test {
+ sql """alter catalog oracle_function_rules set
properties("function_rules" = 'invalid_json')"""
+ exception """Failed to parse push down rules: invalid_json"""
+ }
+
+ // sql """ drop catalog if exists oracle_null_operator; """
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
