This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new c8418d13b5 [improvement](config)Use session variable to replace
configuration for 'enable_function_pushdown' (#11641)
c8418d13b5 is described below
commit c8418d13b5fec81ff0a6a1124799142c2061003b
Author: Jerry Hu <[email protected]>
AuthorDate: Wed Aug 10 19:25:02 2022 +0800
[improvement](config)Use session variable to replace configuration for
'enable_function_pushdown' (#11641)
---
be/src/common/config.h | 2 -
be/src/exec/olap_scan_node.cpp | 2 +-
be/src/runtime/runtime_state.h | 5 +
be/src/vec/exec/volap_scan_node.cpp | 10 +-
.../java/org/apache/doris/qe/SessionVariable.java | 11 ++
gensrc/thrift/PaloInternalService.thrift | 2 +
.../test_string_function_like_pushdown.out | 133 +++++++++++++++++++++
.../test_string_function_like_pushdown.groovy | 90 ++++++++++++++
8 files changed, 247 insertions(+), 8 deletions(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 59e56bcde3..d6eaef6fe4 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -809,8 +809,6 @@ CONF_Int32(quick_compaction_batch_size, "10");
// do compaction min rowsets
CONF_Int32(quick_compaction_min_rowsets, "10");
-CONF_mBool(enable_function_pushdown, "false");
-
// cooldown task configs
CONF_Int32(cooldown_thread_num, "5");
CONF_mInt64(generate_cooldown_task_interval_sec, "20");
diff --git a/be/src/exec/olap_scan_node.cpp b/be/src/exec/olap_scan_node.cpp
index f8d53a9e43..048a922bae 100644
--- a/be/src/exec/olap_scan_node.cpp
+++ b/be/src/exec/olap_scan_node.cpp
@@ -482,7 +482,7 @@ Status OlapScanNode::start_scan(RuntimeState* state) {
// 3.1 Using ColumnValueRange to Build StorageEngine filters
RETURN_IF_ERROR(build_key_ranges_and_filters());
// 3.2 Function pushdown
- if (config::enable_function_pushdown)
RETURN_IF_ERROR(build_function_filters());
+ if (state->enable_function_pushdown())
RETURN_IF_ERROR(build_function_filters());
VLOG_CRITICAL << "Filter idle conjuncts";
// 4. Filter idle conjunct which already trans to olap filters
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 9ab470bf64..f12f41ddee 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -146,6 +146,11 @@ public:
// Returns true if codegen is enabled for this query.
bool codegen_enabled() const { return !_query_options.disable_codegen; }
+ bool enable_function_pushdown() const {
+ return _query_options.__isset.enable_function_pushdown &&
+ _query_options.enable_function_pushdown;
+ }
+
// Create a codegen object in _codegen. No-op if it has already been
called.
// If codegen is enabled for the query, this is created when the runtime
// state is created. If codegen is disabled for the query, this is created
diff --git a/be/src/vec/exec/volap_scan_node.cpp
b/be/src/vec/exec/volap_scan_node.cpp
index 73cba8915e..71f94cb2b5 100644
--- a/be/src/vec/exec/volap_scan_node.cpp
+++ b/be/src/vec/exec/volap_scan_node.cpp
@@ -1698,9 +1698,6 @@ Status VOlapScanNode::_normalize_bloom_filter(VExpr*
expr, VExprContext* expr_ct
Status VOlapScanNode::_normalize_function_filters(VExpr* expr, VExprContext*
expr_ctx,
SlotDescriptor* slot, bool*
push_down) {
- if (!config::enable_function_pushdown) {
- return Status::OK();
- }
bool opposite = false;
VExpr* fn_expr = expr;
if (TExprNodeType::COMPOUND_PRED == expr->node_type() &&
@@ -1794,8 +1791,11 @@ VExpr* VOlapScanNode::_normalize_predicate(RuntimeState*
state, VExpr* conjunct_
if (is_key_column(slot->col_name())) {
RETURN_IF_PUSH_DOWN(_normalize_bloom_filter(
cur_expr, *(_vconjunct_ctx_ptr.get()),
slot, &push_down));
-
RETURN_IF_PUSH_DOWN(_normalize_function_filters(
- cur_expr, *(_vconjunct_ctx_ptr.get()),
slot, &push_down));
+ if (state->enable_function_pushdown()) {
+
RETURN_IF_PUSH_DOWN(_normalize_function_filters(
+ cur_expr,
*(_vconjunct_ctx_ptr.get()), slot,
+ &push_down));
+ }
}
},
*range);
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
index bd2ef62fda..6a44207d42 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java
@@ -209,6 +209,8 @@ public class SessionVariable implements Serializable,
Writable {
public static final String ENABLE_SINGLE_REPLICA_INSERT =
"enable_single_replica_insert";
+ public static final String ENABLE_FUNCTION_PUSHDOWN =
"enable_function_pushdown";
+
// session origin value
public Map<Field, String> sessionOriginValue = new HashMap<Field,
String>();
// check stmt is or not [select /*+ SET_VAR(...)*/ ...]
@@ -524,6 +526,9 @@ public class SessionVariable implements Serializable,
Writable {
@VariableMgr.VarAttr(name = ENABLE_SINGLE_REPLICA_INSERT, needForward =
true)
public boolean enableSingleReplicaInsert = false;
+ @VariableMgr.VarAttr(name = ENABLE_FUNCTION_PUSHDOWN)
+ public boolean enableFunctionPushdown;
+
public String getBlockEncryptionMode() {
return blockEncryptionMode;
}
@@ -913,6 +918,10 @@ public class SessionVariable implements Serializable,
Writable {
this.enableVectorizedEngine = enableVectorizedEngine;
}
+ public boolean getEnableFunctionPushdown() {
+ return this.enableFunctionPushdown;
+ }
+
/**
* getInsertVisibleTimeoutMs.
**/
@@ -1118,6 +1127,8 @@ public class SessionVariable implements Serializable,
Writable {
tResult.setResourceLimit(resourceLimit);
}
+ tResult.setEnableFunctionPushdown(enableFunctionPushdown);
+
return tResult;
}
diff --git a/gensrc/thrift/PaloInternalService.thrift
b/gensrc/thrift/PaloInternalService.thrift
index e2b73efd84..910f700310 100644
--- a/gensrc/thrift/PaloInternalService.thrift
+++ b/gensrc/thrift/PaloInternalService.thrift
@@ -163,6 +163,8 @@ struct TQueryOptions {
// trim tailing spaces while querying external table and stream load
44: optional bool trim_tailing_spaces_for_external_table_query = false
+
+ 45: optional bool enable_function_pushdown;
}
diff --git
a/regression-test/data/query/sql_functions/string_functions/test_string_function_like_pushdown.out
b/regression-test/data/query/sql_functions/string_functions/test_string_function_like_pushdown.out
new file mode 100644
index 0000000000..712b680f96
--- /dev/null
+++
b/regression-test/data/query/sql_functions/string_functions/test_string_function_like_pushdown.out
@@ -0,0 +1,133 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+a
+
+-- !sql --
+a
+ba
+
+-- !sql --
+a
+ab
+accb
+
+-- !sql --
+a
+ab
+accb
+ba
+bab
+
+-- !sql --
+ba
+
+-- !sql --
+ab
+
+-- !sql --
+bab
+
+-- !sql --
+accb
+
+-- !sql --
+ab
+accb
+b
+ba
+bab
+bb
+
+-- !sql --
+ab
+accb
+b
+bab
+bb
+
+-- !sql --
+b
+ba
+bab
+bb
+
+-- !sql --
+b
+bb
+
+-- !sql --
+a
+ab
+accb
+b
+bab
+bb
+
+-- !sql --
+a
+accb
+b
+ba
+bab
+bb
+
+-- !sql --
+a
+ab
+accb
+b
+ba
+bb
+
+-- !sql --
+a
+ab
+b
+ba
+bab
+bb
+
+-- !sql --
+a
+ab
+accb
+b
+ba
+bab
+bb
+
+-- !sql --
+
+-- !sql --
+1 a
+1 b
+2 bab
+2 bb
+3 ba
+4 ab
+5 accb
+
+-- !sql --
+1 a
+1 b
+2 bab
+2 bb
+3 ba
+4 ab
+5 accb
+
+-- !sql --
+1 a
+1 b
+
+-- !sql --
+2 bb
+4 ab
+
+-- !sql --
+2 bb
+
+-- !sql --
+2 bab
+2 bb
+
diff --git
a/regression-test/suites/query/sql_functions/string_functions/test_string_function_like_pushdown.groovy
b/regression-test/suites/query/sql_functions/string_functions/test_string_function_like_pushdown.groovy
new file mode 100644
index 0000000000..8e2f450bb2
--- /dev/null
+++
b/regression-test/suites/query/sql_functions/string_functions/test_string_function_like_pushdown.groovy
@@ -0,0 +1,90 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_string_function_like_pushdown", "query") {
+ sql "set enable_vectorized_engine = true;"
+ sql "set enable_function_pushdown = true;"
+ sql "set batch_size = 4096;"
+
+ def tbName = "test_string_function_like_pushdown"
+ sql "DROP TABLE IF EXISTS ${tbName}"
+ sql """
+ CREATE TABLE IF NOT EXISTS ${tbName} (
+ id int,
+ k varchar(32)
+ )
+ DISTRIBUTED BY HASH(k) BUCKETS 5 properties("replication_num" =
"1");
+ """
+ sql """
+ INSERT INTO ${tbName} VALUES
+ (1, "a"),
+ (1, "b"),
+ (2, "bb"),
+ (2, "bab"),
+ (3, "ba"),
+ (4, "ab"),
+ (5, "accb");
+ """
+ qt_sql "SELECT k FROM ${tbName} WHERE k LIKE \"a\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k LIKE \"%a\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k LIKE \"a%\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k LIKE \"%a%\" ORDER BY k;"
+
+ qt_sql "SELECT k FROM ${tbName} WHERE k LIKE \"_a\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k LIKE \"a_\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k LIKE \"_a_\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k LIKE \"a__b\" ORDER BY k;"
+
+ qt_sql "SELECT k FROM ${tbName} WHERE k NOT LIKE \"a\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k NOT LIKE \"%a\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k NOT LIKE \"a%\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k NOT LIKE \"%a%\" ORDER BY k;"
+
+ qt_sql "SELECT k FROM ${tbName} WHERE k NOT LIKE \"_a\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k NOT LIKE \"a_\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k NOT LIKE \"_a_\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k NOT LIKE \"a__b\" ORDER BY k;"
+
+ qt_sql "SELECT k FROM ${tbName} WHERE k LIKE \"%\" ORDER BY k;"
+ qt_sql "SELECT k FROM ${tbName} WHERE k NOT LIKE \"%\" ORDER BY k;"
+
+ qt_sql """
+ select * from ${tbName} where k like "%" order by id, k;
+ """
+
+ qt_sql """
+ select * from ${tbName} where k like "%" and id > 0 order by id, k;
+ """
+
+ qt_sql """
+ select * from ${tbName} where k like "_" and id = 1 order by id, k;
+ """
+
+ qt_sql """
+ select * from ${tbName} where k like "_b" and id < 5 order by id, k;
+ """
+
+ qt_sql """
+ select * from ${tbName} where k like "_b" and id < 4 order by id, k;
+ """
+
+ qt_sql """
+ select * from ${tbName} where k like "b%b" and id > 1 order by id, k;
+ """
+
+ sql "DROP TABLE ${tbName};"
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
