This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2a57976238c [enhancement](delete-pred) enable delete sub predicate v2
for compaction (#35859)
2a57976238c is described below
commit 2a57976238c2e0a5022c2d258f1c2f5576cbdd70
Author: Siyang Tang <[email protected]>
AuthorDate: Wed Jun 5 09:37:50 2024 +0800
[enhancement](delete-pred) enable delete sub predicate v2 for compaction
(#35859)
This PR enable elete sub predicate v2 for compaction, and legacy version
of delete predicate will be processed in the original way.
---
be/src/olap/delete_handler.cpp | 85 +++++++++++++++++++++---------------------
be/src/olap/delete_handler.h | 14 ++-----
be/src/olap/tablet_reader.cpp | 6 ---
3 files changed, 45 insertions(+), 60 deletions(-)
diff --git a/be/src/olap/delete_handler.cpp b/be/src/olap/delete_handler.cpp
index ca28d07254e..6e390874126 100644
--- a/be/src/olap/delete_handler.cpp
+++ b/be/src/olap/delete_handler.cpp
@@ -21,12 +21,9 @@
#include <gen_cpp/olap_file.pb.h>
#include <thrift/protocol/TDebugProtocol.h>
-#include <algorithm>
#include <boost/regex.hpp>
-#include <limits>
#include <sstream>
#include <string>
-#include <utility>
#include <vector>
#include "common/config.h"
@@ -49,6 +46,47 @@ using ::google::protobuf::RepeatedPtrField;
namespace doris {
using namespace ErrorCode;
+// construct sub condition from TCondition
+std::string construct_sub_predicate(const TCondition& condition) {
+ string op = condition.condition_op;
+ if (op == "<") {
+ op += "<";
+ } else if (op == ">") {
+ op += ">";
+ }
+ string condition_str;
+ if ("IS" == op) {
+ // ATTN: tricky! Surround IS with spaces to make it "special"
+ condition_str = condition.column_name + " IS " +
condition.condition_values[0];
+ } else { // multi-elements IN expr has been processed with InPredicatePB
+ if (op == "*=") {
+ op = "=";
+ } else if (op == "!*=") {
+ op = "!=";
+ }
+ condition_str = condition.column_name + op + "'" +
condition.condition_values[0] + "'";
+ }
+ return condition_str;
+}
+
+// make operators from FE adaptive to BE
+std::string trans_op(const std::string& opt) {
+ std::string op = string(opt);
+ if (op == "<") {
+ op += "<";
+ } else if (op == ">") {
+ op += ">";
+ }
+ if ("IS" != op) {
+ if (op == "*=") {
+ op = "=";
+ } else if (op == "!*=") {
+ op = "!=";
+ }
+ }
+ return op;
+}
+
Status DeleteHandler::generate_delete_predicate(const TabletSchema& schema,
const std::vector<TCondition>&
conditions,
DeletePredicatePB* del_pred) {
@@ -126,45 +164,6 @@ void
DeleteHandler::convert_to_sub_pred_v2(DeletePredicatePB* delete_pred,
}
}
-std::string DeleteHandler::construct_sub_predicate(const TCondition&
condition) {
- string op = condition.condition_op;
- if (op == "<") {
- op += "<";
- } else if (op == ">") {
- op += ">";
- }
- string condition_str;
- if ("IS" == op) {
- // ATTN: tricky! Surround IS with spaces to make it "special"
- condition_str = condition.column_name + " IS " +
condition.condition_values[0];
- } else { // multi-elements IN expr has been processed with InPredicatePB
- if (op == "*=") {
- op = "=";
- } else if (op == "!*=") {
- op = "!=";
- }
- condition_str = condition.column_name + op + "'" +
condition.condition_values[0] + "'";
- }
- return condition_str;
-}
-
-std::string DeleteHandler::trans_op(const std::string& opt) {
- std::string op = string(opt);
- if (op == "<") {
- op += "<";
- } else if (op == ">") {
- op += ">";
- }
- if ("IS" != op) {
- if (op == "*=") {
- op = "=";
- } else if (op == "!*=") {
- op = "!=";
- }
- }
- return op;
-}
-
bool DeleteHandler::is_condition_value_valid(const TabletColumn& column,
const std::string& condition_op,
const string& value_str) {
@@ -395,7 +394,7 @@ Status DeleteHandler::init(TabletSchemaSPtr tablet_schema,
const auto& delete_condition = delete_pred->delete_predicate();
DeleteConditions temp;
temp.filter_version = delete_pred->version().first;
- if (with_sub_pred_v2) {
+ if (with_sub_pred_v2 && !delete_condition.sub_predicates_v2().empty())
{
RETURN_IF_ERROR(_parse_column_pred(tablet_schema,
delete_pred_related_schema,
delete_condition.sub_predicates_v2(), &temp));
} else {
diff --git a/be/src/olap/delete_handler.h b/be/src/olap/delete_handler.h
index bce18669c58..a2b38cd1548 100644
--- a/be/src/olap/delete_handler.h
+++ b/be/src/olap/delete_handler.h
@@ -18,12 +18,9 @@
#pragma once
#include <butil/macros.h>
-#include <stdint.h>
-#include <memory>
+#include <cstdint>
#include <string>
-#include <unordered_map>
-#include <vector>
#include "common/factory_creator.h"
#include "common/status.h"
@@ -88,12 +85,6 @@ private:
const std::string& condition_op,
const std::string& value_str);
- // construct sub condition from TCondition
- static std::string construct_sub_predicate(const TCondition& condition);
-
- // make operators from FE adaptive to BE
- [[nodiscard]] static std::string trans_op(const string& op);
-
// extract 'column_name', 'op' and 'operands' to condition
static Status parse_condition(const DeleteSubPredicatePB& sub_cond,
TCondition* condition);
@@ -107,7 +98,8 @@ public:
// input:
// * schema: tablet's schema, the delete conditions and data rows are
in this schema
// * version: maximum version
- // * with_sub_pred_v2: whether to use delete sub predicate v2 (v2 is
based on PB, v1 is based on condition string)
+ // * with_sub_pred_v2: whether to use delete sub predicate v2 (v2 is
based on PB and use column uid to specify a column,
+ // v1 is based on condition string, and relies on regex for parse)
// return:
// * Status::Error<DELETE_INVALID_PARAMETERS>(): input parameters are
not valid
// * Status::Error<MEM_ALLOC_FAILED>(): alloc memory failed
diff --git a/be/src/olap/tablet_reader.cpp b/be/src/olap/tablet_reader.cpp
index 728e6e3d6d7..f1003314a6f 100644
--- a/be/src/olap/tablet_reader.cpp
+++ b/be/src/olap/tablet_reader.cpp
@@ -643,13 +643,7 @@ Status TabletReader::_init_delete_condition(const
ReaderParams& read_params) {
((read_params.reader_type ==
ReaderType::READER_CUMULATIVE_COMPACTION &&
config::enable_delete_when_cumu_compaction)) ||
read_params.reader_type == ReaderType::READER_CHECKSUM);
- if (_filter_delete) {
- // note(tsy): for compaction, keep delete sub pred v1 temporarily
- return _delete_handler.init(_tablet_schema,
read_params.delete_predicates,
- read_params.version.second, false);
- }
auto* runtime_state = read_params.runtime_state;
- // note(tsy): for query, use session var to enable delete sub pred v2, for
schema change, use v2 directly
bool enable_sub_pred_v2 =
runtime_state == nullptr ? true :
runtime_state->enable_delete_sub_pred_v2();
return _delete_handler.init(_tablet_schema, read_params.delete_predicates,
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
