This is an automated email from the ASF dual-hosted git repository.
kxiao pushed a commit to branch branch-2.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-2.0 by this push:
new a55c7dafba8 [Fix](point query) add query options for short circuit
queries (#32530) (#35548)
a55c7dafba8 is described below
commit a55c7dafba845ad7d9298be4e8543dfb67c89bed
Author: lihangyu <[email protected]>
AuthorDate: Thu May 30 11:49:25 2024 +0800
[Fix](point query) add query options for short circuit queries (#32530)
(#35548)
Some options like `be_exec_version` needed for functions
---
be/src/runtime/runtime_state.h | 1 +
be/src/service/point_query_executor.cpp | 18 ++++++++++++++----
be/src/service/point_query_executor.h | 2 +-
.../java/org/apache/doris/analysis/PrepareStmt.java | 17 +++++++++++++++++
.../java/org/apache/doris/planner/OriginalPlanner.java | 2 ++
.../main/java/org/apache/doris/planner/Planner.java | 6 ++++++
.../main/java/org/apache/doris/qe/PointQueryExec.java | 10 ++++++++++
gensrc/proto/internal_service.proto | 2 ++
.../suites/point_query_p0/test_point_query.groovy | 16 ++++++++++++++++
9 files changed, 69 insertions(+), 5 deletions(-)
diff --git a/be/src/runtime/runtime_state.h b/be/src/runtime/runtime_state.h
index 246d5a54783..a27266614e8 100644
--- a/be/src/runtime/runtime_state.h
+++ b/be/src/runtime/runtime_state.h
@@ -408,6 +408,7 @@ public:
OpentelemetryTracer get_tracer() { return _tracer; }
void set_tracer(OpentelemetryTracer&& tracer) { _tracer =
std::move(tracer); }
+ void set_query_options(const TQueryOptions& query_options) {
_query_options = query_options; }
bool enable_profile() const {
return _query_options.__isset.enable_profile &&
_query_options.enable_profile;
diff --git a/be/src/service/point_query_executor.cpp
b/be/src/service/point_query_executor.cpp
index d9a31b8c9d6..9017b10aef8 100644
--- a/be/src/service/point_query_executor.cpp
+++ b/be/src/service/point_query_executor.cpp
@@ -20,6 +20,7 @@
#include <fmt/format.h>
#include <gen_cpp/Descriptors_types.h>
#include <gen_cpp/Exprs_types.h>
+#include <gen_cpp/PaloInternalService_types.h>
#include <gen_cpp/internal_service.pb.h>
#include <stdlib.h>
@@ -48,9 +49,10 @@ namespace doris {
Reusable::~Reusable() {}
constexpr static int s_preallocted_blocks_num = 32;
Status Reusable::init(const TDescriptorTable& t_desc_tbl, const
std::vector<TExpr>& output_exprs,
- size_t block_size) {
+ const TQueryOptions& query_options, size_t block_size) {
SCOPED_MEM_COUNT(&_mem_size);
_runtime_state = RuntimeState::create_unique();
+ _runtime_state->set_query_options(query_options);
RETURN_IF_ERROR(DescriptorTbl::create(_runtime_state->obj_pool(),
t_desc_tbl, &_desc_tbl));
_runtime_state->set_desc_tbl(_desc_tbl);
_block_pool.resize(block_size);
@@ -184,13 +186,21 @@ Status PointQueryExecutor::init(const
PTabletKeyLookupRequest* request,
reinterpret_cast<const
uint8_t*>(request->output_expr().data()), &len, false,
&t_output_exprs));
_reusable = reusable_ptr;
+ TQueryOptions t_query_options;
+ len = request->query_options().size();
+ if (request->has_query_options()) {
+ RETURN_IF_ERROR(deserialize_thrift_msg(
+ reinterpret_cast<const
uint8_t*>(request->query_options().data()), &len, false,
+ &t_query_options));
+ }
if (uuid != 0) {
// could be reused by requests after, pre allocte more blocks
- RETURN_IF_ERROR(
- reusable_ptr->init(t_desc_tbl, t_output_exprs.exprs,
s_preallocted_blocks_num));
+ RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl,
t_output_exprs.exprs, t_query_options,
+ s_preallocted_blocks_num));
LookupConnectionCache::instance()->add(uuid, reusable_ptr);
} else {
- RETURN_IF_ERROR(reusable_ptr->init(t_desc_tbl,
t_output_exprs.exprs, 1));
+ RETURN_IF_ERROR(
+ reusable_ptr->init(t_desc_tbl, t_output_exprs.exprs,
t_query_options, 1));
}
}
_tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(request->tablet_id());
diff --git a/be/src/service/point_query_executor.h
b/be/src/service/point_query_executor.h
index 768176db20c..73e92c4d276 100644
--- a/be/src/service/point_query_executor.h
+++ b/be/src/service/point_query_executor.h
@@ -71,7 +71,7 @@ public:
}
Status init(const TDescriptorTable& t_desc_tbl, const std::vector<TExpr>&
output_exprs,
- size_t block_size = 1);
+ const TQueryOptions& query_options, size_t block_size = 1);
std::unique_ptr<vectorized::Block> get_block();
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java
b/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java
index 383728b6fb4..bc78d8dde41 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/PrepareStmt.java
@@ -23,6 +23,7 @@ import org.apache.doris.qe.ConnectContext;
import org.apache.doris.thrift.TDescriptorTable;
import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TExprList;
+import org.apache.doris.thrift.TQueryOptions;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
@@ -54,6 +55,8 @@ public class PrepareStmt extends StatementBase {
// outputExprs are heavy work
private ByteString serializedDescTable;
private ByteString serializedOutputExpr;
+ private ByteString serializedQueryOptions;
+
private UUID id;
@@ -129,6 +132,16 @@ public class PrepareStmt extends StatementBase {
}
}
+ public void cacheSerializedQueryOptions(TQueryOptions queryOptions) {
+ try {
+ serializedQueryOptions = ByteString.copyFrom(
+ new TSerializer().serialize(queryOptions));
+ } catch (TException e) {
+ LOG.warn("failed to serilize queryOptions , {}", e.getMessage());
+ Preconditions.checkState(false, e.getMessage());
+ }
+ }
+
public ByteString getSerializedDescTable() {
return serializedDescTable;
}
@@ -141,6 +154,10 @@ public class PrepareStmt extends StatementBase {
return isPointQueryShortCircuit;
}
+ public ByteString getSerializedQueryOptions() {
+ return serializedQueryOptions;
+ }
+
@Override
public void analyze(Analyzer analyzer) throws UserException {
// TODO support more Statement
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
index c7e5a9a046f..a4d5762841f 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/OriginalPlanner.java
@@ -98,6 +98,7 @@ public class OriginalPlanner extends Planner {
public void plan(StatementBase queryStmt, TQueryOptions queryOptions)
throws UserException {
+ this.queryOptions = queryOptions;
createPlanFragments(queryStmt, analyzer, queryOptions);
}
@@ -290,6 +291,7 @@ public class OriginalPlanner extends Planner {
// Cache them for later request better performance
analyzer.getPrepareStmt().cacheSerializedDescriptorTable(olapScanNode.getDescTable());
analyzer.getPrepareStmt().cacheSerializedOutputExprs(rootFragment.getOutputExprs());
+
analyzer.getPrepareStmt().cacheSerializedQueryOptions(queryOptions);
}
} else if (selectStmt.isTwoPhaseReadOptEnabled()) {
// Optimize query like `SELECT ... FROM <tbl> WHERE ... ORDER
BY ... LIMIT ...`
diff --git a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
index 46c7bc91d55..35955f06b8a 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/planner/Planner.java
@@ -49,6 +49,8 @@ public abstract class Planner {
protected boolean isBlockQuery = false;
+ protected TQueryOptions queryOptions;
+
public abstract List<ScanNode> getScanNodes();
public abstract void plan(StatementBase queryStmt,
@@ -115,6 +117,10 @@ public abstract class Planner {
return isBlockQuery;
}
+ public TQueryOptions getQueryOptions() {
+ return queryOptions;
+ }
+
public abstract DescriptorTable getDescTable();
public abstract List<RuntimeFilter> getRuntimeFilters();
diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java
b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java
index 634a4967d85..24a9e6b9e12 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/qe/PointQueryExec.java
@@ -39,6 +39,7 @@ import org.apache.doris.rpc.TCustomProtocolFactory;
import org.apache.doris.system.Backend;
import org.apache.doris.thrift.TExpr;
import org.apache.doris.thrift.TExprList;
+import org.apache.doris.thrift.TQueryOptions;
import org.apache.doris.thrift.TResultBatch;
import org.apache.doris.thrift.TScanRangeLocations;
import org.apache.doris.thrift.TStatusCode;
@@ -70,8 +71,10 @@ public class PointQueryExec implements CoordInterface {
// ByteString serialized for prepared statement
private ByteString serializedDescTable;
private ByteString serializedOutputExpr;
+ private ByteString serializedQueryOptions;
private ArrayList<Expr> outputExprs;
private DescriptorTable descriptorTable;
+ private TQueryOptions queryOptions;
private long tabletID = 0;
private long timeoutMs = Config.point_query_timeout_ms; // default 10s
@@ -106,6 +109,7 @@ public class PointQueryExec implements CoordInterface {
this.equalPredicats = planRoot.getPointQueryEqualPredicates();
this.descriptorTable = planRoot.getDescTable();
this.outputExprs = fragment.getOutputExprs();
+ this.queryOptions = planner.getQueryOptions();
PrepareStmt prepareStmt = analyzer == null ? null :
analyzer.getPrepareStmt();
if (prepareStmt != null && prepareStmt.getPreparedType() ==
PrepareStmt.PreparedType.FULL_PREPARED) {
@@ -114,6 +118,7 @@ public class PointQueryExec implements CoordInterface {
this.serializedDescTable = prepareStmt.getSerializedDescTable();
this.serializedOutputExpr = prepareStmt.getSerializedOutputExprs();
this.isBinaryProtocol = true;
+ this.serializedQueryOptions =
prepareStmt.getSerializedQueryOptions();
} else {
// TODO
// planner.getDescTable().toThrift();
@@ -240,12 +245,17 @@ public class PointQueryExec implements CoordInterface {
serializedOutputExpr = ByteString.copyFrom(
new TSerializer().serialize(exprList));
}
+ if (serializedQueryOptions == null) {
+ serializedQueryOptions = ByteString.copyFrom(
+ new TSerializer().serialize(queryOptions));
+ }
InternalService.PTabletKeyLookupRequest.Builder requestBuilder
= InternalService.PTabletKeyLookupRequest.newBuilder()
.setTabletId(tabletID)
.setDescTbl(serializedDescTable)
.setOutputExpr(serializedOutputExpr)
+ .setQueryOptions(serializedQueryOptions)
.setIsBinaryRow(isBinaryProtocol);
if (cacheID != null) {
InternalService.UUID.Builder uuidBuilder =
InternalService.UUID.newBuilder();
diff --git a/gensrc/proto/internal_service.proto
b/gensrc/proto/internal_service.proto
index 8b0b2d63c9d..6c4bfce35b9 100644
--- a/gensrc/proto/internal_service.proto
+++ b/gensrc/proto/internal_service.proto
@@ -267,6 +267,8 @@ message PTabletKeyLookupRequest {
optional bytes output_expr = 5;
// return binary mysql row format if true
optional bool is_binary_row = 6;
+
+ optional bytes query_options = 8;
}
message PTabletKeyLookupResponse {
diff --git a/regression-test/suites/point_query_p0/test_point_query.groovy
b/regression-test/suites/point_query_p0/test_point_query.groovy
index 5be89b840b1..4933ec32401 100644
--- a/regression-test/suites/point_query_p0/test_point_query.groovy
+++ b/regression-test/suites/point_query_p0/test_point_query.groovy
@@ -257,6 +257,22 @@ suite("test_point_query") {
qt_sql "select /*+ SET_VAR(enable_nereids_planner=false) */ *
from test_query where customer_key = 0"
}
}
+ sql "DROP TABLE IF EXISTS test_ODS_EBA_LLREPORT";
+ sql """
+ CREATE TABLE `test_ODS_EBA_LLREPORT` (
+ `RPTNO` VARCHAR(20) NOT NULL ,
+ `A_ENTTYP` VARCHAR(6) NULL ,
+ `A_INTIME` DATETIME NULL
+ ) ENGINE=OLAP
+ UNIQUE KEY(`RPTNO`)
+ DISTRIBUTED BY HASH(`RPTNO`) BUCKETS 3
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "store_row_column" = "true"
+ );
+ """
+ sql "insert into test_ODS_EBA_LLREPORT(RPTNO) values('567890')"
+ sql "select /*+ SET_VAR(enable_nereids_planner=false) */
substr(RPTNO,2,5) from test_ODS_EBA_LLREPORT where RPTNO = '567890'"
} finally {
set_be_config.call("disable_storage_row_cache", "true")
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
