This is an automated email from the ASF dual-hosted git repository.
eldenmoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 2d093dc6a1f [chore](compile) add compile_check to some files (#53283)
2d093dc6a1f is described below
commit 2d093dc6a1f48ec1a3e1ad076db7c5cd316d96ee
Author: lihangyu <[email protected]>
AuthorDate: Wed Jul 16 11:45:00 2025 +0800
[chore](compile) add compile_check to some files (#53283)
1. add compile_check to some files
be/src/exec/rowid_fetcher.cpp
be/src/vec/json/json_parser.cpp
be/src/vec/json/path_in_data.cpp
be/src/service/point_query_executor.cpp
be/src/vec/data_types/data_type_ipv4.h
2. limit the key length of variant
---
be/src/exec/rowid_fetcher.cpp | 31 +++++++++++++---------
be/src/service/point_query_executor.cpp | 15 +++++++----
be/src/vec/data_types/data_type_ipv4.cpp | 9 +++++++
be/src/vec/data_types/data_type_ipv4.h | 4 +--
be/src/vec/json/json_parser.cpp | 16 +++++++++--
be/src/vec/json/path_in_data.cpp | 14 +++++++---
.../suites/variant_p0/column_name.groovy | 10 +++++++
7 files changed, 74 insertions(+), 25 deletions(-)
diff --git a/be/src/exec/rowid_fetcher.cpp b/be/src/exec/rowid_fetcher.cpp
index 9369ffb43a2..07cd653b175 100644
--- a/be/src/exec/rowid_fetcher.cpp
+++ b/be/src/exec/rowid_fetcher.cpp
@@ -79,6 +79,8 @@
namespace doris {
+#include "common/compile_check_begin.h"
+
Status RowIDFetcher::init() {
DorisNodesInfo nodes_info;
nodes_info.setNodes(_fetch_option.t_fetch_opt.nodes_info);
@@ -247,7 +249,7 @@ Status RowIDFetcher::fetch(const vectorized::ColumnPtr&
column_row_ids,
*vectorized::remove_nullable(column_row_ids).get()));
std::vector<PMultiGetResponse> resps(_stubs.size());
std::vector<brpc::Controller> cntls(_stubs.size());
- bthread::CountdownEvent counter(_stubs.size());
+ bthread::CountdownEvent counter(cast_set<int>(_stubs.size()));
for (size_t i = 0; i < _stubs.size(); ++i) {
cntls[i].set_timeout_ms(config::fetch_rpc_timeout_seconds * 1000);
auto callback = brpc::NewCallback(fetch_callback, &counter);
@@ -268,13 +270,14 @@ Status RowIDFetcher::fetch(const vectorized::ColumnPtr&
column_row_ids,
for (size_t i = 0; i < rows_locs.size(); ++i) {
RowsetId rowset_id;
rowset_id.init(rows_locs[i].rowset_id());
- GlobalRowLoacation grl(rows_locs[i].tablet_id(), rowset_id,
rows_locs[i].segment_id(),
- rows_locs[i].ordinal_id());
+ GlobalRowLoacation grl(rows_locs[i].tablet_id(), rowset_id,
+ cast_set<uint32_t>(rows_locs[i].segment_id()),
+ cast_set<uint32_t>(rows_locs[i].ordinal_id()));
positions[grl] = i;
};
// TODO remove this warning code
if (positions.size() < rows_locs.size()) {
- LOG(WARNING) << "contains duplicated row entry";
+ LOG(WARNING) << "cwntains duplicated row entry";
}
vectorized::IColumn::Permutation permutation;
permutation.reserve(column_row_ids->size());
@@ -360,7 +363,7 @@ Status RowIdStorageReader::read_by_rowids(const
PMultiGetRequest& request,
std::unordered_map<IteratorKey, IteratorItem, HashOfIteratorKey>
iterator_map;
// read row by row
- for (size_t i = 0; i < request.row_locs_size(); ++i) {
+ for (int i = 0; i < request.row_locs_size(); ++i) {
const auto& row_loc = request.row_locs(i);
MonotonicStopWatch watch;
watch.start();
@@ -411,11 +414,12 @@ Status RowIdStorageReader::read_by_rowids(const
PMultiGetRequest& request,
}
segment_v2::SegmentSharedPtr segment = *it;
GlobalRowLoacation row_location(row_loc.tablet_id(),
rowset->rowset_id(),
- row_loc.segment_id(),
row_loc.ordinal_id());
+
cast_set<uint32_t>(row_loc.segment_id()),
+
cast_set<uint32_t>(row_loc.ordinal_id()));
// fetch by row store, more effcient way
if (request.fetch_row_store()) {
CHECK(tablet->tablet_schema()->has_row_store_for_all_columns());
- RowLocation loc(rowset_id, segment->id(), row_loc.ordinal_id());
+ RowLocation loc(rowset_id, segment->id(),
cast_set<uint32_t>(row_loc.ordinal_id()));
std::string* value = response->add_binary_row_data();
RETURN_IF_ERROR(scope_timer_run(
[&]() { return tablet->lookup_row_data({}, loc, rowset,
stats, *value); },
@@ -625,7 +629,7 @@ Status RowIdStorageReader::read_batch_doris_format_row(
}
}
- for (size_t j = 0; j < request_block_desc.row_id_size(); ++j) {
+ for (int j = 0; j < request_block_desc.row_id_size(); ++j) {
auto file_id = request_block_desc.file_id(j);
auto file_mapping = id_file_map->get_file_mapping(file_id);
if (!file_mapping) {
@@ -738,7 +742,7 @@ Status RowIdStorageReader::read_batch_external_row(
return value;
};
- for (size_t j = 0; j < request_block_desc.row_id_size(); ++j) {
+ for (int j = 0; j < request_block_desc.row_id_size(); ++j) {
auto file_id = request_block_desc.file_id(j);
auto file_mapping = id_file_map->get_file_mapping(file_id);
if (!file_mapping) {
@@ -1016,7 +1020,7 @@ Status RowIdStorageReader::read_doris_format_row(
// if row_store_read_struct not empty, means the line we should read from
row_store
if (!row_store_read_struct.default_values.empty()) {
CHECK(tablet->tablet_schema()->has_row_store_for_all_columns());
- RowLocation loc(rowset_id, segment->id(), row_id);
+ RowLocation loc(rowset_id, segment->id(), cast_set<uint32_t>(row_id));
row_store_read_struct.row_store_buffer.clear();
RETURN_IF_ERROR(scope_timer_run(
[&]() {
@@ -1042,12 +1046,15 @@ Status RowIdStorageReader::read_doris_format_row(
iterator_map[iterator_key].segment = segment;
}
segment = iterator_item.segment;
- RETURN_IF_ERROR(segment->seek_and_read_by_rowid(full_read_schema,
&slots[x], row_id,
- column, stats,
iterator_item.iterator));
+ RETURN_IF_ERROR(segment->seek_and_read_by_rowid(full_read_schema,
&slots[x],
+
cast_set<uint32_t>(row_id), column,
+ stats,
iterator_item.iterator));
}
}
return Status::OK();
}
+#include "common/compile_check_end.h"
+
} // namespace doris
diff --git a/be/src/service/point_query_executor.cpp
b/be/src/service/point_query_executor.cpp
index 7d8beee3ce8..b5d91d39b1a 100644
--- a/be/src/service/point_query_executor.cpp
+++ b/be/src/service/point_query_executor.cpp
@@ -33,6 +33,7 @@
#include "cloud/cloud_tablet.h"
#include "cloud/config.h"
+#include "common/cast_set.h"
#include "common/consts.h"
#include "common/status.h"
#include "olap/lru_cache.h"
@@ -63,6 +64,8 @@
namespace doris {
+#include "common/compile_check_begin.h"
+
class PointQueryResultBlockBuffer final : public
vectorized::MySQLResultBlockBuffer {
public:
PointQueryResultBlockBuffer(RuntimeState* state) :
vectorized::MySQLResultBlockBuffer(state) {}
@@ -290,17 +293,17 @@ Status PointQueryExecutor::init(const
PTabletKeyLookupRequest* request,
auto reusable_ptr = std::make_shared<Reusable>();
TDescriptorTable t_desc_tbl;
TExprList t_output_exprs;
- uint32_t len = request->desc_tbl().size();
+ auto len = cast_set<uint32_t>(request->desc_tbl().size());
RETURN_IF_ERROR(
deserialize_thrift_msg(reinterpret_cast<const
uint8_t*>(request->desc_tbl().data()),
&len, false, &t_desc_tbl));
- len = request->output_expr().size();
+ len = cast_set<uint32_t>(request->output_expr().size());
RETURN_IF_ERROR(deserialize_thrift_msg(
reinterpret_cast<const
uint8_t*>(request->output_expr().data()), &len, false,
&t_output_exprs));
_reusable = reusable_ptr;
TQueryOptions t_query_options;
- len = request->query_options().size();
+ len = cast_set<uint32_t>(request->query_options().size());
if (request->has_query_options()) {
RETURN_IF_ERROR(deserialize_thrift_msg(
reinterpret_cast<const
uint8_t*>(request->query_options().data()), &len, false,
@@ -384,7 +387,7 @@ Status PointQueryExecutor::_init_keys(const
PTabletKeyLookupRequest* request) {
// 1. get primary key from conditions
std::vector<OlapTuple> olap_tuples;
olap_tuples.resize(request->key_tuples().size());
- for (size_t i = 0; i < request->key_tuples().size(); ++i) {
+ for (int i = 0; i < request->key_tuples().size(); ++i) {
const KeyTuple& key_tuple = request->key_tuples(i);
for (const std::string& key_col : key_tuple.key_column_rep()) {
olap_tuples[i].add_value(key_col);
@@ -536,7 +539,7 @@ Status PointQueryExecutor::_lookup_row_data() {
// thus missing in include_col_uids and missing_col_uids
for (size_t i = 0; i < _result_block->columns(); ++i) {
auto column = _result_block->get_by_position(i).column;
- int padding_rows = _row_hits - column->size();
+ int padding_rows = _row_hits - cast_set<int>(column->size());
if (padding_rows > 0) {
column->assume_mutable()->insert_many_defaults(padding_rows);
}
@@ -607,4 +610,6 @@ Status PointQueryExecutor::_output_data() {
return Status::OK();
}
+#include "common/compile_check_end.h"
+
} // namespace doris
diff --git a/be/src/vec/data_types/data_type_ipv4.cpp
b/be/src/vec/data_types/data_type_ipv4.cpp
index 170613ce352..e1b026c282d 100644
--- a/be/src/vec/data_types/data_type_ipv4.cpp
+++ b/be/src/vec/data_types/data_type_ipv4.cpp
@@ -28,6 +28,9 @@
#include "vec/io/reader_buffer.h"
namespace doris::vectorized {
+
+#include "common/compile_check_begin.h"
+
bool DataTypeIPv4::equals(const IDataType& rhs) const {
return typeid(rhs) == typeid(*this);
}
@@ -76,4 +79,10 @@ MutableColumnPtr DataTypeIPv4::create_column() const {
return ColumnIPv4::create();
}
+Field DataTypeIPv4::get_field(const TExprNode& node) const {
+ return Field::create_field<TYPE_IPV4>(cast_set<const unsigned
int>(node.ipv4_literal.value));
+}
+
+#include "common/compile_check_end.h"
+
} // namespace doris::vectorized
diff --git a/be/src/vec/data_types/data_type_ipv4.h
b/be/src/vec/data_types/data_type_ipv4.h
index 87cd8af1b65..2bec8aebcae 100644
--- a/be/src/vec/data_types/data_type_ipv4.h
+++ b/be/src/vec/data_types/data_type_ipv4.h
@@ -65,9 +65,7 @@ public:
std::string to_string(const IPv4& value) const;
Status from_string(ReadBuffer& rb, IColumn* column) const override;
- Field get_field(const TExprNode& node) const override {
- return Field::create_field<TYPE_IPV4>(node.ipv4_literal.value);
- }
+ Field get_field(const TExprNode& node) const override;
MutableColumnPtr create_column() const override;
diff --git a/be/src/vec/json/json_parser.cpp b/be/src/vec/json/json_parser.cpp
index be3f69e7fb4..34f61005671 100644
--- a/be/src/vec/json/json_parser.cpp
+++ b/be/src/vec/json/json_parser.cpp
@@ -27,12 +27,14 @@
#include <algorithm>
#include <string_view>
+#include "common/cast_set.h"
#include "common/config.h"
#include "common/status.h"
#include "vec/json/path_in_data.h"
#include "vec/json/simd_json_parser.h"
namespace doris::vectorized {
+#include "common/compile_check_begin.h"
template <typename ParserImpl>
std::optional<ParseResult> JSONDataParser<ParserImpl>::parse(const char*
begin, size_t length,
@@ -90,6 +92,10 @@ void JSONDataParser<ParserImpl>::traverseObject(const
JSONObject& object, ParseC
ctx.values.reserve(ctx.values.size() + object.size());
for (auto it = object.begin(); it != object.end(); ++it) {
const auto& [key, value] = *it;
+ if (key.size() >= std::numeric_limits<uint8_t>::max()) {
+ throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
+ "Key length exceeds maximum allowed size of
255 bytes.");
+ }
ctx.builder.append(key, false);
traverse(value, ctx);
ctx.builder.pop_back();
@@ -126,7 +132,11 @@ void
JSONDataParser<ParserImpl>::traverseObjectAsJsonb(const JSONObject& object,
writer.writeStartObject();
for (auto it = object.begin(); it != object.end(); ++it) {
const auto& [key, value] = *it;
- writer.writeKey(key.data(), key.size());
+ if (key.size() >= std::numeric_limits<uint8_t>::max()) {
+ throw doris::Exception(doris::ErrorCode::INVALID_ARGUMENT,
+ "Key length exceeds maximum allowed size of
255 bytes.");
+ }
+ writer.writeKey(key.data(), cast_set<uint8_t>(key.size()));
traverseAsJsonb(value, writer);
}
writer.writeEndObject();
@@ -173,7 +183,7 @@ void JSONDataParser<ParserImpl>::traverseArrayElement(const
Element& element,
ParseContext element_ctx;
element_ctx.has_nested_in_flatten = ctx.has_nested_in_flatten;
traverse(element, element_ctx);
- auto& [_, paths, values, flatten_nested, has_nested] = element_ctx;
+ auto& [_, paths, values, flatten_nested, __] = element_ctx;
size_t size = paths.size();
size_t keys_to_update = ctx.arrays_by_path.size();
for (size_t i = 0; i < size; ++i) {
@@ -306,5 +316,7 @@ StringRef JSONDataParser<ParserImpl>::getNameOfNested(const
PathInData::Parts& p
return {};
}
+#include "common/compile_check_end.h"
+
template class JSONDataParser<SimdJSONParser>;
} // namespace doris::vectorized
\ No newline at end of file
diff --git a/be/src/vec/json/path_in_data.cpp b/be/src/vec/json/path_in_data.cpp
index 61cd06cbf5b..19bef2c209b 100644
--- a/be/src/vec/json/path_in_data.cpp
+++ b/be/src/vec/json/path_in_data.cpp
@@ -24,9 +24,13 @@
#include <string_view>
+#include "common/cast_set.h"
#include "vec/common/sip_hash.h"
namespace doris::vectorized {
+
+#include "common/compile_check_begin.h"
+
PathInData::PathInData(std::string_view path_) : path(path_) {
const char* begin = path.data();
const char* end = path.data() + path.size();
@@ -51,8 +55,8 @@ PathInData::PathInData(const PathInData& other) :
path(other.path) {
PathInData::PathInData(const std::string& root, const
std::vector<std::string>& paths) {
PathInDataBuilder path_builder;
path_builder.append(root, false);
- for (const std::string& path : paths) {
- path_builder.append(path, false);
+ for (const std::string& p : paths) {
+ path_builder.append(p, false);
}
build_path(path_builder.get_parts());
build_parts(path_builder.get_parts());
@@ -132,7 +136,8 @@ void PathInData::from_protobuf(const
segment_v2::ColumnPathInfo& pb) {
Part part;
part.is_nested = part_info.is_nested();
has_nested |= part.is_nested;
- part.anonymous_array_level = part_info.anonymous_array_level();
+ part.anonymous_array_level =
+ cast_set<uint8_t, uint32_t,
false>(part_info.anonymous_array_level());
// use string_view to ref data in path
part.key = std::string_view {begin, part_info.key().length()};
parts.push_back(part);
@@ -257,4 +262,7 @@ void PathInDataBuilder::pop_back(size_t n) {
assert(n <= parts.size());
parts.resize(parts.size() - n);
}
+
+#include "common/compile_check_end.h"
+
} // namespace doris::vectorized
diff --git a/regression-test/suites/variant_p0/column_name.groovy
b/regression-test/suites/variant_p0/column_name.groovy
index 7cf7fe198b1..93228f0d88d 100644
--- a/regression-test/suites/variant_p0/column_name.groovy
+++ b/regression-test/suites/variant_p0/column_name.groovy
@@ -80,4 +80,14 @@ suite("regression_test_variant_column_name", "variant_type"){
logger.info("""INSERT INTO ${table_name} failed: """ + ex)
assertTrue(ex.toString().contains("may contains duplicated entry"));
}
+
+ // test key length larger than 255 bytes
+ def key = "a"
+ for (int i = 0; i < 256; i++) {
+ key += "a"
+ }
+ test {
+ sql """insert into var_column_name values (8, '{"${key}": "test"}')"""
+ exception "Key length exceeds maximum allowed size of 255 bytes."
+ }
}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
