morningman commented on a change in pull request #8567:
URL: https://github.com/apache/incubator-doris/pull/8567#discussion_r831720169
##########
File path: be/src/common/config.h
##########
@@ -720,6 +720,10 @@ CONF_String(function_service_protocol, "h2:grpc");
// use which load balancer to select server to connect
CONF_String(rpc_load_balancer, "rr");
+// a soft limit of string type length, the hard limit is 2GB - 4, but if too
long will cause very low performance,
+// so we set a soft limit, default is 10MB
+CONF_mInt32(string_type_soft_limit, "10485760");
Review comment:
`string_type_length_soft_limit_bytes`
##########
File path: be/src/olap/row_block2.cpp
##########
@@ -188,15 +190,16 @@ Status RowBlockV2::_copy_data_to_column(int cid,
doris::vectorized::MutableColum
}
case OLAP_FIELD_TYPE_STRING: {
auto column_string = assert_cast<vectorized::ColumnString*>(column);
-
+ size_t limit = std::min<size_t>(config::string_type_soft_limit,
OLAP_STRING_MAX_LENGTH);
for (uint16_t j = 0; j < _selected_size; ++j) {
if (!nullable_mark_array[j]) {
uint16_t row_idx = _selection_vector[j];
auto slice = reinterpret_cast<const
Slice*>(column_block(cid).cell_ptr(row_idx));
- if (LIKELY(slice->size <= MAX_SIZE_OF_VEC_STRING)) {
+ if (LIKELY(slice->size <= limit)) {
column_string->insert_data(slice->data, slice->size);
} else {
- return Status::NotSupported("Not support string len over
than 1MB in vec engine.");
+ return Status::NotSupported(
+ "Not support string len over than 1MB in vec
engine.");
Review comment:
Change `1MB` to `limit`
##########
File path: be/src/common/config.h
##########
@@ -720,6 +720,10 @@ CONF_String(function_service_protocol, "h2:grpc");
// use which load balancer to select server to connect
CONF_String(rpc_load_balancer, "rr");
+// a soft limit of string type length, the hard limit is 2GB - 4, but if too
long will cause very low performance,
+// so we set a soft limit, default is 10MB
+CONF_mInt32(string_type_soft_limit, "10485760");
Review comment:
better use 1MB?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
