This is an automated email from the ASF dual-hosted git repository.
yiguolei pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 60c3de7d7d2 [fix](mysql-buffer) fix special buffer size with nested
type (#45126)
60c3de7d7d2 is described below
commit 60c3de7d7d2cbf66a4eb05978aba29c5191f5bb4
Author: amory <[email protected]>
AuthorDate: Mon Dec 16 14:52:36 2024 +0800
[fix](mysql-buffer) fix special buffer size with nested type (#45126)
### What problem does this PR solve?
in special table with sepcial data insert into
we could make sql with mysql return Malformed packet
```
mysql> select * from use1r;
ERROR 2027 (HY000): Malformed packet
```
this pr can resolve this problem
---
be/src/util/mysql_row_buffer.cpp | 6 +-
.../test_special_buffer_before_nested.out | 4 +
.../test_special_buffer_before_nested.groovy | 181 +++++++++++++++++++++
3 files changed, 190 insertions(+), 1 deletion(-)
diff --git a/be/src/util/mysql_row_buffer.cpp b/be/src/util/mysql_row_buffer.cpp
index 3e20a2d9de7..4fd7de13753 100644
--- a/be/src/util/mysql_row_buffer.cpp
+++ b/be/src/util/mysql_row_buffer.cpp
@@ -107,7 +107,11 @@ MysqlRowBuffer<is_binary_format>::~MysqlRowBuffer() {
template <bool is_binary_format>
void MysqlRowBuffer<is_binary_format>::open_dynamic_mode() {
if (!_dynamic_mode) {
- *_pos++ = NEXT_EIGHT_BYTE;
+ // if _pos now exactly at the end of _buf memory,
+ // we should reserve 1 byte for _dynamic_mode flag byte to avoid *pos
= 254
+ // cause _dynamic_mode flag byte be overwritten
+ reserve(1);
+ *_pos++ = NEXT_EIGHT_BYTE; // *_pos = 254 ; _pos++
// write length when dynamic mode close
_len_pos = (_pos - _buf);
_pos = _pos + 8;
diff --git
a/regression-test/data/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.out
b/regression-test/data/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.out
new file mode 100644
index 00000000000..8bc27332c19
--- /dev/null
+++
b/regression-test/data/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.out
@@ -0,0 +1,4 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- !sql --
+71111112 2024-06-22T00:55:50 prem \N Sarkar sultan \N
[email protected] en 1 PK \N Asia/Karachi
\N
https://aaa.dddddd.cccc/asgte/bbbccc/1997-11-18/abcdefghigklmnopk0f4c9cbff3b2aac.png
1 \N \N \N \N 1 \N 21 \N \N
\N \N 1 \N \N \N R+F-M- \N \N \N
\N \N \N \N \N wlg \N Google \N \N
\N bb851bcb725b4fc8b2495bde4898e5f5 2024-12-01T11:17:42 \N
PK \N 2024-06-24T12:52:56 \N google \N 0 \N
20 1 ["154.81.244.112", "154.80.91.113", "154.80.72.110",
"154.80.73.110", "103.131.212.96", "154.80. [...]
+
diff --git
a/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy
b/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy
new file mode 100644
index 00000000000..b7a8572b898
--- /dev/null
+++
b/regression-test/suites/datatype_p0/nested_types/corner_cases/test_special_buffer_before_nested.groovy
@@ -0,0 +1,181 @@
+import org.apache.commons.lang3.StringUtils
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_special_buffer_before_nested", "p0") {
+ // note: this column order cannot be changed which with data should
trigger mysql row buffer exactly reach 4096 before
+ // open_dynamic_mode to write array
+ sql """ DROP TABLE IF EXISTS test_special_buffer_before_nested """
+ sql """ CREATE TABLE `test_special_buffer_before_nested` (
+ `account_id` bigint NOT NULL,
+ `register_time` datetime NULL,
+ `nickname` varchar(1024) NULL,
+ `facebook_name` varchar(1024) NULL,
+ `google_name` varchar(1024) NULL,
+ `twitter_name` varchar(1024) NULL,
+ `email` varchar(1024) NULL,
+ `language` varchar(1024) NULL,
+ `gender` int NULL,
+ `country` varchar(1024) NULL,
+ `phone` varchar(1024) NULL,
+ `timezone` varchar(1024) NULL,
+ `sponsor` varchar(1024) NULL,
+ `avatar` varchar(65533) NULL,
+ `certification` int NULL,
+ `suspicious_user` int NULL,
+ `audit_user` int NULL,
+ `tt_audit_user` int NULL,
+ `auto_invite_bind` int NULL,
+ `third_party_payment` int NULL,
+ `official_payment` int NULL,
+ `user_security_level` int NULL,
+ `vip_end_time` datetime NULL,
+ `agent` int NULL,
+ `user_group_level` int NULL,
+ `nearby_toggle` int NULL,
+ `open_msg_notification` int NULL,
+ `recall_time` datetime NULL,
+ `first_recharge_time` datetime NULL,
+ `total_recharge_amount` double NULL,
+ `user_rfm_level` varchar(1024) NULL,
+ `recharge_credit_amount` double NULL,
+ `recharge_credit_count` int NULL,
+ `recharge_credit_no_callback_count` int NULL,
+ `total_withdraw_amount` double NULL,
+ `withdraw_list` array<int> NULL,
+ `user_charm_level` int NULL,
+ `user_charm_history_level` int NULL,
+ `user_auto_greetings_status` int NULL,
+ `channel` varchar(1024) NULL,
+ `bundle_id` varchar(1024) NULL,
+ `platform` varchar(1024) NULL,
+ `campaign_group_name` varchar(1024) NULL,
+ `similar_group_id` varchar(1024) NULL,
+ `cancel_account_time` datetime NULL,
+ `log_seq` varchar(1024) NULL,
+ `date_time` datetime NULL,
+ `time_millis` bigint NULL,
+ `first_country` varchar(1024) NULL,
+ `invite_time` datetime NULL,
+ `certification_time` datetime NULL,
+ `certification_url` varchar(65533) NULL,
+ `login_type` varchar(1024) NULL,
+ `vip_start_time` datetime NULL,
+ `vip_type` int NULL,
+ `agent_time` datetime NULL,
+ `recall_count` int NULL,
+ `wake_count` int NULL,
+ `client_ips` array<varchar(1024)> NULL,
+ `client_ip` varchar(1024) NULL,
+ `first_client_ip` varchar(1024) NULL,
+ `device_ids` array<varchar(1024)> NULL,
+ `device_id` varchar(1024) NULL,
+ `first_device_id` varchar(1024) NULL,
+ `phone_model` varchar(1024) NULL,
+ `os_version` varchar(1024) NULL,
+ `mcc_mnc` varchar(1024) NULL,
+ `sys_region` varchar(1024) NULL,
+ `carrier_region` varchar(1024) NULL,
+ `cloud_id` varchar(1024) NULL,
+ `device_name` varchar(1024) NULL,
+ `first_channel` varchar(1024) NULL,
+ `version` varchar(1024) NULL,
+ `first_version` varchar(1024) NULL,
+ `app_name` varchar(1024) NULL,
+ `first_app_name` varchar(1024) NULL,
+ `ad_campaign` varchar(65533) NULL,
+ `last_ad_campaign` varchar(65533) NULL,
+ `ad_campaign_id` varchar(65533) NULL,
+ `last_ad_campaign_id` varchar(65533) NULL,
+ `campaign` varchar(65533) NULL,
+ `last_campaign` varchar(65533) NULL,
+ `campaign_time` datetime NULL,
+ `last_campaign_time` datetime NULL,
+ `ad_name` varchar(65533) NULL,
+ `last_ad_name` varchar(65533) NULL,
+ `ad_advertiser` varchar(1024) NULL,
+ `last_ad_advertiser` varchar(1024) NULL,
+ `ad_advertiser_team` varchar(1024) NULL,
+ `last_ad_advertiser_team` varchar(1024) NULL,
+ `ad_app` varchar(1024) NULL,
+ `last_ad_app` varchar(1024) NULL,
+ `ad_group` varchar(65533) NULL,
+ `last_ad_group` varchar(65533) NULL,
+ `adid` varchar(65533) NULL,
+ `last_adid` varchar(65533) NULL,
+ `creative` varchar(65533) NULL,
+ `last_creative` varchar(65533) NULL,
+ `match_type` varchar(1024) NULL,
+ `last_match_type` varchar(1024) NULL,
+ `network` varchar(1024) NULL,
+ `last_network` varchar(1024) NULL,
+ `ad_country` varchar(1024) NULL,
+ `last_ad_country` varchar(1024) NULL,
+ `last_platform` varchar(1024) NULL,
+ `ad_platform` varchar(1024) NULL,
+ `last_ad_platform` varchar(1024) NULL,
+ `install_begin_time` datetime NULL,
+ `last_install_begin_time` datetime NULL,
+ `installed_at` datetime NULL,
+ `last_installed_at` datetime NULL,
+ `install_finish_time` datetime NULL,
+ `last_install_finish_time` datetime NULL,
+ `click_time` datetime NULL,
+ `last_click_time` datetime NULL,
+ `referral_time` datetime NULL,
+ `last_referral_time` datetime NULL,
+ `registration_cost` double NULL,
+ `abtest_sub` array<int> NULL,
+ `abtest_id` array<int> NULL,
+ `block_level_upgrade` int NULL,
+ `passcard_grade` int NULL,
+ `passcard_grade_end_time` datetime NULL,
+ `passcard_pay` array<datetime> NULL,
+ `passcard_number` datetime NULL,
+ `vcast_grade_charm` int NULL,
+ `vcast_grade_wealth` int NULL,
+ `im_chat_violation` int NULL,
+ `process_time` datetime NULL DEFAULT CURRENT_TIMESTAMP
+ ) ENGINE=OLAP
+ UNIQUE KEY(`account_id`)
+ DISTRIBUTED BY HASH(`account_id`) BUCKETS 64
+ PROPERTIES (
+ "replication_allocation" = "tag.location.default: 1",
+ "min_load_replica_num" = "-1",
+ "is_being_synced" = "false",
+ "storage_medium" = "hdd",
+ "storage_format" = "V2",
+ "inverted_index_storage_format" = "V1",
+ "enable_unique_key_merge_on_write" = "true",
+ "light_schema_change" = "true",
+ "store_row_column" = "true",
+ "row_store_page_size" = "16384",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "enable_mow_light_delete" = "false"
+ );
+ """
+ sql """ truncate table test_special_buffer_before_nested """
+ sql """ insert into test_special_buffer_before_nested
values(71111112,'2024-06-22 00:55:50','prem',NULL,'Sarkar
sultan',NULL,'[email protected]','en',1,'PK',NULL,'Asia/Karachi',NULL,'https://aaa.dddddd.cccc/asgte/bbbccc/1997-11-18/abcdefghigklmnopk0f4c9cbff3b2aac.png',1
+
,NULL,NULL,NULL,NULL,1,NULL,21,NULL,NULL,NULL,NULL,1,NULL,NULL,NULL,'R+F-M-',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'wlg',NULL,'Google',NULL,NULL,NULL,'bb851bcb725b4fc8b2495bde4898e5f5','2024-12-01
11:17:42',NULL,'PK',NULL,'2024-06-24 12:52:56',
+ NULL,'google',NULL,0,NULL,20,1,'["154.81.244.112",
"154.80.91.113", "154.80.72.110", "154.80.73.110", "103.131.212.96",
"154.80.64.213", "103.131.212.19", "154.80.116.29", "154.80.117.215",
"103.131.212.45", "154.80.115.43", "154.80.78.149", "154.80.65.149",
"154.80.83.110", "154.80.67.46", "154.81.243.198", "154.80.83.118",
"154.81.244.140", "154.80.105.177", "154.80.123.254", "154.80.99.141",
"154.80.109.50", "154.80.100.66", "103.131.213.174", "154.80.122.66",
"103.131.213 [...]
+
'154.80.118.95','154.81.244.112','["7e5007a7-09e3-4ffa-888b-4487a654c39b"]','7e5007a7-09e3-4ffa-888b-4487a654c39b','7e5007a7-09e3-4ffa-888b-4487a654c39b','vivo_vivo%2B1814','Android_Android_8.1.0','41001','US',NULL,NULL,NULL,'wlg','3.4.6','3.3.5','com.video.welive','com.video.welive',
+
'EN_wlg_zt_ray_gg_ac_2.5_Sup_all_240409_vo',NULL,'21174044377',NULL,NULL,NULL,NULL,NULL,'b3202c4487fe9728e68d748350b84f86',NULL,'ray',NULL,'zt',NULL,'wlg',NULL,'mix-EN-0409',NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'EN',NULL,NULL,'Google',
+ NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,NULL,'[59,
4]','[17, 1]',NULL,NULL,NULL,NULL,NULL,NULL,NULL,1,'2024-12-02 09:50:52'); """
+ qt_sql """ select * from test_special_buffer_before_nested """
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
