This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 8e53a731ec8 [fix](schema_change) fix incorrect cache usage of schema
change expr leading to schema errors (#56602)
8e53a731ec8 is described below
commit 8e53a731ec82cd546cd436820f0da55ccfd074d5
Author: Luwei <[email protected]>
AuthorDate: Tue Sep 30 19:26:38 2025 +0800
[fix](schema_change) fix incorrect cache usage of schema change expr
leading to schema errors (#56602)
---
.../java/org/apache/doris/alter/RollupJobV2.java | 11 +-
.../org/apache/doris/alter/SchemaChangeJobV2.java | 10 +-
.../test_alter_rollup_table.groovy | 136 +++++++++++++++++++++
3 files changed, 155 insertions(+), 2 deletions(-)
diff --git a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
index c60d7ab2363..7701d04753d 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/RollupJobV2.java
@@ -397,11 +397,12 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
}
tbl.readLock();
- Map<Object, Object> objectPool = new ConcurrentHashMap<Object,
Object>();
String vaultId = tbl.getStorageVaultId();
try {
long expiration = (createTimeMs + timeoutMs) / 1000;
Preconditions.checkState(tbl.getState() == OlapTableState.ROLLUP);
+ // Create object pool per MaterializedIndex
+ Map<Long, Map<Object, Object>> indexObjectPoolMap =
Maps.newHashMap();
for (Map.Entry<Long, MaterializedIndex> entry :
this.partitionIdToRollupIndex.entrySet()) {
long partitionId = entry.getKey();
Partition partition = tbl.getPartition(partitionId);
@@ -413,6 +414,14 @@ public class RollupJobV2 extends AlterJobV2 implements
GsonPostProcessable {
Map<String, Expr> defineExprs = Maps.newHashMap();
MaterializedIndex rollupIndex = entry.getValue();
Map<Long, Long> tabletIdMap =
this.partitionIdToBaseRollupTabletIdMap.get(partitionId);
+
+ // Get or create object pool for this MaterializedIndex
+ Map<Object, Object> objectPool =
indexObjectPoolMap.get(rollupIndex.getId());
+ if (objectPool == null) {
+ objectPool = new ConcurrentHashMap<Object, Object>();
+ indexObjectPoolMap.put(rollupIndex.getId(), objectPool);
+ }
+
for (Tablet rollupTablet : rollupIndex.getTablets()) {
long rollupTabletId = rollupTablet.getId();
long baseTabletId = tabletIdMap.get(rollupTabletId);
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
index e238f310973..526f821c825 100644
--- a/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
+++ b/fe/fe-core/src/main/java/org/apache/doris/alter/SchemaChangeJobV2.java
@@ -485,7 +485,6 @@ public class SchemaChangeJobV2 extends AlterJobV2
implements GsonPostProcessable
}
tbl.readLock();
- Map<Object, Object> objectPool = new ConcurrentHashMap<Object,
Object>();
String vaultId = tbl.getStorageVaultId();
try {
long expiration = (createTimeMs + timeoutMs) / 1000;
@@ -497,6 +496,8 @@ public class SchemaChangeJobV2 extends AlterJobV2
implements GsonPostProcessable
}
Preconditions.checkState(tbl.getState() ==
OlapTableState.SCHEMA_CHANGE);
+ // Create object pool per MaterializedIndex
+ Map<Long, Map<Object, Object>> indexObjectPoolMap =
Maps.newHashMap();
for (long partitionId : partitionIndexMap.rowKeySet()) {
Partition partition = tbl.getPartition(partitionId);
Preconditions.checkNotNull(partition, partitionId);
@@ -509,6 +510,13 @@ public class SchemaChangeJobV2 extends AlterJobV2
implements GsonPostProcessable
for (Map.Entry<Long, MaterializedIndex> entry :
shadowIndexMap.entrySet()) {
long shadowIdxId = entry.getKey();
MaterializedIndex shadowIdx = entry.getValue();
+
+ // Get or create object pool for this MaterializedIndex
+ Map<Object, Object> objectPool =
indexObjectPoolMap.get(shadowIdxId);
+ if (objectPool == null) {
+ objectPool = new ConcurrentHashMap<Object, Object>();
+ indexObjectPoolMap.put(shadowIdxId, objectPool);
+ }
long originIdxId = indexIdMap.get(shadowIdxId);
Map<String, Expr> defineExprs = Maps.newHashMap();
List<Column> fullSchema =
tbl.getSchemaByIndexId(originIdxId, true);
diff --git
a/regression-test/suites/schema_change_p0/test_alter_rollup_table.groovy
b/regression-test/suites/schema_change_p0/test_alter_rollup_table.groovy
new file mode 100644
index 00000000000..fb492c63fe7
--- /dev/null
+++ b/regression-test/suites/schema_change_p0/test_alter_rollup_table.groovy
@@ -0,0 +1,136 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_alter_rollup_table") {
+ def tbName = "test_alter_rollup_table"
+
+ sql "DROP TABLE IF EXISTS ${tbName} FORCE"
+ sql """
+ CREATE TABLE `${tbName}` (
+ `user_id` bigint NOT NULL COMMENT "用户 ID",
+ `event_time` datetime NOT NULL COMMENT "事件时间",
+ `event_date` date NOT NULL,
+ `country` varchar(128) NULL DEFAULT "UNKNOWN",
+ `city` text NULL COMMENT "城市信息",
+ `age` int NULL DEFAULT "0" COMMENT "用户年龄",
+ `is_active` boolean NULL DEFAULT "TRUE" COMMENT "是否活跃",
+ `balance` decimal(18,2) NULL DEFAULT "0.00" COMMENT "账户余额",
+ `score` double NULL COMMENT "浮点分数",
+ `last_login` datetime(3) NULL DEFAULT CURRENT_TIMESTAMP(3) COMMENT
"最后登录时间",
+ `last_ip` ipv4 NULL DEFAULT "0.0.0.0" COMMENT "最近一次登录 IP",
+ `ipv6_addr` ipv6 NULL COMMENT "IPv6 地址",
+ `json_data` json NULL COMMENT "扩展 JSON 信息",
+ `user_metadata` variant NULL COMMENT "存储用户自定义半结构化数据",
+ `seq_col` bigint NULL DEFAULT "0" COMMENT "顺序列，测试 sequence",
+ `auto_inc_col` bigint NOT NULL AUTO_INCREMENT(1) COMMENT "自增列，用于测试",
+ `create_time` datetime(6) NULL DEFAULT CURRENT_TIMESTAMP COMMENT
"创建时间",
+ `update_time` datetime(6) NULL DEFAULT CURRENT_TIMESTAMP COMMENT
"更新时间",
+ `tags` array<varchar(32)> NULL,
+ `metadata` json NULL,
+ `status` int NULL DEFAULT "1",
+ `created_at` datetime NULL DEFAULT CURRENT_TIMESTAMP,
+ `optional_data` varchar(64) NULL,
+ `required_data` int NOT NULL DEFAULT "0",
+ `total_score` bigint NULL DEFAULT "0",
+ `last_status` int NULL DEFAULT "0",
+ `max_value` int NULL DEFAULT "0",
+ `min_value` int NULL DEFAULT "0",
+
+ `col1` int NULL DEFAULT "0",
+ `col2` varchar(32) NULL,
+ `a_very_long_column_name_that_is_just_under_the_limit` int NULL,
+ `long_default` varchar(255) NULL DEFAULT
"a_very_long_default_value_that_is_just_under_the_limit_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz_abcdefghijklmnopqrstuvwxyz",
+ `max_int` int NULL DEFAULT "2147483647",
+ `min_int` int NULL DEFAULT "-2147483648",
+ `high_precision` decimal(38,10) NULL DEFAULT
"1234567890123456789012345678.1234567890",
+ `test_col` int NULL DEFAULT "0",
+ `consistency_check` int NULL DEFAULT "42",
+ INDEX idx_json_data_inverted (`user_metadata`) USING INVERTED
+ ) ENGINE=OLAP
+ DUPLICATE KEY(`user_id`, `event_time`, `event_date`)
+ DISTRIBUTED BY HASH(`user_id`) BUCKETS 3
+ PROPERTIES (
+ "file_cache_ttl_seconds" = "0",
+ "bloom_filter_columns" = "country, city",
+ "is_being_synced" = "false",
+ "storage_medium" = "hdd",
+ "storage_format" = "V2",
+ "inverted_index_storage_format" = "V2",
+ "light_schema_change" = "true",
+ "disable_auto_compaction" = "false",
+ "enable_single_replica_compaction" = "false",
+ "group_commit_interval_ms" = "10000",
+ "group_commit_data_bytes" = "134217728"
+ );
+
+ """
+
+ sql """
+ insert into ${tbName}
+ (user_id, event_date, event_time, country, city, age, is_active,
balance, score, last_ip, json_data, seq_col)
+ values
+ (1001, '2025-09-19', '2025-09-19 10:00:00', 'japan', 'tokyo', 30, 1,
1000.50, 88.8, '192.168.0.1', '{"device":"iphone"}', 1),
+ (1002, '2025-09-19', '2025-09-19 11:30:00', 'usa', null, null, 0,
500.00, null, '10.0.0.2', '{"device":"android"}', 2);
+ """
+
+
+ sql """
+ ALTER TABLE ${tbName}
+ ADD ROLLUP r_event_user
+ (
+ event_date,
+ event_time,
+ user_id,
+ country,
+ city,
+ age,
+ balance
+ );
+ """
+
+ sleep(10000)
+
+ sql """
+ ALTER TABLE ${tbName}
+ ADD ROLLUP r_complex
+ (
+ event_date,
+ event_time,
+ user_id,
+ ipv6_addr,
+ last_ip,
+ json_data,
+ create_time,
+ update_time
+ );
+ """
+
+ sleep(10000)
+
+ sql """
+ ALTER TABLE ${tbName} MODIFY COLUMN event_date DATETIME
+ """
+
+ sql """
+ insert into ${tbName}
+ (user_id, event_date, event_time, country, city, age, is_active,
balance, score, last_ip, json_data, seq_col)
+ values
+ (1003, '2025-09-19', '2025-09-19 10:00:00', 'japan', 'tokyo', 30, 1,
1000.50, 88.8, '192.168.0.1', '{"device":"iphone"}', 1),
+ (1004, '2025-09-19', '2025-09-19 11:30:00', 'usa', null, null, 0,
500.00, null, '10.0.0.2', '{"device":"android"}', 2);
+ """
+}
+
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
