This is an automated email from the ASF dual-hosted git repository.
lijibing pushed a commit to branch high-priority-column
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/high-priority-column by this
push:
new 02756be71ef rebase master (#31243)
02756be71ef is described below
commit 02756be71efa67653e37889895a42c4b12fd5a2b
Author: Jibing-Li <[email protected]>
AuthorDate: Wed Feb 21 23:02:31 2024 +0800
rebase master (#31243)
* [fix](stream-load) print stream load profile for pipeline and pipelinex
(#31198)
* [fix](Nereids): OuterJoinAssoc reject literal condition (#31172)
* [Enhancement](group commit) Add bvar and log for group commit (#31017)
* [feature](cloud) add more cases to cloud_p0 (#31221)
* [fix](regression) cloud_p0 should not enable group_commit (#31232)
* [regression test]Optimize testUniqueModelSchemaKeyChange (#31150)
---------
Co-authored-by: Xin Liao <[email protected]>
Co-authored-by: jakevin <[email protected]>
Co-authored-by: abmdocrt <[email protected]>
Co-authored-by: Yongqiang YANG
<[email protected]>
Co-authored-by: meiyi <[email protected]>
Co-authored-by: kkop <[email protected]>
---
be/src/pipeline/pipeline_fragment_context.cpp | 15 ++
.../pipeline_x/pipeline_x_fragment_context.cpp | 15 ++
be/src/runtime/group_commit_mgr.cpp | 4 +
be/src/runtime/group_commit_mgr.h | 4 +
be/src/vec/sink/group_commit_block_sink.cpp | 8 +-
.../rules/exploration/join/OuterJoinAssoc.java | 2 +-
.../exploration/join/OuterJoinAssocProject.java | 2 +-
.../test_unique_model_schema_key_change.out | 121 +++++++++
.../pipeline/cloud_p0/conf/be_custom.conf | 1 -
.../pipeline/cloud_p0/conf/fe_custom.conf | 3 -
.../cloud_p0/conf/regression-conf-custom.groovy | 2 +-
.../test_unique_model_schema_key_change.groovy | 270 +++++++++++++++++++++
12 files changed, 438 insertions(+), 9 deletions(-)
diff --git a/be/src/pipeline/pipeline_fragment_context.cpp
b/be/src/pipeline/pipeline_fragment_context.cpp
index 6c183a96f64..e251b203794 100644
--- a/be/src/pipeline/pipeline_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_fragment_context.cpp
@@ -887,6 +887,21 @@ void PipelineFragmentContext::_close_fragment_instance() {
_runtime_state->runtime_profile()->total_time_counter()->update(
_fragment_watcher.elapsed_time());
static_cast<void>(send_report(true));
+ if (_is_report_success) {
+ std::stringstream ss;
+ // Compute the _local_time_percent before pretty_print the
runtime_profile
+ // Before add this operation, the print out like that:
+ // UNION_NODE (id=0):(Active: 56.720us, non-child: 00.00%)
+ // After add the operation, the print out like that:
+ // UNION_NODE (id=0):(Active: 56.720us, non-child: 82.53%)
+ // We can easily know the exec node execute time without child time
consumed.
+ _runtime_state->runtime_profile()->compute_time_in_profile();
+ _runtime_state->runtime_profile()->pretty_print(&ss);
+ if (_runtime_state->load_channel_profile()) {
+ _runtime_state->load_channel_profile()->pretty_print(&ss);
+ }
+ LOG(INFO) << ss.str();
+ }
// all submitted tasks done
_exec_env->fragment_mgr()->remove_pipeline_context(
std::dynamic_pointer_cast<PipelineFragmentContext>(shared_from_this()));
diff --git a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp
b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp
index 38b7b7457fe..9087f94db5e 100644
--- a/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp
+++ b/be/src/pipeline/pipeline_x/pipeline_x_fragment_context.cpp
@@ -1244,6 +1244,21 @@ void
PipelineXFragmentContext::_close_fragment_instance() {
Defer defer_op {[&]() { _is_fragment_instance_closed = true; }};
_runtime_profile->total_time_counter()->update(_fragment_watcher.elapsed_time());
static_cast<void>(send_report(true));
+ if (_is_report_success) {
+ std::stringstream ss;
+ // Compute the _local_time_percent before pretty_print the
runtime_profile
+ // Before add this operation, the print out like that:
+ // UNION_NODE (id=0):(Active: 56.720us, non-child: 00.00%)
+ // After add the operation, the print out like that:
+ // UNION_NODE (id=0):(Active: 56.720us, non-child: 82.53%)
+ // We can easily know the exec node execute time without child time
consumed.
+ _runtime_state->runtime_profile()->compute_time_in_profile();
+ _runtime_state->runtime_profile()->pretty_print(&ss);
+ if (_runtime_state->load_channel_profile()) {
+ _runtime_state->load_channel_profile()->pretty_print(&ss);
+ }
+ LOG(INFO) << ss.str();
+ }
// all submitted tasks done
_exec_env->fragment_mgr()->remove_pipeline_context(
std::dynamic_pointer_cast<PipelineXFragmentContext>(shared_from_this()));
diff --git a/be/src/runtime/group_commit_mgr.cpp
b/be/src/runtime/group_commit_mgr.cpp
index 7422890aa41..42c0e83f4b6 100644
--- a/be/src/runtime/group_commit_mgr.cpp
+++ b/be/src/runtime/group_commit_mgr.cpp
@@ -78,6 +78,7 @@ Status LoadBlockQueue::add_block(RuntimeState* runtime_state,
VLOG_DEBUG << "group commit meets commit condition for data size,
label=" << label
<< ", instance_id=" << load_instance_id << ", data_bytes="
<< _data_bytes;
_need_commit = true;
+ data_size_condition = true;
}
_get_cond.notify_all();
return Status::OK();
@@ -417,6 +418,9 @@ Status GroupCommitTable::_finish_group_commit_load(int64_t
db_id, int64_t table_
<< ", exec_plan_fragment status=" << status.to_string()
<< ", commit/abort txn rpc status=" << st.to_string()
<< ", commit/abort txn status=" << result_status.to_string()
+ << ", this group commit includes " <<
load_block_queue->group_commit_load_count << " loads"
+ << ", flush because meet "
+ << (load_block_queue->data_size_condition ? "data size " : "time ") <<
"condition"
<< ", wal space info:" <<
ExecEnv::GetInstance()->wal_mgr()->get_wal_dirs_info_string();
if (state) {
if (!state->get_error_log_file_path().empty()) {
diff --git a/be/src/runtime/group_commit_mgr.h
b/be/src/runtime/group_commit_mgr.h
index e3b28be5804..5357ba208f7 100644
--- a/be/src/runtime/group_commit_mgr.h
+++ b/be/src/runtime/group_commit_mgr.h
@@ -78,6 +78,10 @@ public:
int64_t txn_id;
int64_t schema_version;
bool wait_internal_group_commit_finish = false;
+ bool data_size_condition = false;
+
+ // counts of load in one group commit
+ std::atomic_size_t group_commit_load_count = 0;
// the execute status of this internal group commit
std::mutex mutex;
diff --git a/be/src/vec/sink/group_commit_block_sink.cpp
b/be/src/vec/sink/group_commit_block_sink.cpp
index 6ff9d4a1425..2ce03ba0c1f 100644
--- a/be/src/vec/sink/group_commit_block_sink.cpp
+++ b/be/src/vec/sink/group_commit_block_sink.cpp
@@ -35,6 +35,8 @@
namespace doris {
namespace vectorized {
+bvar::Adder<int64_t> g_group_commit_load_rows("doris_group_commit_load_rows");
+bvar::Adder<int64_t>
g_group_commit_load_bytes("doris_group_commit_load_bytes");
GroupCommitBlockSink::GroupCommitBlockSink(ObjectPool* pool, const
RowDescriptor& row_desc,
const std::vector<TExpr>& texprs,
Status* status)
@@ -48,6 +50,7 @@ GroupCommitBlockSink::~GroupCommitBlockSink() {
if (_load_block_queue) {
_remove_estimated_wal_bytes();
_load_block_queue->remove_load_id(_load_id);
+ _load_block_queue->group_commit_load_count.fetch_add(1);
}
}
@@ -144,12 +147,13 @@ Status GroupCommitBlockSink::send(RuntimeState* state,
vectorized::Block* input_
return status;
}
SCOPED_TIMER(_profile->total_time_counter());
+
// update incrementally so that FE can get the progress.
// the real 'num_rows_load_total' will be set when sink being closed.
state->update_num_rows_load_total(rows);
state->update_num_bytes_load_total(bytes);
- DorisMetrics::instance()->load_rows->increment(rows);
- DorisMetrics::instance()->load_bytes->increment(bytes);
+ g_group_commit_load_rows << rows;
+ g_group_commit_load_bytes << bytes;
std::shared_ptr<vectorized::Block> block;
bool has_filtered_rows = false;
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssoc.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssoc.java
index 2080cfce93d..3208654fa78 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssoc.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssoc.java
@@ -79,7 +79,7 @@ public class OuterJoinAssoc extends OneExplorationRuleFactory
{
.addAll(topJoin.getOtherJoinConjuncts()).build();
Set<Slot> notNullSlots =
ExpressionUtils.inferNotNullSlots(on,
ctx.cascadesContext);
- if (!conditionSlot.equals(notNullSlots)) {
+ if (conditionSlot.isEmpty() ||
!conditionSlot.equals(notNullSlots)) {
return null;
}
}
diff --git
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java
index 512aec7bbc7..5cd22058c04 100644
---
a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java
+++
b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/join/OuterJoinAssocProject.java
@@ -80,7 +80,7 @@ public class OuterJoinAssocProject extends
OneExplorationRuleFactory {
.addAll(topJoin.getOtherJoinConjuncts()).build();
Set<Slot> notNullSlots =
ExpressionUtils.inferNotNullSlots(on,
ctx.cascadesContext);
- if (!conditionSlot.equals(notNullSlots)) {
+ if (conditionSlot.isEmpty() ||
!conditionSlot.equals(notNullSlots)) {
return null;
}
}
diff --git
a/regression-test/data/schema_change_p0/test_unique_model_schema_key_change.out
b/regression-test/data/schema_change_p0/test_unique_model_schema_key_change.out
new file mode 100644
index 00000000000..b8e5753aff9
--- /dev/null
+++
b/regression-test/data/schema_change_p0/test_unique_model_schema_key_change.out
@@ -0,0 +1,121 @@
+-- This file is automatically generated. You should know what you did if you
want to edit this
+-- ! --
+123456789 Alice 广东省 Beijing 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+234567890 Bob 广东省 Shanghai 30 1 13998765432
No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol 广东省 Guangzhou 28 0 13724681357
No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave 广东省 Shenzhen 35 1 13680864279
No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve 广东省 Chengdu 27 0 13572468091 No. 654
Street, Chengdu 2022-05-05T18:00
+678901234 Frank 广东省 Hangzhou 32 1 13467985213
No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace 广东省 Xian 29 0 13333333333 No. 222
Street, Xian 2022-07-07T22:00
+123456689 Alice 四川省 Yaan 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice false Beijing 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+234567890 Bob false Shanghai 30 1 13998765432
No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol false Guangzhou 28 0 13724681357
No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave false Shenzhen 35 1 13680864279
No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve false Chengdu 27 0 13572468091 No. 654
Street, Chengdu 2022-05-05T18:00
+678901234 Frank false Hangzhou 32 1 13467985213
No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace false Xian 29 0 13333333333 No. 222
Street, Xian 2022-07-07T22:00
+123456689 Alice false Yaan 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice 0 Beijing 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+234567890 Bob 0 Shanghai 30 1 13998765432
No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol 0 Guangzhou 28 0 13724681357
No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave 0 Shenzhen 35 1 13680864279
No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve 0 Chengdu 27 0 13572468091 No. 654
Street, Chengdu 2022-05-05T18:00
+678901234 Frank 0 Hangzhou 32 1 13467985213
No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace 0 Xian 29 0 13333333333 No. 222
Street, Xian 2022-07-07T22:00
+123456689 Alice 0 Yaan 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice 999 Beijing 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+234567890 Bob 999 Shanghai 30 1 13998765432
No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol 999 Guangzhou 28 0 13724681357
No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave 999 Shenzhen 35 1 13680864279
No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve 999 Chengdu 27 0 13572468091 No. 654
Street, Chengdu 2022-05-05T18:00
+678901234 Frank 999 Hangzhou 32 1 13467985213
No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace 999 Xian 29 0 13333333333 No. 222
Street, Xian 2022-07-07T22:00
+123456689 Alice 567 Yaan 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice 999 Beijing 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+234567890 Bob 999 Shanghai 30 1 13998765432
No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol 999 Guangzhou 28 0 13724681357
No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave 999 Shenzhen 35 1 13680864279
No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve 999 Chengdu 27 0 13572468091 No. 654
Street, Chengdu 2022-05-05T18:00
+678901234 Frank 999 Hangzhou 32 1 13467985213
No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace 999 Xian 29 0 13333333333 No. 222
Street, Xian 2022-07-07T22:00
+123456689 Alice 2 Yaan 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice 99999991 Beijing 25 0 13812345678
No. 123 Street, Beijing 2022-01-01T10:00
+234567890 Bob 99999991 Shanghai 30 1
13998765432 No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol 99999991 Guangzhou 28 0
13724681357 No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave 99999991 Shenzhen 35 1
13680864279 No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve 99999991 Chengdu 27 0 13572468091
No. 654 Street, Chengdu 2022-05-05T18:00
+678901234 Frank 99999991 Hangzhou 32 1
13467985213 No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace 99999991 Xian 29 0 13333333333
No. 222 Street, Xian 2022-07-07T22:00
+123456689 Alice 88889494646 Yaan 25 0 13812345678
No. 123 Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice 9999 Beijing 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+234567890 Bob 9999 Shanghai 30 1 13998765432
No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol 9999 Guangzhou 28 0 13724681357
No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave 9999 Shenzhen 35 1 13680864279
No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve 9999 Chengdu 27 0 13572468091 No. 654
Street, Chengdu 2022-05-05T18:00
+678901234 Frank 9999 Hangzhou 32 1 13467985213
No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace 9999 Xian 29 0 13333333333 No. 222
Street, Xian 2022-07-07T22:00
+123456689 Alice 555888555 Yaan 25 0 13812345678
No. 123 Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice 16899.6464689000 Beijing 25 0
13812345678 No. 123 Street, Beijing 2022-01-01T10:00
+234567890 Bob 16899.6464689000 Shanghai 30 1
13998765432 No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol 16899.6464689000 Guangzhou 28 0
13724681357 No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave 16899.6464689000 Shenzhen 35 1
13680864279 No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve 16899.6464689000 Chengdu 27 0
13572468091 No. 654 Street, Chengdu 2022-05-05T18:00
+678901234 Frank 16899.6464689000 Hangzhou 32 1
13467985213 No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace 16899.6464689000 Xian 29 0
13333333333 No. 222 Street, Xian 2022-07-07T22:00
+123456689 Alice 16499.6464689000 Yaan 25 0
13812345678 No. 123 Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice 1997-01-01 Beijing 25 0 13812345678
No. 123 Street, Beijing 2022-01-01T10:00
+234567890 Bob 1997-01-01 Shanghai 30 1
13998765432 No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol 1997-01-01 Guangzhou 28 0
13724681357 No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave 1997-01-01 Shenzhen 35 1
13680864279 No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve 1997-01-01 Chengdu 27 0 13572468091
No. 654 Street, Chengdu 2022-05-05T18:00
+678901234 Frank 1997-01-01 Hangzhou 32 1
13467985213 No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace 1997-01-01 Xian 29 0 13333333333
No. 222 Street, Xian 2022-07-07T22:00
+123456689 Alice 2024-01-01 Yaan 25 0 13812345678
No. 123 Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice 1997-01-01T00:00 Beijing 25 0
13812345678 No. 123 Street, Beijing 2022-01-01T10:00
+234567890 Bob 1997-01-01T00:00 Shanghai 30 1
13998765432 No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol 1997-01-01T00:00 Guangzhou 28 0
13724681357 No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave 1997-01-01T00:00 Shenzhen 35 1
13680864279 No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve 1997-01-01T00:00 Chengdu 27 0
13572468091 No. 654 Street, Chengdu 2022-05-05T18:00
+678901234 Frank 1997-01-01T00:00 Hangzhou 32 1
13467985213 No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace 1997-01-01T00:00 Xian 29 0
13333333333 No. 222 Street, Xian 2022-07-07T22:00
+123456689 Alice 2024-01-04T09:00 Yaan 25 0
13812345678 No. 123 Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice 1997-01-01T00:00 Beijing 25 0
13812345678 No. 123 Street, Beijing 2022-01-01T10:00
+234567890 Bob 1997-01-01T00:00 Shanghai 30 1
13998765432 No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol 1997-01-01T00:00 Guangzhou 28 0
13724681357 No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave 1997-01-01T00:00 Shenzhen 35 1
13680864279 No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve 1997-01-01T00:00 Chengdu 27 0
13572468091 No. 654 Street, Chengdu 2022-05-05T18:00
+678901234 Frank 1997-01-01T00:00 Hangzhou 32 1
13467985213 No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace 1997-01-01T00:00 Xian 29 0
13333333333 No. 222 Street, Xian 2022-07-07T22:00
+123456689 Alice 2024-01-04T09:00 Yaan 25 0
13812345678 No. 123 Street, Beijing 2022-01-01T10:00
+
+-- ! --
+123456789 Alice F Beijing 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+234567890 Bob F Shanghai 30 1 13998765432
No. 456 Street, Shanghai 2022-02-02T12:00
+345678901 Carol F Guangzhou 28 0 13724681357
No. 789 Street, Guangzhou 2022-03-03T14:00
+456789012 Dave F Shenzhen 35 1 13680864279
No. 987 Street, Shenzhen 2022-04-04T16:00
+567890123 Eve F Chengdu 27 0 13572468091 No. 654
Street, Chengdu 2022-05-05T18:00
+678901234 Frank F Hangzhou 32 1 13467985213
No. 321 Street, Hangzhou 2022-06-06T20:00
+789012345 Grace F Xian 29 0 13333333333 No. 222
Street, Xian 2022-07-07T22:00
+123456689 Alice T Yaan 25 0 13812345678 No. 123
Street, Beijing 2022-01-01T10:00
+
diff --git a/regression-test/pipeline/cloud_p0/conf/be_custom.conf
b/regression-test/pipeline/cloud_p0/conf/be_custom.conf
index ae08823bfb7..2259c031ce9 100644
--- a/regression-test/pipeline/cloud_p0/conf/be_custom.conf
+++ b/regression-test/pipeline/cloud_p0/conf/be_custom.conf
@@ -22,7 +22,6 @@ enable_merge_on_write_correctness_check=true
enable_debug_points=true
prioritize_query_perf_in_compaction = true
cumulative_compaction_min_deltas = 5
-wait_internal_group_commit_finish=true
#p0 parameter
meta_service_endpoint = 127.0.0.1:5000
cloud_unique_id = cloud_unique_id_compute_node0
diff --git a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
index 9a6c6cdeb54..5a96e7f075a 100644
--- a/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
+++ b/regression-test/pipeline/cloud_p0/conf/fe_custom.conf
@@ -29,9 +29,6 @@ max_query_profile_num=1000
statistics_sql_mem_limit_in_bytes=21474836480
cpu_resource_limit_per_analyze_task=-1
-group_commit_interval_ms_default_value=2
-wait_internal_group_commit_finish=true
-
priority_networks=127.0.0.1/24
cloud_http_port=18030
meta_service_endpoint=127.0.0.1:5000
diff --git
a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
index 7bdcc2a7c9f..fa5769e676e 100644
--- a/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
+++ b/regression-test/pipeline/cloud_p0/conf/regression-conf-custom.groovy
@@ -16,5 +16,5 @@
// under the License.
testGroups = "p0"
-testDirectories =
"ddl_p0,database_p0,load,load_p0/insert,load_p0/stream_load,query_p0,table_p0,account_p0,autobucket,bitmap_functions,bloom_filter_p0,cast_decimal_to_boolean,cast_double_to_decimal,compression_p0,connector_p0,correctness,correctness_p0,csv_header_p0,data_model_p0,database_p0,datatype_p0,delete_p0,demo_p0,empty_relation,export_p0,external_table_p0,fault_injection_p0,flink_connector_p0,insert_overwrite_p0,insert_p0,internal_schema_p0,json_p0,jsonb_p0,mysql_ssl_p0,mysql_tup
[...]
+testDirectories =
"ddl_p0,database_p0,load,load_p0,query_p0,table_p0,account_p0,autobucket,bitmap_functions,bloom_filter_p0,cast_decimal_to_boolean,cast_double_to_decimal,compression_p0,connector_p0,correctness,correctness_p0,csv_header_p0,data_model_p0,database_p0,datatype_p0,delete_p0,demo_p0,empty_relation,export_p0,external_table_p0,fault_injection_p0,flink_connector_p0,insert_overwrite_p0,insert_p0,internal_schema_p0,javaudf_p0,job_p0,json_p0,jsonb_p0,meta_action_p0,metrics_p0,mtmv_
[...]
diff --git
a/regression-test/suites/schema_change_p0/test_unique_model_schema_key_change.groovy
b/regression-test/suites/schema_change_p0/test_unique_model_schema_key_change.groovy
new file mode 100644
index 00000000000..deb094fc284
--- /dev/null
+++
b/regression-test/suites/schema_change_p0/test_unique_model_schema_key_change.groovy
@@ -0,0 +1,270 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_unique_model_schema_key_change","p0") {
+ def tbName = "test_unique_model_schema_key_change"
+
+ //Test the unique model by adding a key column
+ sql """ DROP TABLE IF EXISTS ${tbName} """
+ def initTable = " CREATE TABLE IF NOT EXISTS ${tbName}\n" +
+ " (\n" +
+ " `user_id` LARGEINT NOT NULL COMMENT \"用户id\",\n" +
+ " `username` VARCHAR(50) NOT NULL COMMENT
\"用户昵称\",\n" +
+ " `city` VARCHAR(20) COMMENT \"用户所在城市\",\n" +
+ " `age` SMALLINT COMMENT \"用户年龄\",\n" +
+ " `sex` TINYINT COMMENT \"用户性别\",\n" +
+ " `phone` LARGEINT COMMENT \"用户电话\",\n" +
+ " `address` VARCHAR(500) COMMENT \"用户地址\",\n" +
+ " `register_time` DATETIME COMMENT \"用户注册时间\"\n" +
+ " )\n" +
+ " UNIQUE KEY(`user_id`, `username`)\n" +
+ " DISTRIBUTED BY HASH(`user_id`) BUCKETS 1\n" +
+ " PROPERTIES (\n" +
+ " \"replication_allocation\" = \"tag.location.default:
1\",\n" +
+ " \"enable_unique_key_merge_on_write\" = \"true\"\n" +
+ " );"
+
+ def initTableData = "insert into ${tbName} values(123456789, 'Alice',
'Beijing', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01
10:00:00')," +
+ " (234567890, 'Bob', 'Shanghai', 30, 1,
13998765432, 'No. 456 Street, Shanghai', '2022-02-02 12:00:00')," +
+ " (345678901, 'Carol', 'Guangzhou', 28, 0,
13724681357, 'No. 789 Street, Guangzhou', '2022-03-03 14:00:00')," +
+ " (456789012, 'Dave', 'Shenzhen', 35, 1,
13680864279, 'No. 987 Street, Shenzhen', '2022-04-04 16:00:00')," +
+ " (567890123, 'Eve', 'Chengdu', 27, 0, 13572468091,
'No. 654 Street, Chengdu', '2022-05-05 18:00:00')," +
+ " (678901234, 'Frank', 'Hangzhou', 32, 1,
13467985213, 'No. 321 Street, Hangzhou', '2022-06-06 20:00:00')," +
+ " (789012345, 'Grace', 'Xian', 29, 0, 13333333333,
'No. 222 Street, Xian', '2022-07-07 22:00:00');"
+
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column province VARCHAR(20) KEY
DEFAULT "广东省" AFTER username """
+ def insertSql = "insert into ${tbName} values(123456689, 'Alice', '四川省',
'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00');"
+ def getTableStatusSql = " SHOW ALTER TABLE COLUMN WHERE
IndexName='${tbName}' ORDER BY createtime DESC LIMIT 1 "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+ //Test the unique model by adding a key column with BOOLEAN
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column special_area BOOLEAN KEY
DEFAULT "0" AFTER username """
+ insertSql = "insert into ${tbName} values(123456689, 'Alice', 0, 'Yaan',
25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+
+ //Test the unique model by adding a key column with TINYINT
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column special_area TINYINT KEY
DEFAULT "0" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice', 0, 'Yaan',
25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+
+ //Test the unique model by adding a key column with SMALLINT
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column area_num SMALLINT KEY DEFAULT
"999" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice', 567,
'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+ //Test the unique model by adding a key column with INT
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column house_price INT KEY DEFAULT
"999" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice', 2, 'Yaan',
25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+ //Test the unique model by adding a key column with BIGINT
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column house_price1 BIGINT KEY
DEFAULT "99999991" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice',
88889494646, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing',
'2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+
+ //Test the unique model by adding a key column with LARGEINT
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column car_price LARGEINT KEY DEFAULT
"9999" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice', 555888555,
'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00');"
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+
+
+ //Test the unique model by adding a key column with FLOAT
+ //java.sql.SQLException: errCode = 2, detailMessage = Float or double can
not used as a key, use decimal instead.
+/* sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column phone FLOAT KEY DEFAULT
"166.6" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice', 189.9,
'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00');"
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")*/
+
+
+
+
+ //Test the unique model by adding a key column with DOUBLE
+ //java.sql.SQLException: errCode = 2, detailMessage = Float or double can
not used as a key, use decimal instead.
+/* sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column watch FLOAT KEY DEFAULT
"166.689" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice', 189.479,
'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00');
"
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")*/
+
+
+
+
+ //Test the unique model by adding a key column with DECIMAL
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column watch DECIMAL(38,10) KEY
DEFAULT "16899.6464689" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice',
16499.6464689, 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing',
'2022-01-01 10:00:00');"
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+
+ //Test the unique model by adding a key column with DATE
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column watch DATE KEY DEFAULT
"1997-01-01" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice',
\"2024-01-01\", 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing',
'2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+
+
+ //Test the unique model by adding a key column with DATETIME
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column anniversary DATETIME KEY
DEFAULT "1997-01-01 00:00:00" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice',
\"2024-01-04 09:00:00\", 'Yaan', 25, 0, 13812345678, 'No. 123 Street,
Beijing', '2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+
+ //Test the unique model by adding a key column with DATETIME
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column anniversary DATETIME KEY
DEFAULT "1997-01-01 00:00:00" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice',
\"2024-01-04 09:00:00\", 'Yaan', 25, 0, 13812345678, 'No. 123 Street,
Beijing', '2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+
+ //Test the unique model by adding a key column with CHAR
+ sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column teacher CHAR KEY DEFAULT "F"
AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice', \"T\",
'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00');
"
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")
+
+
+
+ //Test the unique model by adding a key column with STRING
+ //java.sql.SQLException: errCode = 2, detailMessage = String Type should
not be used in key column[comment].
+/* sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column comment STRING KEY DEFAULT
"我是小说家" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice', '我是侦探家',
'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing', '2022-01-01 10:00:00');
"
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")*/
+
+
+ //Test the unique model by adding a key column with bitmap
+ //java.sql.SQLException: errCode = 2, detailMessage = Key column can not
set complex type:device_id
+/* sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column device_id bitmap KEY
DEFAULT "to_bitmap(243)" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice',
to_bitmap(243), 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing',
'2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")*/
+
+
+
+ //Test the unique model by adding a key column with Map
+ //java.sql.SQLException: errCode = 2, detailMessage = Map can only be
used in the non-key column of the duplicate table at present.
+/* sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column m Map<STRING, INT> KEY
DEFAULT "{'a': 100, 'b': 200}" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice', '{'a':
100, 'b': 200}', 'Yaan', 25, 0, 13812345678, 'No. 123 Street, Beijing',
'2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")*/
+
+
+
+ //Test the unique model by adding a key column with JSON
+ //java.sql.SQLException: errCode = 2, detailMessage = JSONB type should
not be used in key column[j].
+/* sql initTable
+ sql initTableData
+ sql """ alter table ${tbName} add column j JSON KEY DEFAULT
"{'a': 100, 'b': 200}" AFTER username """
+ insertSql = " insert into ${tbName} values(123456689, 'Alice',
'{\"k1\":\"v31\", \"k2\": 300}', 'Yaan', 25, 0, 13812345678, 'No. 123 Street,
Beijing', '2022-01-01 10:00:00'); "
+ waitForSchemaChangeDone({
+ sql getTableStatusSql
+ time 60
+ }, insertSql, true,"${tbName}")*/
+
+
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
