This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch branch-3.0
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/branch-3.0 by this push:
new 8ec52e4dfad [fix](cloud-mow) Fix the issue of missing and removing
some old version delete bitmap (#44448)
8ec52e4dfad is described below
commit 8ec52e4dfad26ceb6439de66122bbfaf0d23f90a
Author: huanghaibin <[email protected]>
AuthorDate: Tue Nov 26 23:44:00 2024 +0800
[fix](cloud-mow) Fix the issue of missing and removing some old version
delete bitmap (#44448)
Problem Summary:
pr https://github.com/apache/doris/pull/40204 support removing old
delete bitmap, however it donesn't consider boundary which will lead to
miss some delete bitmap should be removed on be , it only affect local
delete bitmap, the delete bitmap store in fdb is right, and these
missing delete bitmaps can only be deleted by next base compaciton
before this pr.
pick pr:https://github.com/apache/doris/pull/44300
---
be/src/cloud/cloud_cumulative_compaction.cpp | 5 +-
be/src/cloud/cloud_delete_bitmap_action.cpp | 61 ++++++++-
be/src/cloud/cloud_delete_bitmap_action.h | 5 +-
be/src/cloud/cloud_meta_mgr.cpp | 15 +-
be/src/cloud/cloud_meta_mgr.h | 5 +-
be/src/olap/tablet_meta.cpp | 6 +-
be/src/service/http_service.cpp | 13 +-
...compaction_remove_old_version_delete_bitmap.out | 67 +++++++--
...paction_remove_old_version_delete_bitmap.groovy | 151 +++++++++++++++------
9 files changed, 252 insertions(+), 76 deletions(-)
diff --git a/be/src/cloud/cloud_cumulative_compaction.cpp
b/be/src/cloud/cloud_cumulative_compaction.cpp
index 92e0f44dc5b..603c6700ae9 100644
--- a/be/src/cloud/cloud_cumulative_compaction.cpp
+++ b/be/src/cloud/cloud_cumulative_compaction.cpp
@@ -393,12 +393,9 @@ Status
CloudCumulativeCompaction::process_old_version_delete_bitmap() {
rowset->rowset_id().to_string();
DeleteBitmap::BitmapKey start {rowset->rowset_id(), seg_id, 0};
DeleteBitmap::BitmapKey end {rowset->rowset_id(), seg_id,
pre_max_version};
- DeleteBitmap::BitmapKey before_end {rowset->rowset_id(),
seg_id,
- pre_max_version - 1};
auto d = _tablet->tablet_meta()->delete_bitmap().get_agg(
{rowset->rowset_id(), seg_id, pre_max_version});
- to_remove_vec.emplace_back(
- std::make_tuple(_tablet->tablet_id(), start,
before_end));
+
to_remove_vec.emplace_back(std::make_tuple(_tablet->tablet_id(), start, end));
if (d->isEmpty()) {
continue;
}
diff --git a/be/src/cloud/cloud_delete_bitmap_action.cpp
b/be/src/cloud/cloud_delete_bitmap_action.cpp
index 672574a5aa8..86cc535e1bc 100644
--- a/be/src/cloud/cloud_delete_bitmap_action.cpp
+++ b/be/src/cloud/cloud_delete_bitmap_action.cpp
@@ -33,6 +33,7 @@
#include <thread>
#include <utility>
+#include "cloud/cloud_meta_mgr.h"
#include "cloud/cloud_tablet.h"
#include "cloud/cloud_tablet_mgr.h"
#include "common/logging.h"
@@ -78,8 +79,8 @@ static Status _check_param(HttpRequest* req, uint64_t*
tablet_id) {
return Status::OK();
}
-Status CloudDeleteBitmapAction::_handle_show_delete_bitmap_count(HttpRequest*
req,
- std::string*
json_result) {
+Status
CloudDeleteBitmapAction::_handle_show_local_delete_bitmap_count(HttpRequest*
req,
+
std::string* json_result) {
uint64_t tablet_id = 0;
// check & retrieve tablet_id from req if it contains
RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param
failed");
@@ -95,7 +96,49 @@ Status
CloudDeleteBitmapAction::_handle_show_delete_bitmap_count(HttpRequest* re
auto count =
tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count();
auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality();
auto size = tablet->tablet_meta()->delete_bitmap().get_size();
- LOG(INFO) << "show_delete_bitmap_count,tablet_id=" << tablet_id <<
",count=" << count
+ LOG(INFO) << "show_local_delete_bitmap_count,tablet_id=" << tablet_id <<
",count=" << count
+ << ",cardinality=" << cardinality << ",size=" << size;
+
+ rapidjson::Document root;
+ root.SetObject();
+ root.AddMember("delete_bitmap_count", count, root.GetAllocator());
+ root.AddMember("cardinality", cardinality, root.GetAllocator());
+ root.AddMember("size", size, root.GetAllocator());
+
+ // to json string
+ rapidjson::StringBuffer strbuf;
+ rapidjson::PrettyWriter<rapidjson::StringBuffer> writer(strbuf);
+ root.Accept(writer);
+ *json_result = std::string(strbuf.GetString());
+
+ return Status::OK();
+}
+
+Status
CloudDeleteBitmapAction::_handle_show_ms_delete_bitmap_count(HttpRequest* req,
+
std::string* json_result) {
+ uint64_t tablet_id = 0;
+ // check & retrieve tablet_id from req if it contains
+ RETURN_NOT_OK_STATUS_WITH_WARN(_check_param(req, &tablet_id), "check param
failed");
+ if (tablet_id == 0) {
+ return Status::InternalError("check param failed: missing tablet_id");
+ }
+ TabletMetaSharedPtr tablet_meta;
+ auto st = _engine.meta_mgr().get_tablet_meta(tablet_id, &tablet_meta);
+ if (!st.ok()) {
+ LOG(WARNING) << "failed to get_tablet_meta tablet=" << tablet_id
+ << ", st=" << st.to_string();
+ return st;
+ }
+ auto tablet = std::make_shared<CloudTablet>(_engine,
std::move(tablet_meta));
+ st = _engine.meta_mgr().sync_tablet_rowsets(tablet.get(), false, true,
true);
+ if (!st.ok()) {
+ LOG(WARNING) << "failed to sync tablet=" << tablet_id << ", st=" << st;
+ return st;
+ }
+ auto count =
tablet->tablet_meta()->delete_bitmap().get_delete_bitmap_count();
+ auto cardinality = tablet->tablet_meta()->delete_bitmap().cardinality();
+ auto size = tablet->tablet_meta()->delete_bitmap().get_size();
+ LOG(INFO) << "show_ms_delete_bitmap_count,tablet_id=" << tablet_id <<
",count=" << count
<< ",cardinality=" << cardinality << ",size=" << size;
rapidjson::Document root;
@@ -115,9 +158,17 @@ Status
CloudDeleteBitmapAction::_handle_show_delete_bitmap_count(HttpRequest* re
void CloudDeleteBitmapAction::handle(HttpRequest* req) {
req->add_output_header(HttpHeaders::CONTENT_TYPE, HEADER_JSON.data());
- if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_INFO) {
+ if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_LOCAL) {
+ std::string json_result;
+ Status st = _handle_show_local_delete_bitmap_count(req, &json_result);
+ if (!st.ok()) {
+ HttpChannel::send_reply(req, HttpStatus::OK, st.to_json());
+ } else {
+ HttpChannel::send_reply(req, HttpStatus::OK, json_result);
+ }
+ } else if (_delete_bitmap_action_type == DeleteBitmapActionType::COUNT_MS)
{
std::string json_result;
- Status st = _handle_show_delete_bitmap_count(req, &json_result);
+ Status st = _handle_show_ms_delete_bitmap_count(req, &json_result);
if (!st.ok()) {
HttpChannel::send_reply(req, HttpStatus::OK, st.to_json());
} else {
diff --git a/be/src/cloud/cloud_delete_bitmap_action.h
b/be/src/cloud/cloud_delete_bitmap_action.h
index 9321661374c..35739a7373e 100644
--- a/be/src/cloud/cloud_delete_bitmap_action.h
+++ b/be/src/cloud/cloud_delete_bitmap_action.h
@@ -31,7 +31,7 @@ class HttpRequest;
class ExecEnv;
-enum class DeleteBitmapActionType { COUNT_INFO = 1 };
+enum class DeleteBitmapActionType { COUNT_LOCAL = 1, COUNT_MS = 2 };
/// This action is used for viewing the delete bitmap status
class CloudDeleteBitmapAction : public HttpHandlerWithAuth {
@@ -45,7 +45,8 @@ public:
void handle(HttpRequest* req) override;
private:
- Status _handle_show_delete_bitmap_count(HttpRequest* req, std::string*
json_result);
+ Status _handle_show_local_delete_bitmap_count(HttpRequest* req,
std::string* json_result);
+ Status _handle_show_ms_delete_bitmap_count(HttpRequest* req, std::string*
json_result);
private:
CloudStorageEngine& _engine;
diff --git a/be/src/cloud/cloud_meta_mgr.cpp b/be/src/cloud/cloud_meta_mgr.cpp
index 466f7b4ad11..43c3517cd2d 100644
--- a/be/src/cloud/cloud_meta_mgr.cpp
+++ b/be/src/cloud/cloud_meta_mgr.cpp
@@ -385,7 +385,7 @@ Status CloudMetaMgr::get_tablet_meta(int64_t tablet_id,
TabletMetaSharedPtr* tab
}
Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet* tablet, bool
warmup_delta_data,
- bool sync_delete_bitmap) {
+ bool sync_delete_bitmap, bool
full_sync) {
using namespace std::chrono;
TEST_SYNC_POINT_RETURN_WITH_VALUE("CloudMetaMgr::sync_tablet_rowsets",
Status::OK(), tablet);
@@ -411,7 +411,11 @@ Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet*
tablet, bool warmup_delta_
idx->set_partition_id(tablet->partition_id());
{
std::shared_lock rlock(tablet->get_header_lock());
- req.set_start_version(tablet->max_version_unlocked() + 1);
+ if (full_sync) {
+ req.set_start_version(0);
+ } else {
+ req.set_start_version(tablet->max_version_unlocked() + 1);
+ }
req.set_base_compaction_cnt(tablet->base_compaction_cnt());
req.set_cumulative_compaction_cnt(tablet->cumulative_compaction_cnt());
req.set_cumulative_point(tablet->cumulative_layer_point());
@@ -471,7 +475,7 @@ Status CloudMetaMgr::sync_tablet_rowsets(CloudTablet*
tablet, bool warmup_delta_
DeleteBitmap delete_bitmap(tablet_id);
int64_t old_max_version = req.start_version() - 1;
auto st = sync_tablet_delete_bitmap(tablet, old_max_version,
resp.rowset_meta(),
- resp.stats(), req.idx(),
&delete_bitmap);
+ resp.stats(), req.idx(),
&delete_bitmap, full_sync);
if (st.is<ErrorCode::ROWSETS_EXPIRED>() && tried++ < retry_times) {
LOG_WARNING("rowset meta is expired, need to retry")
.tag("tablet", tablet->tablet_id())
@@ -618,12 +622,13 @@ bool
CloudMetaMgr::sync_tablet_delete_bitmap_by_cache(CloudTablet* tablet, int64
Status CloudMetaMgr::sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t
old_max_version,
std::ranges::range auto&&
rs_metas,
const TabletStatsPB& stats,
const TabletIndexPB& idx,
- DeleteBitmap* delete_bitmap) {
+ DeleteBitmap* delete_bitmap,
bool full_sync) {
if (rs_metas.empty()) {
return Status::OK();
}
- if (sync_tablet_delete_bitmap_by_cache(tablet, old_max_version, rs_metas,
delete_bitmap)) {
+ if (!full_sync &&
+ sync_tablet_delete_bitmap_by_cache(tablet, old_max_version, rs_metas,
delete_bitmap)) {
return Status::OK();
} else {
DeleteBitmapPtr new_delete_bitmap =
std::make_shared<DeleteBitmap>(tablet->tablet_id());
diff --git a/be/src/cloud/cloud_meta_mgr.h b/be/src/cloud/cloud_meta_mgr.h
index a657c0fdd8e..c49b036ad90 100644
--- a/be/src/cloud/cloud_meta_mgr.h
+++ b/be/src/cloud/cloud_meta_mgr.h
@@ -58,7 +58,7 @@ public:
Status get_tablet_meta(int64_t tablet_id, std::shared_ptr<TabletMeta>*
tablet_meta);
Status sync_tablet_rowsets(CloudTablet* tablet, bool warmup_delta_data =
false,
- bool sync_delete_bitmap = true);
+ bool sync_delete_bitmap = true, bool full_sync
= false);
Status prepare_rowset(const RowsetMeta& rs_meta,
std::shared_ptr<RowsetMeta>* existed_rs_meta =
nullptr);
@@ -116,7 +116,8 @@ private:
Status sync_tablet_delete_bitmap(CloudTablet* tablet, int64_t
old_max_version,
std::ranges::range auto&& rs_metas, const
TabletStatsPB& stats,
- const TabletIndexPB& idx, DeleteBitmap*
delete_bitmap);
+ const TabletIndexPB& idx, DeleteBitmap*
delete_bitmap,
+ bool full_sync = false);
void check_table_size_correctness(const RowsetMeta& rs_meta);
int64_t get_segment_file_size(const RowsetMeta& rs_meta);
int64_t get_inverted_index_file_szie(const RowsetMeta& rs_meta);
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index 3afafd6ba4a..0ebbaf0a65b 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -1205,9 +1205,13 @@ void
DeleteBitmap::remove_stale_delete_bitmap_from_queue(const std::vector<std::
}
auto start_bmk = std::get<1>(delete_bitmap_tuple);
auto end_bmk = std::get<2>(delete_bitmap_tuple);
+ // the key range of to be removed is [start_bmk,end_bmk),
+ // due to the different definitions of the right boundary,
+ // so use end_bmk as right boundary when removing local delete
bitmap,
+ // use (end_bmk - 1) as right boundary when removing ms delete
bitmap
remove(start_bmk, end_bmk);
to_delete.emplace_back(std::make_tuple(std::get<0>(start_bmk).to_string(), 0,
- std::get<2>(end_bmk)));
+ std::get<2>(end_bmk) -
1));
}
_stale_delete_bitmap.erase(version_str);
}
diff --git a/be/src/service/http_service.cpp b/be/src/service/http_service.cpp
index 55ee57cb8df..a74f00291de 100644
--- a/be/src/service/http_service.cpp
+++ b/be/src/service/http_service.cpp
@@ -407,11 +407,16 @@ void
HttpService::register_cloud_handler(CloudStorageEngine& engine) {
TPrivilegeHier::GLOBAL,
TPrivilegeType::ADMIN));
_ev_http_server->register_handler(HttpMethod::GET,
"/api/compaction/run_status",
run_status_compaction_action);
- CloudDeleteBitmapAction* count_delete_bitmap_action =
- _pool.add(new
CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_INFO, _env, engine,
+ CloudDeleteBitmapAction* count_local_delete_bitmap_action =
+ _pool.add(new
CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_LOCAL, _env, engine,
TPrivilegeHier::GLOBAL,
TPrivilegeType::ADMIN));
- _ev_http_server->register_handler(HttpMethod::GET,
"/api/delete_bitmap/count",
- count_delete_bitmap_action);
+ _ev_http_server->register_handler(HttpMethod::GET,
"/api/delete_bitmap/count_local",
+ count_local_delete_bitmap_action);
+ CloudDeleteBitmapAction* count_ms_delete_bitmap_action =
+ _pool.add(new
CloudDeleteBitmapAction(DeleteBitmapActionType::COUNT_MS, _env, engine,
+ TPrivilegeHier::GLOBAL,
TPrivilegeType::ADMIN));
+ _ev_http_server->register_handler(HttpMethod::GET,
"/api/delete_bitmap/count_ms",
+ count_ms_delete_bitmap_action);
#ifdef ENABLE_INJECTION_POINT
InjectionPointAction* injection_point_action = _pool.add(new
InjectionPointAction);
_ev_http_server->register_handler(HttpMethod::GET,
"/api/injection_point/{op}",
diff --git
a/regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out
b/regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out
index 1c3611fe0b7..37dfa3b93a5 100644
---
a/regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out
+++
b/regression-test/data/compaction/test_cu_compaction_remove_old_version_delete_bitmap.out
@@ -1,29 +1,78 @@
-- This file is automatically generated. You should know what you did if you
want to edit this
-- !sql --
-0 0 0
-1 8 8
+0 0 8
+1 1 1
+2 2 2
+3 3 3
+4 4 4
+5 5 5
+6 6 6
+7 7 7
+8 8 8
-- !sql --
-0 0 0
-1 8 8
+0 0 8
+1 1 1
+2 2 2
+3 3 3
+4 4 4
+5 5 5
+6 6 6
+7 7 7
+8 8 8
-- !sql --
-0 0 0
+0 0 13
1 13 13
+2 2 2
+3 3 3
+4 4 4
+5 5 5
+6 6 6
+7 7 7
+8 8 8
-- !sql --
-0 0 0
+0 0 13
1 13 13
+2 2 2
+3 3 3
+4 4 4
+5 5 5
+6 6 6
+7 7 7
+8 8 8
-- !sql --
-0 0 0
+0 0 18
1 23 23
+2 2 2
+3 3 3
+4 4 4
+5 5 5
+6 6 6
+7 7 7
+8 8 8
-- !sql --
-0 0 0
+0 0 18
1 23 23
+2 2 2
+3 3 3
+4 4 4
+5 5 5
+6 6 6
+7 7 7
+8 8 8
-- !sql --
-0 0 0
+0 5 5
1 28 28
+2 2 2
+3 3 3
+4 4 4
+5 5 5
+6 6 6
+7 7 7
+8 8 8
diff --git
a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy
b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy
index 2219cc175b5..a36cb4579ca 100644
---
a/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy
+++
b/regression-test/suites/compaction/test_cu_compaction_remove_old_version_delete_bitmap.groovy
@@ -123,11 +123,11 @@
suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
} while (running)
}
- def getDeleteBitmapStatus = { be_host, be_http_port, tablet_id ->
+ def getLocalDeleteBitmapStatus = { be_host, be_http_port, tablet_id ->
boolean running = true
StringBuilder sb = new StringBuilder();
sb.append("curl -X GET http://${be_host}:${be_http_port}";)
- sb.append("/api/delete_bitmap/count?tablet_id=")
+ sb.append("/api/delete_bitmap/count_local?tablet_id=")
sb.append(tablet_id)
String command = sb.toString()
@@ -135,7 +135,25 @@
suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
process = command.execute()
code = process.waitFor()
out = process.getText()
- logger.info("Get delete bitmap count status: =" + code + ", out=" +
out)
+ logger.info("Get local delete bitmap count status: =" + code + ",
out=" + out)
+ assertEquals(code, 0)
+ def deleteBitmapStatus = parseJson(out.trim())
+ return deleteBitmapStatus
+ }
+
+ def getMSDeleteBitmapStatus = { be_host, be_http_port, tablet_id ->
+ boolean running = true
+ StringBuilder sb = new StringBuilder();
+ sb.append("curl -X GET http://${be_host}:${be_http_port}";)
+ sb.append("/api/delete_bitmap/count_ms?tablet_id=")
+ sb.append(tablet_id)
+
+ String command = sb.toString()
+ logger.info(command)
+ process = command.execute()
+ code = process.waitFor()
+ out = process.getText()
+ logger.info("Get ms delete bitmap count status: =" + code + ", out="
+ out)
assertEquals(code, 0)
def deleteBitmapStatus = parseJson(out.trim())
return deleteBitmapStatus
@@ -174,21 +192,24 @@
suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
GetDebugPoint().enableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.delete_expired_stale_rowsets")
// 1. test normal
sql "sync"
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,1,'1'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,2,'2'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,3,'3'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,4,'4'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,5,'5'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,6,'6'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,7,'7'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,8,'8'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'1'),(1,1,'1'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'2'),(2,2,'2'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'3'),(3,3,'3'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'4'),(4,4,'4'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'5'),(5,5,'5'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'6'),(6,6,'6'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'7'),(7,7,'7'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'8'),(8,8,'8'); """
qt_sql "select * from ${testTable} order by plan_id"
// trigger compaction to generate base rowset
def tablets = sql_return_maparray """ show tablets from ${testTable};
"""
logger.info("tablets: " + tablets)
- def delete_bitmap_count = 0
+ def local_delete_bitmap_count = 0
+ def ms_delete_bitmap_count = 0
+ def local_delete_bitmap_cardinality = 0;
+ def ms_delete_bitmap_cardinality = 0;
for (def tablet in tablets) {
String tablet_id = tablet.TabletId
def tablet_info = sql_return_maparray """ show tablet
${tablet_id}; """
@@ -197,9 +218,20 @@
suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
getTabletStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
// before compaction, delete_bitmap_count is (rowsets num - 1)
- delete_bitmap_count =
getDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
- assertTrue(delete_bitmap_count == 7)
- logger.info("delete_bitmap_count:" + delete_bitmap_count)
+ local_delete_bitmap_count =
getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+ ms_delete_bitmap_count =
getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+ logger.info("local_delete_bitmap_count:" +
local_delete_bitmap_count)
+ logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
+ assertTrue(local_delete_bitmap_count == 7)
+ assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count)
+
+ local_delete_bitmap_cardinality =
getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+ ms_delete_bitmap_cardinality =
getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+ logger.info("local_delete_bitmap_cardinality:" +
local_delete_bitmap_cardinality)
+ logger.info("ms_delete_bitmap_cardinality:" +
ms_delete_bitmap_cardinality)
+ assertTrue(local_delete_bitmap_cardinality == 7)
+ assertTrue(local_delete_bitmap_cardinality ==
ms_delete_bitmap_cardinality)
+
assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id],
"cumulative", tablet_id).contains("Success"));
@@ -211,11 +243,11 @@
suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
def now = System.currentTimeMillis()
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,9,'9'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,10,'10'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,11,'11'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,12,'12'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,13,'13'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'9'),(1,9,'9'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'10'),(1,10,'10'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'11'),(1,11,'11'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'12'),(1,12,'12'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'13'),(1,13,'13'); """
def time_diff = System.currentTimeMillis() - now
logger.info("time_diff:" + time_diff)
@@ -230,11 +262,21 @@
suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
def tablet_info = sql_return_maparray """ show tablet
${tablet_id}; """
logger.info("tablet: " + tablet_info)
- // before compaction, delete_bitmap_count is (rowsets num - 1)
+ // before compaction, local delete_bitmap_count is (total rowsets
num - 1), ms delete_bitmap_count is new rowset num
String trigger_backend_id = tablet.BackendId
- delete_bitmap_count =
getDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
- logger.info("delete_bitmap_count:" + delete_bitmap_count)
- assertTrue(delete_bitmap_count == 12)
+ local_delete_bitmap_count =
getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+ ms_delete_bitmap_count =
getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+ logger.info("local_delete_bitmap_count:" +
local_delete_bitmap_count)
+ logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
+ assertTrue(local_delete_bitmap_count == 12)
+ assertTrue(ms_delete_bitmap_count == 5)
+
+ local_delete_bitmap_cardinality =
getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+ ms_delete_bitmap_cardinality =
getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+ logger.info("local_delete_bitmap_cardinality:" +
local_delete_bitmap_cardinality)
+ logger.info("ms_delete_bitmap_cardinality:" +
ms_delete_bitmap_cardinality)
+ assertTrue(local_delete_bitmap_cardinality == 17)
+ assertTrue(ms_delete_bitmap_cardinality == 10)
getTabletStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id],
@@ -244,9 +286,19 @@
suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
Thread.sleep(1000)
// after compaction, delete_bitmap_count is 1
- delete_bitmap_count =
getDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
- logger.info("delete_bitmap_count:" + delete_bitmap_count)
- assertTrue(delete_bitmap_count == 1)
+ local_delete_bitmap_count =
getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+ ms_delete_bitmap_count =
getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+ logger.info("local_delete_bitmap_count:" +
local_delete_bitmap_count)
+ logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
+ assertTrue(local_delete_bitmap_count == 1)
+ assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count)
+
+ local_delete_bitmap_cardinality =
getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+ ms_delete_bitmap_cardinality =
getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+ logger.info("local_delete_bitmap_cardinality:" +
local_delete_bitmap_cardinality)
+ logger.info("ms_delete_bitmap_cardinality:" +
ms_delete_bitmap_cardinality)
+ assertTrue(local_delete_bitmap_cardinality == 2)
+ assertTrue(ms_delete_bitmap_cardinality == 2)
}
qt_sql "select * from ${testTable} order by plan_id"
@@ -255,11 +307,11 @@
suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
now = System.currentTimeMillis()
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,19,'19'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,20,'20'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,21,'21'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,22,'22'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,23,'23'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'14'),(1,19,'19'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'15'),(1,20,'20'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'16'),(1,21,'21'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'17'),(1,22,'22'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,0,'18'),(1,23,'23'); """
time_diff = System.currentTimeMillis() - now
logger.info("time_diff:" + time_diff)
@@ -273,9 +325,19 @@
suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
logger.info("tablet: " + tablet_info)
String trigger_backend_id = tablet.BackendId
- delete_bitmap_count =
getDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
- assertTrue(delete_bitmap_count == 6)
- logger.info("delete_bitmap_count:" + delete_bitmap_count)
+ local_delete_bitmap_count =
getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+ ms_delete_bitmap_count =
getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+ logger.info("local_delete_bitmap_count:" +
local_delete_bitmap_count)
+ logger.info("ms_delete_bitmap_count:" + ms_delete_bitmap_count)
+ assertTrue(local_delete_bitmap_count == 6)
+ assertTrue(local_delete_bitmap_count == ms_delete_bitmap_count)
+
+ local_delete_bitmap_cardinality =
getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+ ms_delete_bitmap_cardinality =
getMSDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).cardinality
+ logger.info("local_delete_bitmap_cardinality:" +
local_delete_bitmap_cardinality)
+ logger.info("ms_delete_bitmap_cardinality:" +
ms_delete_bitmap_cardinality)
+ assertTrue(local_delete_bitmap_cardinality == 12)
+ assertTrue(ms_delete_bitmap_cardinality == 12)
getTabletStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
assertTrue(triggerCompaction(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id],
@@ -283,28 +345,29 @@
suite("test_cu_compaction_remove_old_version_delete_bitmap", "nonConcurrent") {
waitForCompaction(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id)
getTabletStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id);
- // update fail, delete_bitmap_count will not change
+ // update fail, local delete_bitmap_count will not change
Thread.sleep(1000)
- delete_bitmap_count =
getDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
- assertTrue(delete_bitmap_count == 6)
- logger.info("delete_bitmap_count:" + delete_bitmap_count)
+ local_delete_bitmap_count =
getLocalDeleteBitmapStatus(backendId_to_backendIP[trigger_backend_id],
backendId_to_backendHttpPort[trigger_backend_id], tablet_id).delete_bitmap_count
+ logger.info("local_delete_bitmap_count:" +
local_delete_bitmap_count)
+ assertTrue(local_delete_bitmap_count == 6)
}
qt_sql "select * from ${testTable} order by plan_id"
now = System.currentTimeMillis()
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,24,'24'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,25,'25'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,26,'26'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,27,'27'); """
- sql """ INSERT INTO ${testTable} VALUES (0,0,'0'),(1,28,'28'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,1,'1'),(1,24,'24'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,3,'2'),(1,25,'25'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,3,'3'),(1,26,'26'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,4,'4'),(1,27,'27'); """
+ sql """ INSERT INTO ${testTable} VALUES (0,5,'5'),(1,28,'28'); """
time_diff = System.currentTimeMillis() - now
logger.info("time_diff:" + time_diff)
assertTrue(time_diff <= timeout, "wait_for_insert_into_values timeout")
qt_sql "select * from ${testTable} order by plan_id"
+
GetDebugPoint().disableDebugPointForAllBEs("CloudCumulativeCompaction.modify_rowsets.update_delete_bitmap_failed")
} finally {
reset_be_param("compaction_promotion_version_count")
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
