This is an automated email from the ASF dual-hosted git repository.
dataroaring pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/doris.git
The following commit(s) were added to refs/heads/master by this push:
new 26a46d8c3f [fix](cooldown) Handle full clone with cooldowned rowsets
(#17069)
26a46d8c3f is described below
commit 26a46d8c3fef64017a2506b15dea85482fbaa8fb
Author: plat1ko <[email protected]>
AuthorDate: Tue Feb 28 11:04:01 2023 +0800
[fix](cooldown) Handle full clone with cooldowned rowsets (#17069)
---
be/src/olap/base_tablet.h | 2 +-
be/src/olap/rowset/beta_rowset.cpp | 6 +-
be/src/olap/rowset/beta_rowset.h | 4 +-
be/src/olap/rowset/rowset_factory.cpp | 5 +-
be/src/olap/rowset/rowset_factory.h | 4 +-
be/src/olap/snapshot_manager.cpp | 6 --
be/src/olap/storage_engine.cpp | 4 +-
be/src/olap/tablet.cpp | 81 +++++---------------
be/src/olap/tablet.h | 14 ++--
be/src/olap/tablet_manager.cpp | 6 --
be/src/olap/tablet_manager.h | 2 -
be/src/olap/tablet_meta.cpp | 1 +
be/src/olap/tablet_meta.h | 5 --
be/src/olap/task/engine_clone_task.cpp | 130 +++++++--------------------------
14 files changed, 66 insertions(+), 204 deletions(-)
diff --git a/be/src/olap/base_tablet.h b/be/src/olap/base_tablet.h
index 8f8c624307..05707e81a6 100644
--- a/be/src/olap/base_tablet.h
+++ b/be/src/olap/base_tablet.h
@@ -73,7 +73,7 @@ protected:
protected:
TabletState _state;
- TabletMetaSharedPtr _tablet_meta;
+ const TabletMetaSharedPtr _tablet_meta;
TabletSchemaSPtr _schema;
DataDir* _data_dir;
diff --git a/be/src/olap/rowset/beta_rowset.cpp
b/be/src/olap/rowset/beta_rowset.cpp
index 5f881dbd20..26d4b0b7ce 100644
--- a/be/src/olap/rowset/beta_rowset.cpp
+++ b/be/src/olap/rowset/beta_rowset.cpp
@@ -98,9 +98,9 @@ std::string BetaRowset::local_segment_path_segcompacted(const
std::string& table
return fmt::format("{}/{}_{}-{}.dat", tablet_path, rowset_id.to_string(),
begin, end);
}
-BetaRowset::BetaRowset(TabletSchemaSPtr schema, const std::string& tablet_path,
- RowsetMetaSharedPtr rowset_meta)
- : Rowset(schema, tablet_path, std::move(rowset_meta)) {
+BetaRowset::BetaRowset(const TabletSchemaSPtr& schema, const std::string&
tablet_path,
+ const RowsetMetaSharedPtr& rowset_meta)
+ : Rowset(schema, tablet_path, rowset_meta) {
if (_rowset_meta->is_local()) {
_rowset_dir = tablet_path;
} else {
diff --git a/be/src/olap/rowset/beta_rowset.h b/be/src/olap/rowset/beta_rowset.h
index af3bbd08c9..c2a86f13c8 100644
--- a/be/src/olap/rowset/beta_rowset.h
+++ b/be/src/olap/rowset/beta_rowset.h
@@ -91,8 +91,8 @@ public:
Status get_segments_size(std::vector<size_t>* segments_size);
protected:
- BetaRowset(TabletSchemaSPtr schema, const std::string& tablet_path,
- RowsetMetaSharedPtr rowset_meta);
+ BetaRowset(const TabletSchemaSPtr& schema, const std::string& tablet_path,
+ const RowsetMetaSharedPtr& rowset_meta);
// init segment groups
Status init() override;
diff --git a/be/src/olap/rowset/rowset_factory.cpp
b/be/src/olap/rowset/rowset_factory.cpp
index 7657e5c4ac..78f6b2326c 100644
--- a/be/src/olap/rowset/rowset_factory.cpp
+++ b/be/src/olap/rowset/rowset_factory.cpp
@@ -28,8 +28,9 @@
namespace doris {
using namespace ErrorCode;
-Status RowsetFactory::create_rowset(TabletSchemaSPtr schema, const
std::string& tablet_path,
- RowsetMetaSharedPtr rowset_meta,
RowsetSharedPtr* rowset) {
+Status RowsetFactory::create_rowset(const TabletSchemaSPtr& schema, const
std::string& tablet_path,
+ const RowsetMetaSharedPtr& rowset_meta,
+ RowsetSharedPtr* rowset) {
if (rowset_meta->rowset_type() == ALPHA_ROWSET) {
return Status::Error<ROWSET_INVALID>();
}
diff --git a/be/src/olap/rowset/rowset_factory.h
b/be/src/olap/rowset/rowset_factory.h
index ea7106935b..6f6b262456 100644
--- a/be/src/olap/rowset/rowset_factory.h
+++ b/be/src/olap/rowset/rowset_factory.h
@@ -31,8 +31,8 @@ class RowsetFactory {
public:
// return OK and set inited rowset in `*rowset`.
// return others if failed to create or init rowset.
- static Status create_rowset(TabletSchemaSPtr schema, const std::string&
tablet_path,
- RowsetMetaSharedPtr rowset_meta,
RowsetSharedPtr* rowset);
+ static Status create_rowset(const TabletSchemaSPtr& schema, const
std::string& tablet_path,
+ const RowsetMetaSharedPtr& rowset_meta,
RowsetSharedPtr* rowset);
// create and init rowset writer.
// return OK and set `*output` to inited rowset writer.
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index 2c348ccc14..6e29a37dd7 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -175,9 +175,6 @@ Status SnapshotManager::convert_rowset_ids(const
std::string& clone_dir, int64_t
// remote rowset
*rowset_meta = visible_rowset;
}
- // FIXME(cyx): Redundant?
- rowset_meta->set_tablet_id(tablet_id);
- rowset_meta->set_tablet_schema_hash(schema_hash);
Version rowset_version = {visible_rowset.start_version(),
visible_rowset.end_version()};
rs_version_map[rowset_version] = rowset_meta;
}
@@ -206,9 +203,6 @@ Status SnapshotManager::convert_rowset_ids(const
std::string& clone_dir, int64_t
// remote rowset
*rowset_meta = stale_rowset;
}
- // FIXME(cyx): Redundant?
- rowset_meta->set_tablet_id(tablet_id);
- rowset_meta->set_tablet_schema_hash(schema_hash);
}
if (!rowset_id_mapping.empty() &&
cloned_tablet_meta_pb.has_delete_bitmap()) {
diff --git a/be/src/olap/storage_engine.cpp b/be/src/olap/storage_engine.cpp
index 089242dbd1..abfc5401e6 100644
--- a/be/src/olap/storage_engine.cpp
+++ b/be/src/olap/storage_engine.cpp
@@ -854,6 +854,7 @@ void StorageEngine::start_delete_unused_rowset() {
if (it->second->is_local()) {
unused_rowsets_copy[it->first] = it->second;
}
+ // remote rowset data will be reclaimed by
`remove_unused_remote_files`
it = _unused_rowsets.erase(it);
} else {
++it;
@@ -861,9 +862,6 @@ void StorageEngine::start_delete_unused_rowset() {
}
}
for (auto it = unused_rowsets_copy.begin(); it !=
unused_rowsets_copy.end(); ++it) {
- // FIXME(cyx): Currently remote unused rowsets are generated by
compaction gc,
- // we cannot remove them directly as other BE may need them.
-
VLOG_NOTICE << "start to remove rowset:" << it->second->rowset_id()
<< ", version:" << it->second->version().first << "-"
<< it->second->version().second;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index e50b188e5a..62d8ceae60 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -194,81 +194,32 @@ void Tablet::save_meta() {
<< ", root=" << _data_dir->path();
}
-Status Tablet::revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>&
rowsets_to_clone,
- const std::vector<Version>&
versions_to_delete) {
- LOG(INFO) << "begin to revise tablet. tablet=" << full_name()
- << ", rowsets_to_clone=" << rowsets_to_clone.size()
- << ", versions_to_delete=" << versions_to_delete.size();
- Status res = Status::OK();
- RowsetVector rs_to_delete, rs_to_add;
-
- for (auto& version : versions_to_delete) {
- auto it = _rs_version_map.find(version);
- DCHECK(it != _rs_version_map.end());
- StorageEngine::instance()->add_unused_rowset(it->second);
- rs_to_delete.push_back(it->second);
- _rs_version_map.erase(it);
- }
-
- for (auto& rs_meta : rowsets_to_clone) {
- Version version = {rs_meta->start_version(), rs_meta->end_version()};
- RowsetSharedPtr rowset;
- res = RowsetFactory::create_rowset(_schema, _tablet_path, rs_meta,
&rowset);
- if (!res.ok()) {
- LOG(WARNING) << "fail to init rowset. version=" << version;
- return res;
- }
- rs_to_add.push_back(rowset);
- _rs_version_map[version] = std::move(rowset);
- }
-
+Status Tablet::revise_tablet_meta(const std::vector<RowsetSharedPtr>& to_add,
+ const std::vector<RowsetSharedPtr>&
to_delete) {
+ LOG(INFO) << "begin to revise tablet. tablet_id=" << tablet_id();
+ delete_rowsets(to_delete, false);
+ add_rowsets(to_add);
+ // FIXME: How to reclaim delete bitmap of deleted rowsets and stale
rowsets?
if (keys_type() == UNIQUE_KEYS && enable_unique_key_merge_on_write()) {
auto new_rowset_tree = std::make_unique<RowsetTree>();
- ModifyRowSetTree(*_rowset_tree, rs_to_delete, rs_to_add,
new_rowset_tree.get());
+ ModifyRowSetTree(*_rowset_tree, to_delete, to_add,
new_rowset_tree.get());
_rowset_tree = std::move(new_rowset_tree);
- for (auto rowset_ptr : rs_to_add) {
- RETURN_IF_ERROR(update_delete_bitmap_without_lock(rowset_ptr));
+ for (auto& rs : to_add) {
+ RETURN_IF_ERROR(update_delete_bitmap_without_lock(rs));
}
}
-
- do {
- // load new local tablet_meta to operate on
- TabletMetaSharedPtr new_tablet_meta(new (nothrow)
TabletMeta(*_tablet_meta));
-
- // delete versions from new local tablet_meta
- for (const Version& version : versions_to_delete) {
- new_tablet_meta->delete_rs_meta_by_version(version, nullptr);
- LOG(INFO) << "delete version from new local tablet_meta when
clone. [table="
- << full_name() << ", version=" << version << "]";
- }
-
- // add new cloned rowset
- for (auto& rs_meta : rowsets_to_clone) {
- new_tablet_meta->add_rs_meta(rs_meta);
- }
- VLOG_NOTICE << "load rowsets successfully when clone. tablet=" <<
full_name()
- << ", added rowset size=" << rowsets_to_clone.size();
- // save and reload tablet_meta
- res = new_tablet_meta->save_meta(_data_dir);
- if (!res.ok()) {
- LOG(WARNING) << "failed to save new local tablet_meta when clone.
res:" << res;
- break;
- }
- _tablet_meta = new_tablet_meta;
- } while (0);
-
// reconstruct from tablet meta
_timestamped_version_tracker.construct_versioned_tracker(_tablet_meta->all_rs_metas());
// clear stale rowset
- for (auto& it : _stale_rs_version_map) {
- StorageEngine::instance()->add_unused_rowset(it.second);
+ for (auto& [v, rs] : _stale_rs_version_map) {
+ StorageEngine::instance()->add_unused_rowset(rs);
}
_stale_rs_version_map.clear();
_tablet_meta->clear_stale_rowset();
+ save_meta();
- LOG(INFO) << "finish to revise tablet. res=" << res << ", "
- << "table=" << full_name();
- return res;
+ LOG(INFO) << "finish to revise tablet. tablet_id=" << tablet_id();
+ return Status::OK();
}
RowsetSharedPtr Tablet::get_rowset(const RowsetId& rowset_id) {
@@ -421,6 +372,7 @@ Status Tablet::modify_rowsets(std::vector<RowsetSharedPtr>&
to_add,
}
void Tablet::add_rowsets(const std::vector<RowsetSharedPtr>& to_add) {
+ if (to_add.empty()) return;
std::vector<RowsetMetaSharedPtr> rs_metas;
rs_metas.reserve(to_add.size());
for (auto& rs : to_add) {
@@ -432,6 +384,7 @@ void Tablet::add_rowsets(const
std::vector<RowsetSharedPtr>& to_add) {
}
void Tablet::delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete,
bool move_to_stale) {
+ if (to_delete.empty()) return;
std::vector<RowsetMetaSharedPtr> rs_metas;
rs_metas.reserve(to_delete.size());
for (auto& rs : to_delete) {
@@ -1688,7 +1641,7 @@ void
Tablet::_init_context_common_fields(RowsetWriterContext& context) {
context.enable_unique_key_merge_on_write =
enable_unique_key_merge_on_write();
}
-Status Tablet::create_rowset(RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr*
rowset) {
+Status Tablet::create_rowset(const RowsetMetaSharedPtr& rowset_meta,
RowsetSharedPtr* rowset) {
return RowsetFactory::create_rowset(tablet_schema(), tablet_path(),
rowset_meta, rowset);
}
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index e59e5cae30..a4114d2ca5 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -81,8 +81,8 @@ public:
void save_meta();
// Used in clone task, to update local meta when finishing a clone job
- Status revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>&
rowsets_to_clone,
- const std::vector<Version>& versions_to_delete);
+ Status revise_tablet_meta(const std::vector<RowsetSharedPtr>& to_add,
+ const std::vector<RowsetSharedPtr>& to_delete);
int64_t cumulative_layer_point() const;
void set_cumulative_layer_point(int64_t new_point);
@@ -293,13 +293,18 @@ public:
Status create_vertical_rowset_writer(RowsetWriterContext& context,
std::unique_ptr<RowsetWriter>*
rowset_writer);
- Status create_rowset(RowsetMetaSharedPtr rowset_meta, RowsetSharedPtr*
rowset);
+ Status create_rowset(const RowsetMetaSharedPtr& rowset_meta,
RowsetSharedPtr* rowset);
// MUST hold EXCLUSIVE `_meta_lock`
void add_rowsets(const std::vector<RowsetSharedPtr>& to_add);
// MUST hold EXCLUSIVE `_meta_lock`
void delete_rowsets(const std::vector<RowsetSharedPtr>& to_delete, bool
move_to_stale);
+ // MUST hold SHARED `_meta_lock`
+ const auto& rowset_map() const { return _rs_version_map; }
+ // MUST hold SHARED `_meta_lock`
+ const auto& stale_rowset_map() const { return _stale_rs_version_map; }
+
////////////////////////////////////////////////////////////////////////////
// begin cooldown functions
////////////////////////////////////////////////////////////////////////////
@@ -397,9 +402,6 @@ public:
for (auto& [v, rs] : _rs_version_map) {
visitor(rs);
}
- for (auto& [v, rs] : _stale_rs_version_map) {
- visitor(rs);
- }
}
inline void increase_io_error_times() { ++_io_error_times; }
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 129f1046cc..9758356f59 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -91,12 +91,6 @@ TabletManager::~TabletManager() {
DEREGISTER_HOOK_METRIC(tablet_meta_mem_consumption);
}
-void TabletManager::add_tablet(const TabletSharedPtr& tablet) {
- auto& tablet_map = _get_tablet_map(tablet->tablet_id());
- std::lock_guard wlock(_get_tablets_shard_lock(tablet->tablet_id()));
- tablet_map[tablet->tablet_id()] = tablet;
-}
-
Status TabletManager::_add_tablet_unlocked(TTabletId tablet_id, const
TabletSharedPtr& tablet,
bool update_meta, bool force) {
Status res = Status::OK();
diff --git a/be/src/olap/tablet_manager.h b/be/src/olap/tablet_manager.h
index 0cda4b6edf..1b7e9cf89b 100644
--- a/be/src/olap/tablet_manager.h
+++ b/be/src/olap/tablet_manager.h
@@ -61,8 +61,6 @@ public:
// task to be fail, even if there is enough space on other disks
Status create_tablet(const TCreateTabletReq& request,
std::vector<DataDir*> stores);
- void add_tablet(const TabletSharedPtr& tablet);
-
// Drop a tablet by description.
// If `is_drop_table_or_partition` is true, we need to remove all remote
rowsets in this tablet.
Status drop_tablet(TTabletId tablet_id, TReplicaId replica_id, bool
is_drop_table_or_partition);
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index d2116b40bd..becf2c5c03 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -261,6 +261,7 @@ TabletMeta::TabletMeta(const TabletMeta& b)
_in_restore_mode(b._in_restore_mode),
_preferred_rowset_type(b._preferred_rowset_type),
_storage_policy_id(b._storage_policy_id),
+ _cooldown_meta_id(b._cooldown_meta_id),
_enable_unique_key_merge_on_write(b._enable_unique_key_merge_on_write),
_delete_bitmap(b._delete_bitmap) {};
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index c0a00d6fa5..0c9ed40136 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -121,7 +121,6 @@ public:
int64_t partition_id() const;
int64_t tablet_id() const;
int64_t replica_id() const;
- void set_replica_id(int64_t replica_id);
int32_t schema_hash() const;
int16_t shard_id() const;
void set_shard_id(int32_t shard_id);
@@ -440,10 +439,6 @@ inline int64_t TabletMeta::replica_id() const {
return _replica_id;
}
-inline void TabletMeta::set_replica_id(int64_t replica_id) {
- _replica_id = replica_id;
-}
-
inline int32_t TabletMeta::schema_hash() const {
return _schema_hash;
}
diff --git a/be/src/olap/task/engine_clone_task.cpp
b/be/src/olap/task/engine_clone_task.cpp
index 464a1433ef..76b5690559 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -308,6 +308,7 @@ Status EngineCloneTask::_make_snapshot(const std::string&
ip, int port, TTableId
request.__set_tablet_id(tablet_id);
request.__set_schema_hash(schema_hash);
request.__set_preferred_snapshot_version(g_Types_constants.TPREFER_SNAPSHOT_REQ_VERSION);
+ request.__set_version(_clone_req.committed_version);
// TODO: missing version composed of singleton delta.
// if not, this place should be rewrote.
request.__isset.missing_version = (!missed_versions.empty());
@@ -563,39 +564,22 @@ Status EngineCloneTask::_finish_incremental_clone(Tablet*
tablet,
<< ", missed_versions_size=" << missed_versions.size();
// check missing versions exist in clone src
- std::vector<RowsetMetaSharedPtr> rowsets_to_clone;
+ std::vector<RowsetSharedPtr> rowsets_to_clone;
for (Version version : missed_versions) {
- RowsetMetaSharedPtr rs_meta =
cloned_tablet_meta->acquire_rs_meta_by_version(version);
+ auto rs_meta = cloned_tablet_meta->acquire_rs_meta_by_version(version);
if (rs_meta == nullptr) {
return Status::InternalError("missed version {} is not found in
cloned tablet meta",
version.to_string());
}
- rowsets_to_clone.push_back(rs_meta);
+ RowsetSharedPtr rs;
+ RETURN_IF_ERROR(tablet->create_rowset(rs_meta, &rs));
+ rowsets_to_clone.push_back(std::move(rs));
}
/// clone_data to tablet
/// For incremental clone, nothing will be deleted.
/// So versions_to_delete is empty.
- std::vector<Version> versions_to_delete;
- return tablet->revise_tablet_meta(rowsets_to_clone, versions_to_delete);
-}
-
-// replace `origin_tablet` with Tablet created by `cloned_tablet_meta`.
-static Status full_clone(Tablet* origin_tablet, const TabletMetaSharedPtr&
cloned_tablet_meta) {
- // keep origin `replica_id` and `tablet_uid`
- cloned_tablet_meta->set_replica_id(origin_tablet->replica_id());
- cloned_tablet_meta->set_tablet_uid(origin_tablet->tablet_uid());
- auto tablet = Tablet::create_tablet_from_meta(cloned_tablet_meta,
origin_tablet->data_dir());
- RETURN_IF_ERROR(tablet->init());
- tablet->save_meta();
- StorageEngine::instance()->tablet_manager()->add_tablet(tablet);
- // reclaim local rowsets in origin tablet
- tablet->traverse_rowsets([](auto& rs) {
- if (rs->is_local()) {
- StorageEngine::instance()->add_unused_rowset(rs);
- }
- });
- return Status::OK();
+ return tablet->revise_tablet_meta(rowsets_to_clone, {});
}
/// This method will do:
@@ -611,103 +595,45 @@ Status EngineCloneTask::_finish_full_clone(Tablet*
tablet,
// For example:
// clone version is 8
//
- // local tablet: [0-1] [2-5] [6-6] [7-7]
+ // local tablet: [0-1] [2-5] [6-6] [7-7] [9-10]
// clone tablet: [0-1] [2-4] [5-6] [7-8]
//
// after compare, the version mark with "x" will be deleted
//
- // local tablet: [0-1] [2-5]x [6-6]x [7-7]x
- // clone tablet: [0-1]x [2-4] [5-6] [7-8]
-
- // All versions deleted from local tablet will be saved in
versions_to_delete
- // And these versions file will be deleted finally.
- std::vector<Version> versions_to_delete;
- std::vector<RowsetMetaSharedPtr> rs_metas_found_in_src;
- for (auto& rs_meta : tablet->tablet_meta()->all_rs_metas()) {
- Version local_version(rs_meta->start_version(),
rs_meta->end_version());
- LOG(INFO) << "check local delta when full clone."
- << "tablet=" << tablet->full_name() << ", local_version=" <<
local_version;
+ // local tablet: [0-1]x [2-5]x [6-6]x [7-7]x [9-10]
+ // clone tablet: [0-1] [2-4] [5-6] [7-8]
+ std::vector<RowsetSharedPtr> to_delete;
+ std::vector<RowsetSharedPtr> to_add;
+ for (auto& [v, rs] : tablet->rowset_map()) {
// if local version cross src latest, clone failed
// if local version is : 0-0, 1-1, 2-10, 12-14, 15-15,16-16
// cloned max version is 13-13, this clone is failed, because could not
// fill local data by using cloned data.
// It should not happen because if there is a hole, the following
delta will not
// do compaction.
- if (local_version.first <= cloned_max_version.second &&
- local_version.second > cloned_max_version.second) {
+ if (v.first <= cloned_max_version.second && v.second >
cloned_max_version.second) {
return Status::InternalError(
"version cross src latest. cloned_max_version={},
local_version={}",
- cloned_max_version.to_string(), local_version.to_string());
- } else if (local_version.second <= cloned_max_version.second) {
- // if local version smaller than src, check if existed in src,
will not clone it
- bool existed_in_src = false;
-
- // if delta labeled with local_version is same with the specified
version in clone header,
- // there is no necessity to clone it.
- for (auto& rs_meta : cloned_tablet_meta->all_rs_metas()) {
- if (rs_meta->version().first == local_version.first &&
- rs_meta->version().second == local_version.second) {
- existed_in_src = true;
- break;
- }
- }
-
- if (existed_in_src) {
- cloned_tablet_meta->delete_rs_meta_by_version(local_version,
-
&rs_metas_found_in_src);
- LOG(INFO) << "version exist in local tablet, no need to clone.
delete it from "
- "clone tablet"
- << ". tablet=" << tablet->full_name() << ",
version='" << local_version;
- } else {
- versions_to_delete.push_back(local_version);
- LOG(INFO) << "version not exist in local tablet. it will be
replaced by other "
- "version. "
- << "delete it from local tablet. "
- << "tablet=" << tablet->full_name() << ","
- << ", version=" << local_version;
- }
+ cloned_max_version.second, v.to_string());
}
- }
- std::vector<RowsetMetaSharedPtr> rowsets_to_clone;
- for (auto& rs_meta : cloned_tablet_meta->all_rs_metas()) {
- if (!rs_meta->is_local()) {
- // MUST clone all cooldowned rowset if there is any cooldowned
rowset to clone to ensure
- // `cooldown_meta_id` is consistent with the cooldowned rowsets
after clone.
- LOG(INFO) << "clone cooldowned rowsets, tablet_id=" <<
tablet->tablet_id();
- RETURN_IF_ERROR(full_clone(tablet, cloned_tablet_meta));
+ if (v.second <= cloned_max_version.second) {
+ to_delete.push_back(rs);
+ } else {
+ // cooldowned rowsets MUST be continuous, so rowsets whose version
> missed version MUST be local rowset
+ DCHECK(rs->is_local());
}
- rowsets_to_clone.push_back(rs_meta);
- LOG(INFO) << "version to be cloned from clone tablet to local tablet: "
- << tablet->full_name() << ", version=" << rs_meta->version();
}
- // clone_data to tablet
- // only replace rowset info, must not modify other info such as alter task
info. for example
- // 1. local tablet finished alter task
- // 2. local tablet has error in push
- // 3. local tablet cloned rowset from other nodes
- // 4. if cleared alter task info, then push will not write to new tablet,
the report info is error
- Status clone_res = tablet->revise_tablet_meta(rowsets_to_clone,
versions_to_delete);
- // in previous step, copy all files from CLONE_DIR to tablet dir
- // but some rowset is useless, so that remove them here
- for (auto& rs_meta_ptr : rs_metas_found_in_src) {
- RowsetSharedPtr rowset_to_remove;
- auto s =
-
RowsetFactory::create_rowset(cloned_tablet_meta->tablet_schema(),
- tablet->tablet_path(),
rs_meta_ptr, &rowset_to_remove);
- if (!s.ok()) {
- LOG(WARNING) << "failed to init rowset to remove: "
- << rs_meta_ptr->rowset_id().to_string();
- continue;
- }
- s = rowset_to_remove->remove();
- if (!s.ok()) {
- LOG(WARNING) << "failed to remove rowset " <<
rs_meta_ptr->rowset_id().to_string()
- << ": " << s;
- }
+ to_add.reserve(cloned_tablet_meta->all_rs_metas().size());
+ for (auto& rs_meta : cloned_tablet_meta->all_rs_metas()) {
+ RowsetSharedPtr rs;
+ RETURN_IF_ERROR(tablet->create_rowset(rs_meta, &rs));
+ to_add.push_back(std::move(rs));
}
- return clone_res;
+
tablet->tablet_meta()->set_cooldown_meta_id(cloned_tablet_meta->cooldown_meta_id());
+ return tablet->revise_tablet_meta(to_add, to_delete);
+ // TODO(plat1ko): write cooldown meta to remote if this replica is
cooldown replica
}
} // namespace doris
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
