This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new a6e2c3e [Bug][Clone] Fix the bug that incremental clone is not
triggered (#5230)
a6e2c3e is described below
commit a6e2c3e3f12508756d746ed97b63bc2792ae7af3
Author: Mingyu Chen <[email protected]>
AuthorDate: Sat Feb 6 22:04:48 2021 +0800
[Bug][Clone] Fix the bug that incremental clone is not triggered (#5230)
In version 0.13, we support a more efficient compaction logic.
This logic will maintain multiple version paths of the tablet.
This can avoid -230 errors and can also support incremental clone.
But the previous incremental clone uses the incremental rowset meta
recorded in `incr_rs_meta`.
At present, the incremental rowset meta recorded in `incr_rs_meta` and the
records
in `stale_rs_meta` are duplicated, and the current clone logic does not
adapt to the
new multi-version path, resulting in many cases not triggering incremental
clone.
This CL mainly modified:
1. Removed `incr_rs_meta` metadata
2. Modified the clone logic. When the clone is incremented, it will try to
read the rowset in `stale_rs_meta`.
3. Delete a lot of code that was previously used for version compatibility.
---
be/src/agent/agent_server.cpp | 7 +-
be/src/agent/task_worker_pool.cpp | 3 +-
be/src/http/action/snapshot_action.cpp | 3 +-
be/src/olap/data_dir.cpp | 155 -------------
be/src/olap/data_dir.h | 3 -
be/src/olap/olap_snapshot_converter.cpp | 278 ----------------------
be/src/olap/olap_snapshot_converter.h | 19 --
be/src/olap/rowset/rowset.cpp | 2 +-
be/src/olap/snapshot_manager.cpp | 204 ++++++++---------
be/src/olap/snapshot_manager.h | 15 +-
be/src/olap/tablet.cpp | 123 ++--------
be/src/olap/tablet.h | 22 +-
be/src/olap/tablet_manager.cpp | 1 -
be/src/olap/tablet_meta.cpp | 53 +----
be/src/olap/tablet_meta.h | 12 +-
be/src/olap/task/engine_clone_task.cpp | 279 +++++++++--------------
be/src/olap/task/engine_clone_task.h | 8 +-
be/test/olap/CMakeLists.txt | 2 +-
be/test/olap/tablet_meta_manager_test.cpp | 3 +-
be/test/olap/test_data/header_without_inc_rs.txt | 144 ++++++++++++
gensrc/proto/olap_file.proto | 2 +
gensrc/thrift/AgentService.thrift | 1 +
22 files changed, 391 insertions(+), 948 deletions(-)
diff --git a/be/src/agent/agent_server.cpp b/be/src/agent/agent_server.cpp
index fe0ce00..6c2882f 100644
--- a/be/src/agent/agent_server.cpp
+++ b/be/src/agent/agent_server.cpp
@@ -198,8 +198,9 @@ void AgentServer::make_snapshot(TAgentResult&
t_agent_result,
const TSnapshotRequest& snapshot_request) {
Status ret_st;
string snapshot_path;
+ bool allow_incremental_clone = false;
OLAPStatus err_code =
- SnapshotManager::instance()->make_snapshot(snapshot_request,
&snapshot_path);
+ SnapshotManager::instance()->make_snapshot(snapshot_request,
&snapshot_path, &allow_incremental_clone);
if (err_code != OLAP_SUCCESS) {
LOG(WARNING) << "fail to make_snapshot. tablet_id=" <<
snapshot_request.tablet_id
<< ", schema_hash=" << snapshot_request.schema_hash
@@ -211,13 +212,11 @@ void AgentServer::make_snapshot(TAgentResult&
t_agent_result,
<< ", schema_hash=" << snapshot_request.schema_hash
<< ", snapshot_path: " << snapshot_path;
t_agent_result.__set_snapshot_path(snapshot_path);
+ t_agent_result.__set_allow_incremental_clone(allow_incremental_clone);
}
ret_st.to_thrift(&t_agent_result.status);
t_agent_result.__set_snapshot_version(snapshot_request.preferred_snapshot_version);
- if (snapshot_request.__isset.allow_incremental_clone) {
-
t_agent_result.__set_allow_incremental_clone(snapshot_request.allow_incremental_clone);
- }
}
void AgentServer::release_snapshot(TAgentResult& t_agent_result, const
std::string& snapshot_path) {
diff --git a/be/src/agent/task_worker_pool.cpp
b/be/src/agent/task_worker_pool.cpp
index e4848c7..86e181d 100644
--- a/be/src/agent/task_worker_pool.cpp
+++ b/be/src/agent/task_worker_pool.cpp
@@ -1348,9 +1348,10 @@ void TaskWorkerPool::_make_snapshot_thread_callback() {
TStatus task_status;
string snapshot_path;
+ bool allow_incremental_clone = false; // not used
std::vector<string> snapshot_files;
OLAPStatus make_snapshot_status =
- SnapshotManager::instance()->make_snapshot(snapshot_request,
&snapshot_path);
+ SnapshotManager::instance()->make_snapshot(snapshot_request,
&snapshot_path, &allow_incremental_clone);
if (make_snapshot_status != OLAP_SUCCESS) {
status_code = make_snapshot_status ==
OLAP_ERR_VERSION_ALREADY_MERGED
?
TStatusCode::OLAP_ERR_VERSION_ALREADY_MERGED
diff --git a/be/src/http/action/snapshot_action.cpp
b/be/src/http/action/snapshot_action.cpp
index 18c3bd6..ccdaea3 100644
--- a/be/src/http/action/snapshot_action.cpp
+++ b/be/src/http/action/snapshot_action.cpp
@@ -100,7 +100,8 @@ int64_t SnapshotAction::make_snapshot(int64_t tablet_id,
int32_t schema_hash,
request.schema_hash = schema_hash;
OLAPStatus res = OLAPStatus::OLAP_SUCCESS;
- res = SnapshotManager::instance()->make_snapshot(request, snapshot_path);
+ bool allow_incremental_clone; // not used
+ res = SnapshotManager::instance()->make_snapshot(request, snapshot_path,
&allow_incremental_clone);
if (res != OLAPStatus::OLAP_SUCCESS) {
LOG(WARNING) << "make snapshot failed. status: " << res << ",
signature: " << tablet_id;
return -1L;
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index d1ae0f8..c451f75 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -446,161 +446,6 @@ OLAPStatus DataDir::_clean_unfinished_converting_data() {
return OLAP_SUCCESS;
}
-// convert old tablet and its files to new tablet meta and rowset format
-// if any error occurred during converting, stop it and break.
-OLAPStatus DataDir::_convert_old_tablet() {
- auto convert_tablet_func = [this](int64_t tablet_id, int32_t schema_hash,
- const std::string& value) -> bool {
- OlapSnapshotConverter converter;
- // convert olap header and files
- OLAPHeaderMessage olap_header_msg;
- TabletMetaPB tablet_meta_pb;
- std::vector<RowsetMetaPB> pending_rowsets;
- bool parsed = olap_header_msg.ParseFromString(value);
- if (!parsed) {
- LOG(FATAL) << "convert olap header to tablet meta failed when load
olap header tablet="
- << tablet_id << "." << schema_hash;
- return false;
- }
- string old_data_path_prefix =
- get_absolute_tablet_path(olap_header_msg.shard_id(),
olap_header_msg.tablet_id(),
- olap_header_msg.schema_hash());
- OLAPStatus status = converter.to_new_snapshot(olap_header_msg,
old_data_path_prefix,
- old_data_path_prefix,
&tablet_meta_pb,
- &pending_rowsets, true);
- if (status != OLAP_SUCCESS) {
- LOG(FATAL) << "convert olap header to tablet meta failed when
convert header and files "
- "tablet="
- << tablet_id << "." << schema_hash;
- return false;
- }
-
- // write pending rowset to olap meta
- for (auto& rowset_pb : pending_rowsets) {
- RowsetId rowset_id;
- rowset_id.init(rowset_pb.rowset_id_v2());
- status = RowsetMetaManager::save(_meta, rowset_pb.tablet_uid(),
rowset_id, rowset_pb);
- if (status != OLAP_SUCCESS) {
- LOG(FATAL)
- << "convert olap header to tablet meta failed when
save rowset meta tablet="
- << tablet_id << "." << schema_hash;
- return false;
- }
- }
-
- // write converted tablet meta to olap meta
- string meta_binary;
- tablet_meta_pb.SerializeToString(&meta_binary);
- status = TabletMetaManager::save(this, tablet_meta_pb.tablet_id(),
- tablet_meta_pb.schema_hash(),
meta_binary);
- if (status != OLAP_SUCCESS) {
- LOG(FATAL) << "convert olap header to tablet meta failed when save
tablet meta tablet="
- << tablet_id << "." << schema_hash;
- return false;
- } else {
- LOG(INFO) << "convert olap header to tablet meta successfully and
save tablet meta to "
- "meta tablet="
- << tablet_id << "." << schema_hash;
- }
- return true;
- };
- OLAPStatus convert_tablet_status =
- TabletMetaManager::traverse_headers(_meta, convert_tablet_func,
OLD_HEADER_PREFIX);
- if (convert_tablet_status != OLAP_SUCCESS) {
- LOG(FATAL) << "there is failure when convert old tablet, data dir:" <<
_path;
- return convert_tablet_status;
- } else {
- LOG(INFO) << "successfully convert old tablet, data dir: " << _path;
- }
- return OLAP_SUCCESS;
-}
-
-OLAPStatus DataDir::remove_old_meta_and_files() {
- // clean old meta(olap header message)
- auto clean_old_meta_files_func = [this](int64_t tablet_id, int32_t
schema_hash,
- const std::string& value) -> bool {
- // convert olap header and files
- OLAPHeaderMessage olap_header_msg;
- TabletMetaPB tablet_meta_pb;
- std::vector<RowsetMetaPB> pending_rowsets;
- bool parsed = olap_header_msg.ParseFromString(value);
- if (!parsed) {
- LOG(FATAL) << "convert olap header to tablet meta failed when load
olap header tablet="
- << tablet_id << "." << schema_hash;
- return true;
- }
- OlapSnapshotConverter converter;
- OLAPStatus status =
- converter.to_tablet_meta_pb(olap_header_msg, &tablet_meta_pb,
&pending_rowsets);
- if (status != OLAP_SUCCESS) {
- LOG(FATAL) << "convert olap header to tablet meta failed when
convert header and files "
- "tablet="
- << tablet_id << "." << schema_hash;
- return true;
- }
-
- TabletSchema tablet_schema;
- tablet_schema.init_from_pb(tablet_meta_pb.schema());
- string data_path_prefix =
- get_absolute_tablet_path(tablet_meta_pb.shard_id(),
tablet_meta_pb.tablet_id(),
- tablet_meta_pb.schema_hash());
-
- // convert visible pdelta file to rowsets and remove old files
- for (auto& visible_rowset : tablet_meta_pb.rs_metas()) {
- RowsetMetaSharedPtr rowset_meta(new AlphaRowsetMeta());
- rowset_meta->init_from_pb(visible_rowset);
-
- RowsetSharedPtr rowset;
- auto s = RowsetFactory::create_rowset(&tablet_schema,
data_path_prefix, rowset_meta,
- &rowset);
- if (s != OLAP_SUCCESS) {
- LOG(INFO) << "errors while init rowset. tablet_path=" <<
data_path_prefix;
- return true;
- }
- std::vector<std::string> old_files;
- if (rowset->remove_old_files(&old_files) != OLAP_SUCCESS) {
- LOG(INFO) << "errors while remove_old_files. tablet_path=" <<
data_path_prefix;
- return true;
- }
- }
-
- // remove incremental dir and pending dir
- std::string pending_delta_path = data_path_prefix +
PENDING_DELTA_PREFIX;
- if (FileUtils::check_exist(pending_delta_path)) {
- LOG(INFO) << "remove pending delta path:" << pending_delta_path;
-
- RETURN_WITH_WARN_IF_ERROR(
- FileUtils::remove_all(pending_delta_path), true,
- "errors while remove pending delta path. tablet_path=" +
data_path_prefix);
- }
-
- std::string incremental_delta_path = data_path_prefix +
INCREMENTAL_DELTA_PREFIX;
- if (FileUtils::check_exist(incremental_delta_path)) {
- LOG(INFO) << "remove incremental delta path:" <<
incremental_delta_path;
-
- RETURN_WITH_WARN_IF_ERROR(
- FileUtils::remove_all(incremental_delta_path), true,
- "errors while remove incremental delta path. tablet_path="
+ data_path_prefix);
- }
-
- TabletMetaManager::remove(this, tablet_id, schema_hash,
OLD_HEADER_PREFIX);
- LOG(INFO) << "successfully clean old tablet meta(olap header) for
tablet=" << tablet_id
- << "." << schema_hash << " tablet_path=" << data_path_prefix;
-
- return true;
- };
- OLAPStatus clean_old_meta_files_status =
TabletMetaManager::traverse_headers(
- _meta, clean_old_meta_files_func, OLD_HEADER_PREFIX);
- if (clean_old_meta_files_status != OLAP_SUCCESS) {
- // If failed to clean meta just skip the error, there will be useless
metas in rocksdb column family
- LOG(WARNING) << "there is failure when clean old tablet meta(olap
header) from data dir:"
- << _path;
- } else {
- LOG(INFO) << "successfully clean old tablet meta(olap header) from
data dir: " << _path;
- }
- return OLAP_SUCCESS;
-}
-
bool DataDir::convert_old_data_success() {
return _convert_old_data_success;
}
diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h
index c05b50e..cd9844f 100644
--- a/be/src/olap/data_dir.h
+++ b/be/src/olap/data_dir.h
@@ -108,8 +108,6 @@ public:
void perform_path_gc_by_tablet();
- OLAPStatus remove_old_meta_and_files();
-
bool convert_old_data_success();
OLAPStatus set_convert_finished();
@@ -144,7 +142,6 @@ private:
Status _read_cluster_id(const std::string& cluster_id_path, int32_t*
cluster_id);
Status _write_cluster_id_to_path(const std::string& path, int32_t
cluster_id);
OLAPStatus _clean_unfinished_converting_data();
- OLAPStatus _convert_old_tablet();
// Check whether has old format (hdr_ start) in olap. When doris updating
to current version,
// it may lead to data missing. When
conf::storage_strict_check_incompatible_old_format is true,
// process will log fatal.
diff --git a/be/src/olap/olap_snapshot_converter.cpp
b/be/src/olap/olap_snapshot_converter.cpp
index bb6353a..4f8c2d1 100644
--- a/be/src/olap/olap_snapshot_converter.cpp
+++ b/be/src/olap/olap_snapshot_converter.cpp
@@ -24,187 +24,6 @@
namespace doris {
-OLAPStatus OlapSnapshotConverter::to_olap_header(const TabletMetaPB&
tablet_meta_pb,
- OLAPHeaderMessage*
olap_header) {
- if (!tablet_meta_pb.schema().has_num_rows_per_row_block()) {
- LOG(FATAL) << "tablet schema does not have num_rows_per_row_block."
- << " tablet id = " << tablet_meta_pb.tablet_id();
- }
-
olap_header->set_num_rows_per_data_block(tablet_meta_pb.schema().num_rows_per_row_block());
- if (!tablet_meta_pb.has_cumulative_layer_point()) {
- LOG(FATAL) << "tablet schema does not have cumulative_layer_point."
- << " tablet id = " << tablet_meta_pb.tablet_id();
- }
- olap_header->set_cumulative_layer_point(-1);
- if (!tablet_meta_pb.schema().has_num_short_key_columns()) {
- LOG(FATAL) << "tablet schema does not have num_short_key_columns."
- << " tablet id = " << tablet_meta_pb.tablet_id();
- }
-
olap_header->set_num_short_key_fields(tablet_meta_pb.schema().num_short_key_columns());
-
- for (auto& column : tablet_meta_pb.schema().column()) {
- ColumnMessage* column_msg = olap_header->add_column();
- to_column_msg(column, column_msg);
- }
-
- if (!tablet_meta_pb.has_creation_time()) {
- LOG(FATAL) << "tablet schema does not have creation_time."
- << " tablet id = " << tablet_meta_pb.tablet_id();
- }
- olap_header->set_creation_time(tablet_meta_pb.creation_time());
- olap_header->set_data_file_type(DataFileType::COLUMN_ORIENTED_FILE);
- if (tablet_meta_pb.schema().has_next_column_unique_id()) {
-
olap_header->set_next_column_unique_id(tablet_meta_pb.schema().next_column_unique_id());
- }
- if (tablet_meta_pb.schema().has_compress_kind()) {
-
olap_header->set_compress_kind(tablet_meta_pb.schema().compress_kind());
- }
- if (tablet_meta_pb.schema().has_bf_fpp()) {
- olap_header->set_bf_fpp(tablet_meta_pb.schema().bf_fpp());
- }
- if (tablet_meta_pb.schema().has_keys_type()) {
- olap_header->set_keys_type(tablet_meta_pb.schema().keys_type());
- }
-
- for (auto& rs_meta : tablet_meta_pb.rs_metas()) {
- // Add delete predicate OLAPHeaderMessage from PDelta.
- PDelta* pdelta = olap_header->add_delta();
- convert_to_pdelta(rs_meta, pdelta);
- if (pdelta->has_delete_condition()) {
- DeletePredicatePB* delete_condition =
olap_header->add_delete_data_conditions();
- *delete_condition = pdelta->delete_condition();
- }
- }
- // not add pending delta, it is useless in clone or backup restore
- for (auto& inc_rs_meta : tablet_meta_pb.inc_rs_metas()) {
- PDelta* pdelta = olap_header->add_incremental_delta();
- convert_to_pdelta(inc_rs_meta, pdelta);
- }
- if (tablet_meta_pb.has_in_restore_mode()) {
- olap_header->set_in_restore_mode(tablet_meta_pb.in_restore_mode());
- }
- if (tablet_meta_pb.has_tablet_id()) {
- olap_header->set_tablet_id(tablet_meta_pb.tablet_id());
- }
- if (tablet_meta_pb.has_schema_hash()) {
- olap_header->set_schema_hash(tablet_meta_pb.schema_hash());
- }
- if (tablet_meta_pb.has_shard_id()) {
- olap_header->set_shard_id(tablet_meta_pb.shard_id());
- }
- return OLAP_SUCCESS;
-}
-
-OLAPStatus OlapSnapshotConverter::to_tablet_meta_pb(const OLAPHeaderMessage&
olap_header,
- TabletMetaPB*
tablet_meta_pb,
- std::vector<RowsetMetaPB>*
pending_rowsets) {
- if (olap_header.has_tablet_id()) {
- tablet_meta_pb->set_tablet_id(olap_header.tablet_id());
- }
- if (olap_header.has_schema_hash()) {
- tablet_meta_pb->set_schema_hash(olap_header.schema_hash());
- }
- if (olap_header.has_shard_id()) {
- tablet_meta_pb->set_shard_id(olap_header.shard_id());
- }
- tablet_meta_pb->set_creation_time(olap_header.creation_time());
- tablet_meta_pb->set_cumulative_layer_point(-1);
-
- TabletSchemaPB* schema = tablet_meta_pb->mutable_schema();
- for (auto& column_msg : olap_header.column()) {
- ColumnPB* column_pb = schema->add_column();
- to_column_pb(column_msg, column_pb);
- }
- if (olap_header.has_keys_type()) {
- schema->set_keys_type(olap_header.keys_type());
- } else {
- // Doris support AGG_KEYS/UNIQUE_KEYS/DUP_KEYS/ three storage model.
- // Among these three model, UNIQUE_KYES/DUP_KEYS is added after
AGG_KEYS.
- // For historical tablet, the keys_type field to indicate storage model
- // may be missed for AGG_KEYS.
- // So upgrade from historical tablet, this situation should be taken
into
- // consideration and set to be AGG_KEYS.
- schema->set_keys_type(KeysType::AGG_KEYS);
- }
-
- schema->set_num_short_key_columns(olap_header.num_short_key_fields());
- schema->set_num_rows_per_row_block(olap_header.num_rows_per_data_block());
- schema->set_compress_kind(olap_header.compress_kind());
- if (olap_header.has_bf_fpp()) {
- schema->set_bf_fpp(olap_header.bf_fpp());
- }
- if (olap_header.has_next_column_unique_id()) {
- schema->set_next_column_unique_id(olap_header.next_column_unique_id());
- }
-
- std::unordered_map<Version, RowsetMetaPB*, HashOfVersion> _rs_version_map;
- const DelPredicateArray& delete_conditions =
olap_header.delete_data_conditions();
- for (auto& delta : olap_header.delta()) {
- RowsetId next_id = StorageEngine::instance()->next_rowset_id();
- RowsetMetaPB* rowset_meta = tablet_meta_pb->add_rs_metas();
- PDelta temp_delta = delta;
- // PDelta is not corresponding with RowsetMeta in DeletePredicate
- // Add delete predicate to PDelta from OLAPHeaderMessage.
- // Only after this, convert from PDelta to RowsetMeta is valid.
- if (temp_delta.start_version() == temp_delta.end_version()) {
- for (auto& del_pred : delete_conditions) {
- if (temp_delta.start_version() == del_pred.version()) {
- DeletePredicatePB* delete_condition =
temp_delta.mutable_delete_condition();
- *delete_condition = del_pred;
- }
- }
- }
- convert_to_rowset_meta(temp_delta, next_id, olap_header.tablet_id(),
- olap_header.schema_hash(), rowset_meta);
- Version rowset_version = {temp_delta.start_version(),
temp_delta.end_version()};
- _rs_version_map[rowset_version] = rowset_meta;
- }
-
- for (auto& inc_delta : olap_header.incremental_delta()) {
- // check if inc delta already exist in delta
- Version rowset_version = {inc_delta.start_version(),
inc_delta.end_version()};
- auto exist_rs = _rs_version_map.find(rowset_version);
- if (exist_rs != _rs_version_map.end()) {
- RowsetMetaPB* rowset_meta = tablet_meta_pb->add_inc_rs_metas();
- *rowset_meta = *(exist_rs->second);
- continue;
- }
- RowsetId next_id = StorageEngine::instance()->next_rowset_id();
- RowsetMetaPB* rowset_meta = tablet_meta_pb->add_inc_rs_metas();
- PDelta temp_inc_delta = inc_delta;
- if (temp_inc_delta.start_version() == temp_inc_delta.end_version()) {
- for (auto& del_pred : delete_conditions) {
- if (temp_inc_delta.start_version() == del_pred.version()) {
- DeletePredicatePB* delete_condition =
temp_inc_delta.mutable_delete_condition();
- *delete_condition = del_pred;
- }
- }
- }
- convert_to_rowset_meta(temp_inc_delta, next_id,
olap_header.tablet_id(),
- olap_header.schema_hash(), rowset_meta);
- }
-
- for (auto& pending_delta : olap_header.pending_delta()) {
- RowsetId next_id = StorageEngine::instance()->next_rowset_id();
- RowsetMetaPB rowset_meta;
- convert_to_rowset_meta(pending_delta, next_id, olap_header.tablet_id(),
- olap_header.schema_hash(), &rowset_meta);
- pending_rowsets->emplace_back(std::move(rowset_meta));
- }
- if (olap_header.has_schema_change_status()) {
- AlterTabletPB* alter_tablet_pb = tablet_meta_pb->mutable_alter_task();
- to_alter_tablet_pb(olap_header.schema_change_status(),
alter_tablet_pb);
- }
- if (olap_header.has_in_restore_mode()) {
- tablet_meta_pb->set_in_restore_mode(olap_header.in_restore_mode());
- }
- tablet_meta_pb->set_tablet_state(TabletStatePB::PB_RUNNING);
- *(tablet_meta_pb->mutable_tablet_uid()) = TabletUid::gen_uid().to_proto();
- VLOG_NOTICE << "convert tablet meta tablet id = " <<
olap_header.tablet_id()
- << " schema hash = " << olap_header.schema_hash() << "
successfully.";
- return OLAP_SUCCESS;
-}
-
OLAPStatus OlapSnapshotConverter::convert_to_pdelta(const RowsetMetaPB&
rowset_meta_pb,
PDelta* delta) {
if (!rowset_meta_pb.has_start_version()) {
@@ -450,103 +269,6 @@ OLAPStatus OlapSnapshotConverter::to_alter_tablet_pb(
return OLAP_SUCCESS;
}
-// from olap header to tablet meta
-OLAPStatus OlapSnapshotConverter::to_new_snapshot(const OLAPHeaderMessage&
olap_header,
- const string&
old_data_path_prefix,
- const string&
new_data_path_prefix,
- TabletMetaPB* tablet_meta_pb,
- std::vector<RowsetMetaPB>*
pending_rowsets,
- bool is_startup) {
- RETURN_NOT_OK(to_tablet_meta_pb(olap_header, tablet_meta_pb,
pending_rowsets));
-
- TabletSchema tablet_schema;
- tablet_schema.init_from_pb(tablet_meta_pb->schema());
-
- // convert visible pdelta file to rowsets
- for (auto& visible_rowset : tablet_meta_pb->rs_metas()) {
- RowsetMetaSharedPtr alpha_rowset_meta(new AlphaRowsetMeta());
- alpha_rowset_meta->init_from_pb(visible_rowset);
- alpha_rowset_meta->set_tablet_uid(tablet_meta_pb->tablet_uid());
- AlphaRowset rowset(&tablet_schema, new_data_path_prefix,
alpha_rowset_meta);
- RETURN_NOT_OK(rowset.init());
- std::vector<std::string> success_files;
- RETURN_NOT_OK(rowset.convert_from_old_files(old_data_path_prefix,
&success_files));
-
_modify_old_segment_group_id(const_cast<RowsetMetaPB&>(visible_rowset));
- }
-
- // convert inc delta file to rowsets
- for (auto& inc_rowset : tablet_meta_pb->inc_rs_metas()) {
- RowsetMetaSharedPtr alpha_rowset_meta(new AlphaRowsetMeta());
- alpha_rowset_meta->init_from_pb(inc_rowset);
- alpha_rowset_meta->set_tablet_uid(tablet_meta_pb->tablet_uid());
- AlphaRowset rowset(&tablet_schema, new_data_path_prefix,
alpha_rowset_meta);
- RETURN_NOT_OK(rowset.init());
- std::vector<std::string> success_files;
- std::string inc_data_path = old_data_path_prefix;
- // in clone case: there is no incremental perfix
- // in start up case: there is incremental prefix
- if (is_startup) {
- inc_data_path = inc_data_path + "/" + INCREMENTAL_DELTA_PREFIX;
- }
- RETURN_NOT_OK(rowset.convert_from_old_files(inc_data_path,
&success_files));
- _modify_old_segment_group_id(const_cast<RowsetMetaPB&>(inc_rowset));
- }
-
- for (auto it = pending_rowsets->begin(); it != pending_rowsets->end();
++it) {
- RowsetMetaSharedPtr alpha_rowset_meta(new AlphaRowsetMeta());
- alpha_rowset_meta->init_from_pb(*it);
- alpha_rowset_meta->set_tablet_uid(tablet_meta_pb->tablet_uid());
- AlphaRowset rowset(&tablet_schema, new_data_path_prefix,
alpha_rowset_meta);
- RETURN_NOT_OK(rowset.init());
- std::vector<std::string> success_files;
- // std::string pending_delta_path = old_data_path_prefix +
PENDING_DELTA_PREFIX;
- // if this is a pending segment group, rowset will add
pending_delta_prefix when
- // construct old file path
- RETURN_NOT_OK(rowset.convert_from_old_files(old_data_path_prefix,
&success_files));
- // pending delta does not have row num, index size, data size info
- // should load the pending delta, get these info and reset rowset
meta's row num
- // data size, index size
- RETURN_NOT_OK(rowset.reset_sizeinfo());
- // pending rowset not have segment group id == -1 problem, not need to
modify sg id in meta
- rowset.to_rowset_pb(&(*it));
- }
- return OLAP_SUCCESS;
-}
-
-// from tablet meta to olap header
-OLAPStatus OlapSnapshotConverter::to_old_snapshot(const TabletMetaPB&
tablet_meta_pb,
- string& new_data_path_prefix,
- string& old_data_path_prefix,
- OLAPHeaderMessage*
olap_header) {
- RETURN_NOT_OK(to_olap_header(tablet_meta_pb, olap_header));
-
- TabletSchema tablet_schema;
- tablet_schema.init_from_pb(tablet_meta_pb.schema());
-
- // convert visible pdelta file to rowsets
- for (auto& visible_rowset : tablet_meta_pb.rs_metas()) {
- RowsetMetaSharedPtr alpha_rowset_meta(new AlphaRowsetMeta());
- alpha_rowset_meta->init_from_pb(visible_rowset);
- AlphaRowset rowset(&tablet_schema, new_data_path_prefix,
alpha_rowset_meta);
- RETURN_NOT_OK(rowset.init());
- RETURN_NOT_OK(rowset.load());
- std::vector<std::string> success_files;
- RETURN_NOT_OK(rowset.convert_to_old_files(old_data_path_prefix,
&success_files));
- }
-
- // convert inc delta file to rowsets
- for (auto& inc_rowset : tablet_meta_pb.inc_rs_metas()) {
- RowsetMetaSharedPtr alpha_rowset_meta(new AlphaRowsetMeta());
- alpha_rowset_meta->init_from_pb(inc_rowset);
- AlphaRowset rowset(&tablet_schema, new_data_path_prefix,
alpha_rowset_meta);
- RETURN_NOT_OK(rowset.init());
- RETURN_NOT_OK(rowset.load());
- std::vector<std::string> success_files;
- RETURN_NOT_OK(rowset.convert_to_old_files(old_data_path_prefix,
&success_files));
- }
- return OLAP_SUCCESS;
-}
-
OLAPStatus OlapSnapshotConverter::save(const string& file_path,
const OLAPHeaderMessage& olap_header) {
DCHECK(!file_path.empty());
diff --git a/be/src/olap/olap_snapshot_converter.h
b/be/src/olap/olap_snapshot_converter.h
index 6b223e3..46bb71e 100644
--- a/be/src/olap/olap_snapshot_converter.h
+++ b/be/src/olap/olap_snapshot_converter.h
@@ -39,15 +39,6 @@ namespace doris {
class OlapSnapshotConverter {
public:
- // convert tablet meta pb to olap header
- // only consider alpha rowset not other rowsets
- OLAPStatus to_olap_header(const TabletMetaPB& tablet_meta_pb,
OLAPHeaderMessage* olap_header);
-
- // convert olap header to tablet meta pb, convert delta to rowsetmetapb
- // pending delta is not in tablet meta any more, so that convert pending
delta to rowset and add it to pending rowsets
- // as a return value
- OLAPStatus to_tablet_meta_pb(const OLAPHeaderMessage& olap_header,
TabletMetaPB* tablet_meta_pb,
- vector<RowsetMetaPB>* pending_rowsets);
OLAPStatus convert_to_pdelta(const RowsetMetaPB& rowset_meta_pb, PDelta*
delta);
@@ -68,16 +59,6 @@ public:
OLAPStatus to_alter_tablet_pb(const SchemaChangeStatusMessage&
schema_change_msg,
AlterTabletPB* alter_tablet_pb);
- // from olap header to tablet meta
- OLAPStatus to_new_snapshot(const OLAPHeaderMessage& olap_header,
- const string& old_data_path_prefix,
- const string& new_data_path_prefix,
TabletMetaPB* tablet_meta_pb,
- vector<RowsetMetaPB>* pending_rowsets, bool
is_startup);
-
- // from tablet meta to olap header
- OLAPStatus to_old_snapshot(const TabletMetaPB& tablet_meta_pb, string&
new_data_path_prefix,
- string& old_data_path_prefix,
OLAPHeaderMessage* olap_header);
-
OLAPStatus save(const string& file_path, const OLAPHeaderMessage&
olap_header);
private:
diff --git a/be/src/olap/rowset/rowset.cpp b/be/src/olap/rowset/rowset.cpp
index b441ce3..56c0e54 100644
--- a/be/src/olap/rowset/rowset.cpp
+++ b/be/src/olap/rowset/rowset.cpp
@@ -53,7 +53,7 @@ OLAPStatus Rowset::load(bool use_cache) {
}
}
// load is done
- LOG(INFO) << "rowset is loaded. rowset version:" << start_version() << "-"
<< end_version()
+ LOG(INFO) << "rowset is loaded. " << rowset_id() << ", rowset version:" <<
rowset_meta()->version()
<< ", state from ROWSET_UNLOADED to ROWSET_LOADED. tabletid:"
<< _rowset_meta->tablet_id();
return OLAP_SUCCESS;
diff --git a/be/src/olap/snapshot_manager.cpp b/be/src/olap/snapshot_manager.cpp
index 71c892c..32adf76 100644
--- a/be/src/olap/snapshot_manager.cpp
+++ b/be/src/olap/snapshot_manager.cpp
@@ -66,7 +66,11 @@ SnapshotManager* SnapshotManager::instance() {
return _s_instance;
}
-OLAPStatus SnapshotManager::make_snapshot(const TSnapshotRequest& request,
string* snapshot_path) {
+OLAPStatus SnapshotManager::make_snapshot(
+ const TSnapshotRequest& request,
+ string* snapshot_path,
+ bool* allow_incremental_clone) {
+
OLAPStatus res = OLAP_SUCCESS;
if (snapshot_path == nullptr) {
LOG(WARNING) << "output parameter cannot be NULL";
@@ -81,12 +85,7 @@ OLAPStatus SnapshotManager::make_snapshot(const
TSnapshotRequest& request, strin
return OLAP_ERR_TABLE_NOT_FOUND;
}
- res = _create_snapshot_files(ref_tablet, request, snapshot_path,
- request.preferred_snapshot_version);
- // if all nodes has been upgraded, it can be removed
- if (request.__isset.missing_version && res == OLAP_SUCCESS) {
-
(const_cast<TSnapshotRequest&>(request)).__set_allow_incremental_clone(true);
- }
+ res = _create_snapshot_files(ref_tablet, request, snapshot_path,
allow_incremental_clone);
if (res != OLAP_SUCCESS) {
LOG(WARNING) << "failed to make snapshot. res=" << res << " tablet="
<< request.tablet_id
@@ -148,7 +147,10 @@ OLAPStatus SnapshotManager::convert_rowset_ids(const
string& clone_dir, int64_t
TabletMetaPB new_tablet_meta_pb;
new_tablet_meta_pb = cloned_tablet_meta_pb;
new_tablet_meta_pb.clear_rs_metas();
+ // inc_rs_meta is deprecated since 0.13.
+ // keep this just for safety
new_tablet_meta_pb.clear_inc_rs_metas();
+ new_tablet_meta_pb.clear_stale_rs_metas();
// should modify tablet id and schema hash because in restore process the
tablet id is not
// equal to tablet id in meta
new_tablet_meta_pb.set_tablet_id(tablet_id);
@@ -156,7 +158,7 @@ OLAPStatus SnapshotManager::convert_rowset_ids(const
string& clone_dir, int64_t
TabletSchema tablet_schema;
tablet_schema.init_from_pb(new_tablet_meta_pb.schema());
- std::unordered_map<Version, RowsetMetaPB*, HashOfVersion> _rs_version_map;
+ std::unordered_map<Version, RowsetMetaPB*, HashOfVersion> rs_version_map;
for (auto& visible_rowset : cloned_tablet_meta_pb.rs_metas()) {
RowsetMetaPB* rowset_meta = new_tablet_meta_pb.add_rs_metas();
RowsetId rowset_id = StorageEngine::instance()->next_rowset_id();
@@ -165,21 +167,19 @@ OLAPStatus SnapshotManager::convert_rowset_ids(const
string& clone_dir, int64_t
rowset_meta->set_tablet_id(tablet_id);
rowset_meta->set_tablet_schema_hash(schema_hash);
Version rowset_version = {visible_rowset.start_version(),
visible_rowset.end_version()};
- _rs_version_map[rowset_version] = rowset_meta;
+ rs_version_map[rowset_version] = rowset_meta;
}
- for (auto& inc_rowset : cloned_tablet_meta_pb.inc_rs_metas()) {
- Version rowset_version = {inc_rowset.start_version(),
inc_rowset.end_version()};
- auto exist_rs = _rs_version_map.find(rowset_version);
- if (exist_rs != _rs_version_map.end()) {
- RowsetMetaPB* rowset_meta = new_tablet_meta_pb.add_inc_rs_metas();
- *rowset_meta = *(exist_rs->second);
+ for (auto& stale_rowset : cloned_tablet_meta_pb.stale_rs_metas()) {
+ Version rowset_version = {stale_rowset.start_version(),
stale_rowset.end_version()};
+ auto exist_rs = rs_version_map.find(rowset_version);
+ if (exist_rs != rs_version_map.end()) {
continue;
}
- RowsetMetaPB* rowset_meta = new_tablet_meta_pb.add_inc_rs_metas();
+ RowsetMetaPB* rowset_meta = new_tablet_meta_pb.add_stale_rs_metas();
RowsetId rowset_id = StorageEngine::instance()->next_rowset_id();
RETURN_NOT_OK(
- _rename_rowset_id(inc_rowset, clone_dir, tablet_schema,
rowset_id, rowset_meta));
+ _rename_rowset_id(stale_rowset, clone_dir, tablet_schema,
rowset_id, rowset_meta));
rowset_meta->set_tablet_id(tablet_id);
rowset_meta->set_tablet_schema_hash(schema_hash);
}
@@ -306,17 +306,19 @@ OLAPStatus SnapshotManager::_link_index_and_data_files(
OLAPStatus SnapshotManager::_create_snapshot_files(const TabletSharedPtr&
ref_tablet,
const TSnapshotRequest&
request,
string* snapshot_path,
- int32_t snapshot_version) {
- LOG(INFO) << "receive a make snapshot request,"
- << " request detail is " <<
apache::thrift::ThriftDebugString(request)
- << " snapshot_path is " << *snapshot_path << " snapshot_version
is "
- << snapshot_version;
+ bool*
allow_incremental_clone) {
+ int32_t snapshot_version = request.preferred_snapshot_version;
+ LOG(INFO) << "receive a make snapshot request"
+ << ", request detail is " <<
apache::thrift::ThriftDebugString(request)
+ << ", snapshot_version is " << snapshot_version;
OLAPStatus res = OLAP_SUCCESS;
if (snapshot_path == nullptr) {
LOG(WARNING) << "output parameter cannot be NULL";
return OLAP_ERR_INPUT_PARAMETER_ERROR;
}
+ // snapshot_id_path:
+ // /data/shard_id/tablet_id/snapshot/time_str/id.timeout/
string snapshot_id_path;
int64_t timeout_s = config::snapshot_expire_time_sec;
if (request.__isset.timeout) {
@@ -329,7 +331,11 @@ OLAPStatus SnapshotManager::_create_snapshot_files(const
TabletSharedPtr& ref_ta
return res;
}
+ // schema_full_path:
+ // /snapshot_id_path/tablet_id/schema_hash/
string schema_full_path = get_schema_hash_full_path(ref_tablet,
snapshot_id_path);
+ // header_path:
+ // /schema_full_path/tablet_id.hdr
string header_path = _get_header_full_path(ref_tablet, schema_full_path);
if (FileUtils::check_exist(schema_full_path)) {
VLOG_TRACE << "remove the old schema_full_path.";
@@ -352,57 +358,71 @@ OLAPStatus SnapshotManager::_create_snapshot_files(const
TabletSharedPtr& ref_ta
break;
}
std::vector<RowsetSharedPtr> consistent_rowsets;
- if (request.__isset.missing_version) {
+
+ /// If set missing_version, try to get all missing version.
+ /// If some of them not exist in tablet, we will fall back to
+ /// make the full snapshot of the tablet.
+ {
ReadLock rdlock(ref_tablet->get_header_lock_ptr());
- for (int64_t missed_version : request.missing_version) {
- Version version = {missed_version, missed_version};
- const RowsetSharedPtr rowset =
ref_tablet->get_inc_rowset_by_version(version);
- if (rowset != nullptr) {
- consistent_rowsets.push_back(rowset);
- } else {
- LOG(WARNING) << "failed to find missed version when
snapshot. "
- << " tablet=" << request.tablet_id
- << " schema_hash=" << request.schema_hash
- << " version=" << version.first << "-" <<
version.second;
+ if (request.__isset.missing_version) {
+ for (int64_t missed_version : request.missing_version) {
+ Version version = {missed_version, missed_version};
+ // find rowset in both rs_meta and stale_rs_meta
+ const RowsetSharedPtr rowset =
ref_tablet->get_rowset_by_version(version, true);
+ if (rowset != nullptr) {
+ consistent_rowsets.push_back(rowset);
+ } else {
+ LOG(WARNING) << "failed to find missed version when
snapshot. "
+ << " tablet=" << request.tablet_id
+ << " schema_hash=" << request.schema_hash
+ << " version=" << version;
+ res = OLAP_ERR_VERSION_NOT_EXIST;
+ break;
+ }
+ }
+ }
+
+ if (res != OLAP_SUCCESS || !request.__isset.missing_version) {
+ /// not all missing versions are found, fall back to full
snapshot.
+ res = OLAP_SUCCESS; // reset res
+ consistent_rowsets.clear(); // reset vector
+
+ ReadLock rdlock(ref_tablet->get_header_lock_ptr());
+ // get latest version
+ const RowsetSharedPtr last_version =
ref_tablet->rowset_with_max_version();
+ if (last_version == nullptr) {
+ LOG(WARNING) << "tablet has not any version. path="
+ << ref_tablet->full_name().c_str();
res = OLAP_ERR_VERSION_NOT_EXIST;
break;
}
- }
- if (res != OLAP_SUCCESS) {
- break;
- }
- ref_tablet->generate_tablet_meta_copy_unlocked(new_tablet_meta);
- } else {
- ReadLock rdlock(ref_tablet->get_header_lock_ptr());
- // get latest version
- const RowsetSharedPtr last_version =
ref_tablet->rowset_with_max_version();
- if (last_version == nullptr) {
- LOG(WARNING) << "tablet has not any version. path="
- << ref_tablet->full_name().c_str();
- res = OLAP_ERR_VERSION_NOT_EXIST;
- break;
- }
- // get snapshot version, use request.version if specified
- int32_t version = last_version->end_version();
- if (request.__isset.version) {
- if (last_version->end_version() < request.version) {
- LOG(WARNING) << "invalid make snapshot request. "
- << " version=" << last_version->end_version()
- << " req_version=" << request.version;
- res = OLAP_ERR_INPUT_PARAMETER_ERROR;
+ // get snapshot version, use request.version if specified
+ int32_t version = last_version->end_version();
+ if (request.__isset.version) {
+ if (last_version->end_version() < request.version) {
+ LOG(WARNING) << "invalid make snapshot request. "
+ << " version=" <<
last_version->end_version()
+ << " req_version=" << request.version;
+ res = OLAP_ERR_INPUT_PARAMETER_ERROR;
+ break;
+ }
+ version = request.version;
+ }
+ // get shortest version path
+ // it very important!!!!
+ // it means 0-version has to be a readable version graph
+ res = ref_tablet->capture_consistent_rowsets(Version(0,
version), &consistent_rowsets);
+ if (res != OLAP_SUCCESS) {
+ LOG(WARNING) << "fail to select versions to span. res=" <<
res;
break;
}
- version = request.version;
- }
- // get shortest version path
- // it very important!!!!
- // it means 0-version has to be a readable version graph
- res = ref_tablet->capture_consistent_rowsets(Version(0, version),
&consistent_rowsets);
- if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to select versions to span. res=" << res;
- break;
+ *allow_incremental_clone = false;
+ } else {
+ *allow_incremental_clone = true;
}
+ // copy the tablet meta to new_tablet_meta inside header lock
+ CHECK(res == OLAP_SUCCESS) << res;
ref_tablet->generate_tablet_meta_copy_unlocked(new_tablet_meta);
}
@@ -425,26 +445,15 @@ OLAPStatus SnapshotManager::_create_snapshot_files(const
TabletSharedPtr& ref_ta
// clear alter task info in snapshot files
new_tablet_meta->delete_alter_task();
- if (request.__isset.missing_version) {
- new_tablet_meta->revise_inc_rs_metas(std::move(rs_metas));
- new_tablet_meta->revise_rs_metas(vector<RowsetMetaSharedPtr>());
- } else {
- // If this is a full clone, then should clear inc rowset metas
because
- // related files is not created
-
new_tablet_meta->revise_inc_rs_metas(vector<RowsetMetaSharedPtr>());
- new_tablet_meta->revise_rs_metas(std::move(rs_metas));
- }
+ // The inc_rs_metas is deprecated since Doris version 0.13.
+ // Clear it for safety reason.
+ // Whether it is incremental or full snapshot, rowset information is
stored in rs_meta.
+ new_tablet_meta->revise_rs_metas(std::move(rs_metas));
if (snapshot_version == g_Types_constants.TSNAPSHOT_REQ_VERSION1) {
// convert beta rowset to alpha rowset
- if (request.__isset.missing_version) {
- res = _convert_beta_rowsets_to_alpha(new_tablet_meta,
-
new_tablet_meta->all_inc_rs_metas(),
- schema_full_path, true);
- } else {
- res = _convert_beta_rowsets_to_alpha(
- new_tablet_meta, new_tablet_meta->all_rs_metas(),
schema_full_path, false);
- }
+ res = _convert_beta_rowsets_to_alpha(
+ new_tablet_meta, new_tablet_meta->all_rs_metas(),
schema_full_path);
if (res != OLAP_SUCCESS) {
break;
}
@@ -467,28 +476,6 @@ OLAPStatus SnapshotManager::_create_snapshot_files(const
TabletSharedPtr& ref_ta
break;
}
- // append a single delta if request.version is end_version of
cumulative delta
- if (request.__isset.version) {
- for (auto& rs : consistent_rowsets) {
- if (rs->end_version() == request.version) {
- if (rs->start_version() != request.version) {
- // visible version in fe is 900
- // A need to clone 900 from B, but B's last version is
901, and 901 is not a visible version
- // and 901 will be reverted
- // since 900 is not the last version in B, 900 maybe
compacted with other versions
- // if A only get 900, then A's last version will be a
cumulative delta
- // many codes in be assumes that the last version is a
single delta
- // both clone and backup restore depend on this logic
- // TODO (yiguolei) fix it in the future
- // res = _append_single_delta(request, data_dir);
- if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to append single delta.
res=" << res;
- }
- }
- break;
- }
- }
- }
} while (0);
if (res != OLAP_SUCCESS) {
@@ -508,8 +495,7 @@ OLAPStatus SnapshotManager::_create_snapshot_files(const
TabletSharedPtr& ref_ta
OLAPStatus SnapshotManager::_convert_beta_rowsets_to_alpha(
const TabletMetaSharedPtr& new_tablet_meta,
- const std::vector<RowsetMetaSharedPtr>& rowset_metas, const
std::string& dst_path,
- bool is_incremental) {
+ const std::vector<RowsetMetaSharedPtr>& rowset_metas, const
std::string& dst_path) {
OLAPStatus res = OLAP_SUCCESS;
RowsetConverter rowset_converter(new_tablet_meta);
std::vector<RowsetMetaSharedPtr> new_rowset_metas;
@@ -540,11 +526,7 @@ OLAPStatus SnapshotManager::_convert_beta_rowsets_to_alpha(
}
}
if (res == OLAP_SUCCESS && modified) {
- if (is_incremental) {
- new_tablet_meta->revise_inc_rs_metas(std::move(new_rowset_metas));
- } else {
- new_tablet_meta->revise_rs_metas(std::move(new_rowset_metas));
- }
+ new_tablet_meta->revise_rs_metas(std::move(new_rowset_metas));
}
return res;
}
diff --git a/be/src/olap/snapshot_manager.h b/be/src/olap/snapshot_manager.h
index ba0d0da..d3685a4 100644
--- a/be/src/olap/snapshot_manager.h
+++ b/be/src/olap/snapshot_manager.h
@@ -45,11 +45,12 @@ namespace doris {
class SnapshotManager {
public:
~SnapshotManager() {}
- // @brief 创建snapshot
- // @param tablet_id [in] 原表的id
- // @param schema_hash [in] 原表的schema，与tablet_id参数合起来唯一确定一张表
- // @param snapshot_path [out] 新生成的snapshot的路径
- OLAPStatus make_snapshot(const TSnapshotRequest& request, std::string*
snapshot_path);
+
+ /// Create a snapshot
+ /// snapshot_path: out param, the dir of snapshot
+ /// allow_incremental_clone: out param, true if it is an incremental clone
+ OLAPStatus make_snapshot(const TSnapshotRequest& request, std::string*
snapshot_path,
+ bool* allow_incremental_clone);
std::string get_schema_hash_full_path(const TabletSharedPtr& ref_tablet,
const std::string& location) const;
@@ -78,7 +79,7 @@ private:
OLAPStatus _create_snapshot_files(const TabletSharedPtr& ref_tablet,
const TSnapshotRequest& request,
std::string* snapshot_path,
- int32_t snapshot_version);
+ bool* allow_incremental_clone);
OLAPStatus _prepare_snapshot_dir(const TabletSharedPtr& ref_tablet,
std::string* snapshot_id_path);
@@ -89,7 +90,7 @@ private:
OLAPStatus _convert_beta_rowsets_to_alpha(const TabletMetaSharedPtr&
new_tablet_meta,
const
vector<RowsetMetaSharedPtr>& rowset_metas,
- const std::string& dst_path,
bool is_incremental);
+ const std::string& dst_path);
private:
static SnapshotManager* _s_instance;
diff --git a/be/src/olap/tablet.cpp b/be/src/olap/tablet.cpp
index eff6a10..27b110e 100644
--- a/be/src/olap/tablet.cpp
+++ b/be/src/olap/tablet.cpp
@@ -104,22 +104,6 @@ OLAPStatus Tablet::_init_once_action() {
_rs_version_map[version] = std::move(rowset);
}
- // init incremental rowset
- for (auto& inc_rs_meta : _tablet_meta->all_inc_rs_metas()) {
- Version version = inc_rs_meta->version();
- RowsetSharedPtr rowset = get_rowset_by_version(version);
- if (rowset == nullptr) {
- res = RowsetFactory::create_rowset(&_schema, _tablet_path,
inc_rs_meta, &rowset);
- if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to init incremental rowset. tablet_id:"
<< tablet_id()
- << ", schema_hash:" << schema_hash() << ",
version=" << version
- << ", res:" << res;
- return res;
- }
- }
- _inc_rs_version_map[version] = std::move(rowset);
- }
-
// init stale rowset
for (auto& stale_rs_meta : _tablet_meta->all_stale_rs_metas()) {
Version version = stale_rs_meta->version();
@@ -154,9 +138,9 @@ void Tablet::save_meta() {
OLAPStatus Tablet::revise_tablet_meta(const std::vector<RowsetMetaSharedPtr>&
rowsets_to_clone,
const std::vector<Version>&
versions_to_delete) {
- LOG(INFO) << "begin to clone data to tablet. tablet=" << full_name()
+ LOG(INFO) << "begin to revise tablet. tablet=" << full_name()
<< ", rowsets_to_clone=" << rowsets_to_clone.size()
- << ", versions_to_delete_size=" << versions_to_delete.size();
+ << ", versions_to_delete=" << versions_to_delete.size();
OLAPStatus res = OLAP_SUCCESS;
do {
// load new local tablet_meta to operate on
@@ -173,6 +157,7 @@ OLAPStatus Tablet::revise_tablet_meta(const
std::vector<RowsetMetaSharedPtr>& ro
<< full_name() << ", version=" << version << "]";
}
+ // add new cloned rowset
for (auto& rs_meta : rowsets_to_clone) {
new_tablet_meta->add_rs_meta(rs_meta);
}
@@ -193,10 +178,6 @@ OLAPStatus Tablet::revise_tablet_meta(const
std::vector<RowsetMetaSharedPtr>& ro
StorageEngine::instance()->add_unused_rowset(it->second);
_rs_version_map.erase(it);
}
- for (auto& it : _inc_rs_version_map) {
- StorageEngine::instance()->add_unused_rowset(it.second);
- }
- _inc_rs_version_map.clear();
for (auto& rs_meta : rowsets_to_clone) {
Version version = {rs_meta->start_version(), rs_meta->end_version()};
@@ -212,9 +193,8 @@ OLAPStatus Tablet::revise_tablet_meta(const
std::vector<RowsetMetaSharedPtr>& ro
// reconstruct from tablet meta
_timestamped_version_tracker.construct_versioned_tracker(_tablet_meta->all_rs_metas());
- LOG(INFO) << "finish to clone data to tablet. res=" << res << ", "
- << "table=" << full_name() << ", "
- << "rowsets_to_clone=" << rowsets_to_clone.size();
+ LOG(INFO) << "finish to revise tablet. res=" << res << ", "
+ << "table=" << full_name();
return res;
}
@@ -293,10 +273,12 @@ void Tablet::modify_rowsets(const
std::vector<RowsetSharedPtr>& to_add,
// snapshot manager may call this api to check if version exists, so that
// the version maybe not exist
-const RowsetSharedPtr Tablet::get_rowset_by_version(const Version& version)
const {
+const RowsetSharedPtr Tablet::get_rowset_by_version(const Version& version,
bool find_in_stale) const {
auto iter = _rs_version_map.find(version);
if (iter == _rs_version_map.end()) {
- VLOG_NOTICE << "no rowset for version:" << version << ", tablet: " <<
full_name();
+ if (find_in_stale) {
+ return get_stale_rowset_by_version(version);
+ }
return nullptr;
}
return iter->second;
@@ -311,18 +293,6 @@ const RowsetSharedPtr
Tablet::get_stale_rowset_by_version(const Version& version
return iter->second;
}
-// This function only be called by SnapshotManager to perform incremental
clone.
-// It will be called under protected of _meta_lock(SnapshotManager will fetch
it manually),
-// so it is no need to lock here.
-const RowsetSharedPtr Tablet::get_inc_rowset_by_version(const Version&
version) const {
- auto iter = _inc_rs_version_map.find(version);
- if (iter == _inc_rs_version_map.end()) {
- VLOG_NOTICE << "no rowset for version:" << version << ", tablet: " <<
full_name();
- return nullptr;
- }
- return iter->second;
-}
-
// Already under _meta_lock
const RowsetSharedPtr Tablet::rowset_with_max_version() const {
Version max_version = _tablet_meta->max_version();
@@ -364,28 +334,13 @@ OLAPStatus Tablet::add_inc_rowset(const RowsetSharedPtr&
rowset) {
RETURN_NOT_OK(_tablet_meta->add_rs_meta(rowset->rowset_meta()));
_rs_version_map[rowset->version()] = rowset;
- _inc_rs_version_map[rowset->version()] = rowset;
_timestamped_version_tracker.add_version(rowset->version());
- RETURN_NOT_OK(_tablet_meta->add_inc_rs_meta(rowset->rowset_meta()));
++_newly_created_rowset_num;
return OLAP_SUCCESS;
}
-void Tablet::_delete_inc_rowset_by_version(const Version& version,
- const VersionHash& version_hash) {
- // delete incremental rowset from map
- _inc_rs_version_map.erase(version);
-
- RowsetMetaSharedPtr rowset_meta =
_tablet_meta->acquire_inc_rs_meta_by_version(version);
- if (rowset_meta == nullptr) {
- return;
- }
- _tablet_meta->delete_inc_rs_meta_by_version(version);
- VLOG_NOTICE << "delete incremental rowset. tablet=" << full_name() << ",
version=" << version;
-}
-
void Tablet::_delete_stale_rowset_by_version(const Version& version) {
RowsetMetaSharedPtr rowset_meta =
_tablet_meta->acquire_stale_rs_meta_by_version(version);
if (rowset_meta == nullptr) {
@@ -395,34 +350,6 @@ void Tablet::_delete_stale_rowset_by_version(const
Version& version) {
VLOG_NOTICE << "delete stale rowset. tablet=" << full_name() << ",
version=" << version;
}
-void Tablet::delete_expired_inc_rowsets() {
- int64_t now = UnixSeconds();
- std::vector<pair<Version, VersionHash>> expired_versions;
- WriteLock wrlock(&_meta_lock);
- for (auto& rs_meta : _tablet_meta->all_inc_rs_metas()) {
- double diff = ::difftime(now, rs_meta->creation_time());
- if (diff >= config::inc_rowset_expired_sec) {
- Version version(rs_meta->version());
- expired_versions.push_back(std::make_pair(version,
rs_meta->version_hash()));
- VLOG_NOTICE << "find expire incremental rowset. tablet=" <<
full_name()
- << ", version=" << version << ", version_hash=" <<
rs_meta->version_hash()
- << ", exist_sec=" << diff;
- }
- }
-
- if (expired_versions.empty()) {
- return;
- }
-
- for (auto& pair : expired_versions) {
- _delete_inc_rowset_by_version(pair.first, pair.second);
- VLOG_NOTICE << "delete expire incremental data. tablet=" << full_name()
- << ", version=" << pair.first;
- }
-
- save_meta();
-}
-
void Tablet::delete_expired_stale_rowset() {
int64_t now = UnixSeconds();
std::vector<pair<Version, VersionHash>> expired_versions;
@@ -804,10 +731,9 @@ void Tablet::calc_missed_versions(int64_t spec_version,
std::vector<Version>* mi
calc_missed_versions_unlocked(spec_version, missed_versions);
}
-// TODO(lingbin): there may be a bug here, should check it.
// for example:
// [0-4][5-5][8-8][9-9]
-// if spec_version = 6, we still return {6, 7} other than {7}
+// if spec_version = 6, we still return {7} other than {6, 7}
void Tablet::calc_missed_versions_unlocked(int64_t spec_version,
std::vector<Version>*
missed_versions) const {
DCHECK(spec_version > 0) << "invalid spec_version: " << spec_version;
@@ -826,7 +752,7 @@ void Tablet::calc_missed_versions_unlocked(int64_t
spec_version,
int64_t last_version = -1;
for (const Version& version : existing_versions) {
if (version.first > last_version + 1) {
- for (int64_t i = last_version + 1; i < version.first; ++i) {
+ for (int64_t i = last_version + 1; i < version.first && i <=
spec_version; ++i) {
missed_versions->emplace_back(Version(i, i));
}
}
@@ -957,10 +883,10 @@ void Tablet::delete_all_files() {
it.second->remove();
}
_rs_version_map.clear();
- for (auto it : _inc_rs_version_map) {
+
+ for (auto it : _stale_rs_version_map) {
it.second->remove();
}
- _inc_rs_version_map.clear();
_stale_rs_version_map.clear();
}
@@ -979,12 +905,6 @@ bool Tablet::check_path(const std::string& path_to_check)
const {
return true;
}
}
- for (auto& inc_version_rowset : _inc_rs_version_map) {
- bool ret = inc_version_rowset.second->check_path(path_to_check);
- if (ret) {
- return true;
- }
- }
for (auto& stale_version_rowset : _stale_rs_version_map) {
bool ret = stale_version_rowset.second->check_path(path_to_check);
if (ret) {
@@ -1010,11 +930,6 @@ bool Tablet::check_rowset_id(const RowsetId& rowset_id) {
return true;
}
}
- for (auto& inc_version_rowset : _inc_rs_version_map) {
- if (inc_version_rowset.second->rowset_id() == rowset_id) {
- return true;
- }
- }
for (auto& stale_version_rowset : _stale_rs_version_map) {
if (stale_version_rowset.second->rowset_id() == rowset_id) {
return true;
@@ -1222,14 +1137,6 @@ bool Tablet::rowset_meta_is_useful(RowsetMetaSharedPtr
rowset_meta) {
find_version = true;
}
}
- for (auto& inc_version_rowset : _inc_rs_version_map) {
- if (inc_version_rowset.second->rowset_id() ==
rowset_meta->rowset_id()) {
- return true;
- }
- if
(inc_version_rowset.second->contains_version(rowset_meta->version())) {
- find_version = true;
- }
- }
for (auto& stale_version_rowset : _stale_rs_version_map) {
if (stale_version_rowset.second->rowset_id() ==
rowset_meta->rowset_id()) {
return true;
@@ -1247,8 +1154,8 @@ bool Tablet::_contains_rowset(const RowsetId rowset_id) {
return true;
}
}
- for (auto& inc_version_rowset : _inc_rs_version_map) {
- if (inc_version_rowset.second->rowset_id() == rowset_id) {
+ for (auto& stale_version_rowset : _stale_rs_version_map) {
+ if (stale_version_rowset.second->rowset_id() == rowset_id) {
return true;
}
}
diff --git a/be/src/olap/tablet.h b/be/src/olap/tablet.h
index c34d909..a903715 100644
--- a/be/src/olap/tablet.h
+++ b/be/src/olap/tablet.h
@@ -98,16 +98,14 @@ public:
void modify_rowsets(const vector<RowsetSharedPtr>& to_add,
const vector<RowsetSharedPtr>& to_delete);
- // _rs_version_map and _inc_rs_version_map should be protected by
_meta_lock
+ // _rs_version_map and _stale_rs_version_map should be protected by
_meta_lock
// The caller must call hold _meta_lock when call this two function.
- const RowsetSharedPtr get_rowset_by_version(const Version& version) const;
- const RowsetSharedPtr get_inc_rowset_by_version(const Version& version)
const;
+ const RowsetSharedPtr get_rowset_by_version(const Version& version, bool
find_is_stale = false) const;
const RowsetSharedPtr get_stale_rowset_by_version(const Version& version)
const;
const RowsetSharedPtr rowset_with_max_version() const;
OLAPStatus add_inc_rowset(const RowsetSharedPtr& rowset);
- void delete_expired_inc_rowsets();
/// Delete stale rowset by timing. This delete policy uses now() minutes
/// config::tablet_rowset_expired_stale_sweep_time_sec to compute the
deadline of expired rowset
/// to delete. When rowset is deleted, it will be added to StorageEngine
unused map and record
@@ -229,7 +227,7 @@ public:
// Check whether the rowset is useful or not, unuseful rowset can be swept
up then.
// Rowset which is under tablet's management is useful, i.e. rowset is in
- // _rs_version_map, _inc_rs_version_map, or _stale_rs_version_map.
+ // _rs_version_map, or _stale_rs_version_map.
// Rowset whose version range is not covered by this tablet is also useful.
bool rowset_meta_is_useful(RowsetMetaSharedPtr rowset_meta);
@@ -259,7 +257,6 @@ private:
void _max_continuous_version_from_beginning_unlocked(Version* version,
VersionHash* v_hash)
const;
RowsetSharedPtr _rowset_with_largest_size();
- void _delete_inc_rowset_by_version(const Version& version, const
VersionHash& version_hash);
/// Delete stale rowset by version. This method not only delete the
version in expired rowset map,
/// but also delete the version in rowset meta vector.
void _delete_stale_rowset_by_version(const Version& version);
@@ -287,18 +284,9 @@ private:
// TODO(lingbin): There is a _meta_lock TabletMeta too, there should be a
comment to
// explain how these two locks work together.
mutable RWMutex _meta_lock;
- // A new load job will produce a new rowset, which will be inserted into
both _rs_version_map
- // and _inc_rs_version_map. Only the most recent rowsets are kept in
_inc_rs_version_map to
- // reduce the amount of data that needs to be copied during the clone task.
- // NOTE: Not all incremental-rowsets are in _rs_version_map. Because after
some rowsets
- // are compacted, they will be remove from _rs_version_map, but it may not
be deleted from
- // _inc_rs_version_map.
- // Which rowsets should be deleted from _inc_rs_version_map is affected by
- // inc_rowset_expired_sec conf. In addition, the deletion is triggered
periodically,
- // So at a certain time point (such as just after a base compaction), some
rowsets in
- // _inc_rs_version_map may do not exist in _rs_version_map.
+ // After version 0.13, all newly created rowsets are saved in
_rs_version_map.
+ // And if rowset being compacted, the old rowsetis will be saved in
_stale_rs_version_map;
std::unordered_map<Version, RowsetSharedPtr, HashOfVersion>
_rs_version_map;
- std::unordered_map<Version, RowsetSharedPtr, HashOfVersion>
_inc_rs_version_map;
// This variable _stale_rs_version_map is used to record these rowsets
which are be compacted.
// These _stale rowsets are been removed when rowsets' pathVersion is
expired,
// this policy is judged and computed by TimestampedVersionTracker.
diff --git a/be/src/olap/tablet_manager.cpp b/be/src/olap/tablet_manager.cpp
index 5a2fb5a..af2bd59 100644
--- a/be/src/olap/tablet_manager.cpp
+++ b/be/src/olap/tablet_manager.cpp
@@ -1008,7 +1008,6 @@ OLAPStatus TabletManager::start_trash_sweep() {
}
for (const auto& tablet : all_tablets) {
- tablet->delete_expired_inc_rowsets();
tablet->delete_expired_stale_rowset();
}
all_tablets.clear();
diff --git a/be/src/olap/tablet_meta.cpp b/be/src/olap/tablet_meta.cpp
index c36900e..a4940ed 100644
--- a/be/src/olap/tablet_meta.cpp
+++ b/be/src/olap/tablet_meta.cpp
@@ -373,11 +373,6 @@ void TabletMeta::init_from_pb(const TabletMetaPB&
tablet_meta_pb) {
}
_rs_metas.push_back(std::move(rs_meta));
}
- for (auto& it : tablet_meta_pb.inc_rs_metas()) {
- RowsetMetaSharedPtr rs_meta(new AlphaRowsetMeta());
- rs_meta->init_from_pb(it);
- _inc_rs_metas.push_back(std::move(rs_meta));
- }
for (auto& it : tablet_meta_pb.stale_rs_metas()) {
RowsetMetaSharedPtr rs_meta(new AlphaRowsetMeta());
@@ -432,9 +427,6 @@ void TabletMeta::to_meta_pb(TabletMetaPB* tablet_meta_pb) {
for (auto& rs : _rs_metas) {
rs->to_rowset_pb(tablet_meta_pb->add_rs_metas());
}
- for (auto rs : _inc_rs_metas) {
- rs->to_rowset_pb(tablet_meta_pb->add_inc_rs_metas());
- }
for (auto rs : _stale_rs_metas) {
rs->to_rowset_pb(tablet_meta_pb->add_stale_rs_metas());
}
@@ -538,27 +530,6 @@ void
TabletMeta::revise_rs_metas(std::vector<RowsetMetaSharedPtr>&& rs_metas) {
_rs_metas = std::move(rs_metas);
}
-void TabletMeta::revise_inc_rs_metas(std::vector<RowsetMetaSharedPtr>&&
rs_metas) {
- WriteLock wrlock(&_meta_lock);
- // delete alter task
- _alter_task.reset();
-
- _inc_rs_metas = std::move(rs_metas);
-}
-
-OLAPStatus TabletMeta::add_inc_rs_meta(const RowsetMetaSharedPtr& rs_meta) {
- // check RowsetMeta is valid
- for (auto rs : _inc_rs_metas) {
- if (rs->version() == rs_meta->version()) {
- LOG(WARNING) << "rowset already exist. rowset_id=" <<
rs->rowset_id();
- return OLAP_ERR_ROWSET_ALREADY_EXIST;
- }
- }
-
- _inc_rs_metas.push_back(rs_meta);
- return OLAP_SUCCESS;
-}
-
void TabletMeta::delete_stale_rs_meta_by_version(const Version& version) {
auto it = _stale_rs_metas.begin();
while (it != _stale_rs_metas.end()) {
@@ -570,8 +541,8 @@ void TabletMeta::delete_stale_rs_meta_by_version(const
Version& version) {
}
}
-RowsetMetaSharedPtr TabletMeta::acquire_stale_rs_meta_by_version(const
Version& version) const {
- for (auto it : _stale_rs_metas) {
+RowsetMetaSharedPtr TabletMeta::acquire_rs_meta_by_version(const Version&
version) const {
+ for (auto it : _rs_metas) {
if (it->version() == version) {
return it;
}
@@ -579,20 +550,8 @@ RowsetMetaSharedPtr
TabletMeta::acquire_stale_rs_meta_by_version(const Version&
return nullptr;
}
-void TabletMeta::delete_inc_rs_meta_by_version(const Version& version) {
- auto it = _inc_rs_metas.begin();
- while (it != _inc_rs_metas.end()) {
- if ((*it)->version() == version) {
- _inc_rs_metas.erase(it);
- break;
- } else {
- it++;
- }
- }
-}
-
-RowsetMetaSharedPtr TabletMeta::acquire_inc_rs_meta_by_version(const Version&
version) const {
- for (auto it : _inc_rs_metas) {
+RowsetMetaSharedPtr TabletMeta::acquire_stale_rs_meta_by_version(const
Version& version) const {
+ for (auto it : _stale_rs_metas) {
if (it->version() == version) {
return it;
}
@@ -730,10 +689,6 @@ bool operator==(const TabletMeta& a, const TabletMeta& b) {
for (int i = 0; i < a._rs_metas.size(); ++i) {
if (a._rs_metas[i] != b._rs_metas[i]) return false;
}
- if (a._inc_rs_metas.size() != b._inc_rs_metas.size()) return false;
- for (int i = 0; i < a._inc_rs_metas.size(); ++i) {
- if (a._inc_rs_metas[i] != b._inc_rs_metas[i]) return false;
- }
if (a._alter_task != b._alter_task) return false;
if (a._in_restore_mode != b._in_restore_mode) return false;
if (a._preferred_rowset_type != b._preferred_rowset_type) return false;
diff --git a/be/src/olap/tablet_meta.h b/be/src/olap/tablet_meta.h
index bfefbd6..1336838 100644
--- a/be/src/olap/tablet_meta.h
+++ b/be/src/olap/tablet_meta.h
@@ -171,13 +171,8 @@ public:
const std::vector<RowsetMetaSharedPtr>& to_delete);
void revise_rs_metas(std::vector<RowsetMetaSharedPtr>&& rs_metas);
- void revise_inc_rs_metas(std::vector<RowsetMetaSharedPtr>&& rs_metas);
-
- inline const std::vector<RowsetMetaSharedPtr>& all_inc_rs_metas() const;
inline const std::vector<RowsetMetaSharedPtr>& all_stale_rs_metas() const;
- OLAPStatus add_inc_rs_meta(const RowsetMetaSharedPtr& rs_meta);
- void delete_inc_rs_meta_by_version(const Version& version);
- RowsetMetaSharedPtr acquire_inc_rs_meta_by_version(const Version& version)
const;
+ RowsetMetaSharedPtr acquire_rs_meta_by_version(const Version& version)
const;
void delete_stale_rs_meta_by_version(const Version& version);
RowsetMetaSharedPtr acquire_stale_rs_meta_by_version(const Version&
version) const;
@@ -222,7 +217,6 @@ private:
TabletSchema _schema;
std::vector<RowsetMetaSharedPtr> _rs_metas;
- std::vector<RowsetMetaSharedPtr> _inc_rs_metas;
// This variable _stale_rs_metas is used to record these rowsets‘ meta
which are be compacted.
// These stale rowsets meta are been removed when rowsets' pathVersion is
expired,
// this policy is judged and computed by TimestampedVersionTracker.
@@ -330,10 +324,6 @@ inline const std::vector<RowsetMetaSharedPtr>&
TabletMeta::all_rs_metas() const
return _rs_metas;
}
-inline const std::vector<RowsetMetaSharedPtr>& TabletMeta::all_inc_rs_metas()
const {
- return _inc_rs_metas;
-}
-
inline const std::vector<RowsetMetaSharedPtr>&
TabletMeta::all_stale_rs_metas() const {
return _stale_rs_metas;
}
diff --git a/be/src/olap/task/engine_clone_task.cpp
b/be/src/olap/task/engine_clone_task.cpp
index 4d346a5..dc3bda4 100644
--- a/be/src/olap/task/engine_clone_task.cpp
+++ b/be/src/olap/task/engine_clone_task.cpp
@@ -80,73 +80,48 @@ OLAPStatus EngineCloneTask::_do_clone() {
if (!migration_rlock.own_lock()) {
return OLAP_ERR_RWLOCK_ERROR;
}
- LOG(INFO) << "clone tablet exist yet, begin to incremental clone. "
- << "signature:" << _signature << ", tablet_id:" <<
_clone_req.tablet_id
- << ", schema_hash:" << _clone_req.schema_hash
- << ", committed_version:" << _clone_req.committed_version;
// get download path
string local_data_path = tablet->tablet_path() + CLONE_PREFIX;
bool allow_incremental_clone = false;
- // check if current tablet has version == 2 and version hash == 0
- // version 2 may be an invalid rowset
- Version clone_version = {_clone_req.committed_version,
_clone_req.committed_version};
- RowsetSharedPtr clone_rowset =
tablet->get_rowset_by_version(clone_version);
- if (clone_rowset == nullptr) {
- // try to incremental clone
- std::vector<Version> missed_versions;
- tablet->calc_missed_versions(_clone_req.committed_version,
&missed_versions);
- LOG(INFO) << "finish to calculate missed versions when clone. "
- << "tablet=" << tablet->full_name()
- << ", committed_version=" << _clone_req.committed_version
- << ", missed_versions_size=" << missed_versions.size();
- // if missed version size is 0, then it is useless to clone from
remote be, it means local data is
- // completed. Or remote be will just return header not the rowset
files. clone will failed.
- if (missed_versions.size() == 0) {
- LOG(INFO) << "missed version size = 0, skip clone and return
success";
- _set_tablet_info(DORIS_SUCCESS, is_new_tablet);
- return OLAP_SUCCESS;
- }
- status = _clone_copy(*(tablet->data_dir()), local_data_path,
&src_host, &src_file_path,
- _error_msgs, &missed_versions,
&allow_incremental_clone);
- } else {
- LOG(INFO) << "current tablet has invalid rowset that's version ==
commit_version but "
- "version hash not equal"
- << " clone req commit_version=" <<
_clone_req.committed_version
- << " tablet info = " << tablet->full_name();
+
+ // try to incremental clone
+ std::vector<Version> missed_versions;
+ tablet->calc_missed_versions(_clone_req.committed_version,
&missed_versions);
+
+ // if missed version size is 0, then it is useless to clone from
remote be, it means local data is
+ // completed. Or remote be will just return header not the rowset
files. clone will failed.
+ if (missed_versions.size() == 0) {
+ LOG(INFO) << "missed version size = 0, skip clone and return
success. tablet id=" << _clone_req.tablet_id;
+ _set_tablet_info(DORIS_SUCCESS, is_new_tablet);
+ return OLAP_SUCCESS;
}
- if (status == DORIS_SUCCESS && allow_incremental_clone) {
+
+ // try to download missing version from src backend.
+ // if tablet on src backend does not contains missing version, it will
download all versions,
+ // and set allow_incremental_clone to false
+ status = _make_and_download_snapshots(*(tablet->data_dir()),
local_data_path, &src_host, &src_file_path,
+ _error_msgs, &missed_versions, &allow_incremental_clone);
+
+ LOG(INFO) << "tablet exist with number of missing version: " <<
missed_versions.size()
+ << ", try to incremental clone succeed: " <<
allow_incremental_clone
+ << ", signature: " << _signature << ", tablet id: " <<
_clone_req.tablet_id
+ << ", schema hash: " << _clone_req.schema_hash
+ << ", clone version: " << _clone_req.committed_version
+ << ", download snapshot: " << status;
+
+ if (status == DORIS_SUCCESS) {
OLAPStatus olap_status =
_finish_clone(tablet.get(), local_data_path,
_clone_req.committed_version,
allow_incremental_clone);
if (olap_status != OLAP_SUCCESS) {
- LOG(WARNING) << "failed to finish incremental clone. [table="
<< tablet->full_name()
+ LOG(WARNING) << "failed to finish clone. [table=" <<
tablet->full_name()
<< " res=" << olap_status << "]";
- _error_msgs->push_back("incremental clone error.");
+ _error_msgs->push_back("clone error.");
status = DORIS_ERROR;
}
- } else {
- bool allow_incremental_clone = false;
- // begin to full clone if incremental failed
- LOG(INFO) << "begin to full clone. [table=" << tablet->full_name();
- status = _clone_copy(*(tablet->data_dir()), local_data_path,
&src_host, &src_file_path,
- _error_msgs, NULL, &allow_incremental_clone);
- if (status == DORIS_SUCCESS) {
- LOG(INFO) << "download successfully when full clone. [table="
<< tablet->full_name()
- << " src_host=" << src_host.host << "
src_file_path=" << src_file_path
- << " local_data_path=" << local_data_path << "]";
-
- OLAPStatus olap_status = _finish_clone(tablet.get(),
local_data_path,
-
_clone_req.committed_version, false);
-
- if (olap_status != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to finish full clone. [table=" <<
tablet->full_name()
- << " res=" << olap_status << "]";
- _error_msgs->push_back("full clone error.");
- status = DORIS_ERROR;
- }
- }
}
+
} else {
LOG(INFO) << "clone tablet not exist, begin clone a new tablet from
remote be. "
<< "signature:" << _signature << ", tablet_id:" <<
_clone_req.tablet_id
@@ -169,7 +144,7 @@ OLAPStatus EngineCloneTask::_do_clone() {
if (status == DORIS_SUCCESS) {
bool allow_incremental_clone = false;
- status = _clone_copy(*store, tablet_dir_stream.str(), &src_host,
&src_file_path,
+ status = _make_and_download_snapshots(*store,
tablet_dir_stream.str(), &src_host, &src_file_path,
_error_msgs, nullptr,
&allow_incremental_clone);
}
@@ -280,7 +255,12 @@ void EngineCloneTask::_set_tablet_info(AgentStatus status,
bool is_new_tablet) {
*_res_status = status;
}
-AgentStatus EngineCloneTask::_clone_copy(DataDir& data_dir, const string&
local_data_path,
+/// This method will do following things:
+/// 1. Make snapshots on source BE.
+/// 2. Download all snapshots to CLONE dir.
+/// 3. Convert rowset ids of downloaded snapshots.
+/// 4. Release the snapshots on source BE.
+AgentStatus EngineCloneTask::_make_and_download_snapshots(DataDir& data_dir,
const string& local_data_path,
TBackend* src_host, string*
snapshot_path,
std::vector<string>* error_msgs,
const std::vector<Version>*
missed_versions,
@@ -308,6 +288,7 @@ AgentStatus EngineCloneTask::_clone_copy(DataDir& data_dir,
const string& local_
<< ", tablet=" << _clone_req.tablet_id
<< ", schema_hash=" << _clone_req.schema_hash
<< ", snapshot_path=" << *snapshot_path << ",
signature=" << _signature;
+ status = DORIS_SUCCESS;
} else {
LOG(WARNING) << "fail to make snapshot, ip=" << src.host << ",
port=" << src.be_port
<< ", tablet=" << _clone_req.tablet_id
@@ -339,14 +320,6 @@ AgentStatus EngineCloneTask::_clone_copy(DataDir&
data_dir, const string& local_
// when there is an error, keep this program executing to release
snapshot
}
- if (status == DORIS_SUCCESS && snapshot_version == 1) {
- auto olap_st = _convert_to_new_snapshot(local_path,
_clone_req.tablet_id);
- if (olap_st != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to convert to new snapshot, path=" <<
local_path
- << ", tablet_id=" << _clone_req.tablet_id << ",
error=" << olap_st;
- status = DORIS_ERROR;
- }
- }
if (status == DORIS_SUCCESS) {
// change all rowset ids because they maybe its id same with local
rowset
auto olap_st = SnapshotManager::instance()->convert_rowset_ids(
@@ -363,10 +336,10 @@ AgentStatus EngineCloneTask::_clone_copy(DataDir&
data_dir, const string& local_
st = _release_snapshot(src.host, src.be_port, *snapshot_path);
if (st.ok()) {
LOG(INFO) << "success to release snapshot, ip=" << src.host << ",
port=" << src.be_port
- << ", snapshot_path=" << snapshot_path;
+ << ", snapshot_path=" << *snapshot_path;
} else {
LOG(WARNING) << "fail to release snapshot, ip=" << src.host << ",
port=" << src.be_port
- << ", snapshot_path=" << snapshot_path << ", error="
<< st.to_string();
+ << ", snapshot_path=" << *snapshot_path << ", error="
<< st.to_string();
// DON'T change the status
}
if (status == DORIS_SUCCESS) {
@@ -538,71 +511,10 @@ Status EngineCloneTask::_download_files(DataDir*
data_dir, const std::string& re
return Status::OK();
}
-OLAPStatus EngineCloneTask::_convert_to_new_snapshot(const string& clone_dir,
int64_t tablet_id) {
- OLAPStatus res = OLAP_SUCCESS;
- // check clone dir existed
- if (!FileUtils::check_exist(clone_dir)) {
- res = OLAP_ERR_DIR_NOT_EXIST;
- LOG(WARNING) << "clone dir not existed when clone. clone_dir=" <<
clone_dir.c_str();
- return res;
- }
-
- // load src header
- string cloned_meta_file = clone_dir + "/" + std::to_string(tablet_id) +
".hdr";
- FileHeader<OLAPHeaderMessage> file_header;
- FileHandler file_handler;
- OLAPHeaderMessage olap_header_msg;
- if (file_handler.open(cloned_meta_file.c_str(), O_RDONLY) != OLAP_SUCCESS)
{
- LOG(WARNING) << "fail to open ordinal file. file=" << cloned_meta_file;
- return OLAP_ERR_IO_ERROR;
- }
-
- // In file_header.unserialize(), it validates file length, signature,
checksum of protobuf.
- if (file_header.unserialize(&file_handler) != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to unserialize tablet_meta. file='" <<
cloned_meta_file;
- return OLAP_ERR_PARSE_PROTOBUF_ERROR;
- }
-
- set<string> clone_files;
-
- RETURN_WITH_WARN_IF_ERROR(
- FileUtils::list_dirs_files(clone_dir, NULL, &clone_files,
Env::Default()),
- OLAP_ERR_DISK_FAILURE, "failed to dir walk when clone. clone_dir="
+ clone_dir);
-
- try {
- olap_header_msg.CopyFrom(file_header.message());
- } catch (...) {
- LOG(WARNING) << "fail to copy protocol buffer object. file='" <<
cloned_meta_file;
- return OLAP_ERR_PARSE_PROTOBUF_ERROR;
- }
- OlapSnapshotConverter converter;
- TabletMetaPB tablet_meta_pb;
- std::vector<RowsetMetaPB> pending_rowsets;
- res = converter.to_new_snapshot(olap_header_msg, clone_dir, clone_dir,
&tablet_meta_pb,
- &pending_rowsets, false);
- if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to convert snapshot to new format. dir='" <<
clone_dir;
- return res;
- }
- std::vector<string> files_to_delete;
- for (auto file_name : clone_files) {
- string full_file_path = clone_dir + "/" + file_name;
- files_to_delete.push_back(full_file_path);
- }
- // remove all files
- RETURN_WITH_WARN_IF_ERROR(FileUtils::remove_paths(files_to_delete),
OLAP_ERR_IO_ERROR,
- "remove paths failed.")
-
- res = TabletMeta::save(cloned_meta_file, tablet_meta_pb);
- if (res != OLAP_SUCCESS) {
- LOG(WARNING) << "fail to save converted tablet meta to dir='" <<
clone_dir;
- return res;
- }
-
- return OLAP_SUCCESS;
-}
-
-// only incremental clone use this method
+/// This method will only be called if tablet already exist in this BE when
doing clone.
+/// This method will do the following things:
+/// 1. Linke all files from CLONE dir to tablet dir if file does not exist in
tablet dir
+/// 2. Call _finish_xx_clone() to revise the tablet meta.
OLAPStatus EngineCloneTask::_finish_clone(Tablet* tablet, const string&
clone_dir,
int64_t committed_version, bool
is_incremental_clone) {
OLAPStatus res = OLAP_SUCCESS;
@@ -623,7 +535,9 @@ OLAPStatus EngineCloneTask::_finish_clone(Tablet* tablet,
const string& clone_di
break;
}
- // load src header
+ // Load src header.
+ // The tablet meta info is downloaded from source BE as .hdr file.
+ // So we load it and generate cloned_tablet_meta.
string cloned_tablet_meta_file =
clone_dir + "/" + std::to_string(tablet->tablet_id()) + ".hdr";
TabletMeta cloned_tablet_meta;
@@ -635,12 +549,11 @@ OLAPStatus EngineCloneTask::_finish_clone(Tablet* tablet,
const string& clone_di
// remove the cloned meta file
FileUtils::remove(cloned_tablet_meta_file);
- // TODO(ygl): convert old format file into rowset
// check all files in /clone and /tablet
set<string> clone_files;
Status ret = FileUtils::list_dirs_files(clone_dir, NULL, &clone_files,
Env::Default());
if (!ret.ok()) {
- LOG(WARNING) << "failed to dir walk when clone. [clone_dir=" <<
clone_dir << "]"
+ LOG(WARNING) << "failed to list clone dir when clone. [clone_dir="
<< clone_dir << "]"
<< " error: " << ret.to_string();
res = OLAP_ERR_DISK_FAILURE;
break;
@@ -650,13 +563,15 @@ OLAPStatus EngineCloneTask::_finish_clone(Tablet* tablet,
const string& clone_di
string tablet_dir = tablet->tablet_path();
ret = FileUtils::list_dirs_files(tablet_dir, NULL, &local_files,
Env::Default());
if (!ret.ok()) {
- LOG(WARNING) << "failed to dir walk when clone. [tablet_dir=" <<
tablet_dir << "]"
+ LOG(WARNING) << "failed to list local tablet dir when clone.
[tablet_dir=" << tablet_dir << "]"
<< " error: " << ret.to_string();
res = OLAP_ERR_DISK_FAILURE;
break;
}
- // link files from clone dir, if file exists, skip it
+ /// Traverse all downloaded clone files in CLONE dir.
+ /// If it does not exist in local tablet dir, link the file to local
tablet dir
+ /// And save all linked files in linked_success_files.
for (const string& clone_file : clone_files) {
if (local_files.find(clone_file) != local_files.end()) {
VLOG_NOTICE << "find same file when clone, skip it. "
@@ -681,9 +596,9 @@ OLAPStatus EngineCloneTask::_finish_clone(Tablet* tablet,
const string& clone_di
}
if (is_incremental_clone) {
- res = _clone_incremental_data(tablet, cloned_tablet_meta,
committed_version);
+ res = _finish_incremental_clone(tablet, cloned_tablet_meta,
committed_version);
} else {
- res = _clone_full_data(tablet,
const_cast<TabletMeta*>(&cloned_tablet_meta));
+ res = _finish_full_clone(tablet,
const_cast<TabletMeta*>(&cloned_tablet_meta));
}
// if full clone success, need to update cumulative layer point
@@ -711,55 +626,75 @@ OLAPStatus EngineCloneTask::_finish_clone(Tablet* tablet,
const string& clone_di
return res;
}
-OLAPStatus EngineCloneTask::_clone_incremental_data(Tablet* tablet,
- const TabletMeta&
cloned_tablet_meta,
- int64_t committed_version)
{
- LOG(INFO) << "begin to incremental clone. tablet=" << tablet->full_name()
- << ", committed_version=" << committed_version;
-
+/// This method will do:
+/// 1. Get missing version from local tablet again and check if they exist in
cloned tablet.
+/// 2. Revise the local tablet meta to add all incremental cloned rowset's
meta.
+OLAPStatus EngineCloneTask::_finish_incremental_clone(Tablet* tablet,
+ const TabletMeta& cloned_tablet_meta,
+ int64_t committed_version) {
+ LOG(INFO) << "begin to finish incremental clone. tablet=" <<
tablet->full_name()
+ << ", clone version=" << committed_version;
+
+ /// Get missing versions again from local tablet.
+ /// We got it before outside the lock, so it has to be got again.
std::vector<Version> missed_versions;
tablet->calc_missed_versions_unlocked(committed_version, &missed_versions);
-
- std::vector<Version> versions_to_delete;
- std::vector<RowsetMetaSharedPtr> rowsets_to_clone;
-
VLOG_NOTICE << "get missed versions again when finish incremental clone. "
- << "tablet=" << tablet->full_name() << ", committed_version=" <<
committed_version
+ << "tablet=" << tablet->full_name() << ", clone version=" <<
committed_version
<< ", missed_versions_size=" << missed_versions.size();
// check missing versions exist in clone src
+ std::vector<RowsetMetaSharedPtr> rowsets_to_clone;
for (Version version : missed_versions) {
- RowsetMetaSharedPtr inc_rs_meta =
- cloned_tablet_meta.acquire_inc_rs_meta_by_version(version);
- if (inc_rs_meta == nullptr) {
+ RowsetMetaSharedPtr rs_meta =
+ cloned_tablet_meta.acquire_rs_meta_by_version(version);
+ if (rs_meta == nullptr) {
LOG(WARNING) << "missed version is not found in cloned tablet
meta."
<< ", missed_version=" << version.first << "-" <<
version.second;
return OLAP_ERR_VERSION_NOT_EXIST;
}
- rowsets_to_clone.push_back(inc_rs_meta);
+ rowsets_to_clone.push_back(rs_meta);
}
- // clone_data to tablet
+ /// clone_data to tablet
+ /// For incremental clone, nothing will be deleted.
+ /// So versions_to_delete is empty.
+ std::vector<Version> versions_to_delete;
OLAPStatus clone_res = tablet->revise_tablet_meta(rowsets_to_clone,
versions_to_delete);
LOG(INFO) << "finish to incremental clone. [tablet=" << tablet->full_name()
<< " res=" << clone_res << "]";
return clone_res;
}
-OLAPStatus EngineCloneTask::_clone_full_data(Tablet* tablet, TabletMeta*
cloned_tablet_meta) {
+/// This method will do:
+/// 1. Compare the version of local tablet and cloned tablet to decide which
version to keep
+/// 2. Revise the local tablet meta
+OLAPStatus EngineCloneTask::_finish_full_clone(Tablet* tablet, TabletMeta*
cloned_tablet_meta) {
Version cloned_max_version = cloned_tablet_meta->max_version();
- LOG(INFO) << "begin to full clone. tablet=" << tablet->full_name()
- << ", cloned_max_version=" << cloned_max_version.first << "-"
- << cloned_max_version.second;
+ LOG(INFO) << "begin to finish full clone. tablet=" << tablet->full_name()
+ << ", cloned_max_version=" << cloned_max_version;
+
+ // Compare the version of local tablet and cloned tablet.
+ // For example:
+ // clone version is 8
+ //
+ // local tablet: [0-1] [2-5] [6-6] [7-7]
+ // clone tablet: [0-1] [2-4] [5-6] [7-8]
+ //
+ // after compare, the version mark with "x" will be deleted
+ //
+ // local tablet: [0-1] [2-5]x [6-6]x [7-7]x
+ // clone tablet: [0-1]x [2-4] [5-6] [7-8]
+
+ // All versions deleted from local tablet will be saved in
versions_to_delete
+ // And these versions file will be deleted finally.
std::vector<Version> versions_to_delete;
std::vector<RowsetMetaSharedPtr> rs_metas_found_in_src;
- // check local versions
for (auto& rs_meta : tablet->tablet_meta()->all_rs_metas()) {
Version local_version(rs_meta->start_version(),
rs_meta->end_version());
LOG(INFO) << "check local delta when full clone."
- << "tablet=" << tablet->full_name() << ", local_version=" <<
local_version.first
- << "-" << local_version.second;
+ << "tablet=" << tablet->full_name() << ", local_version=" <<
local_version;
// if local version cross src latest, clone failed
// if local version is : 0-0, 1-1, 2-10, 12-14, 15-15,16-16
@@ -771,8 +706,7 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet*
tablet, TabletMeta* cloned_
local_version.second > cloned_max_version.second) {
LOG(WARNING) << "stop to full clone, version cross src latest."
<< "tablet=" << tablet->full_name()
- << ", local_version=" << local_version.first << "-"
- << local_version.second;
+ << ", local_version=" << local_version;
return OLAP_ERR_TABLE_VERSION_DUPLICATE_ERROR;
} else if (local_version.second <= cloned_max_version.second) {
@@ -792,27 +726,22 @@ OLAPStatus EngineCloneTask::_clone_full_data(Tablet*
tablet, TabletMeta* cloned_
if (existed_in_src) {
cloned_tablet_meta->delete_rs_meta_by_version(local_version,
&rs_metas_found_in_src);
- LOG(INFO) << "Delta has already existed in local header, no
need to clone."
- << "tablet=" << tablet->full_name() << ", version='"
- << local_version.first << "-" <<
local_version.second;
+ LOG(INFO) << "version exist in local tablet, no need to clone.
delete it from clone tablet"
+ << ". tablet=" << tablet->full_name() << ",
version='" << local_version;
} else {
- // Delta labeled in local_version is not existed in clone
header,
- // some overlapping delta will be cloned to replace it.
- // And also, the specified delta should deleted from local
header.
versions_to_delete.push_back(local_version);
- LOG(INFO) << "Delete delta not included by the clone header,
should delete it from "
- "local header."
+ LOG(INFO) << "version not exist in local tablet. it will be
replaced by other version. "
+ << "delete it from local tablet. "
<< "tablet=" << tablet->full_name() << ","
- << ", version=" << local_version.first << "-" <<
local_version.second;
+ << ", version=" << local_version;
}
}
}
std::vector<RowsetMetaSharedPtr> rowsets_to_clone;
for (auto& rs_meta : cloned_tablet_meta->all_rs_metas()) {
rowsets_to_clone.push_back(rs_meta);
- LOG(INFO) << "Delta to clone."
- << "tablet=" << tablet->full_name() << ", version=" <<
rs_meta->version().first
- << "-" << rs_meta->version().second;
+ LOG(INFO) << "version to be cloned from clone tablet to local tablet: "
+ << tablet->full_name() << ", version=" << rs_meta->version();
}
// clone_data to tablet
diff --git a/be/src/olap/task/engine_clone_task.h
b/be/src/olap/task/engine_clone_task.h
index ad58a9d..dbe3c1f 100644
--- a/be/src/olap/task/engine_clone_task.h
+++ b/be/src/olap/task/engine_clone_task.h
@@ -45,17 +45,15 @@ private:
virtual OLAPStatus _finish_clone(Tablet* tablet, const std::string&
clone_dir,
int64_t committed_version, bool
is_incremental_clone);
- OLAPStatus _clone_incremental_data(Tablet* tablet, const TabletMeta&
cloned_tablet_meta,
+ OLAPStatus _finish_incremental_clone(Tablet* tablet, const TabletMeta&
cloned_tablet_meta,
int64_t committed_version);
- OLAPStatus _clone_full_data(Tablet* tablet, TabletMeta*
cloned_tablet_meta);
+ OLAPStatus _finish_full_clone(Tablet* tablet, TabletMeta*
cloned_tablet_meta);
- AgentStatus _clone_copy(DataDir& data_dir, const string& local_data_path,
TBackend* src_host,
+ AgentStatus _make_and_download_snapshots(DataDir& data_dir, const string&
local_data_path, TBackend* src_host,
string* src_file_path, vector<string>* error_msgs,
const vector<Version>* missing_versions, bool*
allow_incremental_clone);
- OLAPStatus _convert_to_new_snapshot(const string& clone_dir, int64_t
tablet_id);
-
void _set_tablet_info(AgentStatus status, bool is_new_tablet);
// Download tablet files from
diff --git a/be/test/olap/CMakeLists.txt b/be/test/olap/CMakeLists.txt
index ffc4507..c01e925 100644
--- a/be/test/olap/CMakeLists.txt
+++ b/be/test/olap/CMakeLists.txt
@@ -76,7 +76,7 @@ ADD_BE_TEST(rowset/alpha_rowset_test)
ADD_BE_TEST(rowset/beta_rowset_test)
ADD_BE_TEST(rowset/unique_rowset_id_generator_test)
ADD_BE_TEST(rowset/rowset_converter_test)
-ADD_BE_TEST(olap_snapshot_converter_test)
+# ADD_BE_TEST(olap_snapshot_converter_test)
ADD_BE_TEST(txn_manager_test)
ADD_BE_TEST(generic_iterators_test)
ADD_BE_TEST(key_coder_test)
diff --git a/be/test/olap/tablet_meta_manager_test.cpp
b/be/test/olap/tablet_meta_manager_test.cpp
index 0064874..7820ce2 100644
--- a/be/test/olap/tablet_meta_manager_test.cpp
+++ b/be/test/olap/tablet_meta_manager_test.cpp
@@ -36,7 +36,8 @@ using std::string;
namespace doris {
-const std::string meta_path = "./be/test/olap/test_data/header.txt";
+// const std::string meta_path = "./be/test/olap/test_data/header.txt";
+const std::string meta_path =
"./be/test/olap/test_data/header_without_inc_rs.txt";
class TabletMetaManagerTest : public testing::Test {
public:
diff --git a/be/test/olap/test_data/header_without_inc_rs.txt
b/be/test/olap/test_data/header_without_inc_rs.txt
new file mode 100644
index 0000000..96acbb3
--- /dev/null
+++ b/be/test/olap/test_data/header_without_inc_rs.txt
@@ -0,0 +1,144 @@
+{
+ "table_id": 15670,
+ "partition_id": 15671,
+ "tablet_id": 15672,
+ "schema_hash": 567997577,
+ "shard_id": 34,
+ "creation_time": 1553765664,
+ "cumulative_layer_point": 2,
+ "tablet_state": "PB_NOTREADY",
+ "schema": {
+ "keys_type": "AGG_KEYS",
+ "column": [
+ {
+ "unique_id": 0,
+ "name": "k1",
+ "type": "BIGINT",
+ "is_key": true,
+ "aggregation": "NONE",
+ "is_nullable": false,
+ "length": 8,
+ "index_length": 8,
+ "visible": true
+ },
+ {
+ "unique_id": 1,
+ "name": "v1",
+ "type": "HLL",
+ "is_key": false,
+ "aggregation": "HLL_UNION",
+ "is_nullable": false,
+ "default_value": "MA==",
+ "length": 16387,
+ "index_length": 16,
+ "visible": true
+ },
+ {
+ "unique_id": 2,
+ "name": "v2",
+ "type": "INT",
+ "is_key": false,
+ "aggregation": "SUM",
+ "is_nullable": false,
+ "length": 4,
+ "index_length": 4,
+ "visible": true
+ }
+ ],
+ "num_short_key_columns": 1,
+ "num_rows_per_row_block": 1024,
+ "compress_kind": "COMPRESS_LZ4",
+ "next_column_unique_id": 3,
+ "is_in_memory": false,
+ "delete_sign_idx": -1,
+ "sequence_col_idx": -1
+ },
+ "rs_metas": [
+ {
+ "rowset_id": 540072,
+ "tablet_id": 15673,
+ "tablet_schema_hash": 567997577,
+ "rowset_type": "ALPHA_ROWSET",
+ "rowset_state": "VISIBLE",
+ "start_version": 0,
+ "end_version": 1,
+ "version_hash": 0,
+ "num_rows": 0,
+ "total_disk_size": 0,
+ "data_disk_size": 0,
+ "index_disk_size": 0,
+ "empty": true,
+ "creation_time": 1553765664,
+ "num_segments": 0,
+ "alpha_rowset_extra_meta_pb": {
+ "segment_groups": [
+ {
+ "segment_group_id": 0,
+ "num_segments": 0,
+ "index_size": 0,
+ "data_size": 0,
+ "num_rows": 0,
+ "empty": true
+ }
+ ]
+ }
+ },
+ {
+ "rowset_id": 540081,
+ "tablet_id": 15673,
+ "txn_id": 4042,
+ "tablet_schema_hash": 567997577,
+ "rowset_type": "ALPHA_ROWSET",
+ "rowset_state": "VISIBLE",
+ "start_version": 2,
+ "end_version": 2,
+ "version_hash": 8391828013814912580,
+ "num_rows": 3929,
+ "total_disk_size": 84699,
+ "data_disk_size": 84464,
+ "index_disk_size": 235,
+ "empty": false,
+ "load_id": {
+ "hi": -5350970832824939812,
+ "lo": -6717994719194512122
+ },
+ "creation_time": 1553765670,
+ "num_segments": 1,
+ "alpha_rowset_extra_meta_pb": {
+ "segment_groups": [
+ {
+ "segment_group_id": 0,
+ "num_segments": 1,
+ "index_size": 132,
+ "data_size": 576,
+ "num_rows": 5,
+ "zone_maps": [
+ {
+ "min": "MQ==",
+ "max": "NQ==",
+ "null_flag": false
+ },
+ {
+ "min": "MQ==",
+ "max": "Mw==",
+ "null_flag": false
+ },
+ {
+ "min": "J2J1c2gn",
+ "max": "J3RvbSc=",
+ "null_flag": false
+ }
+ ],
+ "empty": false
+ }
+ ]
+ }
+ }
+ ],
+ "in_restore_mode": false,
+ "tablet_uid": {
+ "hi": 10,
+ "lo": 10
+ },
+ "tablet_type": "TABLET_TYPE_DISK"
+}
diff --git a/gensrc/proto/olap_file.proto b/gensrc/proto/olap_file.proto
index 789e45b..48b28ac 100644
--- a/gensrc/proto/olap_file.proto
+++ b/gensrc/proto/olap_file.proto
@@ -211,6 +211,7 @@ message OLAPHeaderMessage {
optional KeysType keys_type = 15; // TabletSchemaPB.keys_type
repeated PDelta delta = 16; // TabletMetaPB.rs_metas
repeated PPendingDelta pending_delta = 17; // need write to olap meta
store
+ // @Deprecated
repeated PDelta incremental_delta = 18; // TabletMetaPB.inc_rs_metas
// if true, this tablet will not do compaction,
@@ -298,6 +299,7 @@ message TabletMetaPB {
optional TabletStatePB tablet_state = 8;
optional TabletSchemaPB schema = 9;
repeated RowsetMetaPB rs_metas = 10;
+ // @Deprecated
repeated RowsetMetaPB inc_rs_metas = 11;
optional AlterTabletPB alter_task = 12;
// if true, this tablet will not do compaction,
diff --git a/gensrc/thrift/AgentService.thrift
b/gensrc/thrift/AgentService.thrift
index 5eef2d1..f7cf318 100644
--- a/gensrc/thrift/AgentService.thrift
+++ b/gensrc/thrift/AgentService.thrift
@@ -204,6 +204,7 @@ struct TSnapshotRequest {
6: optional list<Types.TVersion> missing_version
7: optional bool list_files
// if all nodes has been upgraded, it can be removed.
+ // Deprecated since version 0.13
8: optional bool allow_incremental_clone
9: optional i32 preferred_snapshot_version =
Types.TPREFER_SNAPSHOT_REQ_VERSION
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
