This is an automated email from the ASF dual-hosted git repository.
morningman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push:
new 4571b09 [storage][compatibility] Add meta format detection to
prevent data loss. (#4539)
4571b09 is described below
commit 4571b09dd68fa041b85370c22ad274dbcebb8d70
Author: ZhangYu0123 <[email protected]>
AuthorDate: Sun Sep 13 11:58:22 2020 +0800
[storage][compatibility] Add meta format detection to prevent data loss.
(#4539)
After 0.12 version, doris remove the format convert functiion which can
convert from hdr_ format
to tabletmeta_ format when loading metas, the commit link: 3bca253
When we update doris version and there are old format meta in storage,
BE will not read the old format tablet. It can lead to data loss.
So we add meta format detection function to prevent data loss.
When there are old format meta in olap_meta, BE can find and print log or
exit.
---
be/src/common/config.h | 6 ++++
be/src/olap/data_dir.cpp | 34 ++++++++++++++++++++++
be/src/olap/data_dir.h | 4 +++
be/src/olap/tablet_meta_manager.cpp | 3 +-
docs/en/administrator-guide/config/be_config.md | 9 ++++++
docs/zh-CN/administrator-guide/config/be_config.md | 9 ++++++
6 files changed, 64 insertions(+), 1 deletion(-)
diff --git a/be/src/common/config.h b/be/src/common/config.h
index 08151ad..b074c85 100644
--- a/be/src/common/config.h
+++ b/be/src/common/config.h
@@ -223,6 +223,12 @@ namespace config {
CONF_mInt32(disk_stat_monitor_interval, "5");
CONF_mInt32(unused_rowset_monitor_interval, "30");
CONF_String(storage_root_path, "${DORIS_HOME}/storage");
+
+ // Config is used to check incompatible old format hdr_ format
+ // whether doris uses strict way. When config is true, process will log
fatal
+ // and exit. When config is false, process will only log warning.
+ CONF_Bool(storage_strict_check_incompatible_old_format, "true");
+
// BE process will exit if the percentage of error disk reach this value.
CONF_mInt32(max_percentage_of_error_disk, "0");
// CONF_Int32(default_num_rows_per_data_block, "1024");
diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp
index 82613eb..42afdd8 100644
--- a/be/src/olap/data_dir.cpp
+++ b/be/src/olap/data_dir.cpp
@@ -602,6 +602,36 @@ OLAPStatus DataDir::set_convert_finished() {
return OLAP_SUCCESS;
}
+OLAPStatus DataDir::_check_incompatible_old_format_tablet() {
+
+ auto check_incompatible_old_func = [this](int64_t tablet_id, int32_t
schema_hash,
+ const std::string& value) ->
bool {
+
+ // if strict check incompatible old format, then log fatal
+ if (config::storage_strict_check_incompatible_old_format) {
+ LOG(FATAL) << "There are incompatible old format metas, current
version does not support "
+ << "and it may lead to data missing!!! "
+ << "talbet_id = " << tablet_id << " schema_hash = " <<
schema_hash;
+ } else {
+ LOG(WARNING)
+ << "There are incompatible old format metas, current
version does not support "
+ << "and it may lead to data missing!!! "
+ << "talbet_id = " << tablet_id << " schema_hash = " <<
schema_hash;
+ }
+ return false;
+ };
+
+ // seek old header prefix. when check_incompatible_old_func is called, it
has old format in olap_meta
+ OLAPStatus check_incompatible_old_status =
TabletMetaManager::traverse_headers(
+ _meta, check_incompatible_old_func, OLD_HEADER_PREFIX);
+ if (check_incompatible_old_status != OLAP_SUCCESS) {
+ LOG(WARNING) << "check incompatible old format meta fails, it may lead
to data missing!!! " << _path;
+ } else {
+ LOG(INFO) << "successfully check incompatible old format meta " <<
_path;
+ }
+ return check_incompatible_old_status;
+}
+
// TODO(ygl): deal with rowsets and tablets when load failed
OLAPStatus DataDir::load() {
LOG(INFO) << "start to load tablets from " << _path;
@@ -609,6 +639,10 @@ OLAPStatus DataDir::load() {
// COMMITTED: add to txn manager
// VISIBLE: add to tablet
// if one rowset load failed, then the total data dir will not be loaded
+
+ // necessarily check incompatible old format. when there are old metas, it
may load to data missing
+ _check_incompatible_old_format_tablet();
+
std::vector<RowsetMetaSharedPtr> dir_rowset_metas;
LOG(INFO) << "begin loading rowset from meta";
auto load_rowset_func = [&dir_rowset_metas](TabletUid tablet_uid, RowsetId
rowset_id,
diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h
index be434de..0fc8b4a 100644
--- a/be/src/olap/data_dir.h
+++ b/be/src/olap/data_dir.h
@@ -141,6 +141,10 @@ private:
Status _write_cluster_id_to_path(const std::string& path, int32_t
cluster_id);
OLAPStatus _clean_unfinished_converting_data();
OLAPStatus _convert_old_tablet();
+ // Check whether has old format (hdr_ start) in olap. When doris updating
to current version,
+ // it may lead to data missing. When
conf::storage_strict_check_incompatible_old_format is true,
+ // process will log fatal.
+ OLAPStatus _check_incompatible_old_format_tablet();
void _process_garbage_path(const std::string& path);
diff --git a/be/src/olap/tablet_meta_manager.cpp
b/be/src/olap/tablet_meta_manager.cpp
index abd89d8..74fdc16 100755
--- a/be/src/olap/tablet_meta_manager.cpp
+++ b/be/src/olap/tablet_meta_manager.cpp
@@ -137,7 +137,8 @@ OLAPStatus TabletMetaManager::traverse_headers(OlapMeta*
meta,
std::function<bool(long, long, const std::string&)> const& func, const
string& header_prefix) {
auto traverse_header_func = [&func](const std::string& key, const
std::string& value) -> bool {
std::vector<std::string> parts;
- // key format: "hdr_" + tablet_id + "_" + schema_hash
+ // old format key format: "hdr_" + tablet_id + "_" + schema_hash 0.11
+ // new format key format: "tabletmata_" + tablet_id + "_" +
schema_hash 0.10
split_string<char>(key, '_', &parts);
if (parts.size() != 3) {
LOG(WARNING) << "invalid tablet_meta key:" << key << ", splitted
size:" << parts.size();
diff --git a/docs/en/administrator-guide/config/be_config.md
b/docs/en/administrator-guide/config/be_config.md
index 1beaa46..33a9ef3 100644
--- a/docs/en/administrator-guide/config/be_config.md
+++ b/docs/en/administrator-guide/config/be_config.md
@@ -483,6 +483,15 @@ Indicates how many tablets in this data directory failed
to load. At the same ti
### `storage_root_path`
+### `storage_strict_check_incompatible_old_format`
+* Type: bool
+* Description: Used to check incompatible old format strictly
+* Default value: true
+* Dynamically modify: false
+
+This config is used to check incompatible old format hdr_ format whether doris
uses strict way. When config is true,
+process will log fatal and exit. When config is false, process will only log
warning.
+
### `streaming_load_max_mb`
* Type: int64
diff --git a/docs/zh-CN/administrator-guide/config/be_config.md
b/docs/zh-CN/administrator-guide/config/be_config.md
index a5dd8a9..bde36f4 100644
--- a/docs/zh-CN/administrator-guide/config/be_config.md
+++ b/docs/zh-CN/administrator-guide/config/be_config.md
@@ -482,6 +482,15 @@ load tablets from header failed, failed tablets size: xxx,
path=xxx
### `storage_root_path`
+### `storage_strict_check_incompatible_old_format`
+* 类型：bool
+* 描述：用来检查不兼容的旧版本格式时是否使用严格的验证方式
+* 默认值： true
+* 可动态修改：否
+
+配置用来检查不兼容的旧版本格式时是否使用严格的验证方式，当含有旧版本的 hdr 格式时，使用严谨的方式时，程序会
+打出 fatal log 并且退出运行；否则，程序仅打印 warn log.
+
### `streaming_load_max_mb`
* 类型：int64
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
