kangpinghuang commented on a change in pull request #3641:
URL: https://github.com/apache/incubator-doris/pull/3641#discussion_r428463285
##########
File path: be/src/olap/data_dir.cpp
##########
@@ -709,13 +716,25 @@ OLAPStatus DataDir::load() {
return true;
};
OLAPStatus load_tablet_status = TabletMetaManager::traverse_headers(_meta,
load_tablet_func);
- if (failed_tablet_ids.size() != 0 && !config::ignore_load_tablet_failure) {
- LOG(FATAL) << "load tablets from header failed, failed tablets size: "
<< failed_tablet_ids.size();
+ if (failed_tablet_ids.size() != 0) {
+ LOG(WARNING) << "load tablets from header failed"
+ << ", loaded tablet: " << tablet_ids.size()
+ << ", error tablet: " << failed_tablet_ids.size()
+ << ", path: " << _path;
+ if (!config::ignore_load_tablet_failure) {
+ LOG(FATAL) << "load tablets encounter failure. stop BE process.
path: " << _path;
+ }
}
if (load_tablet_status != OLAP_SUCCESS) {
- LOG(WARNING) << "there is failure when loading tablet headers, path:"
<< _path;
+ LOG(WARNING) << "there is failure when loading tablet headers"
+ << ", loaded tablet: " << tablet_ids.size()
+ << ", error tablet: " << failed_tablet_ids.size()
+ << ", path: " << _path;
} else {
- LOG(INFO) << "load rowset from meta finished, data dir: " << _path;
+ LOG(INFO) << "load rowset from meta finished"
Review comment:
```suggestion
LOG(INFO) << "load tablet meta finished"
```
##########
File path: be/src/tools/meta_tool.cpp
##########
@@ -122,13 +126,131 @@ void delete_meta(DataDir* data_dir) {
std::cout << "delete meta successfully" << std::endl;
}
+Status init_data_dir(const std::string dir, DataDir** ret) {
Review comment:
```suggestion
Status init_data_dir(const std::string& dir, DataDir** ret) {
```
##########
File path: be/src/tools/meta_tool.cpp
##########
@@ -122,13 +126,131 @@ void delete_meta(DataDir* data_dir) {
std::cout << "delete meta successfully" << std::endl;
}
+Status init_data_dir(const std::string dir, DataDir** ret) {
+ std::string root_path;
+ Status st = FileUtils::canonicalize(dir, &root_path);
+ if (!st.ok()) {
+ std::cout << "invalid root path:" << FLAGS_root_path
+ << ", error: " << st.to_string() << std::endl;
+ return Status::InternalError("invalid root path");
+ }
+ doris::StorePath path;
+ auto res = parse_root_path(root_path, &path);
+ if (res != OLAP_SUCCESS) {
+ std::cout << "parse root path failed:" << root_path << std::endl;
+ return Status::InternalError("parse root path failed");
+ }
+
+ *ret = new (std::nothrow) DataDir(path.path, path.capacity_bytes,
path.storage_medium);
+ if (*ret == nullptr) {
+ std::cout << "new data dir failed" << std::endl;
+ return Status::InternalError("new data dir failed");
+ }
+ st = (*ret)->init();
+ if (!st.ok()) {
+ std::cout << "data_dir load failed" << std::endl;
+ return Status::InternalError("data_dir load failed");
+ }
+
+ return Status::OK();
+}
+
+void batch_delete_meta(const std::string tablet_file) {
+ // each line in tablet file indicate a tablet to delete, format is:
+ // data_dir,tablet_id,schema_hash
+ // eg:
+ // /data1/palo.HDD,100010,11212389324
+ // /data2/palo.HDD,100010,23049230234
+ std::ifstream infile(tablet_file);
+ std::string line;
+ int err_num = 0;
+ int delete_num = 0;
+ int total_num = 0;
+ std::unordered_map<std::string, std::unique_ptr<DataDir>> dir_map;
+ while (std::getline(infile, line)) {
+ total_num++;
+ vector<string> v = strings::Split(line, ",");
+ if (v.size() != 3) {
+ std::cout << "invalid line in tablet_file: " << line << std::endl;
+ err_num++;
+ continue;
+ }
+ // 1. get dir
+ std::string dir;
+ Status st = FileUtils::canonicalize(v[0], &dir);
+ if (!st.ok()) {
+ std::cout << "invalid root dir in tablet_file: " << line <<
std::endl;
+ err_num++;
+ continue;
+ }
+
+ if (dir_map.find(dir) == dir_map.end()) {
+ // new data dir, init it
+ DataDir* data_dir_p = nullptr;
+ Status st = init_data_dir(dir, &data_dir_p);
+ if (!st.ok()) {
+ std::cout << "invalid root path:" << FLAGS_root_path
+ << ", error: " << st.to_string() << std::endl;
+ err_num++;
+ continue;
+ }
+ dir_map[dir] = std::unique_ptr<DataDir>(data_dir_p);
+ std::cout << "get a new data dir: " << dir << std::endl;
+ }
+ DataDir* data_dir = dir_map[dir].get();
+ if (data_dir == nullptr) {
+ std::cout << "failed to get data dir: " << line << std::endl;
+ err_num++;
+ continue;
+ }
+
+ // 2. get tablet id/schema_hash
+ int64_t tablet_id;
+ if (!safe_strto64(v[1].c_str(), &tablet_id)) {
+ std::cout << "invalid tablet id: " << line << std::endl;
+ err_num++;
+ continue;
+ }
+ int64_t schema_hash;
+ if (!safe_strto64(v[2].c_str(), &schema_hash)) {
+ std::cout << "invalid schema hash: " << line << std::endl;
+ err_num++;
+ continue;
+ }
+
+ OLAPStatus s = TabletMetaManager::remove(data_dir, tablet_id,
schema_hash);
+ if (s != OLAP_SUCCESS) {
+ std::cout << "delete tablet meta failed for tablet_id:"
+ << FLAGS_tablet_id << ", schema_hash:" << FLAGS_schema_hash
Review comment:
```suggestion
<< tablet_id << ", schema_hash:" << schema_hash
```
##########
File path: docs/zh-CN/administrator-guide/operation/tablet-meta-tool.md
##########
@@ -93,14 +93,39 @@ api：
### 删除 header
-为了实现从某个 be 的某个盘中删除某个 tablet 的功能。
+为了实现从某个 be 的某个盘中删除某个 tablet 的功能。可以单独删除一个 tablet，或者批量删除一组 tablet。
Review comment:
```suggestion
为了实现从某个 be 的某个盘中删除某个 tablet 的元数据功能。可以单独删除一个 tablet元数据，或者批量删除一组 tablet元数据。
```
##########
File path: be/src/tools/meta_tool.cpp
##########
@@ -122,13 +126,131 @@ void delete_meta(DataDir* data_dir) {
std::cout << "delete meta successfully" << std::endl;
}
+Status init_data_dir(const std::string dir, DataDir** ret) {
+ std::string root_path;
+ Status st = FileUtils::canonicalize(dir, &root_path);
+ if (!st.ok()) {
+ std::cout << "invalid root path:" << FLAGS_root_path
+ << ", error: " << st.to_string() << std::endl;
+ return Status::InternalError("invalid root path");
+ }
+ doris::StorePath path;
+ auto res = parse_root_path(root_path, &path);
+ if (res != OLAP_SUCCESS) {
+ std::cout << "parse root path failed:" << root_path << std::endl;
+ return Status::InternalError("parse root path failed");
+ }
+
+ *ret = new (std::nothrow) DataDir(path.path, path.capacity_bytes,
path.storage_medium);
+ if (*ret == nullptr) {
+ std::cout << "new data dir failed" << std::endl;
+ return Status::InternalError("new data dir failed");
+ }
+ st = (*ret)->init();
+ if (!st.ok()) {
+ std::cout << "data_dir load failed" << std::endl;
+ return Status::InternalError("data_dir load failed");
+ }
+
+ return Status::OK();
+}
+
+void batch_delete_meta(const std::string tablet_file) {
+ // each line in tablet file indicate a tablet to delete, format is:
+ // data_dir,tablet_id,schema_hash
+ // eg:
+ // /data1/palo.HDD,100010,11212389324
+ // /data2/palo.HDD,100010,23049230234
+ std::ifstream infile(tablet_file);
+ std::string line;
Review comment:
```suggestion
std::string line = "";
```
##########
File path: be/src/tools/meta_tool.cpp
##########
@@ -122,13 +126,131 @@ void delete_meta(DataDir* data_dir) {
std::cout << "delete meta successfully" << std::endl;
}
+Status init_data_dir(const std::string dir, DataDir** ret) {
Review comment:
and why here use DataDir**?
I think DataDir* or std::unique_ptr<DataDir>* is ok
##########
File path: be/src/tools/meta_tool.cpp
##########
@@ -122,13 +126,131 @@ void delete_meta(DataDir* data_dir) {
std::cout << "delete meta successfully" << std::endl;
}
+Status init_data_dir(const std::string dir, DataDir** ret) {
+ std::string root_path;
+ Status st = FileUtils::canonicalize(dir, &root_path);
+ if (!st.ok()) {
+ std::cout << "invalid root path:" << FLAGS_root_path
+ << ", error: " << st.to_string() << std::endl;
+ return Status::InternalError("invalid root path");
+ }
+ doris::StorePath path;
+ auto res = parse_root_path(root_path, &path);
+ if (res != OLAP_SUCCESS) {
+ std::cout << "parse root path failed:" << root_path << std::endl;
+ return Status::InternalError("parse root path failed");
+ }
+
+ *ret = new (std::nothrow) DataDir(path.path, path.capacity_bytes,
path.storage_medium);
+ if (*ret == nullptr) {
+ std::cout << "new data dir failed" << std::endl;
+ return Status::InternalError("new data dir failed");
+ }
+ st = (*ret)->init();
+ if (!st.ok()) {
+ std::cout << "data_dir load failed" << std::endl;
+ return Status::InternalError("data_dir load failed");
+ }
+
+ return Status::OK();
+}
+
+void batch_delete_meta(const std::string tablet_file) {
Review comment:
```suggestion
void batch_delete_meta(const std::string& tablet_file) {
```
##########
File path: docs/zh-CN/administrator-guide/operation/tablet-meta-tool.md
##########
@@ -93,14 +93,39 @@ api：
### 删除 header
-为了实现从某个 be 的某个盘中删除某个 tablet 的功能。
+为了实现从某个 be 的某个盘中删除某个 tablet 的功能。可以单独删除一个 tablet，或者批量删除一组 tablet。
-命令:
+删除单个tablet：
Review comment:
```suggestion
删除单个tablet元数据：
```
##########
File path: docs/zh-CN/administrator-guide/operation/tablet-meta-tool.md
##########
@@ -93,14 +93,39 @@ api：
### 删除 header
-为了实现从某个 be 的某个盘中删除某个 tablet 的功能。
+为了实现从某个 be 的某个盘中删除某个 tablet 的功能。可以单独删除一个 tablet，或者批量删除一组 tablet。
-命令:
+删除单个tablet：
```
./lib/meta_tool --operation=delete_meta --root_path=/path/to/root_path
--tablet_id=xxx --schema_hash=xxx
```
+删除一组tablet：
Review comment:
```suggestion
删除一组tablet元数据：
```
##########
File path: docs/en/administrator-guide/operation/tablet-meta-tool.md
##########
@@ -93,14 +93,39 @@ Order:
### Delete header
-In order to realize the function of deleting a tablet from a disk of a be.
+In order to realize the function of deleting a tablet from a disk of a BE.
Support single delete and batch delete.
-Order:
+Single delete:
```
./lib/meta_tool --operation=delete_meta --root_path=/path/to/root_path
--tablet_id=xxx --schema_hash=xxx`
```
+Batch delete:
+
+```
+./lib/meta_tool --operation=batch_delete_meta
--tablet_file=/path/to/tablet_file.txt
Review comment:
will you need to support delete tablet data meanwhile?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
