kangpinghuang commented on a change in pull request #777: Optimize path gc
URL: https://github.com/apache/incubator-doris/pull/777#discussion_r267725197
##########
File path: be/src/olap/data_dir.cpp
##########
@@ -823,4 +824,174 @@ OLAPStatus DataDir::load() {
return OLAP_SUCCESS;
}
+void DataDir::add_pending_ids(const std::string& id) {
+ WriteLock wr_lock(&_pending_path_mutex);
+ _pending_path_ids.insert(id);
+}
+
+void DataDir::remove_pending_ids(const std::string& id) {
+ WriteLock wr_lock(&_pending_path_mutex);
+ _pending_path_ids.erase(id);
+}
+
+// path consumer
+void DataDir::perform_path_gc() {
+ // init the set of valid path
+ // validate the path in data dir
+ std::unique_lock<std::mutex> lck(_check_path_mutex);
+ cv.wait(lck, [this]{return _scanned;});
+ _scanned = false;
+ LOG(INFO) << "start to path gc.";
+ int counter = 0;
+ for (int index = 0; index < _all_check_paths.size();) {
+ ++counter;
+ if (config::path_gc_check_step > 0 && counter %
config::path_gc_check_step == 0) {
+ usleep(config::path_gc_check_step_interval_ms * 1000);
+ }
+ auto path_iter = std::next(_all_check_paths.begin(), index);
+ if (path_iter == _all_check_paths.end()) {
+ break;
+ }
+ std::string path = *path_iter;
+ TTabletId tablet_id = -1;
+ TSchemaHash schema_hash = -1;
+ bool is_valid =
StorageEngine::instance()->tablet_manager()->get_tablet_id_and_schema_hash_from_path(path,
+ &tablet_id, &schema_hash);
+ std::set<std::string> paths;
+ paths.insert(path);
+ if (!is_valid) {
+ LOG(WARNING) << "unknow path:" << path;
+ _remove_check_paths_no_lock(paths);
+ continue;
+ } else {
+ if (tablet_id >0 && schema_hash >0) {
Review comment:
ok
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
