kangpinghuang commented on a change in pull request #777: Optimize path gc
URL: https://github.com/apache/incubator-doris/pull/777#discussion_r267725555
##########
File path: be/src/olap/data_dir.cpp
##########
@@ -823,4 +824,174 @@ OLAPStatus DataDir::load() {
return OLAP_SUCCESS;
}
+void DataDir::add_pending_ids(const std::string& id) {
+ WriteLock wr_lock(&_pending_path_mutex);
+ _pending_path_ids.insert(id);
+}
+
+void DataDir::remove_pending_ids(const std::string& id) {
+ WriteLock wr_lock(&_pending_path_mutex);
+ _pending_path_ids.erase(id);
+}
+
+// path consumer
+void DataDir::perform_path_gc() {
+ // init the set of valid path
+ // validate the path in data dir
+ std::unique_lock<std::mutex> lck(_check_path_mutex);
+ cv.wait(lck, [this]{return _scanned;});
+ _scanned = false;
+ LOG(INFO) << "start to path gc.";
+ int counter = 0;
+ for (int index = 0; index < _all_check_paths.size();) {
+ ++counter;
+ if (config::path_gc_check_step > 0 && counter %
config::path_gc_check_step == 0) {
+ usleep(config::path_gc_check_step_interval_ms * 1000);
+ }
+ auto path_iter = std::next(_all_check_paths.begin(), index);
+ if (path_iter == _all_check_paths.end()) {
+ break;
+ }
+ std::string path = *path_iter;
+ TTabletId tablet_id = -1;
+ TSchemaHash schema_hash = -1;
+ bool is_valid =
StorageEngine::instance()->tablet_manager()->get_tablet_id_and_schema_hash_from_path(path,
+ &tablet_id, &schema_hash);
+ std::set<std::string> paths;
+ paths.insert(path);
+ if (!is_valid) {
+ LOG(WARNING) << "unknow path:" << path;
+ _remove_check_paths_no_lock(paths);
+ continue;
+ } else {
+ if (tablet_id >0 && schema_hash >0) {
+ // tablet schema hash path or rowset file path
+ TabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(tablet_id, schema_hash);
+ if (tablet == nullptr) {
+ std::string tablet_path_id = TABLET_ID_PREFIX +
std::to_string(tablet_id);
+ bool exist_in_pending = _check_pending_ids(tablet_path_id);
+ if (!exist_in_pending) {
+ _process_garbage_path(path);
+ _remove_check_paths_no_lock(paths);
+ continue;
+ }
+ } else {
+ bool valid = tablet->check_path(path);
+ if (!valid) {
+ RowsetId rowset_id = -1;
+ bool is_rowset_file =
StorageEngine::instance()->tablet_manager()
+ ->get_rowset_id_from_path(path, &rowset_id);
+ if (is_rowset_file) {
+ std::string rowset_path_id = ROWSET_ID_PREFIX +
std::to_string(rowset_id);
+ bool exist_in_pending =
_check_pending_ids(rowset_path_id);
+ if (!exist_in_pending) {
+ _process_garbage_path(path);
+ _remove_check_paths_no_lock(paths);
+ continue;
+ }
+ }
+ }
+ }
+ } else if (tablet_id >0 && schema_hash <= 0) {
+ // tablet id path
+ if (FileUtils::is_dir(path)) {
+ bool exist =
StorageEngine::instance()->tablet_manager()->check_tablet_id_exist(tablet_id);
+ if (!exist) {
+ std::string tablet_path_id = TABLET_ID_PREFIX +
std::to_string(tablet_id);
+ bool exist_in_pending =
_check_pending_ids(tablet_path_id);
+ if (!exist_in_pending) {
+ _process_garbage_path(path);
+ _remove_check_paths_no_lock(paths);
+ continue;
+ }
+ }
+ } else {
+ LOG(WARNING) << "unknown path:" << path;
+ _remove_check_paths_no_lock(paths);
+ continue;
+ }
+ } else {
+ LOG(WARNING) << "unknown path:" << path;
+ _remove_check_paths_no_lock(paths);
+ continue;
+ }
+ }
+ ++index;
+ }
+ _all_check_paths.clear();
+ LOG(INFO) << "finished one time path gc.";
+}
+
+// path producer
+void DataDir::perform_path_scan() {
+ std::unique_lock<std::mutex> lck(_check_path_mutex);
+ _scanned = true;
+ LOG(INFO) << "start to scan data dir path:" << _path;
+ std::set<std::string> shards;
+ std::string data_path = _path + DATA_PREFIX;
+ if (dir_walk(data_path, &shards, nullptr) != OLAP_SUCCESS) {
+ LOG(WARNING) << "fail to walk dir. [path=" << data_path << "]";
+ lck.unlock();
+ cv.notify_one();
+ return;
+ }
+ for (const auto& shard : shards) {
+ std::string shard_path = data_path + "/" + shard;
+ std::set<std::string> tablet_ids;
+ if (dir_walk(shard_path, &tablet_ids, nullptr) != OLAP_SUCCESS) {
+ LOG(WARNING) << "fail to walk dir. [path=" << shard_path << "]";
+ continue;
+ }
+ for (const auto& tablet_id : tablet_ids) {
+ std::string tablet_id_path = shard_path + "/" + tablet_id;
+ _all_check_paths.insert(tablet_id_path);
+ std::set<std::string> schema_hashes;
+ if (dir_walk(tablet_id_path, &schema_hashes, nullptr) !=
OLAP_SUCCESS) {
+ LOG(WARNING) << "fail to walk dir. [path=" << tablet_id_path
<< "]";
+ continue;
+ }
+ for (const auto& schema_hash : schema_hashes) {
+ std::string tablet_schema_hash_path = tablet_id_path + "/" +
schema_hash;
+ _all_check_paths.insert(tablet_schema_hash_path);
+ std::set<std::string> rowset_files;
+ if (dir_walk(tablet_schema_hash_path, nullptr, &rowset_files)
!= OLAP_SUCCESS) {
+ LOG(WARNING) << "fail to walk dir. [path=" <<
tablet_schema_hash_path << "]";
+ continue;
+ }
+ for (const auto& rowset_file : rowset_files) {
+ std::string rowset_file_path = tablet_schema_hash_path +
"/" + rowset_file;
+ _all_check_paths.insert(rowset_file_path);
+ }
+ }
+ }
+ }
+ LOG(INFO) << "scan data dir path:" << _path << " finished. path size:" <<
_all_check_paths.size();
+ lck.unlock();
+ cv.notify_one();
+}
+
+void DataDir::_process_garbage_path(const std::string& path) {
+ if (check_dir_existed(path)) {
+ LOG(INFO) << "collect garbage dir path:" << path;
+ OLAPStatus status = remove_all_dir(path);
+ if (status != OLAP_SUCCESS) {
+ LOG(WARNING) << "remove garbage dir path:" << path << " failed";
+ }
+ }
+}
+
+bool DataDir::_check_pending_ids(const std::string& id) {
+ ReadLock rd_lock(&_pending_path_mutex);
+ return _pending_path_ids.find(id) != _pending_path_ids.end();
+}
+
+void DataDir::_remove_check_paths_no_lock(const std::set<std::string> paths) {
Review comment:
ok
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
