imay commented on a change in pull request #777: Optimize path gc
URL: https://github.com/apache/incubator-doris/pull/777#discussion_r268102176
##########
File path: be/src/olap/data_dir.cpp
##########
@@ -823,4 +827,174 @@ OLAPStatus DataDir::load() {
return OLAP_SUCCESS;
}
+void DataDir::add_pending_ids(const std::string& id) {
+ WriteLock wr_lock(&_pending_path_mutex);
+ _pending_path_ids.insert(id);
+}
+
+void DataDir::remove_pending_ids(const std::string& id) {
+ WriteLock wr_lock(&_pending_path_mutex);
+ _pending_path_ids.erase(id);
+}
+
+// path consumer
+void DataDir::perform_path_gc() {
+ // init the set of valid path
+ // validate the path in data dir
+ std::unique_lock<std::mutex> lck(_check_path_mutex);
+ cv.wait(lck, [this]{return _scanned;});
+ _scanned = false;
+ LOG(INFO) << "start to path gc.";
+ int counter = 0;
+ for (int index = 0; index < _all_check_paths.size();) {
+ ++counter;
+ if (config::path_gc_check_step > 0 && counter %
config::path_gc_check_step == 0) {
+ usleep(config::path_gc_check_step_interval_ms * 1000);
+ }
+ auto path_iter = std::next(_all_check_paths.begin(), index);
+ if (path_iter == _all_check_paths.end()) {
+ break;
+ }
+ std::string path = *path_iter;
+ TTabletId tablet_id = -1;
+ TSchemaHash schema_hash = -1;
+ bool is_valid =
_tablet_manager->get_tablet_id_and_schema_hash_from_path(path,
+ &tablet_id, &schema_hash);
+ std::set<std::string> paths;
+ paths.insert(path);
+ if (!is_valid) {
+ LOG(WARNING) << "unknown path:" << path;
+ _remove_check_paths_no_lock(paths);
+ continue;
+ } else {
+ if (tablet_id > 0 && schema_hash > 0) {
+ // tablet schema hash path or rowset file path
+ TabletSharedPtr tablet =
_tablet_manager->get_tablet(tablet_id, schema_hash);
+ if (tablet == nullptr) {
+ std::string tablet_path_id = TABLET_ID_PREFIX +
std::to_string(tablet_id);
+ bool exist_in_pending = _check_pending_ids(tablet_path_id);
+ if (!exist_in_pending) {
+ _process_garbage_path(path);
+ _remove_check_paths_no_lock(paths);
+ continue;
+ }
+ } else {
+ bool valid = tablet->check_path(path);
+ if (!valid) {
+ RowsetId rowset_id = -1;
+ bool is_rowset_file = _tablet_manager
+ ->get_rowset_id_from_path(path, &rowset_id);
+ if (is_rowset_file) {
+ std::string rowset_path_id = ROWSET_ID_PREFIX +
std::to_string(rowset_id);
+ bool exist_in_pending =
_check_pending_ids(rowset_path_id);
+ if (!exist_in_pending) {
+ _process_garbage_path(path);
+ _remove_check_paths_no_lock(paths);
+ continue;
+ }
+ }
+ }
+ }
+ } else if (tablet_id > 0 && schema_hash <= 0) {
+ // tablet id path
+ if (FileUtils::is_dir(path)) {
+ bool exist =
_tablet_manager->check_tablet_id_exist(tablet_id);
+ if (!exist) {
+ std::string tablet_path_id = TABLET_ID_PREFIX +
std::to_string(tablet_id);
+ bool exist_in_pending =
_check_pending_ids(tablet_path_id);
+ if (!exist_in_pending) {
+ _process_garbage_path(path);
+ _remove_check_paths_no_lock(paths);
+ continue;
+ }
+ }
+ } else {
+ LOG(WARNING) << "unknown path:" << path;
+ _remove_check_paths_no_lock(paths);
+ continue;
+ }
+ } else {
+ LOG(WARNING) << "unknown path:" << path;
+ _remove_check_paths_no_lock(paths);
+ continue;
+ }
+ }
+ ++index;
+ }
+ _all_check_paths.clear();
+ LOG(INFO) << "finished one time path gc.";
+}
+
+// path producer
+void DataDir::perform_path_scan() {
+ std::unique_lock<std::mutex> lck(_check_path_mutex);
Review comment:
{
unique_lock
}
cv.notify_one()
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
