This is an automated email from the ASF dual-hosted git repository. zhaoc pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-doris.git
The following commit(s) were added to refs/heads/master by this push: new 948f497 Fix some warning when compile type is debug using -Werror flag(because of use deprecated funcrtions) (#1953) 948f497 is described below commit 948f497dd408b891aa0baa7964ab834d6fa4747d Author: yangzhg <780531...@qq.com> AuthorDate: Fri Oct 11 17:21:41 2019 +0800 Fix some warning when compile type is debug using -Werror flag(because of use deprecated funcrtions) (#1953) --- be/src/agent/cgroups_mgr.cpp | 173 ++++++++++++++++++++++--------------------- be/src/olap/data_dir.cpp | 63 ++++++---------- be/src/olap/data_dir.h | 5 +- be/src/util/disk_info.cpp | 1 + 4 files changed, 112 insertions(+), 130 deletions(-) diff --git a/be/src/agent/cgroups_mgr.cpp b/be/src/agent/cgroups_mgr.cpp index 8a2faac..9495271 100644 --- a/be/src/agent/cgroups_mgr.cpp +++ b/be/src/agent/cgroups_mgr.cpp @@ -18,12 +18,13 @@ #include "agent/cgroups_mgr.h" #include <fstream> #include <future> -#include <linux/magic.h> +#include <linux/magic.h> #include <map> #include <unistd.h> #include <asm/unistd.h> #include <sstream> #include <sys/stat.h> +#include <sys/sysmacros.h> #include <sys/vfs.h> #include "boost/filesystem.hpp" #include "common/logging.h" @@ -46,12 +47,12 @@ static CgroupsMgr *s_global_cg_mgr; const std::string CgroupsMgr::_s_system_user = "system"; const std::string CgroupsMgr::_s_system_group = "normal"; -std::map<TResourceType::type, std::string> CgroupsMgr::_s_resource_cgroups = - {{TResourceType::type::TRESOURCE_CPU_SHARE, "cpu.shares"}, +std::map<TResourceType::type, std::string> CgroupsMgr::_s_resource_cgroups = + {{TResourceType::type::TRESOURCE_CPU_SHARE, "cpu.shares"}, {TResourceType::type::TRESOURCE_IO_SHARE, "blkio.weight"}}; -CgroupsMgr::CgroupsMgr(ExecEnv* exec_env, const string& root_cgroups_path) - : _exec_env(exec_env), +CgroupsMgr::CgroupsMgr(ExecEnv* exec_env, const string& root_cgroups_path) + : _exec_env(exec_env), _root_cgroups_path(root_cgroups_path), _is_cgroups_init_success(false), _cur_version(-1) { @@ -64,31 +65,31 @@ CgroupsMgr::~CgroupsMgr() { } AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fetched_resource) { - + std::lock_guard<std::mutex> lck(_update_cgroups_mtx); if (!_is_cgroups_init_success) { return AgentStatus::DORIS_ERROR; } - - + + if (_cur_version >= new_fetched_resource.resourceVersion) { return AgentStatus::DORIS_SUCCESS; } - - const std::map<std::string, TUserResource>& new_user_resource + + const std::map<std::string, TUserResource>& new_user_resource = new_fetched_resource.resourceByUser; - + if (!_local_users.empty()) { std::set<std::string>::const_iterator old_it = _local_users.begin(); for (; old_it != _local_users.end(); ++old_it) { if (new_user_resource.count(*old_it) == 0) { this->delete_user_cgroups(*old_it); - } + } } } // Clear local users set, because it will be inserted again - _local_users.clear(); + _local_users.clear(); std::map<std::string, TUserResource>::const_iterator new_it = new_user_resource.begin(); for (; new_it != new_user_resource.end(); ++new_it) { @@ -102,7 +103,7 @@ AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fe if (_s_resource_cgroups.count(resource_it->first) > 0) { user_share[_s_resource_cgroups[resource_it->first]] = resource_it->second; - } + } } modify_user_cgroups(user_name, user_share, level_share); @@ -111,46 +112,46 @@ AgentStatus CgroupsMgr::update_local_cgroups(const TFetchResourceResult& new_fe // Insert user to local user's set _local_users.insert(user_name); } - + // Using resource version, not subscribe version _cur_version = new_fetched_resource.resourceVersion; return AgentStatus::DORIS_SUCCESS; } -void CgroupsMgr::_config_user_disk_throttle(std::string user_name, +void CgroupsMgr::_config_user_disk_throttle(std::string user_name, const std::map<TResourceType::type, int32_t>& resource_share) { - int64_t hdd_read_iops = _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_IOPS, + int64_t hdd_read_iops = _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_IOPS, resource_share); - int64_t hdd_write_iops = _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_IOPS, + int64_t hdd_write_iops = _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_IOPS, resource_share); - int64_t hdd_read_mbps = _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_MBPS, + int64_t hdd_read_mbps = _get_resource_value(TResourceType::type::TRESOURCE_HDD_READ_MBPS, resource_share); - int64_t hdd_write_mbps = _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_MBPS, + int64_t hdd_write_mbps = _get_resource_value(TResourceType::type::TRESOURCE_HDD_WRITE_MBPS, resource_share); - int64_t ssd_read_iops = _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_IOPS, + int64_t ssd_read_iops = _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_IOPS, resource_share); - int64_t ssd_write_iops = _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_IOPS, + int64_t ssd_write_iops = _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_IOPS, resource_share); - int64_t ssd_read_mbps = _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_MBPS, + int64_t ssd_read_mbps = _get_resource_value(TResourceType::type::TRESOURCE_SSD_READ_MBPS, resource_share); - int64_t ssd_write_mbps = _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_MBPS, + int64_t ssd_write_mbps = _get_resource_value(TResourceType::type::TRESOURCE_SSD_WRITE_MBPS, resource_share); - - _config_disk_throttle(user_name, "", hdd_read_iops, hdd_write_iops, - hdd_read_mbps, hdd_write_mbps, - ssd_read_iops, ssd_write_iops, + + _config_disk_throttle(user_name, "", hdd_read_iops, hdd_write_iops, + hdd_read_mbps, hdd_write_mbps, + ssd_read_iops, ssd_write_iops, ssd_read_mbps, ssd_write_mbps); - _config_disk_throttle(user_name, "low", hdd_read_iops, hdd_write_iops, - hdd_read_mbps, hdd_write_mbps, - ssd_read_iops, ssd_write_iops, + _config_disk_throttle(user_name, "low", hdd_read_iops, hdd_write_iops, + hdd_read_mbps, hdd_write_mbps, + ssd_read_iops, ssd_write_iops, ssd_read_mbps, ssd_write_mbps); - _config_disk_throttle(user_name, "normal", hdd_read_iops, hdd_write_iops, - hdd_read_mbps, hdd_write_mbps, - ssd_read_iops, ssd_write_iops, + _config_disk_throttle(user_name, "normal", hdd_read_iops, hdd_write_iops, + hdd_read_mbps, hdd_write_mbps, + ssd_read_iops, ssd_write_iops, ssd_read_mbps, ssd_write_mbps); - _config_disk_throttle(user_name, "high", hdd_read_iops, hdd_write_iops, - hdd_read_mbps, hdd_write_mbps, - ssd_read_iops, ssd_write_iops, + _config_disk_throttle(user_name, "high", hdd_read_iops, hdd_write_iops, + hdd_read_mbps, hdd_write_mbps, + ssd_read_iops, ssd_write_iops, ssd_read_mbps, ssd_write_mbps); } @@ -166,32 +167,32 @@ int64_t CgroupsMgr::_get_resource_value(const TResourceType::type resource_type, AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name, std::string level, - int64_t hdd_read_iops, + int64_t hdd_read_iops, int64_t hdd_write_iops, - int64_t hdd_read_mbps, - int64_t hdd_write_mbps, - int64_t ssd_read_iops, - int64_t ssd_write_iops, - int64_t ssd_read_mbps, + int64_t hdd_read_mbps, + int64_t hdd_write_mbps, + int64_t ssd_read_iops, + int64_t ssd_write_iops, + int64_t ssd_read_mbps, int64_t ssd_write_mbps) { - string cgroups_path = this->_root_cgroups_path + "/" + user_name + "/" + level; + string cgroups_path = this->_root_cgroups_path + "/" + user_name + "/" + level; string read_bps_path = cgroups_path + "/blkio.throttle.read_bps_device"; string write_bps_path = cgroups_path + "/blkio.throttle.write_bps_device"; string read_iops_path = cgroups_path + "/blkio.throttle.read_iops_device"; string write_iops_path = cgroups_path + "/blkio.throttle.write_iops_device"; - if (!is_file_exist(cgroups_path.c_str())) { + if (!is_file_exist(cgroups_path.c_str())) { if (!boost::filesystem::create_directory(cgroups_path)) { LOG(ERROR) << "Create cgroups: " << cgroups_path << " failed"; return AgentStatus::DORIS_ERROR; } } - + // add olap engine data path here auto stores = StorageEngine::instance()->get_stores(); // buld load data path, it is alreay in data path // _exec_env->load_path_mgr()->get_load_data_path(&data_paths); - + stringstream ctrl_cmd; for (auto store : stores) { // check disk type @@ -215,7 +216,7 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name, minor_number = (minor_number / 16) * 16; if (read_iops != -1) { ctrl_cmd << major_number << ":" - << minor_number << " " + << minor_number << " " << read_iops; _echo_cmd_to_cgroup(ctrl_cmd, read_iops_path); ctrl_cmd.clear(); @@ -223,7 +224,7 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name, } if (write_iops != -1) { ctrl_cmd << major_number << ":" - << minor_number << " " + << minor_number << " " << write_iops; _echo_cmd_to_cgroup(ctrl_cmd, write_iops_path); ctrl_cmd.clear(); @@ -231,7 +232,7 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name, } if (read_mbps != -1) { ctrl_cmd << major_number << ":" - << minor_number << " " + << minor_number << " " << (read_mbps << 20); _echo_cmd_to_cgroup(ctrl_cmd, read_bps_path); ctrl_cmd.clear(); @@ -239,7 +240,7 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name, } if (write_mbps != -1) { ctrl_cmd << major_number << ":" - << minor_number << " " + << minor_number << " " << (write_mbps << 20); _echo_cmd_to_cgroup(ctrl_cmd, write_bps_path); ctrl_cmd.clear(); @@ -249,12 +250,12 @@ AgentStatus CgroupsMgr::_config_disk_throttle(std::string user_name, return AgentStatus::DORIS_SUCCESS; } -AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name, - const map<string, int32_t>& user_share, +AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name, + const map<string, int32_t>& user_share, const map<string, int32_t>& level_share) { // Check if the user's cgroups exists, if not create it - string user_cgroups_path = this->_root_cgroups_path + "/" + user_name; - if (!is_file_exist(user_cgroups_path.c_str())) { + string user_cgroups_path = this->_root_cgroups_path + "/" + user_name; + if (!is_file_exist(user_cgroups_path.c_str())) { if (!boost::filesystem::create_directory(user_cgroups_path)) { LOG(ERROR) << "Create cgroups for user " << user_name << " failed"; return AgentStatus::DORIS_ERROR; @@ -262,7 +263,7 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name, } // Traverse the user resource share map to append share value to cgroup's file - for (map<string, int32_t>::const_iterator user_resource = user_share.begin(); + for (map<string, int32_t>::const_iterator user_resource = user_share.begin(); user_resource != user_share.end(); ++user_resource){ string resource_file_name = user_resource->first; int32_t user_share_weight = user_resource->second; @@ -271,11 +272,11 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name, std::ofstream user_cgroups(user_resource_path.c_str(), std::ios::out | std::ios::app); if (!user_cgroups.is_open()) { return AgentStatus::DORIS_ERROR; - } + } user_cgroups << user_share_weight << std::endl; user_cgroups.close(); LOG(INFO) << "Append " << user_share_weight << " to " << user_resource_path; - for (map<string, int32_t>::const_iterator level_resource = level_share.begin(); + for (map<string, int32_t>::const_iterator level_resource = level_share.begin(); level_resource != level_share.end(); ++level_resource){ // Append resource share to level shares string level_name = level_resource->first; @@ -288,9 +289,9 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name, } } - // Append the share_weight value to the file + // Append the share_weight value to the file string level_resource_path = level_cgroups_path + "/" + resource_file_name; - std::ofstream level_cgroups(level_resource_path.c_str(), + std::ofstream level_cgroups(level_resource_path.c_str(), std::ios::out | std::ios::app); if (!level_cgroups.is_open()) { return AgentStatus::DORIS_ERROR; @@ -299,7 +300,7 @@ AgentStatus CgroupsMgr::modify_user_cgroups(const string& user_name, level_cgroups.close(); LOG(INFO) << "Append " << level_share_weight << " to " << level_resource_path; - } + } } return AgentStatus::DORIS_SUCCESS; } @@ -316,14 +317,14 @@ AgentStatus CgroupsMgr::init_cgroups() { if (fs_type.f_type != CGROUP_SUPER_MAGIC) { LOG(ERROR) << _root_cgroups_path << " is not a cgroups file system."; _is_cgroups_init_success = false; - return AgentStatus::DORIS_ERROR; + return AgentStatus::DORIS_ERROR; } #endif // Check if current user have write permission to cgroup folder if (access(_root_cgroups_path.c_str(), W_OK) != 0) { - LOG(ERROR) << "Doris does not have write permission to " + LOG(ERROR) << "Doris does not have write permission to " << _root_cgroups_path; - _is_cgroups_init_success = false; + _is_cgroups_init_success = false; return AgentStatus::DORIS_ERROR; } // If root folder exists, then delete all subfolders under it @@ -332,10 +333,10 @@ AgentStatus CgroupsMgr::init_cgroups() { for (; item_begin != item_end; item_begin++) { if (is_directory(item_begin->path().string().c_str())) { // Delete the sub folder - if (delete_user_cgroups(item_begin->path().filename().string()) + if (delete_user_cgroups(item_begin->path().filename().string()) != AgentStatus::DORIS_SUCCESS) { - LOG(ERROR) << "Could not clean subfolder " - << item_begin->path().string(); + LOG(ERROR) << "Could not clean subfolder " + << item_begin->path().string(); _is_cgroups_init_success = false; return AgentStatus::DORIS_ERROR; } @@ -346,11 +347,11 @@ AgentStatus CgroupsMgr::init_cgroups() { _is_cgroups_init_success = true; return AgentStatus::DORIS_SUCCESS; } else { - LOG(ERROR) << "Could not find a valid cgroups path for resource isolation," + LOG(ERROR) << "Could not find a valid cgroups path for resource isolation," << "current value is " << _root_cgroups_path; _is_cgroups_init_success = false; return AgentStatus::DORIS_ERROR; - } + } } #define gettid() syscall(__NR_gettid) @@ -361,7 +362,7 @@ void CgroupsMgr::apply_cgroup(const string& user_name, const string& level) { s_global_cg_mgr->assign_to_cgroups(user_name, level); } -AgentStatus CgroupsMgr::assign_to_cgroups(const string& user_name, +AgentStatus CgroupsMgr::assign_to_cgroups(const string& user_name, const string& level) { if (!_is_cgroups_init_success) { return AgentStatus::DORIS_ERROR; @@ -370,16 +371,16 @@ AgentStatus CgroupsMgr::assign_to_cgroups(const string& user_name, return assign_thread_to_cgroups(tid, user_name, level); } -AgentStatus CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id, - const string& user_name, +AgentStatus CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id, + const string& user_name, const string& level) { if (!_is_cgroups_init_success) { return AgentStatus::DORIS_ERROR; } string tasks_path = _root_cgroups_path + "/" + user_name + "/" + level + "/tasks"; if (!is_file_exist(_root_cgroups_path + "/" + user_name)) { - tasks_path = this->_root_cgroups_path + "/" - + _default_user_name + "/" + tasks_path = this->_root_cgroups_path + "/" + + _default_user_name + "/" + _default_level + "/tasks"; } else if (!is_file_exist(_root_cgroups_path + "/" + user_name + "/" + level)) { tasks_path = this->_root_cgroups_path + "/" + user_name + "/tasks"; @@ -387,7 +388,7 @@ AgentStatus CgroupsMgr::assign_thread_to_cgroups(int64_t thread_id, if (!is_file_exist(tasks_path.c_str())) { LOG(ERROR) << "Cgroups path " << tasks_path << " not exist!"; return AgentStatus::DORIS_ERROR; - } + } std::ofstream tasks(tasks_path.c_str(), std::ios::out | std::ios::app); if (!tasks.is_open()) { // This means doris could not open this file. May be it does not have access to it @@ -419,25 +420,25 @@ AgentStatus CgroupsMgr::delete_user_cgroups(const string& user_name) { if (this->drop_cgroups(user_cgroups_path) < 0) { return AgentStatus::DORIS_ERROR; } - } + } return AgentStatus::DORIS_SUCCESS; } AgentStatus CgroupsMgr::drop_cgroups(const string& deleted_cgroups_path) { // Try to delete the cgroups folder - // If failed then there maybe exist active tasks under it and try to relocate them - // Currently, try 10 times to relocate and delete the cgroups. - int32_t i = 0; - while (is_file_exist(deleted_cgroups_path) - && rmdir(deleted_cgroups_path.c_str()) < 0 - && i < this->_drop_retry_times) { + // If failed then there maybe exist active tasks under it and try to relocate them + // Currently, try 10 times to relocate and delete the cgroups. + int32_t i = 0; + while (is_file_exist(deleted_cgroups_path) + && rmdir(deleted_cgroups_path.c_str()) < 0 + && i < this->_drop_retry_times) { this->relocate_tasks(deleted_cgroups_path, this->_root_cgroups_path); ++i; #ifdef BE_TEST boost::filesystem::remove_all(deleted_cgroups_path); #endif if (i == this->_drop_retry_times){ - LOG(ERROR) << "drop cgroups under path: " << deleted_cgroups_path + LOG(ERROR) << "drop cgroups under path: " << deleted_cgroups_path << " failed."; return AgentStatus::DORIS_ERROR; } @@ -454,7 +455,7 @@ AgentStatus CgroupsMgr::relocate_tasks(const string& src_cgroups, const string& } std::ofstream dest_tasks(dest_tasks_path.c_str(), std::ios::out | std::ios::app); if (!dest_tasks) { - return AgentStatus::DORIS_ERROR; + return AgentStatus::DORIS_ERROR; } int64_t taskid; while (src_tasks >> taskid) { @@ -469,7 +470,7 @@ AgentStatus CgroupsMgr::relocate_tasks(const string& src_cgroups, const string& } void CgroupsMgr::_echo_cmd_to_cgroup(stringstream& ctrl_cmd, string& cgroups_path) { - std::ofstream cgroups_stream(cgroups_path.c_str(), + std::ofstream cgroups_stream(cgroups_path.c_str(), std::ios::out | std::ios::app); if (cgroups_stream.is_open()) { cgroups_stream << ctrl_cmd.str() << std::endl; @@ -489,7 +490,7 @@ bool CgroupsMgr::is_directory(const char* file_path) { } } -bool CgroupsMgr::is_file_exist(const char* file_path) { +bool CgroupsMgr::is_file_exist(const char* file_path) { struct stat file_stat; if (stat(file_path, &file_stat) != 0) { return false; diff --git a/be/src/olap/data_dir.cpp b/be/src/olap/data_dir.cpp index f86f21f..dba6d76 100755 --- a/be/src/olap/data_dir.cpp +++ b/be/src/olap/data_dir.cpp @@ -92,7 +92,10 @@ Status DataDir::init() { LOG(WARNING) << "fail to allocate memory. size=" << TEST_FILE_BUF_SIZE; return Status::InternalError("No memory"); } - RETURN_IF_ERROR(_check_path_exist()); + if (!check_dir_existed(_path)) { + LOG(WARNING) << "opendir failed, path=" << _path; + return Status::InternalError("opendir failed"); + } std::string align_tag_path = _path + ALIGN_TAG_PREFIX; if (access(align_tag_path.c_str(), F_OK) == 0) { LOG(WARNING) << "align tag was found, path=" << _path; @@ -109,28 +112,6 @@ Status DataDir::init() { return Status::OK(); } -Status DataDir::_check_path_exist() { - DIR* dirp = opendir(_path.c_str()); - if (dirp == nullptr) { - char buf[64]; - LOG(WARNING) << "opendir failed, path=" << _path - << ", errno=" << errno << ", errmsg=" << strerror_r(errno, buf, 64); - return Status::InternalError("opendir failed"); - } - struct dirent dirent; - struct dirent* result = nullptr; - if (readdir_r(dirp, &dirent, &result) != 0) { - char buf[64]; - LOG(WARNING) << "readdir failed, path=" << _path - << ", errno=" << errno << ", errmsg=" << strerror_r(errno, buf, 64); - closedir(dirp); - return Status::InternalError("readdir failed"); - } - // opendir and closedir should be called both or not. - closedir(dirp); - return Status::OK(); -} - Status DataDir::_init_cluster_id() { std::string cluster_id_path = _path + CLUSTER_ID_PREFIX; if (access(cluster_id_path.c_str(), F_OK) != 0) { @@ -566,7 +547,7 @@ OLAPStatus DataDir::_convert_old_tablet() { OLAPStatus status = converter.to_new_snapshot(olap_header_msg, old_data_path_prefix, old_data_path_prefix, *this, &tablet_meta_pb, &pending_rowsets, true); if (status != OLAP_SUCCESS) { - LOG(FATAL) << "convert olap header to tablet meta failed when convert header and files tablet=" + LOG(FATAL) << "convert olap header to tablet meta failed when convert header and files tablet=" << tablet_id << "." << schema_hash; return false; } @@ -577,7 +558,7 @@ OLAPStatus DataDir::_convert_old_tablet() { rowset_id.init(rowset_pb.rowset_id_v2()); status = RowsetMetaManager::save(_meta, rowset_pb.tablet_uid(), rowset_id, rowset_pb); if (status != OLAP_SUCCESS) { - LOG(FATAL) << "convert olap header to tablet meta failed when save rowset meta tablet=" + LOG(FATAL) << "convert olap header to tablet meta failed when save rowset meta tablet=" << tablet_id << "." << schema_hash; return false; } @@ -588,16 +569,16 @@ OLAPStatus DataDir::_convert_old_tablet() { tablet_meta_pb.SerializeToString(&meta_binary); status = TabletMetaManager::save(this, tablet_meta_pb.tablet_id(), tablet_meta_pb.schema_hash(), meta_binary); if (status != OLAP_SUCCESS) { - LOG(FATAL) << "convert olap header to tablet meta failed when save tablet meta tablet=" + LOG(FATAL) << "convert olap header to tablet meta failed when save tablet meta tablet=" << tablet_id << "." << schema_hash; return false; } else { - LOG(INFO) << "convert olap header to tablet meta successfully and save tablet meta to meta tablet=" + LOG(INFO) << "convert olap header to tablet meta successfully and save tablet meta to meta tablet=" << tablet_id << "." << schema_hash; } return true; }; - OLAPStatus convert_tablet_status = TabletMetaManager::traverse_headers(_meta, + OLAPStatus convert_tablet_status = TabletMetaManager::traverse_headers(_meta, convert_tablet_func, OLD_HEADER_PREFIX); if (convert_tablet_status != OLAP_SUCCESS) { LOG(FATAL) << "there is failure when convert old tablet, data dir:" << _path; @@ -609,7 +590,7 @@ OLAPStatus DataDir::_convert_old_tablet() { } OLAPStatus DataDir::remove_old_meta_and_files() { - // clean old meta(olap header message) + // clean old meta(olap header message) auto clean_old_meta_files_func = [this](int64_t tablet_id, int32_t schema_hash, const std::string& value) -> bool { // convert olap header and files @@ -625,7 +606,7 @@ OLAPStatus DataDir::remove_old_meta_and_files() { OlapSnapshotConverter converter; OLAPStatus status = converter.to_tablet_meta_pb(olap_header_msg, &tablet_meta_pb, &pending_rowsets); if (status != OLAP_SUCCESS) { - LOG(FATAL) << "convert olap header to tablet meta failed when convert header and files tablet=" + LOG(FATAL) << "convert olap header to tablet meta failed when convert header and files tablet=" << tablet_id << "." << schema_hash; return true; } @@ -672,7 +653,7 @@ OLAPStatus DataDir::remove_old_meta_and_files() { } TabletMetaManager::remove(this, tablet_id, schema_hash, OLD_HEADER_PREFIX); - LOG(INFO) << "successfully clean old tablet meta(olap header) for tablet=" + LOG(INFO) << "successfully clean old tablet meta(olap header) for tablet=" << tablet_id << "." << schema_hash << " tablet_path=" << data_path_prefix; @@ -733,7 +714,7 @@ OLAPStatus DataDir::load() { // if one rowset load failed, then the total data dir will not be loaded std::vector<RowsetMetaSharedPtr> dir_rowset_metas; LOG(INFO) << "begin loading rowset from meta"; - auto load_rowset_func = [this, &dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id, + auto load_rowset_func = [this, &dir_rowset_metas](TabletUid tablet_uid, RowsetId rowset_id, const std::string& meta_str) -> bool { RowsetMetaSharedPtr rowset_meta(new AlphaRowsetMeta()); @@ -777,7 +758,7 @@ OLAPStatus DataDir::load() { LOG(INFO) << "load rowset from meta finished, data dir: " << _path; } - // tranverse rowset + // tranverse rowset // 1. add committed rowset to txn map // 2. add visible rowset to tablet // ignore any errors when load tablet or rowset, because fe will repair them after report @@ -808,20 +789,20 @@ OLAPStatus DataDir::load() { && rowset_meta->tablet_uid() == tablet->tablet_uid()) { OLAPStatus commit_txn_status = _txn_manager->commit_txn( _meta, - rowset_meta->partition_id(), rowset_meta->txn_id(), - rowset_meta->tablet_id(), rowset_meta->tablet_schema_hash(), + rowset_meta->partition_id(), rowset_meta->txn_id(), + rowset_meta->tablet_id(), rowset_meta->tablet_schema_hash(), rowset_meta->tablet_uid(), rowset_meta->load_id(), rowset, true); if (commit_txn_status != OLAP_SUCCESS && commit_txn_status != OLAP_ERR_PUSH_TRANSACTION_ALREADY_EXIST) { LOG(WARNING) << "failed to add committed rowset: " << rowset_meta->rowset_id() - << " to tablet: " << rowset_meta->tablet_id() + << " to tablet: " << rowset_meta->tablet_id() << " for txn: " << rowset_meta->txn_id(); } else { LOG(INFO) << "successfully to add committed rowset: " << rowset_meta->rowset_id() - << " to tablet: " << rowset_meta->tablet_id() + << " to tablet: " << rowset_meta->tablet_id() << " schema hash: " << rowset_meta->tablet_schema_hash() << " for txn: " << rowset_meta->txn_id(); } - } else if (rowset_meta->rowset_state() == RowsetStatePB::VISIBLE + } else if (rowset_meta->rowset_state() == RowsetStatePB::VISIBLE && rowset_meta->tablet_uid() == tablet->tablet_uid()) { OLAPStatus publish_status = tablet->add_rowset(rowset, false); if (publish_status != OLAP_SUCCESS && publish_status != OLAP_ERR_PUSH_VERSION_ALREADY_EXIST) { @@ -833,7 +814,7 @@ OLAPStatus DataDir::load() { } } else { LOG(WARNING) << "find invalid rowset: " << rowset_meta->rowset_id() - << " with tablet id: " << rowset_meta->tablet_id() + << " with tablet id: " << rowset_meta->tablet_id() << " tablet uid: " << rowset_meta->tablet_uid() << " schema hash: " << rowset_meta->tablet_schema_hash() << " txn: " << rowset_meta->txn_id() @@ -949,7 +930,7 @@ void DataDir::perform_path_gc_by_rowsetid() { if (is_rowset_file) { TabletSharedPtr tablet = _tablet_manager->get_tablet(tablet_id, schema_hash); if (tablet != nullptr) { - if (!tablet->check_rowset_id(rowset_id) + if (!tablet->check_rowset_id(rowset_id) && !StorageEngine::instance()->check_rowset_id_in_unused_rowsets(rowset_id)) { _process_garbage_path(path); } @@ -1057,7 +1038,7 @@ Status DataDir::update_capacity() { bool DataDir::reach_capacity_limit(int64_t incoming_data_size) { double used_pct = (_disk_capacity_bytes - _available_bytes + incoming_data_size) / (double) _disk_capacity_bytes; int64_t left_bytes = _disk_capacity_bytes - _available_bytes - incoming_data_size; - + if (used_pct >= config::storage_flood_stage_usage_percent / 100.0 && left_bytes <= config::storage_flood_stage_left_capacity_bytes) { LOG(WARNING) << "reach capacity limit. used pct: " << used_pct << ", left bytes: " << left_bytes diff --git a/be/src/olap/data_dir.h b/be/src/olap/data_dir.h index 8316811..429f0de 100644 --- a/be/src/olap/data_dir.h +++ b/be/src/olap/data_dir.h @@ -92,7 +92,7 @@ public: void find_tablet_in_trash(int64_t tablet_id, std::vector<std::string>* paths); static std::string get_root_path_from_schema_hash_path_in_trash(const std::string& schema_hash_dir_in_trash); - + // load data from meta and data files OLAPStatus load(); @@ -129,7 +129,6 @@ public: private: std::string _cluster_id_path() const { return _path + CLUSTER_ID_PREFIX; } Status _init_cluster_id(); - Status _check_path_exist(); Status _init_extension_and_capacity(); Status _init_file_system(); Status _init_meta(); @@ -137,7 +136,7 @@ private: Status _check_disk(); OLAPStatus _read_and_write_test_file(); Status _read_cluster_id(const std::string& path, int32_t* cluster_id); - Status _write_cluster_id_to_path(const std::string& path, int32_t cluster_id); + Status _write_cluster_id_to_path(const std::string& path, int32_t cluster_id); OLAPStatus _clean_unfinished_converting_data(); OLAPStatus _convert_old_tablet(); diff --git a/be/src/util/disk_info.cpp b/be/src/util/disk_info.cpp index 5767de3..5628740 100644 --- a/be/src/util/disk_info.cpp +++ b/be/src/util/disk_info.cpp @@ -24,6 +24,7 @@ #include <sys/vfs.h> #include <sys/types.h> #include <sys/stat.h> +#include <sys/sysmacros.h> #include <boost/algorithm/string.hpp> #include <boost/algorithm/string/join.hpp> --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@doris.apache.org For additional commands, e-mail: commits-h...@doris.apache.org