dataroaring commented on code in PR #57072:
URL: https://github.com/apache/doris/pull/57072#discussion_r2453729648
##########
be/src/io/cache/fs_file_cache_storage.cpp:
##########
@@ -424,122 +420,30 @@ Status
FSFileCacheStorage::collect_directory_entries(const std::filesystem::path
}
Status FSFileCacheStorage::upgrade_cache_dir_if_necessary() const {
- /*
- * If use version2 but was version 1, do upgrade:
- *
- * Action I:
- * version 1.0: cache_base_path / key / offset
- * version 2.0: cache_base_path / key_prefix / key / offset
- *
- * Action II:
- * add '_0' to hash dir
- *
- * Note: This is a sync operation with tons of IOs, so it may affect BE
- * boot time heavily. Fortunately, Action I & II will only happen when
- * upgrading (once in the cluster life time).
- */
-
std::string version;
- std::error_code ec;
int rename_count = 0;
int failure_count = 0;
auto start_time = std::chrono::steady_clock::now();
RETURN_IF_ERROR(read_file_cache_version(&version));
- LOG(INFO) << "Checking cache version upgrade. Current version: " << version
- << ", target version: 2.0, need upgrade: "
- << (USE_CACHE_VERSION2 && version != "2.0");
- if (USE_CACHE_VERSION2 && version != "2.0") {
- // move directories format as version 2.0
- std::vector<std::string> file_list;
- file_list.reserve(10000);
- RETURN_IF_ERROR(collect_directory_entries(_cache_base_path,
file_list));
-
- // this directory_iterator should be a problem in concurrent access
- for (const auto& file_path : file_list) {
- try {
- if (std::filesystem::is_directory(file_path)) {
- std::string cache_key =
std::filesystem::path(file_path).filename().native();
- if (cache_key.size() > KEY_PREFIX_LENGTH) {
- if (cache_key.find('_') == std::string::npos) {
- cache_key += "_0";
- }
- std::string key_prefix =
- Path(_cache_base_path) / cache_key.substr(0,
KEY_PREFIX_LENGTH);
- bool exists = false;
- auto exists_status = fs->exists(key_prefix, &exists);
- if (!exists_status.ok()) {
- LOG(WARNING) << "Failed to check directory
existence: " << key_prefix
- << ", error: " <<
exists_status.to_string();
- ++failure_count;
- continue;
- }
- if (!exists) {
- auto create_status =
fs->create_directory(key_prefix);
- if (!create_status.ok() &&
- create_status.code() !=
TStatusCode::type::ALREADY_EXIST) {
- LOG(WARNING) << "Failed to create directory: "
<< key_prefix
- << ", error: " <<
create_status.to_string();
- ++failure_count;
- continue;
- }
- }
- auto rename_status = Status::OK();
- const std::string new_file_path = key_prefix + "/" +
cache_key;
- TEST_SYNC_POINT_CALLBACK(
-
"FSFileCacheStorage::upgrade_cache_dir_if_necessary_rename",
- &file_path, &new_file_path);
- rename_status = fs->rename(file_path, new_file_path);
- if (rename_status.ok() ||
- rename_status.code() ==
TStatusCode::type::DIRECTORY_NOT_EMPTY) {
- ++rename_count;
- } else {
- LOG(WARNING)
- << "Failed to rename directory from " <<
file_path << " to "
- << new_file_path << ", error: " <<
rename_status.to_string();
- ++failure_count;
- continue;
- }
- }
- }
- } catch (const std::exception& e) {
- LOG(WARNING) << "Error occurred while upgrading file cache
directory: " << file_path
- << " err: " << e.what();
- ++failure_count;
- }
- }
-
- std::vector<std::string> rebuilt_file_list;
- rebuilt_file_list.reserve(10000);
- RETURN_IF_ERROR(collect_directory_entries(_cache_base_path,
rebuilt_file_list));
-
- for (const auto& key_it : rebuilt_file_list) {
- if (!std::filesystem::is_directory(key_it)) {
- // maybe version hits file
- continue;
- }
- try {
- if (Path(key_it).filename().native().size() !=
KEY_PREFIX_LENGTH) {
- LOG(WARNING) << "Unknown directory " << key_it << ", try
to remove it";
- auto delete_status = fs->delete_directory(key_it);
- if (!delete_status.ok()) {
- LOG(WARNING) << "Failed to delete unknown directory: "
<< key_it
- << ", error: " <<
delete_status.to_string();
- ++failure_count;
- continue;
- }
- }
- } catch (const std::exception& e) {
- LOG(WARNING) << "Error occurred while upgrading file cache
directory: " << key_it
- << " err: " << e.what();
- ++failure_count;
- }
- }
- if (auto st = write_file_cache_version(); !st.ok()) {
- return Status::InternalError("Failed to write version hints for
file cache, err={}",
- st.to_string());
- }
+ if (version == "1.0") {
+ LOG(ERROR) << "Cache version upgrade issue: Cannot upgrade directly
from 1.0 to 3.0.Please "
+ "upgrade to 2.0 first,or clear the file cache directory
to start anew "
+ "(LOSING ALL THE CACHE).";
Review Comment:
Please suggest doris version.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
