kangpinghuang commented on a change in pull request #669: Convert old files
when start be
URL: https://github.com/apache/incubator-doris/pull/669#discussion_r261913757
##########
File path: be/src/olap/task/engine_clone_task.cpp
##########
@@ -549,9 +561,71 @@ AgentStatus EngineCloneTask::_clone_copy(
return status;
}
+OLAPStatus EngineCloneTask::_convert_to_new_snapshot(DataDir& data_dir, const
string& clone_dir, int64_t tablet_id) {
+ OLAPStatus res = OLAP_SUCCESS;
+ // check clone dir existed
+ if (!check_dir_existed(clone_dir)) {
+ res = OLAP_ERR_DIR_NOT_EXIST;
+ OLAP_LOG_WARNING("clone dir not existed when clone. [clone_dir=%s]",
+ clone_dir.c_str());
+ return res;
+ }
+
+ // load src header
+ string cloned_meta_file = clone_dir + "/" + std::to_string(tablet_id) +
".hdr";
+ FileHeader<OLAPHeaderMessage> file_header;
+ FileHandler file_handler;
+ OLAPHeaderMessage olap_header_msg;
+ if (file_handler.open(cloned_meta_file.c_str(), O_RDONLY) != OLAP_SUCCESS)
{
+ LOG(WARNING) << "fail to open ordinal file. file=" << cloned_meta_file;
+ return OLAP_ERR_IO_ERROR;
+ }
+
+ // In file_header.unserialize(), it validates file length, signature,
checksum of protobuf.
+ if (file_header.unserialize(&file_handler) != OLAP_SUCCESS) {
+ LOG(WARNING) << "fail to unserialize tablet_meta. file='" <<
cloned_meta_file;
+ return OLAP_ERR_PARSE_PROTOBUF_ERROR;
+ }
+
+ set<string> clone_files;
+ if ((res = dir_walk(clone_dir, NULL, &clone_files)) != OLAP_SUCCESS) {
+ LOG(WARNING) << "failed to dir walk when clone. [clone_dir=" <<
clone_dir << "]";
+ return res;
+ }
+
+ try {
+ olap_header_msg.CopyFrom(file_header.message());
+ } catch (...) {
+ LOG(WARNING) << "fail to copy protocol buffer object. file='" <<
cloned_meta_file;
+ return OLAP_ERR_PARSE_PROTOBUF_ERROR;
+ }
+ OlapSnapshotConverter converter;
+ TabletMetaPB tablet_meta_pb;
+ vector<RowsetMetaPB> pending_rowsets;
+ res = converter.to_new_snapshot(olap_header_msg, clone_dir,
&tablet_meta_pb, clone_dir, data_dir, &pending_rowsets);
+ if (res != OLAP_SUCCESS) {
+ LOG(WARNING) << "fail to convert snapshot to new format. dir='" <<
clone_dir;
+ return res;
+ }
+ vector<string> files_to_delete;
+ for (auto& file_name : clone_files) {
+ files_to_delete.push_back(file_name);
+ }
Review comment:
try files_to_delete.push_back(flles_to_delete.end(), clone_files.begin(),
clone_files.end());
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
