yiguolei commented on a change in pull request #669: Convert old files when
start be
URL: https://github.com/apache/incubator-doris/pull/669#discussion_r261940066
##########
File path: be/src/olap/storage_engine.cpp
##########
@@ -117,8 +119,179 @@ StorageEngine::~StorageEngine() {
clear();
}
+// convert old tablet and its files to new tablet meta and rowset format
+OLAPStatus StorageEngine::_convert_old_tablet(DataDir* data_dir) {
+ auto convert_tablet_func = [this, data_dir](long tablet_id,
+ long schema_hash, const std::string& value) -> bool {
+ OlapSnapshotConverter converter;
+ // convert olap header and files
+ OLAPHeaderMessage olap_header_msg;
+ TabletMetaPB tablet_meta_pb;
+ vector<RowsetMetaPB> pending_rowsets;
+ bool parsed = olap_header_msg.ParseFromString(value);
+ if (!parsed) {
+ LOG(WARNING) << "convert olap header to tablet meta failed when
load olap header tablet="
+ << tablet_id << "." << schema_hash;
+ return false;
+ }
+ string old_data_path_prefix =
data_dir->get_absolute_tablet_path(olap_header_msg, true);
+ OLAPStatus status = converter.to_new_snapshot(olap_header_msg,
old_data_path_prefix,
+ &tablet_meta_pb, old_data_path_prefix, *data_dir,
&pending_rowsets);
+ if (status != OLAP_SUCCESS) {
+ LOG(WARNING) << "convert olap header to tablet meta failed when
convert header and files tablet="
+ << tablet_id << "." << schema_hash;
+ return false;
+ }
+
+ // write pending rowset to olap meta
+ for (auto& rowset_pb : pending_rowsets) {
+ string meta_binary;
+ rowset_pb.SerializeToString(&meta_binary);
+ status = RowsetMetaManager::save(data_dir->get_meta(),
rowset_pb.rowset_id() , meta_binary);
+ if (status != OLAP_SUCCESS) {
+ LOG(WARNING) << "convert olap header to tablet meta failed
when save rowset meta tablet="
+ << tablet_id << "." << schema_hash;
+ return false;
+ }
+ }
+
+ // write converted tablet meta to olap meta
+ string meta_binary;
+ tablet_meta_pb.SerializeToString(&meta_binary);
+ status = TabletMetaManager::save(data_dir, tablet_meta_pb.tablet_id(),
tablet_meta_pb.schema_hash(), meta_binary);
+ if (status != OLAP_SUCCESS) {
+ LOG(WARNING) << "convert olap header to tablet meta failed when
save tablet meta tablet="
+ << tablet_id << "." << schema_hash;
+ return false;
+ }
+ return true;
+ };
+ OLAPStatus convert_tablet_status =
TabletMetaManager::traverse_headers(data_dir->get_meta(), convert_tablet_func,
OLD_HEADER_PREFIX);
+ if (convert_tablet_status != OLAP_SUCCESS) {
+ LOG(WARNING) << "there is failure when convert old tablet, data dir:"
<< data_dir->path();
+ return convert_tablet_status;
+ } else {
+ LOG(INFO) << "successfully convert old tablet, data dir: " <<
data_dir->path();
+ }
+ return OLAP_SUCCESS;
+}
+
+OLAPStatus StorageEngine::_clean_unfinished_converting_data(DataDir* data_dir)
{
+ auto clean_unifinished_tablet_meta_func = [this, data_dir](long tablet_id,
+ long schema_hash, const std::string& value) -> bool {
+ TabletMetaManager::remove(data_dir, tablet_id, schema_hash,
HEADER_PREFIX);
+ LOG(INFO) << "successfully clean temp tablet meta for tablet="
+ << tablet_id << "." << schema_hash
+ << "from data dir: " << data_dir->path();
+ return true;
+ };
+ OLAPStatus clean_unfinished_meta_status =
TabletMetaManager::traverse_headers(data_dir->get_meta(),
+ clean_unifinished_tablet_meta_func, HEADER_PREFIX);
+ if (clean_unfinished_meta_status != OLAP_SUCCESS) {
+ // If failed to clean meta just skip the error, there will be useless
metas in rocksdb column family
+ LOG(WARNING) << "there is failure when clean temp tablet meta from
data dir:" << data_dir->path();
+ } else {
+ LOG(INFO) << "successfully clean temp tablet meta from data dir: " <<
data_dir->path();
+ }
+ auto clean_unifinished_rowset_meta_func = [this, data_dir](RowsetId
rowset_id, const std::string& value) -> bool {
+ RowsetMetaManager::remove(data_dir->get_meta(), rowset_id);
+ LOG(INFO) << "successfully clean temp rowset meta for rowset id ="
+ << rowset_id << " from data dir: " << data_dir->path();
+ return true;
+ };
+ OLAPStatus clean_unfinished_rowset_meta_status =
RowsetMetaManager::traverse_rowset_metas(data_dir->get_meta(),
+ clean_unifinished_rowset_meta_func);
+ if (clean_unfinished_rowset_meta_status != OLAP_SUCCESS) {
+ // If failed to clean meta just skip the error, there will be useless
metas in rocksdb column family
+ LOG(WARNING) << "there is failure when clean temp rowset meta from
data dir:" << data_dir->path();
+ } else {
+ LOG(INFO) << "successfully clean temp rowset meta from data dir: " <<
data_dir->path();
+ }
+ return OLAP_SUCCESS;
+}
+
+OLAPStatus StorageEngine::_remove_old_meta_and_files(DataDir* data_dir) {
+ // clean old meta(olap header message)
+ auto clean_old_meta_func = [this, data_dir](long tablet_id,
+ long schema_hash, const std::string& value) -> bool {
+ TabletMetaManager::remove(data_dir, tablet_id, schema_hash,
OLD_HEADER_PREFIX);
+ LOG(INFO) << "successfully clean old tablet meta(olap header) for
tablet="
+ << tablet_id << "." << schema_hash
+ << "from data dir: " << data_dir->path();
+ return true;
+ };
+ OLAPStatus clean_old_meta_status =
TabletMetaManager::traverse_headers(data_dir->get_meta(),
+ clean_old_meta_func, OLD_HEADER_PREFIX);
+ if (clean_old_meta_status != OLAP_SUCCESS) {
+ // If failed to clean meta just skip the error, there will be useless
metas in rocksdb column family
+ LOG(WARNING) << "there is failure when clean old tablet meta(olap
header) from data dir:" << data_dir->path();
+ } else {
+ LOG(INFO) << "successfully clean old tablet meta(olap header) from
data dir: " << data_dir->path();
+ }
+
+ // clean old files because they have hard links in new file name format
+ auto clean_old_files_func = [this, data_dir](long tablet_id,
+ long schema_hash, const std::string& value) -> bool {
+ TabletMetaPB tablet_meta_pb;
+ bool parsed = tablet_meta_pb.ParseFromString(value);
+ if (!parsed) {
+ // if errors when load, just skip it
+ LOG(WARNING) << "failed to load tablet meta from meta store to
tablet=" << tablet_id << "." << schema_hash;
+ return true;
+ }
+
+ TabletSchema tablet_schema;
+ tablet_schema.init_from_pb(tablet_meta_pb.schema());
+ string data_path_prefix =
data_dir->get_absolute_tablet_path(&tablet_meta_pb, true);
+
+ // convert visible pdelta file to rowsets and remove old files
+ for (auto& visible_rowset : tablet_meta_pb.rs_metas()) {
+ RowsetMetaSharedPtr alpha_rowset_meta(new AlphaRowsetMeta());
+ alpha_rowset_meta->init_from_pb(visible_rowset);
+ AlphaRowset rowset(&tablet_schema, data_path_prefix, data_dir,
alpha_rowset_meta);
+ std::vector<std::string> old_files;
+ rowset.remove_old_files(&old_files);
+ }
+
+ // convert inc delta file to rowsets and remove old files
+ for (auto& inc_rowset : tablet_meta_pb.inc_rs_metas()) {
+ RowsetMetaSharedPtr alpha_rowset_meta(new AlphaRowsetMeta());
+ alpha_rowset_meta->init_from_pb(inc_rowset);
+ AlphaRowset rowset(&tablet_schema, data_path_prefix, data_dir,
alpha_rowset_meta);
+ std::vector<std::string> old_files;
+ rowset.remove_old_files(&old_files);
+ }
+ return true;
+ };
+ OLAPStatus clean_old_tablet_status =
TabletMetaManager::traverse_headers(data_dir->get_meta(), clean_old_files_func,
HEADER_PREFIX);
Review comment:
done
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
