chaoyli commented on a change in pull request #596: Add implementation to tablet
URL: https://github.com/apache/incubator-doris/pull/596#discussion_r252130144
##########
File path: be/src/olap/tablet.cpp
##########
@@ -638,45 +744,213 @@ OLAPStatus Tablet::register_tablet_into_dir() {
return _data_dir->register_tablet(this);
}
-OLAPStatus Tablet::clear_schema_change_info(
- AlterTabletType* type,
- bool only_one,
- bool check_only) {
- if (!check_only) {
- WriteLock w(&_meta_lock);
- _unprotect_clear_schema_change_info(type, only_one, check_only);
- } else {
- ReadLock r(&_meta_lock);
- _unprotect_clear_schema_change_info(type, only_one, check_only);
+bool Tablet::version_for_delete_predicate(const Version& version) {
+ return _tablet_meta->version_for_delete_predicate(version);
+}
+
+bool Tablet::version_for_load_deletion(const Version& version) {
+ RowsetSharedPtr rowset = _rs_version_map.at(version);
+ return rowset->delete_flag();
+}
+
+const RowsetSharedPtr Tablet::get_rowset_by_version(const Version& version)
const {
+ RowsetSharedPtr rowset = _rs_version_map.at(version);
+ return rowset;
+}
+
+const RowsetSharedPtr Tablet::rowset_with_max_version() const {
+ Version max_version = _tablet_meta->max_version();
+ RowsetSharedPtr rowset = _rs_version_map.at(max_version);
+ return rowset;
+}
+
+std::string Tablet::storage_root_path_name() const {
+ return _data_dir->path();
+}
+
+std::string Tablet::tablet_path() const {
+ return _tablet_path;
+}
+
+const uint32_t Tablet::calc_cumulative_compaction_score() const {
+ uint32_t score = 0;
+ bool base_rowset_exist = false;
+ const int32_t point = cumulative_layer_point();
+ for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ if (rs_meta->start_version() >= point) {
+ score++;
+ }
+ if (rs_meta->start_version() == 0) {
+ base_rowset_exist = true;
+ }
}
- return OLAP_SUCCESS;
+
+ // base不存在可能是tablet正在做alter table，先不选它，设score=0
+ return base_rowset_exist ? score : 0;
}
-OLAPStatus Tablet::_unprotect_clear_schema_change_info(
- AlterTabletType* type,
- bool only_one,
- bool check_only) {
- OLAPStatus res = OLAP_SUCCESS;
+const uint32_t Tablet::calc_base_compaction_score() const {
+ uint32_t score = 0;
+ const int32_t point = cumulative_layer_point();
+ bool base_rowset_exist = false;
+ for (auto& rs_meta : _tablet_meta->all_rs_metas()) {
+ if (rs_meta->start_version() < point) {
+ score++;
+ }
+ if (rs_meta->start_version() == 0) {
+ base_rowset_exist = true;
+ }
+ }
+ score = score < config::base_compaction_num_cumulative_deltas ? 0 : score;
- vector<Version> versions_to_be_changed;
- if (this->get_schema_change_request(NULL,
- NULL,
- &versions_to_be_changed,
- NULL)) {
- if (versions_to_be_changed.size() != 0) {
- OLAP_LOG_WARNING("schema change is not allowed now, "
- "until previous schema change is done.
[tablet='%s']",
- full_name().c_str());
- return OLAP_ERR_PREVIOUS_SCHEMA_CHANGE_NOT_FINISHED;
+ // base不存在可能是tablet正在做alter table，先不选它，设score=0
+ return base_rowset_exist ? score : 0;
+}
+
+bool Tablet::has_expired_incremental_rowset() {
+ bool exist = false;
+ time_t now = time(NULL);
+ ReadLock rdlock(&_meta_lock);
+ for (auto& rs_meta : _tablet_meta->all_inc_rs_metas()) {
+ double diff = difftime(now, rs_meta->creation_time());
+ if (diff >= config::inc_rowset_expired_sec) {
+ exist = true;
+ break;
}
}
+ return exist;
+}
- if (!check_only) {
- VLOG(3) << "broke old schema change chain";
- clear_schema_change_request();
+void Tablet::delete_expired_incremental_rowset() {
+ time_t now = time(NULL);
+ std::vector<std::pair<Version, VersionHash>> expired_versions;
+ std::vector<string> files_to_remove;
+ WriteLock wrlock(&_meta_lock);
+ for (auto& rs_meta : _tablet_meta->all_inc_rs_metas()) {
+ double diff = difftime(now, rs_meta->creation_time());
+ if (diff >= config::inc_rowset_expired_sec) {
+ Version version(rs_meta->start_version(), rs_meta->end_version());
+ expired_versions.push_back(std::make_pair(version,
rs_meta->version_hash()));
+ VLOG(3) << "find expire incremental rowset. tablet=" <<
full_name() << ", "
+ << "version=" << rs_meta->start_version() << "-" <<
rs_meta->end_version() << ", "
+ << "exist_sec=" << diff;
+ }
}
+ if (expired_versions.empty()) { return; }
+
+ for (auto& pair: expired_versions) {
+ _delete_incremental_rowset(pair.first, pair.second, &files_to_remove);
+ VLOG(3) << "delete expire incremental data. table=" << full_name() <<
", "
+ << "version=" << pair.first.first << "-" << pair.first.second;
+ }
+
+ if (save_tablet_meta() != OLAP_SUCCESS) {
+ LOG(FATAL) << "fail to save tablet_meta when delete expire incremental
data."
+ << "tablet=" << full_name();
+ }
+ remove_files(files_to_remove);
+}
+
+OLAPStatus Tablet::clone_data(const TabletMeta& tablet_meta,
+ const std::vector<RowsetMetaSharedPtr>&
rowsets_to_clone,
+ const std::vector<Version>& versions_to_delete) {
+ LOG(INFO) << "begin to clone data to tablet. tablet=" << full_name()
+ << ", rowsets_to_clone=" << rowsets_to_clone.size()
+ << ", versions_to_delete_size=" << versions_to_delete.size();
+ OLAPStatus res = OLAP_SUCCESS;
+ do {
+ // load new local tablet_meta to operate on
+ TabletMeta new_tablet_meta;
+ TabletMetaManager::get_header(_data_dir, tablet_id(), schema_hash(),
&new_tablet_meta);
+
+ // delete versions from new local tablet_meta
+ for (const Version& version : versions_to_delete) {
+ res = new_tablet_meta.delete_rowset_by_version(version);
+ if (res != OLAP_SUCCESS) {
+ LOG(WARNING) << "failed to delete version from new local
tablet meta. tablet=" << full_name()
+ << ", version=" << version.first << "-" <<
version.second;
+ break;
+ }
+ if (new_tablet_meta.version_for_delete_predicate(version)) {
+ new_tablet_meta.remove_delete_predicate_by_version(version);
+ }
+ LOG(INFO) << "delete version from new local tablet_meta when
clone. [table='" << full_name()
+ << "', version=" << version.first << "-" <<
version.second << "]";
+ }
+
+ if (res != OLAP_SUCCESS) {
+ break;
+ }
+
+ for (auto& rs_meta : rowsets_to_clone) {
+ Version version(rs_meta->start_version(), rs_meta->end_version());
+ new_tablet_meta.add_rs_meta(rs_meta);
+
+ // add delete conditions to new local tablet_meta, if it exists in
tablet_meta
+ if (version.first == version.second) {
+ for (auto it = tablet_meta.delete_predicates().begin();
+ it != tablet_meta.delete_predicates().end(); ++it) {
+ if (it->version() == version.first) {
+ // add it
+ new_tablet_meta.add_delete_predicate(*it,
version.first);
+ LOG(INFO) << "add delete condition when clone.
[table=" << full_name()
+ << " version=" << it->version() << "]";
+ break;
+ }
+ }
+ }
+ }
+
+ if (res != OLAP_SUCCESS) {
+ break;
+ }
+
+ VLOG(3) << "load indices successfully when clone. tablet=" <<
full_name()
Review comment:
OK
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
