morningman commented on a change in pull request #5143:
URL: https://github.com/apache/incubator-doris/pull/5143#discussion_r548980516
##########
File path: be/src/olap/data_dir.cpp
##########
@@ -1018,4 +1018,163 @@ void DataDir::disks_compaction_score_increment(int64_t
delta) {
void DataDir::disks_compaction_num_increment(int64_t delta) {
disks_compaction_num->increment(delta);
}
+
+void DataDir::init_compaction_heap() {
+ for (std::set<TabletInfo>::iterator it = _tablet_set.begin(); it !=
_tablet_set.end(); it++) {
+ TabletSharedPtr tablet =
StorageEngine::instance()->tablet_manager()->get_tablet(
+ it->tablet_id, it->schema_hash);
+ if (tablet != nullptr) {
+ push_tablet_into_compaction_heap(CompactionType::BASE_COMPACTION,
tablet);
+
push_tablet_into_compaction_heap(CompactionType::CUMULATIVE_COMPACTION, tablet);
+ }
+ }
+}
+
+void DataDir::push_tablet_into_compaction_heap(CompactionType compaction_type,
+ TabletSharedPtr tablet) {
+ OlapStopWatch watch;
+ if (compaction_type == CompactionType::BASE_COMPACTION) {
+ std::unique_lock<std::mutex> lock(_base_compaction_heap_mutex);
+ std::vector<TabletSharedPtr>::iterator it =
+ find(_base_compaction_heap.begin(),
_base_compaction_heap.end(), tablet);
+ if (it == _base_compaction_heap.end()) {
+ _base_compaction_heap.push_back(tablet);
+ }
+ std::make_heap(_base_compaction_heap.begin(),
_base_compaction_heap.end(),
+ TabletScoreComparator(CompactionType::BASE_COMPACTION));
Review comment:
why not just using a priority queue?
##########
File path: be/src/exec/olap_scanner.cpp
##########
@@ -538,6 +538,14 @@ Status OlapScanner::close(RuntimeState* state) {
_reader.reset();
Expr::close(_conjunct_ctxs, state);
_is_closed = true;
+ if (config::scan_count_push_tablet_into_compaction_heap != 0 &&
+ _tablet->query_scan_count->value() %
config::scan_count_push_tablet_into_compaction_heap ==
+ 0) {
+
_tablet->data_dir()->push_tablet_into_compaction_heap(CompactionType::BASE_COMPACTION,
Review comment:
Have you test the performance of `push_tablet_into_compaction_heap`.
This is on the critical path of READ, and the default
`scan_count_push_tablet_into_compaction_heap` is 1,
so the tablet will call this method every time it being read.
If the query concurrency is high, i doubt it may impact the performance.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
