[
https://issues.apache.org/jira/browse/KUDU-2638?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16724859#comment-16724859
]
jiaqiyang commented on KUDU-2638:
---------------------------------
in this code :
{code:java}
// code placeholder
MaintenanceOp* MaintenanceManager::FindBestOp() {
TRACE_EVENT0("maintenance", "MaintenanceManager::FindBestOp");
size_t free_threads = num_threads_ - running_ops_;
if (free_threads == 0) {
VLOG_AND_TRACE("maintenance", 1) << LogPrefix()
<< "There are no free threads, so we can't
run anything.";
return nullptr;
}
int64_t low_io_most_logs_retained_bytes = 0;
MaintenanceOp* low_io_most_logs_retained_bytes_op = nullptr;
uint64_t most_mem_anchored = 0;
MaintenanceOp* most_mem_anchored_op = nullptr;
int64_t most_logs_retained_bytes = 0;
int64_t most_logs_retained_bytes_ram_anchored = 0;
MaintenanceOp* most_logs_retained_bytes_op = nullptr;
int64_t most_data_retained_bytes = 0;
MaintenanceOp* most_data_retained_bytes_op = nullptr;
double best_perf_improvement = 0;
MaintenanceOp* best_perf_improvement_op = nullptr;
for (OpMapTy::value_type &val : ops_) {
MaintenanceOp* op(val.first);
MaintenanceOpStats& stats(val.second);
VLOG_WITH_PREFIX(3) << "Considering MM op " << op->name();
// Update op stats.
stats.Clear();
op->UpdateStats(&stats);
if (op->cancelled() || !stats.valid() || !stats.runnable()) {
continue;
}
if (stats.logs_retained_bytes() > low_io_most_logs_retained_bytes &&
op->io_usage() == MaintenanceOp::LOW_IO_USAGE) {
low_io_most_logs_retained_bytes_op = op;
low_io_most_logs_retained_bytes = stats.logs_retained_bytes();
VLOG_AND_TRACE("maintenance", 2) << LogPrefix() << "Op " << op->name() <<
" can free "
<< stats.logs_retained_bytes() << "
bytes of logs";
}
if (stats.ram_anchored() > most_mem_anchored) {
most_mem_anchored_op = op;
most_mem_anchored = stats.ram_anchored();
}
// We prioritize ops that can free more logs, but when it's the same we
pick the one that
// also frees up the most memory.
if (stats.logs_retained_bytes() > 0 &&
(stats.logs_retained_bytes() > most_logs_retained_bytes ||
(stats.logs_retained_bytes() == most_logs_retained_bytes &&
stats.ram_anchored() > most_logs_retained_bytes_ram_anchored)))
{
most_logs_retained_bytes_op = op;
most_logs_retained_bytes = stats.logs_retained_bytes();
most_logs_retained_bytes_ram_anchored = stats.ram_anchored();
}
if (stats.data_retained_bytes() > most_data_retained_bytes) {
most_data_retained_bytes_op = op;
most_data_retained_bytes = stats.data_retained_bytes();
VLOG_AND_TRACE("maintenance", 2) << LogPrefix() << "Op " << op->name() <<
" can free "
<< stats.data_retained_bytes() << "
bytes of data";
}
if ((!best_perf_improvement_op) ||
(stats.perf_improvement() > best_perf_improvement)) {
best_perf_improvement_op = op;
best_perf_improvement = stats.perf_improvement();
}
}
// Look at ops that we can run quickly that free up log retention.
if (low_io_most_logs_retained_bytes_op) {
if (low_io_most_logs_retained_bytes > 0) {
VLOG_AND_TRACE("maintenance", 1) << LogPrefix()
<< "Performing " <<
low_io_most_logs_retained_bytes_op->name() << ", "
<< "because it can free up more logs "
<< "at " << low_io_most_logs_retained_bytes
<< " bytes with a low IO cost";
return low_io_most_logs_retained_bytes_op;
}
}
// Look at free memory. If it is dangerously low, we must select something
// that frees memory-- the op with the most anchored memory.
double capacity_pct;
if (memory_pressure_func_(&capacity_pct)) {
if (!most_mem_anchored_op) {
std::string msg = StringPrintf("we have exceeded our soft memory limit "
"(current capacity is %.2f%%). However, there are no ops currently "
"runnable which would free memory.", capacity_pct);
LOG_WITH_PREFIX(INFO) << msg;
return nullptr;
}
VLOG_AND_TRACE("maintenance", 1) << LogPrefix() << "We have exceeded our
soft memory limit "
<< "(current capacity is " << capacity_pct << "%). Running the op "
<< "which anchors the most memory: " <<
most_mem_anchored_op->name();
return most_mem_anchored_op;
}
if (most_logs_retained_bytes_op &&
most_logs_retained_bytes / 1024 / 1024 >=
FLAGS_log_target_replay_size_mb) {
VLOG_AND_TRACE("maintenance", 1) << LogPrefix()
<< "Performing " << most_logs_retained_bytes_op->name() << ", "
<< "because it can free up more logs (" << most_logs_retained_bytes
<< " bytes)";
return most_logs_retained_bytes_op;
}
// Look at ops that we can run quickly that free up data on disk.
if (most_data_retained_bytes_op &&
most_data_retained_bytes > FLAGS_data_gc_min_size_mb * 1024 * 1024) {
if (!best_perf_improvement_op || best_perf_improvement <= 0 ||
rand_.NextDoubleFraction() <= FLAGS_data_gc_prioritization_prob) {
VLOG_AND_TRACE("maintenance", 1) << LogPrefix()
<< "Performing " << most_data_retained_bytes_op->name() <<
", "
<< "because it can free up more data "
<< "at " << most_data_retained_bytes << " bytes";
return most_data_retained_bytes_op;
}
VLOG(1) << "Skipping data GC due to prioritizing perf improvement";
}
if (best_perf_improvement_op && best_perf_improvement > 0) {
VLOG_AND_TRACE("maintenance", 1) << LogPrefix() << "Performing "
<< best_perf_improvement_op->name() << ", "
<< "because it had the best perf_improvement score, "
<< "at " << best_perf_improvement;
return best_perf_improvement_op;
}
return nullptr;
}
{code}
when restart kudu cluster should not FindBestOp is most_data_retained_bytes_op
> kudu cluster restart very long time to reused
> ---------------------------------------------
>
> Key: KUDU-2638
> URL: https://issues.apache.org/jira/browse/KUDU-2638
> Project: Kudu
> Issue Type: Improvement
> Reporter: jiaqiyang
> Priority: Major
> Fix For: n/a
>
> Attachments: kudu16.tc.tablet.png, tserverLog.tar.gz
>
>
> when restart my kudu cluster ;all tablet not avalible:
> run kudu cluster ksck show that:
> Table Summary
>
>
> Name | Status | Total Tablets | Healthy | Under-replicated | Unavailable
> --------------------------------------------------------------------------------+------------
> t1 | HEALTHY | 1 | 1 | 0 | 0
> t2 | UNAVAILABLE | 5 | 0 | 1 | 4
> t3 | UNAVAILABLE | 6 | 2 | 0 | 4
> t3 | UNAVAILABLE | 3 | 0 | 0 | 3
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
