neverchanje commented on a change in pull request #592:
URL: https://github.com/apache/incubator-pegasus/pull/592#discussion_r487676315
##########
File path: src/server/hotspot_partition_calculator.cpp
##########
@@ -34,67 +35,84 @@ DSN_DEFINE_int64("pegasus.collector",
"eliminate outdated historical "
"data");
-void hotspot_partition_calculator::data_aggregate(const std::vector<row_data>
&partitions)
+void hotspot_partition_calculator::data_aggregate(const std::vector<row_data>
&partition_stats)
{
- while (_partition_stat_histories.size() > FLAGS_max_hotspot_store_size -
1) {
- _partition_stat_histories.pop();
+ while (_partitions_stat_histories.size() >= FLAGS_max_hotspot_store_size) {
+ _partitions_stat_histories.pop_front();
}
- std::vector<hotspot_partition_data> temp(partitions.size());
- // TODO refactor the data structure
- for (int i = 0; i < partitions.size(); i++) {
- temp[i] = std::move(hotspot_partition_data(partitions[i]));
+ std::vector<hotspot_partition_stat> temp;
+ for (const auto &partition_stat : partition_stats) {
+ temp.emplace_back(hotspot_partition_stat(partition_stat));
}
- _partition_stat_histories.emplace(temp);
+ _partitions_stat_histories.emplace_back(temp);
}
void hotspot_partition_calculator::init_perf_counter(int partition_count)
{
- std::string counter_name;
- std::string counter_desc;
+ std::string read_counter_name, write_counter_name;
+ std::string read_counter_desc, write_counter_desc;
for (int i = 0; i < partition_count; i++) {
- string partition_desc = _app_name + '.' + std::to_string(i);
- counter_name = fmt::format("app.stat.hotspots@{}", partition_desc);
- counter_desc = fmt::format("statistic the hotspots of app {}",
partition_desc);
- _hot_points[i].init_app_counter(
- "app.pegasus", counter_name.c_str(), COUNTER_TYPE_NUMBER,
counter_desc.c_str());
+ string read_partition_desc = _app_name + '.' + "read." +
std::to_string(i);
+ read_counter_name = fmt::format("app.stat.hotspots@{}",
read_partition_desc);
+ read_counter_desc = fmt::format("statistic the hotspots of app {}",
read_partition_desc);
+
_hot_points[i].emplace_back(std::make_unique<dsn::perf_counter_wrapper>());
+ _hot_points[i][READ_HOTSPOT_DATA]->init_app_counter("app.pegasus",
+
read_counter_name.c_str(),
+
COUNTER_TYPE_NUMBER,
+
read_counter_desc.c_str());
+ string write_partition_desc = _app_name + '.' + "write." +
std::to_string(i);
+ write_counter_name = fmt::format("app.stat.hotspots@{}",
write_partition_desc);
+ write_counter_desc = fmt::format("statistic the hotspots of app {}",
write_partition_desc);
+
_hot_points[i].emplace_back(std::make_unique<dsn::perf_counter_wrapper>());
+ _hot_points[i][WRITE_HOTSPOT_DATA]->init_app_counter("app.pegasus",
+
write_counter_name.c_str(),
+
COUNTER_TYPE_NUMBER,
+
write_counter_desc.c_str());
}
}
void hotspot_partition_calculator::data_analyse()
{
- dassert(_partition_stat_histories.back().size() == _hot_points.size(),
- "partition counts error, please check");
- std::vector<double> data_samples;
- data_samples.reserve(_partition_stat_histories.size() *
_hot_points.size());
- auto temp_data = _partition_stat_histories;
- double table_qps_sum = 0, standard_deviation = 0, table_qps_avg = 0;
- int sample_count = 0;
- while (!temp_data.empty()) {
- for (const auto &partition_data : temp_data.front()) {
- if (partition_data.total_qps - 1.00 > 0) {
- data_samples.push_back(partition_data.total_qps);
- table_qps_sum += partition_data.total_qps;
- sample_count++;
+ dcheck_eq(_partitions_stat_histories.back().size(), _hot_points.size());
+ for (int data_type = 0; data_type <= 1; data_type++) {
+ // 0: READ_HOTSPOT_DATA; 1: WRITE_HOTSPOT_DATA
+ double table_qps_sum = 0, standard_deviation = 0, table_qps_avg = 0;
+ int sample_count = 0;
+ for (const auto &one_partition_stat_histories :
_partitions_stat_histories) {
+ for (const auto &partition_stat : one_partition_stat_histories) {
+ if (partition_stat.total_qps[data_type] > 1.00) {
Review comment:
I still have no idea in what condition `total_qps[data_type]==0` has
problem? Why skip 0-qps partition? Could you show one example?
##########
File path: src/server/hotspot_partition_calculator.cpp
##########
@@ -34,67 +35,84 @@ DSN_DEFINE_int64("pegasus.collector",
"eliminate outdated historical "
"data");
-void hotspot_partition_calculator::data_aggregate(const std::vector<row_data>
&partitions)
+void hotspot_partition_calculator::data_aggregate(const std::vector<row_data>
&partition_stats)
{
- while (_partition_stat_histories.size() > FLAGS_max_hotspot_store_size -
1) {
- _partition_stat_histories.pop();
+ while (_partitions_stat_histories.size() >= FLAGS_max_hotspot_store_size) {
+ _partitions_stat_histories.pop_front();
}
- std::vector<hotspot_partition_data> temp(partitions.size());
- // TODO refactor the data structure
- for (int i = 0; i < partitions.size(); i++) {
- temp[i] = std::move(hotspot_partition_data(partitions[i]));
+ std::vector<hotspot_partition_stat> temp;
+ for (const auto &partition_stat : partition_stats) {
+ temp.emplace_back(hotspot_partition_stat(partition_stat));
}
- _partition_stat_histories.emplace(temp);
+ _partitions_stat_histories.emplace_back(temp);
}
void hotspot_partition_calculator::init_perf_counter(int partition_count)
{
- std::string counter_name;
- std::string counter_desc;
+ std::string read_counter_name, write_counter_name;
+ std::string read_counter_desc, write_counter_desc;
for (int i = 0; i < partition_count; i++) {
- string partition_desc = _app_name + '.' + std::to_string(i);
- counter_name = fmt::format("app.stat.hotspots@{}", partition_desc);
- counter_desc = fmt::format("statistic the hotspots of app {}",
partition_desc);
- _hot_points[i].init_app_counter(
- "app.pegasus", counter_name.c_str(), COUNTER_TYPE_NUMBER,
counter_desc.c_str());
+ string read_partition_desc = _app_name + '.' + "read." +
std::to_string(i);
+ read_counter_name = fmt::format("app.stat.hotspots@{}",
read_partition_desc);
+ read_counter_desc = fmt::format("statistic the hotspots of app {}",
read_partition_desc);
+
_hot_points[i].emplace_back(std::make_unique<dsn::perf_counter_wrapper>());
+ _hot_points[i][READ_HOTSPOT_DATA]->init_app_counter("app.pegasus",
+
read_counter_name.c_str(),
+
COUNTER_TYPE_NUMBER,
+
read_counter_desc.c_str());
+ string write_partition_desc = _app_name + '.' + "write." +
std::to_string(i);
+ write_counter_name = fmt::format("app.stat.hotspots@{}",
write_partition_desc);
+ write_counter_desc = fmt::format("statistic the hotspots of app {}",
write_partition_desc);
+
_hot_points[i].emplace_back(std::make_unique<dsn::perf_counter_wrapper>());
+ _hot_points[i][WRITE_HOTSPOT_DATA]->init_app_counter("app.pegasus",
+
write_counter_name.c_str(),
+
COUNTER_TYPE_NUMBER,
+
write_counter_desc.c_str());
}
}
void hotspot_partition_calculator::data_analyse()
{
- dassert(_partition_stat_histories.back().size() == _hot_points.size(),
- "partition counts error, please check");
- std::vector<double> data_samples;
- data_samples.reserve(_partition_stat_histories.size() *
_hot_points.size());
- auto temp_data = _partition_stat_histories;
- double table_qps_sum = 0, standard_deviation = 0, table_qps_avg = 0;
- int sample_count = 0;
- while (!temp_data.empty()) {
- for (const auto &partition_data : temp_data.front()) {
- if (partition_data.total_qps - 1.00 > 0) {
- data_samples.push_back(partition_data.total_qps);
- table_qps_sum += partition_data.total_qps;
- sample_count++;
+ dcheck_eq(_partitions_stat_histories.back().size(), _hot_points.size());
+ for (int data_type = 0; data_type <= 1; data_type++) {
+ // 0: READ_HOTSPOT_DATA; 1: WRITE_HOTSPOT_DATA
+ double table_qps_sum = 0, standard_deviation = 0, table_qps_avg = 0;
+ int sample_count = 0;
+ for (const auto &one_partition_stat_histories :
_partitions_stat_histories) {
+ for (const auto &partition_stat : one_partition_stat_histories) {
+ if (partition_stat.total_qps[data_type] > 1.00) {
+ table_qps_sum += partition_stat.total_qps[data_type];
+ sample_count++;
+ }
}
}
- temp_data.pop();
- }
- if (sample_count == 0) {
- ddebug("_partition_stat_histories size == 0");
- return;
- }
- table_qps_avg = table_qps_sum / sample_count;
- for (const auto &data_sample : data_samples) {
- standard_deviation += pow((data_sample - table_qps_avg), 2);
- }
- standard_deviation = sqrt(standard_deviation / sample_count);
- const auto &anly_data = _partition_stat_histories.back();
- for (int i = 0; i < _hot_points.size(); i++) {
- double hot_point = (anly_data[i].total_qps - table_qps_avg) /
standard_deviation;
- // perf_counter->set can only be unsigned __int64
- // use ceil to guarantee conversion results
- hot_point = ceil(std::max(hot_point, double(0)));
- _hot_points[i]->set(hot_point);
+
+ if (sample_count <= 1) {
+ ddebug("_partitions_stat_histories size <= 1");
+ return;
Review comment:
Why skip when sample_count == 1? If there's one partition has high qps,
it means there's hotspot.
##########
File path: src/server/hotspot_partition_calculator.h
##########
@@ -17,13 +17,20 @@
#pragma once
-#include "hotspot_partition_data.h"
+#include "hotspot_partition_stat.h"
#include <gtest/gtest_prod.h>
#include <dsn/perf_counter/perf_counter.h>
namespace pegasus {
namespace server {
+// stores the whole histories of all partitions in one table
+typedef std::list<std::vector<hotspot_partition_stat>> stat_histories;
+// hot_partition_counters c[index_of_partitions][type_of_read(0)/write(1)_stat]
+// so if we have n partitions, we will get 2*n hot_partition_counters, to
demonstrate both
+// read/write hotspot value
+typedef std::vector<std::vector<std::unique_ptr<dsn::perf_counter_wrapper>>>
hot_partition_counters;
Review comment:
```suggestion
typedef std::vector<dsn::perf_counter_wrapper[2]> hot_partition_counters;
```
I think this is better. You don't need to wrap std::unique_ptr around
perf_counter_wrapper. Because perf_counter_wrapper has only a pointer inside.
##########
File path: src/server/hotspot_partition_stat.h
##########
@@ -0,0 +1,30 @@
+// Copyright (c) 2017, Xiaomi, Inc. All rights reserved.
+// This source code is licensed under the Apache License Version 2.0, which
+// can be found in the LICENSE file in the root directory of this source tree.
Review comment:
```suggestion
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
```
##########
File path: src/server/hotspot_partition_calculator.cpp
##########
@@ -34,67 +35,84 @@ DSN_DEFINE_int64("pegasus.collector",
"eliminate outdated historical "
"data");
-void hotspot_partition_calculator::data_aggregate(const std::vector<row_data>
&partitions)
+void hotspot_partition_calculator::data_aggregate(const std::vector<row_data>
&partition_stats)
{
- while (_partition_stat_histories.size() > FLAGS_max_hotspot_store_size -
1) {
- _partition_stat_histories.pop();
+ while (_partitions_stat_histories.size() >= FLAGS_max_hotspot_store_size) {
+ _partitions_stat_histories.pop_front();
}
- std::vector<hotspot_partition_data> temp(partitions.size());
- // TODO refactor the data structure
- for (int i = 0; i < partitions.size(); i++) {
- temp[i] = std::move(hotspot_partition_data(partitions[i]));
+ std::vector<hotspot_partition_stat> temp;
+ for (const auto &partition_stat : partition_stats) {
+ temp.emplace_back(hotspot_partition_stat(partition_stat));
}
- _partition_stat_histories.emplace(temp);
+ _partitions_stat_histories.emplace_back(temp);
}
void hotspot_partition_calculator::init_perf_counter(int partition_count)
{
- std::string counter_name;
- std::string counter_desc;
+ std::string read_counter_name, write_counter_name;
+ std::string read_counter_desc, write_counter_desc;
for (int i = 0; i < partition_count; i++) {
- string partition_desc = _app_name + '.' + std::to_string(i);
- counter_name = fmt::format("app.stat.hotspots@{}", partition_desc);
- counter_desc = fmt::format("statistic the hotspots of app {}",
partition_desc);
- _hot_points[i].init_app_counter(
- "app.pegasus", counter_name.c_str(), COUNTER_TYPE_NUMBER,
counter_desc.c_str());
+ string read_partition_desc = _app_name + '.' + "read." +
std::to_string(i);
+ read_counter_name = fmt::format("app.stat.hotspots@{}",
read_partition_desc);
+ read_counter_desc = fmt::format("statistic the hotspots of app {}",
read_partition_desc);
+
_hot_points[i].emplace_back(std::make_unique<dsn::perf_counter_wrapper>());
+ _hot_points[i][READ_HOTSPOT_DATA]->init_app_counter("app.pegasus",
+
read_counter_name.c_str(),
+
COUNTER_TYPE_NUMBER,
+
read_counter_desc.c_str());
+ string write_partition_desc = _app_name + '.' + "write." +
std::to_string(i);
+ write_counter_name = fmt::format("app.stat.hotspots@{}",
write_partition_desc);
+ write_counter_desc = fmt::format("statistic the hotspots of app {}",
write_partition_desc);
+
_hot_points[i].emplace_back(std::make_unique<dsn::perf_counter_wrapper>());
+ _hot_points[i][WRITE_HOTSPOT_DATA]->init_app_counter("app.pegasus",
+
write_counter_name.c_str(),
+
COUNTER_TYPE_NUMBER,
+
write_counter_desc.c_str());
}
}
void hotspot_partition_calculator::data_analyse()
{
- dassert(_partition_stat_histories.back().size() == _hot_points.size(),
- "partition counts error, please check");
- std::vector<double> data_samples;
- data_samples.reserve(_partition_stat_histories.size() *
_hot_points.size());
- auto temp_data = _partition_stat_histories;
- double table_qps_sum = 0, standard_deviation = 0, table_qps_avg = 0;
- int sample_count = 0;
- while (!temp_data.empty()) {
- for (const auto &partition_data : temp_data.front()) {
- if (partition_data.total_qps - 1.00 > 0) {
- data_samples.push_back(partition_data.total_qps);
- table_qps_sum += partition_data.total_qps;
- sample_count++;
+ dcheck_eq(_partitions_stat_histories.back().size(), _hot_points.size());
+ for (int data_type = 0; data_type <= 1; data_type++) {
+ // 0: READ_HOTSPOT_DATA; 1: WRITE_HOTSPOT_DATA
+ double table_qps_sum = 0, standard_deviation = 0, table_qps_avg = 0;
+ int sample_count = 0;
+ for (const auto &one_partition_stat_histories :
_partitions_stat_histories) {
+ for (const auto &partition_stat : one_partition_stat_histories) {
+ if (partition_stat.total_qps[data_type] > 1.00) {
+ table_qps_sum += partition_stat.total_qps[data_type];
+ sample_count++;
+ }
}
}
- temp_data.pop();
- }
- if (sample_count == 0) {
- ddebug("_partition_stat_histories size == 0");
- return;
- }
- table_qps_avg = table_qps_sum / sample_count;
- for (const auto &data_sample : data_samples) {
- standard_deviation += pow((data_sample - table_qps_avg), 2);
- }
- standard_deviation = sqrt(standard_deviation / sample_count);
- const auto &anly_data = _partition_stat_histories.back();
- for (int i = 0; i < _hot_points.size(); i++) {
- double hot_point = (anly_data[i].total_qps - table_qps_avg) /
standard_deviation;
- // perf_counter->set can only be unsigned __int64
- // use ceil to guarantee conversion results
- hot_point = ceil(std::max(hot_point, double(0)));
- _hot_points[i]->set(hot_point);
+
+ if (sample_count <= 1) {
+ ddebug("_partitions_stat_histories size <= 1");
+ return;
+ }
+ table_qps_avg = table_qps_sum / sample_count;
+ for (const auto &one_partition_stat_histories :
_partitions_stat_histories) {
+ for (const auto &partition_stat : one_partition_stat_histories) {
+ if (partition_stat.total_qps[data_type] > 1.00) {
+ standard_deviation +=
+ pow((partition_stat.total_qps[data_type] -
table_qps_avg), 2);
+ }
+ }
+ }
+ standard_deviation = sqrt(standard_deviation / (sample_count - 1));
Review comment:
```suggestion
standard_deviation = sqrt(standard_deviation / (sample_count - 1));
```
`(sample_count - 1)` Why sub 1?
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
