This is an automated email from the ASF dual-hosted git repository.
laiyingchun pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new d6c15b835 refactor(conf): use DSN_DEFINE_int32 to load int32 type of
configs (part 2) (#1346)
d6c15b835 is described below
commit d6c15b8350fab79151823e967ce1f7ddea56fcec
Author: Yingchun Lai <[email protected]>
AuthorDate: Tue Feb 14 12:07:08 2023 +0800
refactor(conf): use DSN_DEFINE_int32 to load int32 type of configs (part 2)
(#1346)
---
src/client/replication_ddl_client.h | 3 +-
src/failure_detector/test/failure_detector.cpp | 6 +-
src/geo/lib/geo_client.cpp | 74 ++++++++++++++--------
src/geo/lib/geo_client.h | 11 ----
src/geo/test/geo_test.cpp | 7 +-
src/meta/meta_data.cpp | 35 +++++-----
src/meta/meta_options.cpp | 21 ------
src/meta/meta_options.h | 4 --
src/meta/meta_server_failure_detector.cpp | 8 ++-
src/meta/meta_state_service_zookeeper.cpp | 5 +-
src/meta/partition_guardian.cpp | 6 +-
src/meta/server_state.cpp | 20 ++++--
src/meta/test/meta_app_operation_test.cpp | 2 +-
src/perf_counter/perf_counter_atomic.cpp | 73 +++++++++++++++++++++
src/perf_counter/perf_counter_atomic.h | 48 ++------------
src/perf_counter/test/perf_counter_test.cpp | 13 ++--
src/runtime/env.sim.cpp | 19 +++---
src/runtime/env.sim.h | 2 -
src/runtime/rpc/asio_net_provider.cpp | 8 +--
src/runtime/rpc/network.cpp | 6 --
src/runtime/rpc/network.h | 1 -
src/runtime/simulator.cpp | 6 +-
src/server/available_detector.cpp | 14 ++--
src/server/available_detector.h | 1 -
src/server/pegasus_manual_compact_service.cpp | 17 +++--
src/server/pegasus_manual_compact_service.h | 1 -
src/server/test/manual_compact_service_test.cpp | 11 ++--
src/test/kill_test/process_kill_testor.cpp | 65 ++++++++++---------
src/test/kill_test/process_kill_testor.h | 8 ---
src/test/pressure_test/main.cpp | 44 +++++--------
.../distributed_lock_service_zookeeper.cpp | 5 +-
src/zookeeper/zookeeper_session.cpp | 10 ++-
src/zookeeper/zookeeper_session_mgr.cpp | 2 -
src/zookeeper/zookeeper_session_mgr.h | 2 -
34 files changed, 290 insertions(+), 268 deletions(-)
diff --git a/src/client/replication_ddl_client.h
b/src/client/replication_ddl_client.h
index 9aa1d32b3..82432ca90 100644
--- a/src/client/replication_ddl_client.h
+++ b/src/client/replication_ddl_client.h
@@ -68,7 +68,8 @@ public:
bool is_stateless,
bool success_if_exist = true);
- // reserve_seconds == 0 means use default value in configuration
'hold_seconds_for_dropped_app'
+ // 'reserve_seconds' == 0 means use default value in configuration
+ // FLAGS_hold_seconds_for_dropped_app.
dsn::error_code drop_app(const std::string &app_name, int reserve_seconds);
dsn::error_code recall_app(int32_t app_id, const std::string
&new_app_name);
diff --git a/src/failure_detector/test/failure_detector.cpp
b/src/failure_detector/test/failure_detector.cpp
index e173e8937..19dded7ea 100644
--- a/src/failure_detector/test/failure_detector.cpp
+++ b/src/failure_detector/test/failure_detector.cpp
@@ -44,6 +44,8 @@
#include "runtime/rpc/rpc_address.h"
#include <vector>
+DSN_DECLARE_int32(max_succssive_unstable_restart);
+
using namespace dsn;
using namespace dsn::fd;
@@ -203,7 +205,7 @@ public:
error_code start(const std::vector<std::string> &args) override
{
_opts.stable_rs_min_running_seconds = 10;
- _opts.max_succssive_unstable_restart = 10;
+ FLAGS_max_succssive_unstable_restart = 10;
_master_fd = new master_fd_test();
_master_fd->set_options(&_opts);
@@ -619,7 +621,7 @@ TEST(fd, update_stability)
replication::fd_suboptions opts;
opts.stable_rs_min_running_seconds = 5;
- opts.max_succssive_unstable_restart = 2;
+ FLAGS_max_succssive_unstable_restart = 2;
fd->set_options(&opts);
replication::meta_server_failure_detector::stability_map *smap =
diff --git a/src/geo/lib/geo_client.cpp b/src/geo/lib/geo_client.cpp
index c81af5588..83daf855c 100644
--- a/src/geo/lib/geo_client.cpp
+++ b/src/geo/lib/geo_client.cpp
@@ -35,6 +35,7 @@
#include "runtime/serverlet.h"
#include "runtime/service_app.h"
#include "runtime/rpc/rpc_address.h"
+#include "utils/flags.h"
#include "utils/fmt_logging.h"
#include "utils/errors.h"
@@ -43,6 +44,30 @@
namespace pegasus {
namespace geo {
+DSN_DEFINE_int32(geo_client.lib,
+ min_level,
+ 12,
+ "Min cell level for a scan. Cell id at this level is the
hash-key in Pegasus. "
+ "min_level is immutable after geo_client data has been
inserted into DB. "
+ "Edge length at level 12 is about 2 km");
+DSN_DEFINE_int32(geo_client.lib,
+ max_level,
+ 16,
+ "Max cell level for a scan. Cell id at this level is the
prefix of sort-key "
+ "in Pegasus, and it's convenient for scan operation.
max_level is mutable "
+ "at any time, and geo_client-lib users can change it to a
appropriate "
+ "value to improve performance in their scenario. Edge length
at level 16 "
+ "is about 150 m");
+DSN_DEFINE_group_validator(min_max_level, [](std::string &message) -> bool {
+ if (FLAGS_min_level >= FLAGS_max_level) {
+ message = fmt::format("[geo_client.lib].min_level({}) should be < "
+ "[geo_client.lib].max_level({})",
+ FLAGS_min_level,
+ FLAGS_max_level);
+ return false;
+ }
+ return true;
+});
struct SearchResultNearer
{
@@ -74,14 +99,6 @@ geo_client::geo_client(const char *config_file,
_geo_data_client = pegasus_client_factory::get_client(cluster_name,
geo_app_name);
CHECK_NOTNULL(_geo_data_client, "init pegasus _geo_data_client failed");
- _min_level = (int32_t)dsn_config_get_value_uint64(
- "geo_client.lib", "min_level", 12, "min cell level for scan");
-
- _max_level = (int32_t)dsn_config_get_value_uint64(
- "geo_client.lib", "max_level", 16, "max cell level for scan");
-
- CHECK_LT(_min_level, _max_level);
-
uint32_t latitude_index = (uint32_t)dsn_config_get_value_uint64(
"geo_client.lib", "latitude_index", 5, "latitude index in value");
@@ -94,14 +111,14 @@ geo_client::geo_client(const char *config_file,
dsn::error_s geo_client::set_max_level(int level)
{
- if (level <= _min_level) {
+ if (level <= FLAGS_min_level) {
return dsn::FMT_ERR(dsn::ERR_INVALID_PARAMETERS,
- "level({}) must be larger than _min_level({})",
+ "level({}) must be larger than
FLAGS_min_level({})",
level,
- _min_level);
+ FLAGS_min_level);
}
- _max_level = level;
+ FLAGS_max_level = level;
return dsn::error_s::ok();
}
@@ -561,7 +578,7 @@ void geo_client::gen_search_cap(const S2LatLng &latlng,
double radius_m, S2Cap &
void geo_client::gen_cells_covered_by_cap(const S2Cap &cap, S2CellUnion &cids)
{
S2RegionCoverer rc;
- rc.mutable_options()->set_fixed_level(_min_level);
+ rc.mutable_options()->set_fixed_level(FLAGS_min_level);
cids = rc.GetCovering(cap);
}
@@ -593,7 +610,8 @@ void geo_client::async_get_result_from_cells(const
S2CellUnion &cids,
for (const auto &cid : cids) {
if (cap_ptr->Contains(S2Cell(cid))) {
- // for the full contained cell, scan all data in this cell(which
is at the `_min_level`)
+ // for the full contained cell, scan all data in this cell(which
is at the
+ // FLAGS_min_level)
results->emplace_back(std::list<SearchResult>());
scan_count->fetch_add(1);
start_scan(cid.ToString(),
@@ -605,15 +623,16 @@ void geo_client::async_get_result_from_cells(const
S2CellUnion &cids,
single_scan_finish_callback,
results->back());
} else {
- // for the partial contained cell, scan cells covered by the cap
at the `_max_level`
- // which is more accurate than the ones at `_min_level`, but it
will cost more time on
- // calculating here.
- std::string hash_key = cid.parent(_min_level).ToString();
+ // for the partial contained cell, scan cells covered by the cap
at the FLAGS_max_level
+ // which is more accurate than the ones at FLAGS_min_level, but it
will cost more time
+ // on calculating here.
+ std::string hash_key = cid.parent(FLAGS_min_level).ToString();
std::pair<std::string, std::string> start_stop_sort_keys;
S2CellId pre;
- // traverse all sub cell ids of `cid` on `_max_level` along the
Hilbert curve, to find
- // the needed ones.
- for (S2CellId cur = cid.child_begin(_max_level); cur !=
cid.child_end(_max_level);
+ // traverse all sub cell ids of `cid` on FLAGS_max_level along the
Hilbert curve, to
+ // find the needed ones.
+ for (S2CellId cur = cid.child_begin(FLAGS_max_level);
+ cur != cid.child_end(FLAGS_max_level);
cur = cur.next()) {
if (cap_ptr->MayIntersect(S2Cell(cur))) {
// only cells whose any vertex is contained by the cap is
needed
@@ -646,7 +665,8 @@ void geo_client::async_get_result_from_cells(const
S2CellUnion &cids,
}
CHECK(!start_stop_sort_keys.first.empty(), "");
- // the last sub slice of current `cid` on `_max_level` in Hilbert
curve covered by `cap`
+ // the last sub slice of current `cid` on FLAGS_max_level in
Hilbert curve covered by
+ // `cap`
if (start_stop_sort_keys.second.empty()) {
start_stop_sort_keys.second = gen_stop_sort_key(pre, hash_key);
results->emplace_back(std::list<SearchResult>());
@@ -710,14 +730,14 @@ bool geo_client::generate_geo_keys(const std::string
&hash_key,
// generate hash key
S2CellId leaf_cell_id = S2Cell(latlng).id();
- S2CellId parent_cell_id = leaf_cell_id.parent(_min_level);
- geo_hash_key = parent_cell_id.ToString(); // [0,5]{1}/[0,3]{_min_level}
+ S2CellId parent_cell_id = leaf_cell_id.parent(FLAGS_min_level);
+ geo_hash_key = parent_cell_id.ToString(); //
[0,5]{1}/[0,3]{FLAGS_min_level}
// generate sort key
dsn::blob sort_key_postfix;
pegasus_generate_key(sort_key_postfix, hash_key, sort_key);
geo_sort_key = leaf_cell_id.ToString().substr(geo_hash_key.length()) + ":"
+
- sort_key_postfix.to_string(); //
[0,3]{30-_min_level}:combine_keys
+ sort_key_postfix.to_string(); //
[0,3]{30-FLAGS_min_level}:combine_keys
return true;
}
@@ -726,8 +746,8 @@ bool geo_client::restore_origin_keys(const std::string
&geo_sort_key,
std::string &origin_hash_key,
std::string &origin_sort_key)
{
- // geo_sort_key: [0,3]{30-_min_level}:combine_keys
- int cid_prefix_len = 30 - _min_level + 1; // '1' is for ':' in geo_sort_key
+ // geo_sort_key: [0,3]{30-FLAGS_min_level}:combine_keys
+ int cid_prefix_len = 30 - FLAGS_min_level + 1; // '1' is for ':' in
geo_sort_key
if (geo_sort_key.length() <= cid_prefix_len) {
return false;
}
diff --git a/src/geo/lib/geo_client.h b/src/geo/lib/geo_client.h
index c55bc11bf..c10a4eca2 100644
--- a/src/geo/lib/geo_client.h
+++ b/src/geo/lib/geo_client.h
@@ -472,17 +472,6 @@ private:
std::list<SearchResult> &result);
private:
- // cell id at this level is the hash-key in pegasus
- // `_min_level` is immutable after geo_client data has been inserted into
DB.
- int _min_level = 12; // edge length at level 12 is about 2km
-
- // cell id at this level is the prefix of sort-key in pegasus, and
- // it's convenient for scan operation
- // `_max_level` is mutable at any time, and geo_client-lib users can
change it to a appropriate
- // value
- // to improve performance in their scenario.
- int _max_level = 16; // edge length at level 16 is about 150m
-
dsn::task_tracker _tracker;
latlng_codec _codec;
diff --git a/src/geo/test/geo_test.cpp b/src/geo/test/geo_test.cpp
index 222296108..6a0982190 100644
--- a/src/geo/test/geo_test.cpp
+++ b/src/geo/test/geo_test.cpp
@@ -30,10 +30,13 @@
#include "common/replication_other_types.h"
#include "client/replication_ddl_client.h"
#include "base/pegasus_const.h"
+#include "utils/flags.h"
namespace pegasus {
namespace geo {
+DSN_DECLARE_int32(min_level);
+
class geo_client_test : public ::testing::Test
{
public:
@@ -52,8 +55,6 @@ public:
pegasus_client *common_data_client() { return
_geo_client->_common_data_client; }
pegasus::geo::geo_client *geo_client() { return _geo_client.get(); }
- int min_level() { return _geo_client->_min_level; }
-
bool generate_geo_keys(const std::string &hash_key,
const std::string &sort_key,
const std::string &value,
@@ -405,7 +406,7 @@ TEST_F(geo_client_test, generate_and_restore_geo_keys)
ASSERT_TRUE(
generate_geo_keys(test_hash_key, test_sort_key, test_value,
geo_hash_key, geo_sort_key));
- ASSERT_EQ(min_level() + 2, geo_hash_key.length());
+ ASSERT_EQ(FLAGS_min_level + 2, geo_hash_key.length());
ASSERT_EQ(leaf_cell_id.substr(0, geo_hash_key.length()), geo_hash_key);
ASSERT_EQ(leaf_cell_id.substr(geo_hash_key.length()),
geo_sort_key.substr(0, leaf_cell_id.length() -
geo_hash_key.length()));
diff --git a/src/meta/meta_data.cpp b/src/meta/meta_data.cpp
index c9c9674da..e70a04a56 100644
--- a/src/meta/meta_data.cpp
+++ b/src/meta/meta_data.cpp
@@ -55,35 +55,36 @@
namespace dsn {
namespace replication {
-// There is an option `max_replicas_in_group` which restricts the max replica
count of the whole
+// There is an option FLAGS_max_replicas_in_group which restricts the max
replica count of the whole
// cluster. It's a cluster-level option. However, now that it's allowed to
update the replication
// factor of each table, this cluster-level option should be replaced.
//
-// Conceptually `max_replicas_in_group` is the total number of alive and
dropped replicas. Its
-// default value is 4. For a table that has replication factor 3, that
`max_replicas_in_group`
+// Conceptually FLAGS_max_replicas_in_group is the total number of alive and
dropped replicas. Its
+// default value is 4. For a table that has replication factor 3, that
FLAGS_max_replicas_in_group
// is set to 4 means 3 alive replicas plus a dropped replica.
//
-// `max_replicas_in_group` can also be loaded from configuration file, which
means its default
-// value will be overridden. The value of `max_replicas_in_group` will be
assigned to another
+// FLAGS_max_replicas_in_group can also be loaded from configuration file,
which means its default
+// value will be overridden. The value of FLAGS_max_replicas_in_group will be
assigned to another
// static variable `MAX_REPLICA_COUNT_IN_GRROUP`, whose default value is also
4.
//
// For unit tests, `MAX_REPLICA_COUNT_IN_GRROUP` is set to the default value
4; for production
-// environments, `MAX_REPLICA_COUNT_IN_GRROUP` is set to 3 since
`max_replicas_in_group` is
+// environments, `MAX_REPLICA_COUNT_IN_GRROUP` is set to 3 since
FLAGS_max_replicas_in_group is
// configured as 3 in `.ini` file.
//
-// Since the cluster-level option `max_replicas_in_group` contains the alive
and dropped replicas,
-// we can use the replication factor of each table as the number of alive
replicas, and introduce
-// another option `max_reserved_dropped_replicas` representing the max
reserved number allowed for
-// dropped replicas.
+// Since the cluster-level option FLAGS_max_replicas_in_group contains the
alive and dropped
+// replicas, we can use the replication factor of each table as the number of
alive replicas, and
+// introduce another option FLAGS_max_reserved_dropped_replicas representing
the max reserved number
+// allowed for dropped replicas.
//
-// If `max_reserved_dropped_replicas` is set to 1, there is at most one
dropped replicas reserved,
-// which means, once the number of alive replicas reaches max_replica_count,
at most one dropped
-// replica can be reserved and others will be eliminated; If
`max_reserved_dropped_replicas` is
-// set to 0, however, none of dropped replicas can be reserved.
+// If FLAGS_max_reserved_dropped_replicas is set to 1, there is at most one
dropped replicas
+// reserved, which means, once the number of alive replicas reaches
max_replica_count, at most one
+// dropped replica can be reserved and others will be eliminated; If
+// FLAGS_max_reserved_dropped_replicas is set to 0, however, none of dropped
replicas can be
+// reserved.
//
-// To be consistent with `max_replicas_in_group`, default value of
`max_reserved_dropped_replicas`
-// is set to 1 so that the unit tests can be passed. For production
environments, it should be set
-// to 0.
+// To be consistent with FLAGS_max_replicas_in_group, default value of
+// FLAGS_max_reserved_dropped_replicas is set to 1 so that the unit tests can
be passed. For
+// production environments, it should be set to 0.
DSN_DEFINE_uint32(meta_server,
max_reserved_dropped_replicas,
1,
diff --git a/src/meta/meta_options.cpp b/src/meta/meta_options.cpp
index 1f922581e..e778a346f 100644
--- a/src/meta/meta_options.cpp
+++ b/src/meta/meta_options.cpp
@@ -95,22 +95,11 @@ void meta_options::initialize()
false,
"whether to recover from replica server when no apps in remote
storage");
- hold_seconds_for_dropped_app =
- dsn_config_get_value_uint64("meta_server",
- "hold_seconds_for_dropped_app",
- 604800,
- "how long to hold data for dropped apps");
-
add_secondary_enable_flow_control =
dsn_config_get_value_bool("meta_server",
"add_secondary_enable_flow_control",
false,
"enable flow control for add secondary
proposal");
- add_secondary_max_count_for_one_node = dsn_config_get_value_uint64(
- "meta_server",
- "add_secondary_max_count_for_one_node",
- 10,
- "add secondary max count for one node when flow control enabled");
/// failure detector options
_fd_opts.distributed_lock_service_type =
@@ -130,13 +119,6 @@ void meta_options::initialize()
600,
"min running seconds for a stable replica
server");
- _fd_opts.max_succssive_unstable_restart = dsn_config_get_value_uint64(
- "meta_server",
- "max_succssive_unstable_restart",
- 5,
- "meta server will treat an rs unstable so as to reject it's beacons "
- "if its succssively restarting count exceeds this value");
-
/// load balancer options
_lb_opts.server_load_balancer_type =
dsn_config_get_value_string("meta_server",
@@ -148,9 +130,6 @@ void meta_options::initialize()
"replica_assign_delay_ms_for_dropouts",
300000,
"replica_assign_delay_ms_for_dropouts,
default is 300000");
- _lb_opts.max_replicas_in_group = dsn_config_get_value_uint64(
- "meta_server", "max_replicas_in_group", 4, "max replicas(alive & dead)
in a group");
-
_lb_opts.balancer_in_turn = dsn_config_get_value_bool(
"meta_server", "balancer_in_turn", false, "balance the apps
one-by-one/concurrently");
_lb_opts.only_primary_balancer = dsn_config_get_value_bool(
diff --git a/src/meta/meta_options.h b/src/meta/meta_options.h
index 46efef8ce..a227ecf83 100644
--- a/src/meta/meta_options.h
+++ b/src/meta/meta_options.h
@@ -61,7 +61,6 @@ public:
std::vector<std::string> distributed_lock_service_args;
uint64_t stable_rs_min_running_seconds;
- int32_t max_succssive_unstable_restart;
};
class lb_suboptions
@@ -69,7 +68,6 @@ class lb_suboptions
public:
std::string server_load_balancer_type;
uint64_t replica_assign_delay_ms_for_dropouts;
- int32_t max_replicas_in_group;
bool balancer_in_turn;
bool only_primary_balancer;
@@ -86,10 +84,8 @@ public:
uint64_t node_live_percentage_threshold_for_update;
meta_function_level::type meta_function_level_on_start;
bool recover_from_replica_server;
- int32_t hold_seconds_for_dropped_app;
bool add_secondary_enable_flow_control;
- int32_t add_secondary_max_count_for_one_node;
fd_suboptions _fd_opts;
lb_suboptions _lb_opts;
diff --git a/src/meta/meta_server_failure_detector.cpp
b/src/meta/meta_server_failure_detector.cpp
index 57eac0556..3fec4a440 100644
--- a/src/meta/meta_server_failure_detector.cpp
+++ b/src/meta/meta_server_failure_detector.cpp
@@ -37,6 +37,12 @@
#include "utils/fmt_logging.h"
#include "utils/string_conv.h"
+DSN_DEFINE_int32(meta_server,
+ max_succssive_unstable_restart,
+ 5,
+ "meta server will treat a rs unstable so as to reject it is
beacons if "
+ "its successively restarting count exceeds this value.");
+
namespace dsn {
namespace replication {
@@ -242,7 +248,7 @@ bool
meta_server_failure_detector::update_stability_stat(const fd::beacon_msg &b
} else {
LOG_WARNING("{}: possible encounter a staled message, ignore it",
beacon.from_addr);
}
- return w.unstable_restart_count <
_fd_opts->max_succssive_unstable_restart;
+ return w.unstable_restart_count < FLAGS_max_succssive_unstable_restart;
}
}
diff --git a/src/meta/meta_state_service_zookeeper.cpp
b/src/meta/meta_state_service_zookeeper.cpp
index e2408a9fc..ff5e1ad49 100644
--- a/src/meta/meta_state_service_zookeeper.cpp
+++ b/src/meta/meta_state_service_zookeeper.cpp
@@ -38,6 +38,7 @@
#include <boost/lexical_cast.hpp>
#include "meta_state_service_zookeeper.h"
+#include "utils/flags.h"
#include "zookeeper/zookeeper_session_mgr.h"
#include "zookeeper/zookeeper_session.h"
#include "zookeeper/zookeeper_error.h"
@@ -45,6 +46,8 @@
namespace dsn {
namespace dist {
+DSN_DECLARE_int32(timeout_ms);
+
class zoo_transaction : public meta_state_service::transaction_entries
{
public:
@@ -163,7 +166,7 @@ error_code meta_state_service_zookeeper::initialize(const
std::vector<std::strin
ref_this(this),
std::placeholders::_1));
if (_zoo_state != ZOO_CONNECTED_STATE) {
- _notifier.wait_for(zookeeper_session_mgr::instance().timeout());
+ _notifier.wait_for(FLAGS_timeout_ms);
if (_zoo_state != ZOO_CONNECTED_STATE)
return ERR_TIMEOUT;
}
diff --git a/src/meta/partition_guardian.cpp b/src/meta/partition_guardian.cpp
index 49a30948b..270994433 100644
--- a/src/meta/partition_guardian.cpp
+++ b/src/meta/partition_guardian.cpp
@@ -23,13 +23,15 @@
namespace dsn {
namespace replication {
+
+DSN_DEFINE_int32(meta_server, max_replicas_in_group, 4, "max replicas(alive &
dead) in a group");
+
partition_guardian::partition_guardian(meta_service *svc) : _svc(svc)
{
if (svc != nullptr) {
_replica_assign_delay_ms_for_dropouts =
svc->get_meta_options()._lb_opts.replica_assign_delay_ms_for_dropouts;
- config_context::MAX_REPLICA_COUNT_IN_GRROUP =
- svc->get_meta_options()._lb_opts.max_replicas_in_group;
+ config_context::MAX_REPLICA_COUNT_IN_GRROUP =
FLAGS_max_replicas_in_group;
} else {
_replica_assign_delay_ms_for_dropouts = 0;
}
diff --git a/src/meta/server_state.cpp b/src/meta/server_state.cpp
index c766d439e..1dff37f56 100644
--- a/src/meta/server_state.cpp
+++ b/src/meta/server_state.cpp
@@ -88,6 +88,14 @@ DSN_DEFINE_group_validator(min_max_allowed_replica_count,
[](std::string &messag
return true;
});
+DSN_DEFINE_int32(meta_server,
+ hold_seconds_for_dropped_app,
+ 604800,
+ "how long to hold data for dropped apps");
+DSN_DEFINE_int32(meta_server,
+ add_secondary_max_count_for_one_node,
+ 10,
+ "add secondary max count for one node when flow control
enabled");
static const char *lock_state = "lock";
static const char *unlock_state = "unlock";
@@ -145,7 +153,7 @@ void server_state::register_cli_commands()
} else {
if (args[0] == "DEFAULT") {
_add_secondary_max_count_for_one_node =
-
_meta_svc->get_meta_options().add_secondary_max_count_for_one_node;
+ FLAGS_add_secondary_max_count_for_one_node;
} else {
int32_t v = 0;
if (!dsn::buf2int32(args[0], v) || v < 0) {
@@ -165,8 +173,7 @@ void server_state::initialize(meta_service *meta_svc, const
std::string &apps_ro
_apps_root = apps_root;
_add_secondary_enable_flow_control =
_meta_svc->get_meta_options().add_secondary_enable_flow_control;
- _add_secondary_max_count_for_one_node =
- _meta_svc->get_meta_options().add_secondary_max_count_for_one_node;
+ _add_secondary_max_count_for_one_node =
FLAGS_add_secondary_max_count_for_one_node;
_dead_partition_count.init_app_counter("eon.server_state",
"dead_partition_count",
@@ -433,6 +440,8 @@ error_code server_state::initialize_default_apps()
app_info default_app;
for (int i = 0; i < sections.size(); i++) {
+ // TODO(yingchun): dose it mean sections[i] equals to
"meta_server.apps" or
+ // "replication.app" ?
if (strstr(sections[i], "meta_server.apps") == sections[i] ||
utils::equals(sections[i], "replication.app")) {
const char *s = sections[i];
@@ -440,6 +449,8 @@ error_code server_state::initialize_default_apps()
default_app.status = app_status::AS_CREATING;
default_app.app_id = _all_apps.size() + 1;
+ // TODO(yingchun): the old configuration launch methods should be
kept to launch repeat
+ // configs.
default_app.app_name = dsn_config_get_value_string(s, "app_name",
"", "app name");
if (default_app.app_name.length() == 0) {
LOG_WARNING("'[{}] app_name' not specified, ignore this
section", s);
@@ -1213,8 +1224,7 @@ void server_state::drop_app(dsn::message_ex *msg)
request.options.reserve_seconds > 0) {
app->expire_second = app->drop_second +
request.options.reserve_seconds;
} else {
- app->expire_second = app->drop_second +
-
_meta_svc->get_meta_options().hold_seconds_for_dropped_app;
+ app->expire_second = app->drop_second +
FLAGS_hold_seconds_for_dropped_app;
}
app->helpers->pending_response = msg;
CHECK_EQ(app->helpers->partitions_in_progress.load(), 0);
diff --git a/src/meta/test/meta_app_operation_test.cpp
b/src/meta/test/meta_app_operation_test.cpp
index 33246addf..9bdeba6b4 100644
--- a/src/meta/test/meta_app_operation_test.cpp
+++ b/src/meta/test/meta_app_operation_test.cpp
@@ -103,7 +103,7 @@ public:
// dropped app can only be find by app_id
auto app = _ss->get_app(app_id);
- // hold_seconds_for_dropped_app = 604800 in unit test config
+ // FLAGS_hold_seconds_for_dropped_app = 604800 in unit test config
// make app expired immediatly
app->expire_second -= 604800;
}
diff --git a/src/perf_counter/perf_counter_atomic.cpp
b/src/perf_counter/perf_counter_atomic.cpp
new file mode 100644
index 000000000..19650a3b9
--- /dev/null
+++ b/src/perf_counter/perf_counter_atomic.cpp
@@ -0,0 +1,73 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+#include "perf_counter/perf_counter_atomic.h"
+
+#include "utils/flags.h"
+
+namespace dsn {
+
+DSN_DEFINE_int32(components.pegasus_perf_counter_number_percentile_atomic,
+ counter_computation_interval_seconds,
+ 10,
+ "The interval seconds of the system to compute the
percentiles of the "
+ "pegasus_perf_counter_number_percentile_atomic counters");
+
+perf_counter_number_percentile_atomic::perf_counter_number_percentile_atomic(
+ const char *app,
+ const char *section,
+ const char *name,
+ dsn_perf_counter_type_t type,
+ const char *dsptr,
+ bool use_timer)
+ : perf_counter(app, section, name, type, dsptr), _tail(0)
+{
+ _results[COUNTER_PERCENTILE_50] = 0;
+ _results[COUNTER_PERCENTILE_90] = 0;
+ _results[COUNTER_PERCENTILE_95] = 0;
+ _results[COUNTER_PERCENTILE_99] = 0;
+ _results[COUNTER_PERCENTILE_999] = 0;
+
+ if (!use_timer) {
+ return;
+ }
+
+ _timer.reset(new
boost::asio::deadline_timer(tools::shared_io_service::instance().ios));
+ _timer->expires_from_now(
+ boost::posix_time::seconds(::rand() %
FLAGS_counter_computation_interval_seconds + 1));
+ _timer->async_wait(std::bind(
+ &perf_counter_number_percentile_atomic::on_timer, this, _timer,
std::placeholders::_1));
+}
+
+void perf_counter_number_percentile_atomic::on_timer(
+ std::shared_ptr<boost::asio::deadline_timer> timer, const
boost::system::error_code &ec)
+{
+ // as the callback is not in tls context, so the log system calls like
LOG_INFO, CHECK
+ // will cause a lock
+ if (!ec) {
+ calc(boost::make_shared<compute_context>());
+
+ timer->expires_from_now(
+
boost::posix_time::seconds(FLAGS_counter_computation_interval_seconds));
+ timer->async_wait(std::bind(
+ &perf_counter_number_percentile_atomic::on_timer, this, timer,
std::placeholders::_1));
+ } else if (boost::system::errc::operation_canceled != ec) {
+ CHECK(false, "on_timer error!!!");
+ }
+}
+
+} // namespace dsn
diff --git a/src/perf_counter/perf_counter_atomic.h
b/src/perf_counter/perf_counter_atomic.h
index 90328fb1a..61c7f7a56 100644
--- a/src/perf_counter/perf_counter_atomic.h
+++ b/src/perf_counter/perf_counter_atomic.h
@@ -15,6 +15,8 @@
// specific language governing permissions and limitations
// under the License.
+#pragma once
+
#include <atomic>
#include <boost/make_shared.hpp>
@@ -26,6 +28,7 @@
#include "utils/shared_io_service.h"
#include "utils/time_utils.h"
#include "utils/utils.h"
+#include "utils/flags.h"
namespace dsn {
@@ -214,31 +217,7 @@ public:
const char *name,
dsn_perf_counter_type_t type,
const char *dsptr,
- bool use_timer = true)
- : perf_counter(app, section, name, type, dsptr), _tail(0)
- {
- _results[COUNTER_PERCENTILE_50] = 0;
- _results[COUNTER_PERCENTILE_90] = 0;
- _results[COUNTER_PERCENTILE_95] = 0;
- _results[COUNTER_PERCENTILE_99] = 0;
- _results[COUNTER_PERCENTILE_999] = 0;
-
- if (!use_timer) {
- return;
- }
-
- _counter_computation_interval_seconds =
(int)dsn_config_get_value_uint64(
- "components.pegasus_perf_counter_number_percentile_atomic",
- "counter_computation_interval_seconds",
- 10,
- "period (seconds) the system computes the percentiles of the "
- "pegasus_perf_counter_number_percentile_atomic counters");
- _timer.reset(new
boost::asio::deadline_timer(tools::shared_io_service::instance().ios));
- _timer->expires_from_now(
- boost::posix_time::seconds(::rand() %
_counter_computation_interval_seconds + 1));
- _timer->async_wait(std::bind(
- &perf_counter_number_percentile_atomic::on_timer, this, _timer,
std::placeholders::_1));
- }
+ bool use_timer = true);
~perf_counter_number_percentile_atomic(void)
{
@@ -439,29 +418,12 @@ private:
}
void on_timer(std::shared_ptr<boost::asio::deadline_timer> timer,
- const boost::system::error_code &ec)
- {
- // as the callback is not in tls context, so the log system calls like
LOG_INFO, CHECK
- // will cause a lock
- if (!ec) {
- calc(boost::make_shared<compute_context>());
-
- timer->expires_from_now(
-
boost::posix_time::seconds(_counter_computation_interval_seconds));
-
timer->async_wait(std::bind(&perf_counter_number_percentile_atomic::on_timer,
- this,
- timer,
- std::placeholders::_1));
- } else if (boost::system::errc::operation_canceled != ec) {
- CHECK(false, "on_timer error!!!");
- }
- }
+ const boost::system::error_code &ec);
std::shared_ptr<boost::asio::deadline_timer> _timer;
std::atomic<uint64_t> _tail; // should use unsigned int to avoid out of
bound
int64_t _samples[MAX_QUEUE_LENGTH];
int64_t _results[COUNTER_PERCENTILE_COUNT];
- int _counter_computation_interval_seconds;
};
#pragma pack(pop)
diff --git a/src/perf_counter/test/perf_counter_test.cpp
b/src/perf_counter/test/perf_counter_test.cpp
index f4de5c309..7f9403deb 100644
--- a/src/perf_counter/test/perf_counter_test.cpp
+++ b/src/perf_counter/test/perf_counter_test.cpp
@@ -40,12 +40,18 @@
#include <vector>
#include "perf_counter/perf_counter_atomic.h"
+#include "utils/flags.h"
using namespace dsn;
using namespace dsn::tools;
const int count_times = 10000;
+DSN_DEFINE_int32(components.simple_perf_counter,
+ counter_computation_interval_seconds_for_testing,
+ 3,
+ "period");
+
static void adder_function(perf_counter_ptr pc, int id, const std::vector<int>
&vec)
{
for (int i = id; i < 10000; i += 10)
@@ -90,9 +96,6 @@ TEST(perf_counter, perf_counter_atomic)
vec[i] = rand() % 100;
}
std::vector<int> gen_numbers{1, 5, 1043};
- int sleep_interval = (int)dsn_config_get_value_uint64(
- "components.simple_perf_counter",
"counter_computation_interval_seconds", 3, "period");
-
perf_counter_ptr counter = new perf_counter_number_atomic(
"", "", "", dsn_perf_counter_type_t::COUNTER_TYPE_NUMBER, "");
perf_counter_inc_dec(counter);
@@ -113,11 +116,11 @@ TEST(perf_counter, perf_counter_atomic)
counter = new perf_counter_number_percentile_atomic(
"", "", "", dsn_perf_counter_type_t::COUNTER_TYPE_NUMBER_PERCENTILES,
"");
- std::this_thread::sleep_for(std::chrono::seconds(sleep_interval));
+ std::this_thread::sleep_for(
+
std::chrono::seconds(FLAGS_counter_computation_interval_seconds_for_testing));
for (auto &count : gen_numbers) {
for (unsigned int i = 0; i != count; ++i)
counter->set(rand() % 10000);
- // std::this_thread::sleep_for(std::chrono::seconds(sleep_interval));
for (int i = 0; i != COUNTER_PERCENTILE_COUNT; ++i)
LOG_INFO("{}",
counter->get_percentile((dsn_perf_counter_percentile_type_t)i));
}
diff --git a/src/runtime/env.sim.cpp b/src/runtime/env.sim.cpp
index fb2a8dddc..9534227d1 100644
--- a/src/runtime/env.sim.cpp
+++ b/src/runtime/env.sim.cpp
@@ -37,16 +37,20 @@
#include "scheduler.h"
#include "utils/rand.h"
+#include "utils/flags.h"
namespace dsn {
namespace tools {
-/*static*/ int sim_env_provider::_seed;
+DSN_DEFINE_int32(tools.simulator,
+ random_seed,
+ 0,
+ "random seed for the simulator, 0 for random seed");
void sim_env_provider::on_worker_start(task_worker *worker)
{
rand::reseed_thread_local_rng(
- (_seed + worker->index() + worker->index() *
worker->pool_spec().pool_code) ^
+ (FLAGS_random_seed + worker->index() + worker->index() *
worker->pool_spec().pool_code) ^
worker->index());
}
@@ -54,16 +58,11 @@ sim_env_provider::sim_env_provider(env_provider
*inner_provider) : env_provider(
{
task_worker::on_start.put_front(on_worker_start,
"sim_env_provider::on_worker_start");
- _seed =
- (int)dsn_config_get_value_uint64("tools.simulator",
- "random_seed",
- 0,
- "random seed for the simulator, 0 for
random random seed");
- if (_seed == 0) {
- _seed = std::random_device{}();
+ if (FLAGS_random_seed == 0) {
+ FLAGS_random_seed = std::random_device{}();
}
- LOG_ERROR("simulation.random seed for this round is {}", _seed);
+ LOG_INFO("simulation.random seed for this round is {}", FLAGS_random_seed);
}
} // namespace tools
diff --git a/src/runtime/env.sim.h b/src/runtime/env.sim.h
index c0ca2abf8..5a79a29c7 100644
--- a/src/runtime/env.sim.h
+++ b/src/runtime/env.sim.h
@@ -44,11 +44,9 @@ class sim_env_provider : public env_provider
{
public:
sim_env_provider(env_provider *inner_provider);
- static int seed() { return _seed; }
private:
static void on_worker_start(task_worker *worker);
- static int _seed;
};
}
} // end namespace
diff --git a/src/runtime/rpc/asio_net_provider.cpp
b/src/runtime/rpc/asio_net_provider.cpp
index bf583800d..a43c44c34 100644
--- a/src/runtime/rpc/asio_net_provider.cpp
+++ b/src/runtime/rpc/asio_net_provider.cpp
@@ -322,12 +322,6 @@ void asio_udp_provider::do_receive()
error_code asio_udp_provider::start(rpc_channel channel, int port, bool
client_only)
{
_is_client = client_only;
- int io_service_worker_count =
- (int)dsn_config_get_value_uint64("network",
- "io_service_worker_count",
- 1,
- "thread number for io service (timer
and boost network)");
-
CHECK_EQ(channel, RPC_CHANNEL_UDP);
if (client_only) {
@@ -381,7 +375,7 @@ error_code asio_udp_provider::start(rpc_channel channel,
int port, bool client_o
}
}
- for (int i = 0; i < io_service_worker_count; i++) {
+ for (int i = 0; i < FLAGS_io_service_worker_count; i++) {
_workers.push_back(std::make_shared<std::thread>([this, i]() {
task::set_tls_dsn_context(node(), nullptr);
diff --git a/src/runtime/rpc/network.cpp b/src/runtime/rpc/network.cpp
index 3c4435054..08c584af9 100644
--- a/src/runtime/rpc/network.cpp
+++ b/src/runtime/rpc/network.cpp
@@ -506,12 +506,6 @@ network::network(rpc_engine *srv, network *inner_provider)
{
_message_buffer_block_size = 1024 * 64;
_max_buffer_block_count_per_send = 64; // TODO: windows, how about the
other platforms?
- _send_queue_threshold =
- (int)dsn_config_get_value_uint64("network",
- "send_queue_threshold",
- 4 * 1024,
- "send queue size above which
throttling is applied");
-
_unknown_msg_header_format = network_header_format::from_string(
dsn_config_get_value_string(
"network",
diff --git a/src/runtime/rpc/network.h b/src/runtime/rpc/network.h
index a9fdbd359..63c04135c 100644
--- a/src/runtime/rpc/network.h
+++ b/src/runtime/rpc/network.h
@@ -137,7 +137,6 @@ protected:
network_header_format _unknown_msg_header_format; // default is
NET_HDR_INVALID
int _message_buffer_block_size;
int _max_buffer_block_count_per_send;
- int _send_queue_threshold;
private:
friend class rpc_engine;
diff --git a/src/runtime/simulator.cpp b/src/runtime/simulator.cpp
index a1cf97ec7..b8cb25785 100644
--- a/src/runtime/simulator.cpp
+++ b/src/runtime/simulator.cpp
@@ -40,10 +40,13 @@
#include "env.sim.h"
#include "runtime/task/task_engine.sim.h"
#include "sim_clock.h"
+#include "utils/flags.h"
namespace dsn {
namespace tools {
+DSN_DECLARE_int32(random_seed);
+
/*static*/
void simulator::register_checker(const std::string &name, checker::factory f)
{
@@ -121,8 +124,7 @@ void simulator::install(service_spec &spec)
void simulator::on_system_exit(sys_exit_type st)
{
- LOG_ERROR("system exits, you can replay this process using random seed {}",
- sim_env_provider::seed());
+ LOG_INFO("system exits, you can replay this process using random seed {}",
FLAGS_random_seed);
}
void simulator::run()
diff --git a/src/server/available_detector.cpp
b/src/server/available_detector.cpp
index c4c242428..02906e7b7 100644
--- a/src/server/available_detector.cpp
+++ b/src/server/available_detector.cpp
@@ -27,12 +27,18 @@
#include "base/pegasus_key_schema.h"
#include "result_writer.h"
+#include "utils/flags.h"
namespace pegasus {
namespace server {
DEFINE_TASK_CODE(LPC_DETECT_AVAILABLE, TASK_PRIORITY_COMMON,
::dsn::THREAD_POOL_DEFAULT)
+DSN_DEFINE_int32(pegasus.collector,
+ available_detect_alert_fail_count,
+ 30,
+ "available detect alert fail count");
+
available_detector::available_detector()
: _client(nullptr),
_ddl_client(nullptr),
@@ -68,10 +74,6 @@ available_detector::available_detector()
"available_detect_interval_seconds",
3, // default value 3s
"detect interval seconds");
- _alert_fail_count =
(int32_t)dsn_config_get_value_uint64("pegasus.collector",
-
"available_detect_alert_fail_count",
- 30,
- "available detect
alert fail count");
_detect_timeout =
(uint32_t)dsn_config_get_value_uint64("pegasus.collector",
"available_detect_timeout",
@@ -350,9 +352,9 @@ void available_detector::on_detect(int32_t idx)
void available_detector::check_and_send_email(std::atomic<int> *cnt, int32_t
idx)
{
bool send_email = false;
- if (cnt->load() >= _alert_fail_count) {
+ if (cnt->load() >= FLAGS_available_detect_alert_fail_count) {
::dsn::utils::auto_lock<::dsn::utils::ex_lock_nr> l(_alert_lock);
- if (cnt->load() >= _alert_fail_count) {
+ if (cnt->load() >= FLAGS_available_detect_alert_fail_count) {
for (auto i = 0; i < _partition_count; i++) {
std::atomic<int> &c = (*_fail_count[i]);
c.store(0);
diff --git a/src/server/available_detector.h b/src/server/available_detector.h
index 48dae71a4..9af26f757 100644
--- a/src/server/available_detector.h
+++ b/src/server/available_detector.h
@@ -62,7 +62,6 @@ private:
std::shared_ptr<replication_ddl_client> _ddl_client;
std::vector<dsn::rpc_address> _meta_list;
uint32_t _detect_interval_seconds;
- int32_t _alert_fail_count;
::dsn::utils::ex_lock_nr _alert_lock;
// for record partition fail times.
std::vector<std::shared_ptr<std::atomic<int32_t>>> _fail_count;
diff --git a/src/server/pegasus_manual_compact_service.cpp
b/src/server/pegasus_manual_compact_service.cpp
index 2429e85c2..fd757fc53 100644
--- a/src/server/pegasus_manual_compact_service.cpp
+++ b/src/server/pegasus_manual_compact_service.cpp
@@ -33,6 +33,12 @@ namespace server {
DEFINE_TASK_CODE(LPC_MANUAL_COMPACT, TASK_PRIORITY_COMMON, THREAD_POOL_COMPACT)
+DSN_DEFINE_int32(pegasus.server,
+ manual_compact_min_interval_seconds,
+ 0,
+ "minimal interval time in seconds to start a new manual
compaction, <= 0 "
+ "means no interval limit");
+
pegasus_manual_compact_service::pegasus_manual_compact_service(pegasus_server_impl
*app)
: replica_base(*app),
_app(app),
@@ -43,13 +49,6 @@
pegasus_manual_compact_service::pegasus_manual_compact_service(pegasus_server_im
_manual_compact_last_finish_time_ms(0),
_manual_compact_last_time_used_ms(0)
{
- _manual_compact_min_interval_seconds =
(int32_t)dsn_config_get_value_uint64(
- "pegasus.server",
- "manual_compact_min_interval_seconds",
- 0,
- "minimal interval time in seconds to start a new manual compaction, "
- "<= 0 means no interval limit");
-
_pfc_manual_compact_enqueue_count.init_app_counter("app.pegasus",
"manual.compact.enqueue.count",
COUNTER_TYPE_NUMBER,
@@ -260,10 +259,10 @@ bool
pegasus_manual_compact_service::check_manual_compact_state()
{
uint64_t not_enqueue = 0;
uint64_t now = now_timestamp();
- if (_manual_compact_min_interval_seconds <= 0 || // no interval limit
+ if (FLAGS_manual_compact_min_interval_seconds <= 0 || // no interval limit
_manual_compact_last_finish_time_ms.load() == 0 || // has not
compacted yet
now - _manual_compact_last_finish_time_ms.load() >
- (uint64_t)_manual_compact_min_interval_seconds * 1000) { //
interval past
+ (uint64_t)FLAGS_manual_compact_min_interval_seconds * 1000) { //
interval past
// when _manual_compact_enqueue_time_ms is `not_enqueue`(which is 0),
return true to allow a
// compact task enqueue, and update the value to `now`,
// otherwise, return false to not allow, and keep the old value.
diff --git a/src/server/pegasus_manual_compact_service.h
b/src/server/pegasus_manual_compact_service.h
index 31c02d299..9fd016b7f 100644
--- a/src/server/pegasus_manual_compact_service.h
+++ b/src/server/pegasus_manual_compact_service.h
@@ -87,7 +87,6 @@ private:
#ifdef PEGASUS_UNIT_TEST
uint64_t _mock_now_timestamp = 0;
#endif
- int32_t _manual_compact_min_interval_seconds;
// manual compact state
std::atomic<bool> _disabled;
diff --git a/src/server/test/manual_compact_service_test.cpp
b/src/server/test/manual_compact_service_test.cpp
index 928289778..5a43f8bea 100644
--- a/src/server/test/manual_compact_service_test.cpp
+++ b/src/server/test/manual_compact_service_test.cpp
@@ -25,6 +25,8 @@
namespace pegasus {
namespace server {
+DSN_DECLARE_int32(manual_compact_min_interval_seconds);
+
class manual_compact_service_test : public pegasus_server_test_base
{
public:
@@ -93,11 +95,6 @@ public:
manual_compact_svc->_manual_compact_last_time_used_ms.store(finish -
start);
manual_compact_svc->_manual_compact_enqueue_time_ms.store(0);
}
-
- void set_manual_compact_interval(int sec)
- {
- manual_compact_svc->_manual_compact_min_interval_seconds = sec;
- }
};
TEST_F(manual_compact_service_test, check_compact_disabled)
@@ -275,7 +272,7 @@ TEST_F(manual_compact_service_test,
extract_manual_compact_opts)
TEST_F(manual_compact_service_test, check_manual_compact_state_0_interval)
{
- set_manual_compact_interval(0);
+ FLAGS_manual_compact_min_interval_seconds = 0;
uint64_t first_time = 1500000000;
set_mock_now(first_time);
@@ -291,7 +288,7 @@ TEST_F(manual_compact_service_test,
check_manual_compact_state_0_interval)
TEST_F(manual_compact_service_test, check_manual_compact_state_1h_interval)
{
- set_manual_compact_interval(3600);
+ FLAGS_manual_compact_min_interval_seconds = 3600;
uint64_t first_time = 1500000000;
set_mock_now(first_time);
diff --git a/src/test/kill_test/process_kill_testor.cpp
b/src/test/kill_test/process_kill_testor.cpp
index af24dbbb2..7c6b9c6eb 100644
--- a/src/test/kill_test/process_kill_testor.cpp
+++ b/src/test/kill_test/process_kill_testor.cpp
@@ -38,9 +38,34 @@
#include "killer_handler.h"
#include "killer_handler_shell.h"
#include "process_kill_testor.h"
+#include "utils/flags.h"
namespace pegasus {
namespace test {
+
+DSN_DEFINE_int32(section, total_meta_count, 0, "total meta count");
+DSN_DEFINE_int32(section, total_replica_count, 0, "total replica count");
+DSN_DEFINE_int32(section, total_zookeeper_count, 0, "total zookeeper count");
+DSN_DEFINE_int32(section,
+ kill_replica_max_count,
+ FLAGS_total_replica_count,
+ "replica killed max count");
+DSN_DEFINE_int32(section, kill_meta_max_count, FLAGS_total_meta_count, "meta
killed max count");
+DSN_DEFINE_int32(section,
+ kill_zookeeper_max_count,
+ FLAGS_total_zookeeper_count,
+ "zookeeper killed max count");
+DSN_DEFINE_group_validator(kill_test_role_count, [](std::string &message) ->
bool {
+ if (FLAGS_total_meta_count == 0 && FLAGS_total_replica_count == 0 &&
+ FLAGS_total_zookeeper_count == 0) {
+ message = fmt::format("[section].total_meta_count, total_replica_count
and "
+ "total_zookeeper_count should not all be 0.");
+ return false;
+ }
+
+ return true;
+});
+
process_kill_testor::process_kill_testor(const char *config_file) :
kill_testor(config_file)
{
register_kill_handlers();
@@ -59,24 +84,6 @@ process_kill_testor::process_kill_testor(const char
*config_file) : kill_testor(
_job_index_to_kill.resize(JOB_LENGTH);
_sleep_time_before_recover_seconds = (uint32_t)dsn_config_get_value_uint64(
section, "sleep_time_before_recover_seconds", 30, "sleep time before
recover seconds");
-
- _total_meta_count =
- (int32_t)dsn_config_get_value_uint64(section, "total_meta_count", 0,
"total meta count");
- _total_replica_count = (int32_t)dsn_config_get_value_uint64(
- section, "total_replica_count", 0, "total replica count");
- _total_zookeeper_count = (int32_t)dsn_config_get_value_uint64(
- section, "total_zookeeper_count", 0, "total zookeeper count");
-
- if (_total_meta_count == 0 && _total_replica_count == 0 &&
_total_zookeeper_count == 0) {
- CHECK(false, "total number of meta/replica/zookeeper is 0");
- }
-
- _kill_replica_max_count = (int32_t)dsn_config_get_value_uint64(
- section, "kill_replica_max_count", _total_replica_count, "replica
killed max count");
- _kill_meta_max_count = (int32_t)dsn_config_get_value_uint64(
- section, "kill_meta_max_count", _total_meta_count, "meta killed max
count");
- _kill_zk_max_count = (int32_t)dsn_config_get_value_uint64(
- section, "kill_zookeeper_max_count", _total_zookeeper_count,
"zookeeper killed max count");
}
process_kill_testor::~process_kill_testor() {}
@@ -117,20 +124,20 @@ void process_kill_testor::run()
}
if (kill_round == 0) {
- LOG_INFO("Number of meta-server: {}", _total_meta_count);
- LOG_INFO("Number of replica-server: {}", _total_replica_count);
- LOG_INFO("Number of zookeeper: {}", _total_zookeeper_count);
+ LOG_INFO("Number of meta-server: {}", FLAGS_total_meta_count);
+ LOG_INFO("Number of replica-server: {}", FLAGS_total_replica_count);
+ LOG_INFO("Number of zookeeper: {}", FLAGS_total_zookeeper_count);
}
kill_round += 1;
int meta_cnt = 0;
int replica_cnt = 0;
int zk_cnt = 0;
while ((meta_cnt == 0 && replica_cnt == 0 && zk_cnt == 0) ||
- (meta_cnt == _total_meta_count && replica_cnt ==
_total_replica_count &&
- zk_cnt == _total_zookeeper_count)) {
- meta_cnt = generate_one_number(0, _kill_meta_max_count);
- replica_cnt = generate_one_number(0, _kill_replica_max_count);
- zk_cnt = generate_one_number(0, _kill_zk_max_count);
+ (meta_cnt == FLAGS_total_meta_count && replica_cnt ==
FLAGS_total_replica_count &&
+ zk_cnt == FLAGS_total_zookeeper_count)) {
+ meta_cnt = generate_one_number(0, FLAGS_kill_meta_max_count);
+ replica_cnt = generate_one_number(0, FLAGS_kill_replica_max_count);
+ zk_cnt = generate_one_number(0, FLAGS_kill_zookeeper_max_count);
}
LOG_INFO("************************");
LOG_INFO("Round [{}]", kill_round);
@@ -157,7 +164,7 @@ bool process_kill_testor::kill(int meta_cnt, int
replica_cnt, int zookeeper_cnt)
{
std::vector<int> kill_counts = {meta_cnt, replica_cnt, zookeeper_cnt};
std::vector<int> total_count = {
- _total_meta_count, _total_replica_count, _total_zookeeper_count};
+ FLAGS_total_meta_count, FLAGS_total_replica_count,
FLAGS_total_zookeeper_count};
std::vector<int> random_idxs;
generate_random(random_idxs, JOB_LENGTH, META, ZOOKEEPER);
for (auto id : random_idxs) {
@@ -230,14 +237,14 @@ bool process_kill_testor::check_coredump()
bool has_core = false;
// make sure all generated core are logged
- for (int i = 1; i <= _total_meta_count; ++i) {
+ for (int i = 1; i <= FLAGS_total_meta_count; ++i) {
if (_killer_handler->has_meta_dumped_core(i)) {
LOG_ERROR("meta server {} generate core dump", i);
has_core = true;
}
}
- for (int i = 1; i <= _total_replica_count; ++i) {
+ for (int i = 1; i <= FLAGS_total_replica_count; ++i) {
if (_killer_handler->has_replica_dumped_core(i)) {
LOG_ERROR("replica server {} generate core dump", i);
has_core = true;
diff --git a/src/test/kill_test/process_kill_testor.h
b/src/test/kill_test/process_kill_testor.h
index 32c35f09a..39d26bf4f 100644
--- a/src/test/kill_test/process_kill_testor.h
+++ b/src/test/kill_test/process_kill_testor.h
@@ -52,14 +52,6 @@ private:
bool verifier_process_alive();
std::shared_ptr<killer_handler> _killer_handler;
- int32_t _total_meta_count;
- int32_t _total_replica_count;
- int32_t _total_zookeeper_count;
-
- int32_t _kill_replica_max_count;
- int32_t _kill_meta_max_count;
- int32_t _kill_zk_max_count;
-
std::vector<job_type> _job_types;
int64_t kill_round;
diff --git a/src/test/pressure_test/main.cpp b/src/test/pressure_test/main.cpp
index 451b9c9c5..2f8b926bd 100644
--- a/src/test/pressure_test/main.cpp
+++ b/src/test/pressure_test/main.cpp
@@ -27,17 +27,19 @@
#include "utils/fmt_logging.h"
#include "utils/rand.h"
#include "runtime/task/async_calls.h"
-
#include "pegasus/client.h"
+#include "utils/flags.h"
using namespace std;
using namespace ::pegasus;
DEFINE_TASK_CODE(LPC_DEFAUT_TASK, TASK_PRIORITY_COMMON,
dsn::THREAD_POOL_DEFAULT)
-static int32_t hashkey_len;
-static int32_t sortkey_len;
-static int32_t value_len;
+DSN_DEFINE_int32(pressureclient, qps, 0, "qps of pressure client");
+DSN_DEFINE_int32(pressureclient, hashkey_len, 64, "hashkey length");
+DSN_DEFINE_int32(pressureclient, sortkey_len, 64, "sortkey length");
+DSN_DEFINE_int32(pressureclient, value_len, 64, "value length");
+DSN_DEFINE_validator(qps, [](int32_t value) -> bool { return value > 0; });
// generate hashkey/sortkey between [0, ****key_limit]
static int64_t hashkey_limit;
@@ -47,7 +49,6 @@ static int64_t sortkey_limit;
static pegasus_client *pg_client = nullptr;
static string cluster_name;
static string app_name;
-static int32_t qps = 0;
static string op_name; // set/get/scan/del
// fill string in prefix, until with size(len)
std::string fill_string(const std::string &str, int len)
@@ -59,20 +60,20 @@ std::string fill_string(const std::string &str, int len)
std::string get_hashkey()
{
std::string key = to_string(dsn::rand::next_u64(0, hashkey_limit));
- if (key.size() >= hashkey_len) {
+ if (key.size() >= FLAGS_hashkey_len) {
return key;
} else {
- return fill_string(key, hashkey_len);
+ return fill_string(key, FLAGS_hashkey_len);
}
}
std::string get_sortkey()
{
std::string key = to_string(dsn::rand::next_u64(0, sortkey_limit));
- if (key.size() >= sortkey_len) {
+ if (key.size() >= FLAGS_sortkey_len) {
return key;
} else {
- return fill_string(key, sortkey_len);
+ return fill_string(key, FLAGS_sortkey_len);
}
}
@@ -101,7 +102,7 @@ std::string get_value(const std::string &hashkey, const
std::string &sortkey, in
bool verify(const std::string &hashkey, const std::string &sortkey, const
std::string &value)
{
- return (value == get_value(hashkey, sortkey, value_len));
+ return (value == get_value(hashkey, sortkey, FLAGS_value_len));
}
void test_set(int32_t qps)
@@ -117,7 +118,7 @@ void test_set(int32_t qps)
while (cnt > 0) {
std::string hashkey = get_hashkey();
std::string sortkey = get_sortkey();
- std::string value = get_value(hashkey, sortkey, value_len);
+ std::string value = get_value(hashkey, sortkey,
FLAGS_value_len);
pg_client->async_set(hashkey, sortkey, value);
cnt -= 1;
}
@@ -150,7 +151,7 @@ void test_get(int32_t qps)
"hashkey({}) - sortkey({}) - value({}), but
value({})",
hashkey,
sortkey,
- get_value(hashkey, sortkey, value_len),
+ get_value(hashkey, sortkey, FLAGS_value_len),
val);
} else if (ec == PERR_NOT_FOUND) {
// don't output info
@@ -227,9 +228,6 @@ int main(int argc, const char **argv)
app_name = dsn_config_get_value_string("pressureclient", "app_name",
"temp", "app name");
- qps =
- (int32_t)dsn_config_get_value_uint64("pressureclient", "qps", 0, "qps
of pressure client");
-
op_name = dsn_config_get_value_string("pressureclient", "operation_name",
"", "operation name");
hashkey_limit =
@@ -238,27 +236,17 @@ int main(int argc, const char **argv)
sortkey_limit =
(int64_t)dsn_config_get_value_uint64("pressureclient",
"sortkey_limit", 0, "sortkey limit");
- hashkey_len =
- (int32_t)dsn_config_get_value_uint64("pressureclient", "hashkey_len",
64, "hashkey length");
-
- sortkey_len =
- (int32_t)dsn_config_get_value_uint64("pressureclient", "sortkey_len",
64, "sortkey length");
-
- value_len =
- (int32_t)dsn_config_get_value_uint64("pressureclient", "value_len",
64, "value length");
-
- CHECK_GT(qps, 0);
CHECK(!op_name.empty(), "must assign operation name");
- LOG_INFO("pressureclient {} qps = {}", op_name, qps);
+ LOG_INFO("pressureclient {} qps = {}", op_name, FLAGS_qps);
pg_client = pegasus_client_factory::get_client(cluster_name.c_str(),
app_name.c_str());
CHECK_NOTNULL(pg_client, "initialize pg_client failed");
auto it = _all_funcs.find(op_name);
if (it != _all_funcs.end()) {
- LOG_INFO("start pressureclient with {} qps({})", op_name, qps);
- it->second(qps);
+ LOG_INFO("start pressureclient with {} qps({})", op_name, FLAGS_qps);
+ it->second(FLAGS_qps);
} else {
CHECK(false, "Unknown operation name({})", op_name);
}
diff --git a/src/zookeeper/distributed_lock_service_zookeeper.cpp
b/src/zookeeper/distributed_lock_service_zookeeper.cpp
index 2f5c6250b..3cdfc8ccb 100644
--- a/src/zookeeper/distributed_lock_service_zookeeper.cpp
+++ b/src/zookeeper/distributed_lock_service_zookeeper.cpp
@@ -46,10 +46,13 @@
#include "lock_types.h"
#include "zookeeper_error.h"
+#include "utils/flags.h"
namespace dsn {
namespace dist {
+DSN_DECLARE_int32(timeout_ms);
+
std::string distributed_lock_service_zookeeper::LOCK_NODE_PREFIX = "LOCKNODE";
distributed_lock_service_zookeeper::distributed_lock_service_zookeeper() :
ref_counter()
@@ -102,7 +105,7 @@ error_code
distributed_lock_service_zookeeper::initialize(const std::vector<std:
lock_srv_ptr(this),
std::placeholders::_1));
if (_zoo_state != ZOO_CONNECTED_STATE) {
- _waiting_attach.wait_for(zookeeper_session_mgr::instance().timeout());
+ _waiting_attach.wait_for(FLAGS_timeout_ms);
if (_zoo_state != ZOO_CONNECTED_STATE) {
LOG_WARNING(
"attach to zookeeper session timeout, distributed lock service
initialized failed");
diff --git a/src/zookeeper/zookeeper_session.cpp
b/src/zookeeper/zookeeper_session.cpp
index 370c69fd7..826a964a6 100644
--- a/src/zookeeper/zookeeper_session.cpp
+++ b/src/zookeeper/zookeeper_session.cpp
@@ -52,6 +52,12 @@ DSN_DEFINE_string(security,
namespace dsn {
namespace dist {
+// TODO(yingchun): to keep compatibility, the global name is FLAGS_timeout_ms.
The name is not very
+// suitable, maybe improve the macro to us another global name.
+DSN_DEFINE_int32(zookeeper,
+ timeout_ms,
+ 30000,
+ "The timeout of accessing ZooKeeper, in milliseconds");
zookeeper_session::zoo_atomic_packet::zoo_atomic_packet(unsigned int size)
{
@@ -160,7 +166,7 @@ int zookeeper_session::attach(void *callback_owner, const
state_callback &cb)
sasl_params.mechlist = "GSSAPI";
_handle =
zookeeper_init_sasl(zookeeper_session_mgr::instance().zoo_hosts(),
global_watcher,
-
zookeeper_session_mgr::instance().timeout(),
+ FLAGS_timeout_ms,
nullptr,
this,
0,
@@ -169,7 +175,7 @@ int zookeeper_session::attach(void *callback_owner, const
state_callback &cb)
} else {
_handle =
zookeeper_init(zookeeper_session_mgr::instance().zoo_hosts(),
global_watcher,
-
zookeeper_session_mgr::instance().timeout(),
+ FLAGS_timeout_ms,
nullptr,
this,
0);
diff --git a/src/zookeeper/zookeeper_session_mgr.cpp
b/src/zookeeper/zookeeper_session_mgr.cpp
index ffad1f2ee..5eb86b53e 100644
--- a/src/zookeeper/zookeeper_session_mgr.cpp
+++ b/src/zookeeper/zookeeper_session_mgr.cpp
@@ -45,8 +45,6 @@ namespace dist {
zookeeper_session_mgr::zookeeper_session_mgr()
{
_zoo_hosts = dsn_config_get_value_string("zookeeper", "hosts_list", "",
"zookeeper_hosts");
- _timeout_ms = dsn_config_get_value_uint64(
- "zookeeper", "timeout_ms", 30000, "zookeeper_timeout_milliseconds");
_zoo_logfile = dsn_config_get_value_string("zookeeper", "logfile", "",
"zookeeper logfile");
FILE *fp = fopen(_zoo_logfile.c_str(), "a");
diff --git a/src/zookeeper/zookeeper_session_mgr.h
b/src/zookeeper/zookeeper_session_mgr.h
index de9d86f0a..e3959547e 100644
--- a/src/zookeeper/zookeeper_session_mgr.h
+++ b/src/zookeeper/zookeeper_session_mgr.h
@@ -60,7 +60,6 @@ class zookeeper_session_mgr : public
utils::singleton<zookeeper_session_mgr>
public:
zookeeper_session *get_session(const service_app_info &info);
const char *zoo_hosts() const { return _zoo_hosts.c_str(); }
- int timeout() const { return _timeout_ms; }
const char *zoo_logfile() const { return _zoo_logfile.c_str(); }
private:
@@ -69,7 +68,6 @@ private:
utils::ex_lock_nr _store_lock;
std::string _zoo_hosts;
- int _timeout_ms;
std::string _zoo_logfile;
friend class utils::singleton<zookeeper_session_mgr>;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
