This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 16caf01f5 fix(bulk_load): rocksDB parameter allow_ingest_behind was
lost after replica migration (#1651)
16caf01f5 is described below
commit 16caf01f51e8bcb3bb032190cf911bc1acf0e04b
Author: ninsmiracle <[email protected]>
AuthorDate: Wed Nov 29 12:17:56 2023 +0800
fix(bulk_load): rocksDB parameter allow_ingest_behind was lost after
replica migration (#1651)
https://github.com/apache/incubator-pegasus/issues/1650
While reopening a replica with empty envs(currently envs would be empty
ONLY when reopening), put the last allow_ingest_behind value to the current
option.
---
src/server/pegasus_server_impl.cpp | 32 +++++++++++++++++++++++++-------
src/server/pegasus_server_impl.h | 11 +++++++++--
2 files changed, 34 insertions(+), 9 deletions(-)
diff --git a/src/server/pegasus_server_impl.cpp
b/src/server/pegasus_server_impl.cpp
index 11a1d14db..6664fc588 100644
--- a/src/server/pegasus_server_impl.cpp
+++ b/src/server/pegasus_server_impl.cpp
@@ -1667,8 +1667,8 @@ dsn::error_code pegasus_server_impl::start(int argc, char
**argv)
// We don't use `loaded_data_cf_opts` directly because
pointer-typed options will
// only be initialized with default values when calling
'LoadLatestOptions', see
// 'rocksdb/utilities/options_util.h'.
- reset_rocksdb_options(loaded_data_cf_opts, &_table_data_cf_opts);
- _db_opts.allow_ingest_behind = parse_allow_ingest_behind(envs);
+ reset_rocksdb_options(
+ loaded_data_cf_opts, loaded_db_opt, envs,
&_table_data_cf_opts, &_db_opts);
}
} else {
// When create new DB, we have to create a new column family to store
meta data (meta column
@@ -3134,8 +3134,11 @@ bool pegasus_server_impl::set_usage_scenario(const
std::string &usage_scenario)
}
}
-void pegasus_server_impl::reset_rocksdb_options(const
rocksdb::ColumnFamilyOptions &base_opts,
- rocksdb::ColumnFamilyOptions
*target_opts)
+void pegasus_server_impl::reset_rocksdb_options(const
rocksdb::ColumnFamilyOptions &base_cf_opts,
+ const rocksdb::DBOptions
&base_db_opt,
+ const std::map<std::string,
std::string> &envs,
+ rocksdb::ColumnFamilyOptions
*target_cf_opts,
+ rocksdb::DBOptions
*target_db_opt)
{
LOG_INFO_PREFIX("Reset rocksdb envs options");
// Reset rocksdb option includes two aspects:
@@ -3144,11 +3147,13 @@ void pegasus_server_impl::reset_rocksdb_options(const
rocksdb::ColumnFamilyOptio
// ROCKSDB_STATIC_OPTIONS
// aspect 1:
- reset_usage_scenario_options(base_opts, target_opts);
+ reset_usage_scenario_options(base_cf_opts, target_cf_opts);
// aspect 2:
- target_opts->num_levels = base_opts.num_levels;
- target_opts->write_buffer_size = base_opts.write_buffer_size;
+ target_cf_opts->num_levels = base_cf_opts.num_levels;
+ target_cf_opts->write_buffer_size = base_cf_opts.write_buffer_size;
+
+ reset_allow_ingest_behind_option(base_db_opt, envs, target_db_opt);
}
void pegasus_server_impl::reset_usage_scenario_options(
@@ -3169,6 +3174,19 @@ void pegasus_server_impl::reset_usage_scenario_options(
target_opts->max_write_buffer_number = base_opts.max_write_buffer_number;
}
+void pegasus_server_impl::reset_allow_ingest_behind_option(
+ const rocksdb::DBOptions &base_db_opt,
+ const std::map<std::string, std::string> &envs,
+ rocksdb::DBOptions *target_db_opt)
+{
+ if (envs.empty()) {
+ // for reopen db during load balance learning
+ target_db_opt->allow_ingest_behind = base_db_opt.allow_ingest_behind;
+ } else {
+ target_db_opt->allow_ingest_behind = parse_allow_ingest_behind(envs);
+ }
+}
+
void pegasus_server_impl::recalculate_data_cf_options(
const rocksdb::ColumnFamilyOptions &cur_data_cf_opts)
{
diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h
index 02c6421c7..131903085 100644
--- a/src/server/pegasus_server_impl.h
+++ b/src/server/pegasus_server_impl.h
@@ -366,8 +366,15 @@ private:
void reset_usage_scenario_options(const rocksdb::ColumnFamilyOptions
&base_opts,
rocksdb::ColumnFamilyOptions
*target_opts);
- void reset_rocksdb_options(const rocksdb::ColumnFamilyOptions &base_opts,
- rocksdb::ColumnFamilyOptions *target_opts);
+ void reset_allow_ingest_behind_option(const rocksdb::DBOptions
&base_db_opt,
+ const std::map<std::string,
std::string> &envs,
+ rocksdb::DBOptions *target_db_opt);
+
+ void reset_rocksdb_options(const rocksdb::ColumnFamilyOptions
&base_cf_opts,
+ const rocksdb::DBOptions &base_db_opt,
+ const std::map<std::string, std::string> &envs,
+ rocksdb::ColumnFamilyOptions *target_cf_opts,
+ rocksdb::DBOptions *target_db_opt);
// return true if successfully set
bool set_options(const std::unordered_map<std::string, std::string>
&new_options);
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
