This is an automated email from the ASF dual-hosted git repository. yuchenhe pushed a commit to branch v2.1 in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
commit a83343d4a54d835d41180e5a6e1ca93591a76c8b Author: Zhang Yifan <[email protected]> AuthorDate: Mon Sep 7 09:50:28 2020 +0800 fix: Use default options to open db when latest option file has incompatible db options (#593) --- src/server/pegasus_server_impl.cpp | 78 ++++++++++++++++++++++++++++---------- src/server/pegasus_server_impl.h | 3 ++ 2 files changed, 61 insertions(+), 20 deletions(-) diff --git a/src/server/pegasus_server_impl.cpp b/src/server/pegasus_server_impl.cpp index b61bf31..1889e98 100644 --- a/src/server/pegasus_server_impl.cpp +++ b/src/server/pegasus_server_impl.cpp @@ -1330,9 +1330,18 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache // Here we create a `tmp_data_cf_opts` because we don't want to modify `_data_cf_opts`, which // will be used elsewhere. rocksdb::ColumnFamilyOptions tmp_data_cf_opts = _data_cf_opts; + bool has_incompatible_db_options = false; if (db_exist) { + // When DB exists, meta CF and data CF must be present. bool missing_meta_cf = true; bool missing_data_cf = true; + if (check_column_families(path, &missing_meta_cf, &missing_data_cf) != ::dsn::ERR_OK) { + derror_replica("check column families failed"); + return ::dsn::ERR_LOCAL_APP_FAILURE; + } + dassert_replica(!missing_meta_cf, "You must upgrade Pegasus server from 2.0"); + dassert_replica(!missing_data_cf, "Missing data column family"); + // Load latest options from option file stored in the db directory. rocksdb::DBOptions loaded_db_opt; std::vector<rocksdb::ColumnFamilyDescriptor> loaded_cf_descs; @@ -1344,28 +1353,31 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache &loaded_cf_descs, /*ignore_unknown_options=*/true); if (!status.ok()) { - derror_replica("load latest option file failed."); - return ::dsn::ERR_LOCAL_APP_FAILURE; - } - for (int i = 0; i < loaded_cf_descs.size(); ++i) { - if (loaded_cf_descs[i].name == META_COLUMN_FAMILY_NAME) { - missing_meta_cf = false; - } else if (loaded_cf_descs[i].name == DATA_COLUMN_FAMILY_NAME) { - missing_data_cf = false; - loaded_data_cf_opts = loaded_cf_descs[i].options; - } else { - derror_replica("unknown column family name."); + // Here we ignore an invalid argument error related to `pegasus_data_version` and + // `pegasus_data` options, which were used in old version rocksdbs (before 2.1.0). + if (status.code() != rocksdb::Status::kInvalidArgument || + status.ToString().find("pegasus_data") == std::string::npos) { + derror_replica("load latest option file failed: {}.", status.ToString()); return ::dsn::ERR_LOCAL_APP_FAILURE; } + has_incompatible_db_options = true; + dwarn_replica("The latest option file has incompatible db options: {}, use default " + "options to open db.", + status.ToString()); + } + + if (!has_incompatible_db_options) { + for (int i = 0; i < loaded_cf_descs.size(); ++i) { + if (loaded_cf_descs[i].name == DATA_COLUMN_FAMILY_NAME) { + loaded_data_cf_opts = loaded_cf_descs[i].options; + } + } + // Reset usage scenario related options according to loaded_data_cf_opts. + // We don't use `loaded_data_cf_opts` directly because pointer-typed options will + // only be initialized with default values when calling 'LoadLatestOptions', see + // 'rocksdb/utilities/options_util.h'. + reset_usage_scenario_options(loaded_data_cf_opts, &tmp_data_cf_opts); } - // When DB exists, meta CF and data CF must be present. - dassert_replica(!missing_meta_cf, "You must upgrade Pegasus server from 2.0"); - dassert_replica(!missing_data_cf, "Missing data column family"); - // Reset usage scenario related options according to loaded_data_cf_opts. - // We don't use `loaded_data_cf_opts` directly because pointer-typed options will only be - // initialized with default values when calling 'LoadLatestOptions', see - // 'rocksdb/utilities/options_util.h'. - reset_usage_scenario_options(loaded_data_cf_opts, &tmp_data_cf_opts); } else { // When create new DB, we have to create a new column family to store meta data (meta column // family). @@ -1376,7 +1388,7 @@ void pegasus_server_impl::on_clear_scanner(const int64_t &args) { _context_cache {{DATA_COLUMN_FAMILY_NAME, tmp_data_cf_opts}, {META_COLUMN_FAMILY_NAME, _meta_cf_opts}}); auto s = rocksdb::CheckOptionsCompatibility( path, rocksdb::Env::Default(), _db_opts, column_families, /*ignore_unknown_options=*/true); - if (!s.ok() && !s.IsNotFound()) { + if (!s.ok() && !s.IsNotFound() && !has_incompatible_db_options) { derror_replica("rocksdb::CheckOptionsCompatibility failed, error = {}", s.ToString()); return ::dsn::ERR_LOCAL_APP_FAILURE; } @@ -2574,6 +2586,32 @@ bool pegasus_server_impl::set_options( } } +::dsn::error_code pegasus_server_impl::check_column_families(const std::string &path, + bool *missing_meta_cf, + bool *missing_data_cf) +{ + *missing_meta_cf = true; + *missing_data_cf = true; + std::vector<std::string> column_families; + auto s = rocksdb::DB::ListColumnFamilies(rocksdb::DBOptions(), path, &column_families); + if (!s.ok()) { + derror_replica("rocksdb::DB::ListColumnFamilies failed, error = {}", s.ToString()); + return ::dsn::ERR_LOCAL_APP_FAILURE; + } + + for (const auto &column_family : column_families) { + if (column_family == META_COLUMN_FAMILY_NAME) { + *missing_meta_cf = false; + } else if (column_family == DATA_COLUMN_FAMILY_NAME) { + *missing_data_cf = false; + } else { + derror_replica("unknown column family name: {}", column_family); + return ::dsn::ERR_LOCAL_APP_FAILURE; + } + } + return ::dsn::ERR_OK; +} + uint64_t pegasus_server_impl::do_manual_compact(const rocksdb::CompactRangeOptions &options) { // wait flush before compact to make all data compacted. diff --git a/src/server/pegasus_server_impl.h b/src/server/pegasus_server_impl.h index 7165f76..009f97f 100644 --- a/src/server/pegasus_server_impl.h +++ b/src/server/pegasus_server_impl.h @@ -310,6 +310,9 @@ private: return false; } + ::dsn::error_code + check_column_families(const std::string &path, bool *missing_meta_cf, bool *miss_data_cf); + void release_db(); void release_db(rocksdb::DB *db, const std::vector<rocksdb::ColumnFamilyHandle *> &handles); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
