--- src/osaf/consensus/Makefile | 18 ++ src/osaf/consensus/keyvalue.cc | 174 +++++++++++++++++++ src/osaf/consensus/keyvalue.h | 57 +++++++ src/osaf/consensus/plugins/etcd.plugin | 220 ++++++++++++++++++++++++ src/osaf/consensus/plugins/sample.plugin | 163 ++++++++++++++++++ src/osaf/consensus/service.cc | 277 +++++++++++++++++++++++++++++++ src/osaf/consensus/service.h | 75 +++++++++ 7 files changed, 984 insertions(+) create mode 100644 src/osaf/consensus/Makefile create mode 100644 src/osaf/consensus/keyvalue.cc create mode 100644 src/osaf/consensus/keyvalue.h create mode 100644 src/osaf/consensus/plugins/etcd.plugin create mode 100644 src/osaf/consensus/plugins/sample.plugin create mode 100644 src/osaf/consensus/service.cc create mode 100644 src/osaf/consensus/service.h
diff --git a/src/osaf/consensus/Makefile b/src/osaf/consensus/Makefile new file mode 100644 index 000000000..a2c8bc9dd --- /dev/null +++ b/src/osaf/consensus/Makefile @@ -0,0 +1,18 @@ +# -*- OpenSAF -*- +# +# (C) Copyright 2018 The OpenSAF Foundation +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY +# or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed +# under the GNU Lesser General Public License Version 2.1, February 1999. +# The complete license can be accessed from the following location: +# http://opensource.org/licenses/lgpl-license.php +# See the Copying file included with the OpenSAF distribution for full +# licensing terms. +# +# Author(s): Ericsson AB +# + +all: + $(MAKE) -C ../.. lib/libconsensus.la diff --git a/src/osaf/consensus/keyvalue.cc b/src/osaf/consensus/keyvalue.cc new file mode 100644 index 000000000..860a8657f --- /dev/null +++ b/src/osaf/consensus/keyvalue.cc @@ -0,0 +1,174 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2018 The OpenSAF Foundation + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ +#include "osaf/consensus/keyvalue.h" +#include "base/logtrace.h" +#include "base/getenv.h" +#include "base/conf.h" + +int KeyValue::Execute(const std::string& command, std::string& output) { + TRACE_ENTER(); + constexpr size_t buf_size = 128; + std::array<char, buf_size> buffer; + FILE* pipe = popen(command.c_str(), "r"); + if (pipe == nullptr) { + return 1; + } + output = ""; + while (feof(pipe) == 0) { + if (fgets(buffer.data(), buf_size, pipe) != nullptr) { + output += buffer.data(); + } + } + const int exit_code = pclose(pipe); + if (output.empty() == false && isspace(output.back()) != 0) { + // remove newline at end of output + output.pop_back(); + } + TRACE("Executed '%s', returning %d", command.c_str(), exit_code); + return exit_code; +} + +SaAisErrorT KeyValue::Get(const std::string& key, std::string& value) { + TRACE_ENTER(); + + const std::string kv_store_cmd = base::GetEnv( + "FMS_KEYVALUE_STORE_PLUGIN_CMD", ""); + const std::string command(kv_store_cmd + " get " + key); + int rc = KeyValue::Execute(command, value); + TRACE("Read '%s'", value.c_str()); + + if (rc == 0) { + return SA_AIS_OK; + } else { + return SA_AIS_ERR_FAILED_OPERATION; + } +} + +SaAisErrorT KeyValue::Set(const std::string& key, const std::string& value) { + TRACE_ENTER(); + + const std::string kv_store_cmd = base::GetEnv( + "FMS_KEYVALUE_STORE_PLUGIN_CMD", ""); + const std::string command(kv_store_cmd + " set " + key + " " + value); + std::string output; + int rc = KeyValue::Execute(command, output); + + if (rc == 0) { + return SA_AIS_OK; + } else { + return SA_AIS_ERR_FAILED_OPERATION; + } +} + +SaAisErrorT KeyValue::Erase(const std::string& key) { + TRACE_ENTER(); + + const std::string kv_store_cmd = base::GetEnv( + "FMS_KEYVALUE_STORE_PLUGIN_CMD", ""); + const std::string command(kv_store_cmd + " erase " + key); + std::string output; + int rc = KeyValue::Execute(command, output); + + if (rc == 0) { + return SA_AIS_OK; + } else { + return SA_AIS_ERR_FAILED_OPERATION; + } +} + +SaAisErrorT KeyValue::Lock(const std::string& owner, + const unsigned int timeout) { + TRACE_ENTER(); + + const std::string kv_store_cmd = base::GetEnv( + "FMS_KEYVALUE_STORE_PLUGIN_CMD", ""); + const std::string command(kv_store_cmd + " lock " + owner + " " + + std::to_string(timeout)); + std::string output; + int rc = KeyValue::Execute(command, output); + + if (rc == 0) { + return SA_AIS_OK; + } else { + return SA_AIS_ERR_TRY_AGAIN; + } +} + +SaAisErrorT KeyValue::Unlock(const std::string& owner) { + TRACE_ENTER(); + + const std::string kv_store_cmd = base::GetEnv( + "FMS_KEYVALUE_STORE_PLUGIN_CMD", ""); + const std::string command(kv_store_cmd + " unlock " + owner); + std::string output; + int rc = Execute(command, output); + + if (rc == 0) { + return SA_AIS_OK; + } else if (rc == 1) { + LOG_ER("Lock is owned by another node"); + return SA_AIS_ERR_INVALID_PARAM; + } else { + return SA_AIS_ERR_TRY_AGAIN; + } +} + +bool KeyValue::IsLockedByThisNode() { + TRACE_ENTER(); + + const std::string kv_store_cmd = base::GetEnv( + "FMS_KEYVALUE_STORE_PLUGIN_CMD", ""); + const std::string command(kv_store_cmd + " lock_owner"); + std::string output; + int rc = KeyValue::Execute(command, output); + + if (rc == 0) { + TRACE("Lock owner is %s", output.c_str()); + if (output.compare(base::Conf::NodeName()) == 0) { + return true; + } + } + + return false; +} + +void threadFunction(const std::string& key, + const ConsensusCallback& callback, + const uint32_t user_defined) { + TRACE_ENTER(); + + const std::string kv_store_cmd = base::GetEnv( + "FMS_KEYVALUE_STORE_PLUGIN_CMD", ""); + const std::string command(kv_store_cmd + " watch " + key); + std::string value; + int rc = KeyValue::Execute(command, value); + TRACE("Read '%s'", value.c_str()); + + if (rc == 0) { + callback(value, user_defined); + } else { + LOG_ER("Failed to watch %s", key.c_str()); + } +} + +void KeyValue::Watch(const std::string& key, + const ConsensusCallback callback, + const uint32_t user_defined) { + std::thread t(threadFunction, key, callback, user_defined); + t.detach(); + return; +} diff --git a/src/osaf/consensus/keyvalue.h b/src/osaf/consensus/keyvalue.h new file mode 100644 index 000000000..55d096a61 --- /dev/null +++ b/src/osaf/consensus/keyvalue.h @@ -0,0 +1,57 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2018 The OpenSAF Foundation + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ +#ifndef OSAF_CONSENSUS_KEYVALUE_H_ +#define OSAF_CONSENSUS_KEYVALUE_H_ + +#include <saAis.h> +#include <string> +#include <functional> +#include <thread> + +typedef std::function<void(const std::string& new_value, + const uint32_t user_defined)> ConsensusCallback; + +class KeyValue { + public: + // Retrieve value of key + static SaAisErrorT Get(const std::string& key, std::string& value); + + // Set key to value + static SaAisErrorT Set(const std::string& key, const std::string& value); + + // Erase key + static SaAisErrorT Erase(const std::string& key); + + // Obtain lock, default timeout is 20 seconds + static SaAisErrorT Lock(const std::string& owner, + const unsigned int timeout = 20); + + // Release lock + static SaAisErrorT Unlock(const std::string& owner); + + // Is locked by this node? + static bool IsLockedByThisNode(); + + // starts a thread to watch key and call callback if values changes + static void Watch(const std::string& key, ConsensusCallback callback, + const uint32_t user_defined); + + // internal use + static int Execute(const std::string& command, std::string& output); +}; + +#endif // OSAF_CONSENSUS_KEYVALUE_H_ diff --git a/src/osaf/consensus/plugins/etcd.plugin b/src/osaf/consensus/plugins/etcd.plugin new file mode 100644 index 000000000..2ace5c73b --- /dev/null +++ b/src/osaf/consensus/plugins/etcd.plugin @@ -0,0 +1,220 @@ +#!/usr/bin/env bash + +readonly keyname="opensaf_consensus_lock" +readonly etcd_timeout="5s" + +# get +# retrieve <value> of <key> from key-value store +# params: +# $1 - <key> +# returns: +# 0 - success, <value> is echoed to stdout +# non-zero - failure +get() { + local -r key=$1 + + if value=$(etcdctl --timeout $etcd_timeout get "$key" 2>&1) + then + echo "$value" + return 0 + else + return 1 + fi +} + +# set +# set <key> to <value> in key-value store +# params: +# $1 - <key> +# $2 - <value> +# returns: +# 0 - success +# non-zero - failure +set() { + local -r key=$1 + local -r value=$2 + + if etcdctl --timeout $etcd_timeout set "$key" "$value" 1>& /dev/null + then + return 0 + else + return 1 + fi +} + +# erase +# erase <key> in key-value store +# params: +# $1 - <key> +# returns: +# 0 - success +# non-zero - failure +erase() { + local -r key=$1 + + if etcdctl --timeout $etcd_timeout rm "$key" 1>& /dev/null + then + return 0 + else + return 1 + fi +} + +# lock +# params: +# $1 - <owner>, owner of the lock is set to this +# $2 - <timeout>, will automatically unlock after <timeout> seconds +# returns: +# 0 - success +# non-zero - failure +lock() { + local -r owner=$1 + local -r timeout=$2 + + #implementation here + if etcdctl --timeout $etcd_timeout mk "$keyname" "$owner" --ttl "$timeout" >& /dev/null + then + return 0 + else + current_owner=$(etcdctl get $keyname) + # see if we already hold the lock + if [ "$current_owner" == "$owner" ]; then + return 0 + fi + return 1 + fi +} + +# get +# retrieve <owner> of lock +# params: +# none +# returns: +# 0 - success, <owner> is echoed to stdout +# non-zero - failure or not locked +lock_owner() { + get $keyname + return $? +} + +# unlock +# params: +# $1 - owner +# $2 - <forced> +# - (optional parameter) +# - if set 'true', will unlock even if lock is not held by node +# - defaults to 'false' +# returns: +# 0 - success +# 1 - the lock is owned by someone else +# 2 or above - other failure +# +unlock() { + local -r owner=$1 + local -r forced=${2:-false} + + if [ "$forced" = false ]; then + # check we own the lock + current_owner=$(etcdctl --timeout $etcd_timeout get $keyname 2>&1) + if [ "$owner" != "$current_owner" ]; then + return 1 + fi + fi + + if etcdctl --timeout $etcd_timeout rm $keyname >& /dev/null + then + return 0 + else + return 2 + fi +} + +# watch +# watch <key> in key-value store +# params: +# $1 - <key> +# returns: +# 0 - success, <new_value> is echoed to stdout +# non-zero - failure +watch() { + local -r key=$1 + + if value=$(etcdctl --timeout $etcd_timeout watch "$key" 2>&1) + then + # if the key is removed, then "PrevNode.Value: <value>" is returned + echo "$value" + return 0 + else + return 1 + fi +} + + +# argument parsing +case "$1" in + get) + if [ "$#" -ne 2 ]; then + echo "Usage: $0 get <key>" + exit 1 + fi + get "$2" + exit $? + ;; + set) + if [ "$#" -ne 3 ]; then + echo "Usage: $0 set <key> <value>" + exit 1 + fi + set "$2" "$3" + exit $? + ;; + erase) + if [ "$#" -ne 2 ]; then + echo "Usage: $0 erase <key>" + exit 1 + fi + erase "$2" + exit $? + ;; + lock) + if [ "$#" -ne 3 ]; then + echo "Usage: $0 lock <owner> <timeout>" + exit 1 + fi + lock "$2" "$3" + exit $? + ;; + lock_owner) + if [ "$#" -ne 1 ]; then + echo "Usage: $0 lock_owner" + exit 1 + fi + lock_owner + exit $? + ;; + unlock) + if [ "$#" -eq 2 ]; then + unlock "$2" + exit $? + elif [ "$#" -eq 3 ] && [ "$3" == "--force" ]; then + unlock "$2" 1 + exit $? + else + echo "Usage: $0 unlock <owner> [--force]" + exit 1 + fi + ;; + watch) + if [ "$#" -ne 2 ]; then + echo "Usage: $0 watch <key>" + exit 1 + fi + watch "$2" + exit $? + ;; + *) + echo "Usage: $0 {get|set|erase|lock|unlock|lock_owner|watch}" + ;; +esac + +exit 1 diff --git a/src/osaf/consensus/plugins/sample.plugin b/src/osaf/consensus/plugins/sample.plugin new file mode 100644 index 000000000..e0563f72e --- /dev/null +++ b/src/osaf/consensus/plugins/sample.plugin @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +readonly keyname="opensaf_consensus_lock" + +# get +# retrieve <value> of <key> from key-value store +# params: +# $1 - <key> +# returns: +# 0 - success, <value> is echoed to stdout +# non-zero - failure +get() { + local -r key=$1 + ... +} + +# set +# set <key> to <value> in key-value store +# params: +# $1 - <key> +# $2 - <value> +# returns: +# 0 - success +# non-zero - failure +set() { + local -r key=$1 + local -r value=$2 + ... +} + +# erase +# erase <key> in key-value store +# params: +# $1 - <key> +# returns: +# 0 - success +# non-zero - failure +erase() { + local -r key=$1 + ... +} + +# lock +# params: +# $1 - <owner>, owner of the lock is set to this +# $2 - <timeout>, will automatically unlock after <timeout> seconds +# returns: +# 0 - success +# non-zero - failure +lock() { + local -r owner=$1 + local -r timeout=$2 + ... +} + +# get +# retrieve <owner> of lock +# params: +# none +# returns: +# 0 - success, <owner> is echoed to stdout +# non-zero - failure or not locked +lock_owner() { + ... +} + +# unlock +# params: +# $1 - owner +# $2 - <forced> +# - (optional parameter) +# - if set 'true', will unlock even if lock is not held by node +# - defaults to 'false' +# returns: +# 0 - success +# 1 - the lock is owned by someone else +# 2 or above - other failure# +unlock() { + local -r owner=$1 + local -r forced=${2:-false} + ... +} + +# watch +# watch <key> in key-value store +# params: +# $1 - <key> +# returns: +# 0 - success, <new_value> is echoed to stdout +# non-zero - failure +watch() { + local -r key=$1 + .. +} + +# argument parsing +case "$1" in + get) + if [ "$#" -ne 2 ]; then + echo "Usage: $0 get <key>" + exit 1 + fi + get "$2" + exit $? + ;; + set) + if [ "$#" -ne 3 ]; then + echo "Usage: $0 set <key> <value>" + exit 1 + fi + set "$2" "$3" + exit $? + ;; + erase) + if [ "$#" -ne 2 ]; then + echo "Usage: $0 erase <key>" + exit 1 + fi + erase "$2" + exit $? + ;; + lock) + if [ "$#" -ne 3 ]; then + echo "Usage: $0 lock <owner> <timeout>" + exit 1 + fi + lock "$2" "$3" + exit $? + ;; + lock_owner) + if [ "$#" -ne 1 ]; then + echo "Usage: $0 lock_owner" + exit 1 + fi + lock_owner + exit $? + ;; + unlock) + if [ "$#" -eq 2 ]; then + unlock "$2" + exit $? + elif [ "$#" -eq 3 ] && [ "$3" == "--force" ]; then + unlock "$2" 1 + exit $? + else + echo "Usage: $0 unlock <owner> [--force]" + exit 1 + fi + ;; + watch) + if [ "$#" -ne 2 ]; then + echo "Usage: $0 watch <key>" + exit 1 + fi + watch "$2" + exit $? + ;; + *) + echo "Usage: $0 {get|set|erase|lock|unlock|lock_owner|watch}" + ;; +esac + +exit 1 diff --git a/src/osaf/consensus/service.cc b/src/osaf/consensus/service.cc new file mode 100644 index 000000000..6131bbe67 --- /dev/null +++ b/src/osaf/consensus/service.cc @@ -0,0 +1,277 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2018 The OpenSAF Foundation + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ +#include "osaf/consensus/service.h" +#include <unistd.h> +#include <climits> +#include <thread> +#include "base/logtrace.h" +#include "base/conf.h" +#include "base/getenv.h" +#include "base/ncssysf_def.h" + +SaAisErrorT Consensus::BeginActivePromotion() { + TRACE_ENTER(); + SaAisErrorT rc; + + if (use_consensus_ == false) { + return SA_AIS_OK; + } + + uint32_t retries = 0; + rc = KeyValue::Lock(base::Conf::NodeName(), kLockTimeout); + while (rc == SA_AIS_ERR_TRY_AGAIN && retries < kMaxRetry) { + TRACE("Waiting for lock"); + ++retries; + std::this_thread::sleep_for(kSleepInterval); + rc = KeyValue::Lock(base::Conf::NodeName(), kLockTimeout); + } + + LOG_IN("Node %s obtained lock", base::Conf::NodeName().c_str()); + + // check current active node + std::string current; + bool current_valid = false; + rc = KeyValue::Get(keyname, current); + if (rc == SA_AIS_OK) { + current_valid = true; + LOG_NO("Current active controller is %s", current.c_str()); + } + + LOG_NO("Setting active controller to %s", base::Conf::NodeName().c_str()); + rc = KeyValue::Set(keyname, base::Conf::NodeName()); + + if (current_valid == true) { + if (current != base::Conf::NodeName()) { + // fence the old active controller + FenceNode(current); + } + } + return rc; +} + +SaAisErrorT Consensus::EndActivePromotion() { + TRACE_ENTER(); + if (use_consensus_ == false) { + return SA_AIS_OK; + } + + bool locked = KeyValue::IsLockedByThisNode(); + if (locked == false) { + LOG_ER("Lock unexpectedly released"); + } else { + uint32_t retries = 0; + SaAisErrorT rc; + rc = KeyValue::Unlock(base::Conf::NodeName()); + while (rc == SA_AIS_ERR_TRY_AGAIN && retries < kMaxRetry) { + LOG_IN("Trying to unlock"); + ++retries; + std::this_thread::sleep_for(kSleepInterval); + rc = KeyValue::Unlock(base::Conf::NodeName()); + } + if (rc == SA_AIS_OK) { + LOG_IN("Released lock"); + } else { + LOG_ER("Unlock failed"); + } + } + return SA_AIS_OK; +} + +SaAisErrorT Consensus::Demote(const std::string& node = "") { + TRACE_ENTER(); + SaAisErrorT rc; + if (use_consensus_ == false) { + return SA_AIS_OK; + } + + uint32_t retries = 0; + rc = KeyValue::Lock(base::Conf::NodeName(), kLockTimeout); + while (rc == SA_AIS_ERR_TRY_AGAIN && retries < kMaxRetry) { + LOG_IN("Waiting for lock"); + ++retries; + std::this_thread::sleep_for(kSleepInterval); + rc = KeyValue::Lock(base::Conf::NodeName(), kLockTimeout); + } + + // check current active node + std::string current; + rc = KeyValue::Get(keyname, current); + if (rc == SA_AIS_OK) { + LOG_NO("Demoting %s as active controller", current.c_str()); + if (node.empty() == false && node != current) { + FenceNode(node); + } + // if Demote() was called as DemoteCurrentActive() from fmd, + // then fmd will fence the node itself + + rc = KeyValue::Erase(keyname); + if (rc != SA_AIS_OK) { + LOG_ER("Failed to clear active controller in KeyValue"); + } + LOG_NO("Node %s demoted", current.c_str()); + } + + retries = 0; + rc = KeyValue::Unlock(base::Conf::NodeName()); + while (rc == SA_AIS_ERR_TRY_AGAIN && retries < kMaxRetry) { + LOG_IN("Trying to unlock"); + ++retries; + std::this_thread::sleep_for(kSleepInterval); + rc = KeyValue::Unlock(base::Conf::NodeName()); + } + if (rc == SA_AIS_OK) { + LOG_IN("Released lock"); + } else { + LOG_ER("Unlock failed"); + } + + return SA_AIS_OK; +} + +SaAisErrorT Consensus::DemoteCurrentActive() { + TRACE_ENTER(); + return Demote(); +} + +SaAisErrorT Consensus::DemoteThisNode() { + TRACE_ENTER(); + return Demote(base::Conf::NodeName()); +} + +bool Consensus::IsEnabled() const { + return use_consensus_; +} + +bool Consensus::IsWritable() const { + TRACE_ENTER(); + if (use_consensus_ == false) { + return true; + } + + SaAisErrorT rc; + rc = KeyValue::Set(test_keyname, base::Conf::NodeName()); + if (rc == SA_AIS_OK) { + return true; + } else { + return false; + } +} + +bool Consensus::IsRemoteFencingEnabled() const { + return use_remote_fencing_; +} + +std::string Consensus::CurrentActive() const { + TRACE_ENTER(); + SaAisErrorT rc; + bool success = false; + if (use_consensus_ == false) { + return ""; + } + + uint32_t retries = 0; + rc = KeyValue::Lock(base::Conf::NodeName(), kLockTimeout); + while (rc == SA_AIS_ERR_TRY_AGAIN && retries < kMaxRetry) { + LOG_IN("Waiting for lock"); + ++retries; + std::this_thread::sleep_for(kSleepInterval); + rc = KeyValue::Lock(base::Conf::NodeName(), kLockTimeout); + } + + // check current active node + std::string current; + rc = KeyValue::Get(keyname, current); + if (rc == SA_AIS_OK) { + success = true; + LOG_IN("Current active controller is %s", current.c_str()); + } + + retries = 0; + rc = KeyValue::Unlock(base::Conf::NodeName()); + while (rc == SA_AIS_ERR_TRY_AGAIN && retries < kMaxRetry) { + LOG_IN("Trying to unlock"); + ++retries; + std::this_thread::sleep_for(kSleepInterval); + rc = KeyValue::Unlock(base::Conf::NodeName()); + } + if (rc == SA_AIS_OK) { + LOG_IN("Released lock"); + } else { + LOG_ER("Unlock failed"); + } + + if (success == true) { + return current; + } else { + return ""; + } +} + +Consensus::Consensus() { + TRACE_ENTER(); + + uint32_t split_brain_enable = base::GetEnv("FMS_SPLIT_BRAIN_PREVENTION", 0); + std::string kv_store_cmd = base::GetEnv("FMS_KEYVALUE_STORE_PLUGIN_CMD", ""); + uint32_t use_remote_fencing = base::GetEnv("FMS_USE_REMOTE_FENCING" , 0); + + if (split_brain_enable == 1 && kv_store_cmd.empty() == false) { + use_consensus_ = true; + } else { + use_consensus_ = false; + } + + if (use_remote_fencing == 1) { + use_remote_fencing_ = true; + } + + // needed for base::Conf::NodeName() later + base::Conf::InitNodeName(); + + if (use_consensus_ == true) { + LOG_IN("Split brain prevention is enabled"); + } else { + LOG_IN("Split brain prevention is disabled"); + } +} + +Consensus::~Consensus() { +} + +bool Consensus::FenceNode(const std::string& node) { + if (use_remote_fencing_ == true) { + LOG_WA("Fencing remote node %s", node.c_str()); + // @todo currently passing UINT_MAX as node ID, since + // we can't always obtain a valid node ID? + opensaf_reboot(UINT_MAX, node.c_str(), + "Fencing remote node"); + + return true; + } else { + LOG_WA("Fencing is not enabled. Node %s will not be fenced", node.c_str()); + return false; + } +} + +void Consensus::MonitorActive(ConsensusCallback callback, + const uint32_t user_defined) { + TRACE_ENTER(); + if (use_consensus_ == false) { + return; + } + + KeyValue::Watch(keyname, callback, user_defined); +} diff --git a/src/osaf/consensus/service.h b/src/osaf/consensus/service.h new file mode 100644 index 000000000..ae031cc1f --- /dev/null +++ b/src/osaf/consensus/service.h @@ -0,0 +1,75 @@ +/* -*- OpenSAF -*- + * + * (C) Copyright 2018 The OpenSAF Foundation + * + * This program is distributed in the hope that it will be useful, but + * WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY + * or FITNESS FOR A PARTICULAR PURPOSE. This file and program are licensed + * under the GNU Lesser General Public License Version 2.1, February 1999. + * The complete license can be accessed from the following location: + * http://opensource.org/licenses/lgpl-license.php + * See the Copying file included with the OpenSAF distribution for full + * licensing terms. + * + * Author(s): Ericsson AB + * + */ +#ifndef OSAF_CONSENSUS_SERVICE_H_ +#define OSAF_CONSENSUS_SERVICE_H_ + +#include <chrono> +#include <string> +#include "saAis.h" +#include "osaf/consensus/keyvalue.h" + +class Consensus { + public: + // Obtain lock, set active controller to this node + SaAisErrorT BeginActivePromotion(); + + // Release lock + SaAisErrorT EndActivePromotion(); + + // Obtain lock, clear current active controller, release lock + SaAisErrorT DemoteCurrentActive(); + + // Obtain lock, clear this node as active controller, release lock + SaAisErrorT DemoteThisNode(); + + // Returns active controller as known by the consensus service + std::string CurrentActive() const; + + // If the active controller is changed as known by the consensus service, + // then callback will be run from a new thread, with <user_defined> returned + // in the callback + void MonitorActive(ConsensusCallback callback, const uint32_t user_defined); + + // Is consensus service enabled? + bool IsEnabled() const; + + // Is the key-value store writable? + bool IsWritable() const; + + // Is remote fencing enabled? + bool IsRemoteFencingEnabled() const; + + Consensus(); + virtual ~Consensus(); + + Consensus(const Consensus&) = delete; + Consensus& operator=(const Consensus&) = delete; + + private: + bool use_consensus_ = false; + bool use_remote_fencing_ = false; + const std::string keyname = "opensaf_active_controller"; + const std::string test_keyname = "opensaf_write_test"; + const std::chrono::milliseconds kSleepInterval = + std::chrono::milliseconds(100); // in ms + static constexpr uint32_t kLockTimeout = 30; + static constexpr uint32_t kMaxRetry = 600; + SaAisErrorT Demote(const std::string& node); + bool FenceNode(const std::string& node); +}; + +#endif // OSAF_CONSENSUS_SERVICE_H_ -- 2.14.1 ------------------------------------------------------------------------------ Check out the vibrant tech community on one of the world's most engaging tech sites, Slashdot.org! http://sdm.link/slashdot _______________________________________________ Opensaf-devel mailing list Opensaf-devel@lists.sourceforge.net https://lists.sourceforge.net/lists/listinfo/opensaf-devel
