Greg Padgett has uploaded a new change for review. Change subject: agent: enable local maintenance mode ......................................................................
agent: enable local maintenance mode The HA agent needs to be aware of local maintenance mode in order to not run the engine VM on the local host. This provides a method of setting a local maintenance flag which is stored on the host and then read back by the agent, which takes appropriate action to stop the VM. A client method has been provided to set this mode: HAClient.set_maintenance_mode(mode, value) Where mode is a constant in HAClient.MaintenanceMode, and value is a value parsable as a boolean (True, 'true', 'yes', 1, False, etc.) Change-Id: I60e62cd03daec812e16a3a5bbd2d3d896e41402d Bug-Url: https://bugzilla.redhat.com/1015724 Signed-off-by: Greg Padgett <[email protected]> --- M build/var_subst.inc M configure.ac M ovirt-hosted-engine-ha.spec.in M ovirt_hosted_engine_ha/agent/hosted_engine.py M ovirt_hosted_engine_ha/client/client.py M ovirt_hosted_engine_ha/env/Makefile.am M ovirt_hosted_engine_ha/env/config.py M ovirt_hosted_engine_ha/env/constants.py.in A ovirt_hosted_engine_ha/env/ha.conf M ovirt_hosted_engine_ha/lib/util.py 10 files changed, 139 insertions(+), 15 deletions(-) git pull ssh://gerrit.ovirt.org:29418/ovirt-hosted-engine-ha refs/changes/78/20278/1 diff --git a/build/var_subst.inc b/build/var_subst.inc index 9f2b9a9..9b02e79 100644 --- a/build/var_subst.inc +++ b/build/var_subst.inc @@ -28,6 +28,7 @@ -e "s,[@]ENGINE_HA_LIBDIR[@],$(engine_ha_libdir),g" \ -e "s,[@]ENGINE_HA_LOGDIR[@],$(engine_ha_logdir),g" \ -e "s,[@]ENGINE_HA_RUNDIR[@],$(engine_ha_rundir),g" \ + -e "s,[@]ENGINE_HA_STATEDIR[@],$(engine_ha_statedir),g" \ -e "s,[@]ENGINE_SETUP_BINDIR[@],$(engine_setup_bindir),g" CONFIGSUBST = $(top_builddir)/config.status --file=- diff --git a/configure.ac b/configure.ac index ed46bdb..03b07f1 100644 --- a/configure.ac +++ b/configure.ac @@ -70,6 +70,7 @@ AC_SUBST([engine_ha_libdir], ['${pythondir}/ovirt_hosted_engine_ha']) AC_SUBST([engine_ha_logdir], ['${localstatedir}/log/${PACKAGE_NAME}']) AC_SUBST([engine_ha_rundir], ['${localstatedir}/run/${PACKAGE_NAME}']) +AC_SUBST([engine_ha_statedir], ['${localstatedir}/lib/${PACKAGE_NAME}']) AC_SUBST([engine_setup_bindir], ['${sbindir}']) AM_CONDITIONAL([PYTHON_SYNTAX_CHECK], [test "${enable_python_syntax_check}" = "yes"]) diff --git a/ovirt-hosted-engine-ha.spec.in b/ovirt-hosted-engine-ha.spec.in index 610c34e..d57eb98 100644 --- a/ovirt-hosted-engine-ha.spec.in +++ b/ovirt-hosted-engine-ha.spec.in @@ -24,6 +24,7 @@ %global engine_ha_libdir %{python_sitelib}/ovirt_hosted_engine_ha %global engine_ha_logdir @ENGINE_HA_LOGDIR@ %global engine_ha_rundir @ENGINE_HA_RUNDIR@ +%global engine_ha_statedir @ENGINE_HA_STATEDIR@ %global vdsm_user @VDSM_USER@ %global vdsm_group @VDSM_GROUP@ @@ -81,6 +82,7 @@ install -dDm 0700 %{buildroot}%{engine_ha_logdir} install -dDm 0700 %{buildroot}%{engine_ha_rundir} +install -dDm 0700 %{buildroot}%{engine_ha_statedir} %if 0%{?with_systemd} # Install the systemd scripts @@ -131,6 +133,9 @@ %dir %{engine_ha_logdir} %ghost %dir %{engine_ha_rundir} +%dir %{engine_ha_statedir} +%config(noreplace) %{engine_ha_statedir}/ha.conf + %post %if 0%{?with_systemd} diff --git a/ovirt_hosted_engine_ha/agent/hosted_engine.py b/ovirt_hosted_engine_ha/agent/hosted_engine.py index f4533e4..20ba538 100644 --- a/ovirt_hosted_engine_ha/agent/hosted_engine.py +++ b/ovirt_hosted_engine_ha/agent/hosted_engine.py @@ -89,6 +89,11 @@ PENDING = 'PENDING' ACQUIRED = 'ACQUIRED' + class MaintenanceMode(object): + NONE = 'NONE' + GLOBAL = 'GLOBAL' + LOCAL = 'LOCAL' + def __init__(self, shutdown_requested_callback): """ Initialize hosted engine monitoring logic. shutdown_requested_callback @@ -592,6 +597,10 @@ if self._rinfo['bad-health-failure-time']: score = 0 + # Hosts in local maintenance mode should not run the vm + if self._get_maintenance_mode() == self.MaintenanceMode.LOCAL: + score = 0 + ts = int(time.time()) data = ("{md_parse_vers}|{md_feature_vers}|{ts_int}" "|{host_id}|{score}|{engine_status}|{name}" @@ -764,7 +773,7 @@ rinfo['best-score'] = stats['score'] rinfo['best-score-host-id'] = host_id - rinfo['maintenance'] = self._global_stats.get('maintenance', False) + rinfo['maintenance'] = self._get_maintenance_mode() self._rinfo.update(rinfo) @@ -790,6 +799,28 @@ except KeyError: self._log.error("Invalid engine status: %s", status, exc_info=True) return 0 + + def _get_maintenance_mode(self): + """ + Returns maintenance mode: + NONE - no maintenance, function as usual + GLOBAL - HA system in maintenance, ignore VM state and score + LOCAL - local host in maintenance, zero score and shut down vm + If both LOCAL and GLOBAL modes are set, LOCAL will be returned + because is more invasive to the host HA state. + """ + try: + if util.to_bool(self._config.get(config.HA, + config.LOCAL_MAINTENANCE)): + return self.MaintenanceMode.LOCAL + elif self._global_stats.get('maintenance', False): + return self.MaintenanceMode.GLOBAL + else: + return self.MaintenanceMode.NONE + except ValueError: + self._log.error("Invalid value for maintenance setting", + exc_info=True) + return self.MaintenanceMode.NONE def _handle_entry(self): """ @@ -826,9 +857,12 @@ # FIXME remote db down, other statuses - if self._rinfo['maintenance']: - self._log.info("HA maintenance enabled") + if self._rinfo['maintenance'] == self.MaintenanceMode.GLOBAL: + self._log.info("Global HA maintenance enabled") return self.States.MAINTENANCE, True + elif self._rinfo['maintenance'] == self.MaintenanceMode.LOCAL: + self._log.info("Local HA maintenance enabled") + return self.States.OFF, True if self._rinfo['best-score-host-id'] != local_host_id: self._log.info("Engine down, local host does not have best score", @@ -942,9 +976,12 @@ self._log.error("Engine vm unexpectedly running on other host") return self.States.OFF, True - if self._rinfo['maintenance']: - self._log.info("HA maintenance enabled") + if self._rinfo['maintenance'] == self.MaintenanceMode.GLOBAL: + self._log.info("Global HA maintenance enabled") return self.States.MAINTENANCE, True + elif self._rinfo['maintenance'] == self.MaintenanceMode.LOCAL: + self._log.info("Local HA maintenance enabled") + return self.States.STOP, False best_host_id = self._rinfo['best-score-host-id'] if (best_host_id != local_host_id @@ -1120,8 +1157,8 @@ MAINTENANCE state. Allow arbitrary HA VM state while in maintenance mode (i.e. ignore it), and re-init in ENTRY state once complete. """ - if self._rinfo['maintenance']: - self._log.info("HA maintenance enabled", + if self._rinfo['maintenance'] == self.MaintenanceMode.GLOBAL: + self._log.info("Global HA maintenance enabled", extra=self._get_lf_args(self.LF_MAINTENANCE)) return self.States.MAINTENANCE, True else: diff --git a/ovirt_hosted_engine_ha/client/client.py b/ovirt_hosted_engine_ha/client/client.py index 36b478b..bffd75c 100644 --- a/ovirt_hosted_engine_ha/client/client.py +++ b/ovirt_hosted_engine_ha/client/client.py @@ -27,6 +27,7 @@ from ..env import path from ..lib import brokerlink from ..lib import metadata +from ..lib import util from ..lib.exceptions import MetadataError @@ -40,6 +41,23 @@ """ ALL = 'ALL' HOST = 'HOST' + GLOBAL = 'GLOBAL' + + class GlobalMdFlags(object): + """ + Constants used to refer to global metadata flags: + MAINTENANCE - maintenance flag + Note that the value here must equal a key in metadata.global_flags + """ + MAINTENANCE = 'maintenance' + + class MaintenanceMode(object): + """ + Constants used in calls to set maintenance mode: + LOCAL - local host maintenance + GLOBAL - global maintenance + """ + LOCAL = 'LOCAL' GLOBAL = 'GLOBAL' def __init__(self, log=False): @@ -195,3 +213,18 @@ score = md['score'] return score + + def set_maintenance_mode(self, mode, value): + if mode == self.MaintenanceMode.GLOBAL: + self.set_global_md_flag(self.GlobalMdFlags.MAINTENANCE, + str(value)) + + elif mode == self.MaintenanceMode.LOCAL: + if self._config is None: + self._config = config.Config() + self._config.set(config.HA, + config.LOCAL_MAINTENANCE, + str(util.to_bool(value))) + + else: + raise Exception("Invalid maintenance mode: {0}".format(mode)) diff --git a/ovirt_hosted_engine_ha/env/Makefile.am b/ovirt_hosted_engine_ha/env/Makefile.am index 2ccd1d0..c7c4e4d 100644 --- a/ovirt_hosted_engine_ha/env/Makefile.am +++ b/ovirt_hosted_engine_ha/env/Makefile.am @@ -39,6 +39,12 @@ constants.py \ $(NULL) +hastatedir = $(engine_ha_statedir) + +dist_hastate_DATA = \ + ha.conf \ + $(NULL) + EXTRA_DIST = \ constants.py.in \ $(NULL) diff --git a/ovirt_hosted_engine_ha/env/config.py b/ovirt_hosted_engine_ha/env/config.py index 2919a5b..a11056c 100644 --- a/ovirt_hosted_engine_ha/env/config.py +++ b/ovirt_hosted_engine_ha/env/config.py @@ -17,6 +17,8 @@ # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA # +import fcntl + from . import constants # constants for hosted-engine.conf options @@ -32,20 +34,30 @@ VM = 'vm' VM_UUID = 'vmId' +# constants for ha.conf options +HA = 'ha' +LOCAL_MAINTENANCE = 'local_maintenance' + class Config(object): + static_files = { + ENGINE: constants.ENGINE_SETUP_CONF_FILE, + VM: constants.VM_CONF_FILE, + } + # Config files in dynamic_files may change at runtime and are re-read + # whenever configuration values are retrieved from them. + dynamic_files = { + HA: constants.HA_AGENT_CONF_FILE, + } + def __init__(self): self._config = {} - self._files = { - ENGINE: constants.ENGINE_SETUP_CONF_FILE, - VM: constants.VM_CONF_FILE} + self._load(Config.static_files) - self.load() - - def load(self): + def _load(self, files): conf = {} - for type, fname in self._files.iteritems(): + for type, fname in files.iteritems(): with open(fname, 'r') as f: for line in f: tokens = line.split('=', 1) @@ -54,9 +66,36 @@ self._config[type] = conf def get(self, type, key): + if type in Config.dynamic_files.keys(): + self._load(dict([(type, Config.dynamic_files[type])])) + try: return self._config[type][key] except KeyError: unknown = "unknown (type={0})".format(type) raise Exception("Configuration value not found: file={0}, key={1}" .format(self._files.get(type, unknown), key)) + + def set(self, type, key, value): + """ + Writes 'key=value' to the config file for 'type'. + Note that this method is not thread safe. + """ + if type not in Config.dynamic_files: + raise Exception("Configuration type {0} cannot be updated" + .format(type)) + + with open(Config.dynamic_files[type], 'r+') as f: + fcntl.flock(f, fcntl.LOCK_EX) + + # self._load() can re-open the exclusively-locked file because + # it's being called from the same process as the lock holder + self._load(dict([(type, Config.dynamic_files[type])])) + self._config[type][key] = str(value) + + text = '' + for k, v in self._config[type].iteritems(): + text += '{k}={v}\n'.format(k=k, v=v) + + f.write(text) + f.truncate() diff --git a/ovirt_hosted_engine_ha/env/constants.py.in b/ovirt_hosted_engine_ha/env/constants.py.in index 2fb1de2..58259e8 100644 --- a/ovirt_hosted_engine_ha/env/constants.py.in +++ b/ovirt_hosted_engine_ha/env/constants.py.in @@ -34,6 +34,7 @@ ENGINE_SETUP_CONF_FILE = '/etc/ovirt-hosted-engine/hosted-engine.conf' VM_CONF_FILE = '/etc/ovirt-hosted-engine/vm.conf' +HA_AGENT_CONF_FILE = '@ENGINE_HA_STATEDIR@/ha.conf' SD_MOUNT_PARENT = '/rhev/data-center/mnt' SD_METADATA_DIR = 'ha_agent' diff --git a/ovirt_hosted_engine_ha/env/ha.conf b/ovirt_hosted_engine_ha/env/ha.conf new file mode 100644 index 0000000..f671fbc --- /dev/null +++ b/ovirt_hosted_engine_ha/env/ha.conf @@ -0,0 +1 @@ +local_maintenance=False diff --git a/ovirt_hosted_engine_ha/lib/util.py b/ovirt_hosted_engine_ha/lib/util.py index 3b513b6..99b28e5 100644 --- a/ovirt_hosted_engine_ha/lib/util.py +++ b/ovirt_hosted_engine_ha/lib/util.py @@ -79,4 +79,4 @@ elif first in ('f', 'n', '0'): return False else: - raise Exception("Invalid value for boolean: {0}".format(string)) + raise ValueError("Invalid value for boolean: {0}".format(string)) -- To view, visit http://gerrit.ovirt.org/20278 To unsubscribe, visit http://gerrit.ovirt.org/settings Gerrit-MessageType: newchange Gerrit-Change-Id: I60e62cd03daec812e16a3a5bbd2d3d896e41402d Gerrit-PatchSet: 1 Gerrit-Project: ovirt-hosted-engine-ha Gerrit-Branch: master Gerrit-Owner: Greg Padgett <[email protected]> _______________________________________________ Engine-patches mailing list [email protected] http://lists.ovirt.org/mailman/listinfo/engine-patches
