wido commented on code in PR #14320: URL: https://github.com/apache/cloudstack/pull/14320#discussion_r4269100796
########## scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.py: ########## @@ -0,0 +1,131 @@ +#!/usr/bin/env python3 +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Ceph RBD flavor of kvmheartbeat.sh/kvmsmpheartbeat.sh. +# +# There is no shared POSIX mount point to write a heartbeat file to when the +# primary storage pool is Ceph RBD, so the heartbeat timestamp is instead +# stored as a small RADOS object (one object per host) in the same RBD pool. +# Any host with a working path to the Ceph cluster can write/read this object, +# which gives the same semantics as the NFS/SharedMountPoint heartbeat file. +# +# The object is written and read with the Python bindings of librados +# (python3-rados), see kvmha_rbd.py. The cephx key is not an argument, as it would be visible in the process list: it is +# read from the environment variable CEPH_KEY. + +import argparse +import os +import re +import subprocess +import sys +import time + +from kvmha_rbd import BindingsError, Ceph, CephError, KEY_ENV, silence_standard_error + + +class Parser(argparse.ArgumentParser): + def error(self, message): + self.print_usage(sys.stderr) + sys.exit(1) + + +def main(): + parser = Parser(description="Writes or checks the heartbeat of a KVM host on Ceph RBD storage", add_help=False) + parser.add_argument("-s", dest="mon_hosts", default="", help="ceph monitor host(s), comma separated") + parser.add_argument("-o", dest="pool", default="", help="ceph/rbd pool name") + parser.add_argument("-n", dest="user", default="", help="cephx auth user (optional, the key is then in " + KEY_ENV + ")") + parser.add_argument("-h", dest="host", default="", help="host") + parser.add_argument("-r", dest="check", action="store_true", help="read the heartbeat, instead of writing it") + parser.add_argument("-c", dest="cleanup", action="store_true", help="cleanup: reboot the host") + parser.add_argument("-t", dest="interval", default="", help="maximum age of the heartbeat in seconds, when it is read") + parser.add_argument("--help", action="help") + args = parser.parse_args() + + key = os.environ.pop(KEY_ENV, "") + if not args.mon_hosts or not args.pool: + sys.exit(1) + # the host IP names the heartbeat object, so it is required except for the self-fencing (-c) + if not args.cleanup and not args.host: + sys.exit(1) + if args.user and not key: + sys.exit(1) + if args.check and not re.fullmatch(r"[0-9]+", args.interval): + sys.exit(1) + + if args.cleanup: + reboot() + return + + heartbeat_object = "KVMHA-hb-" + args.host + silence_standard_error() + try: + ceph = Ceph(args.mon_hosts, args.pool, args.user, key) + except BindingsError as error: + if args.check: + # not a proof that the host is dead, so DEAD is not reported + print("=====> Unable to check the heartbeat, %s <======" % error) + sys.exit(2) + sys.exit(1) + try: + if args.check: + check(ceph, heartbeat_object, args.pool, int(args.interval)) + return + try: + ceph.write(heartbeat_object, "%d\n" % time.time()) + except CephError: + sys.exit(1) + finally: + ceph.close() + + +def check(ceph, heartbeat_object, pool, interval): + now = int(time.time()) + try: + heartbeat = ceph.read(heartbeat_object).strip() + except CephError: Review Comment: Any read error votes DEAD here, including the checking host not being able to reach Ceph at all. The activity script in this PR reports that same situation as unknown (exit 2). `Ceph.read` already distinguishes `ObjectNotFound`, so the missing object could stay DEAD and other errors could exit 2, consistent with the activity check. ########## scripts/vm/hypervisor/kvm/kvmha_rbd.py: ########## @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Access to Ceph for kvmheartbeat_rbd.py and kvmvmactivity_rbd.py. +# +# The objects of the heartbeat and of the activity state, and the watchers of the +# volumes, are read and written over one connection to the Ceph cluster with the +# Python bindings of librados and librbd (the packages python3-rados and python3-rbd). +# An operation which does not finish in TIMEOUT seconds fails, so that it does not +# block forever if the cluster can not be reached. +# +# The cephx key is not an argument of the scripts, as it would be visible in the +# process list: the scripts read it from the environment variable CEPH_KEY, and give +# it to the bindings in their configuration. +# +# Nothing is written to the standard error output: the agent merges it with the +# standard output and reads only the first line of the result, so a message of +# librados would hide the result of the script. + +import os + +try: + import rados +except ImportError: + rados = None + +try: + import rbd +except ImportError: + rbd = None + +KEY_ENV = "CEPH_KEY" +TIMEOUT = 30 +MAX_OBJECT_SIZE = 1024 * 1024 + + +def silence_standard_error(): + """Sends the standard error output, which librados may write to, to /dev/null.""" + try: + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, 2) + os.close(devnull) + except OSError: + pass + + +class BindingsError(Exception): + """The Python bindings of librados or librbd are not installed or can not be used.""" + + +class CephError(Exception): + """The cluster could not be connected, or an object could not be read, written or found out if it exists.""" + + +class WatchersError(Exception): + """The watchers of a volume could not be listed: it is not known whether the host is alive.""" + + +class Ceph: + def __init__(self, mon_hosts, pool, user, key, watchers=False): + """ + Connects to the cluster. Raises BindingsError if the bindings can not be used, in which case nothing can + be done. If the cluster can not be connected, the operations raise CephError. + + :param watchers: whether the watchers of volumes are listed, which needs librbd + """ + if rados is None: + raise BindingsError("the package python3-rados is not installed") + if watchers and (rbd is None or not hasattr(rbd.Image, "watchers_list")): + raise BindingsError("the package python3-rbd is not installed, or is too old to list the watchers of a volume") + self.pool = pool + self.cluster = None + self.ioctx = None + conf = {"mon_host": mon_hosts, "rados_osd_op_timeout": str(TIMEOUT), "rados_mon_op_timeout": str(TIMEOUT), + "client_mount_timeout": str(TIMEOUT), "log_to_stderr": "false", "err_to_stderr": "false"} + if user: + conf["key"] = key + cluster = rados.Rados(rados_id=user if user else None, conf=conf) + try: + cluster.connect(timeout=TIMEOUT) + self.ioctx = cluster.open_ioctx(pool) + self.cluster = cluster + except rados.Error: + cluster.shutdown() + + def close(self): + if self.ioctx is not None: + self.ioctx.close() + if self.cluster is not None: + self.cluster.shutdown() + self.ioctx = None + self.cluster = None + + def _connected(self, name): + if self.ioctx is None: + raise CephError(name) + return self.ioctx + + def exists(self, name): + """Returns whether the object exists, or raises CephError if it is not known.""" + ioctx = self._connected(name) + try: + ioctx.stat(name) + return True + except rados.ObjectNotFound: + return False + except rados.Error: + raise CephError(name) + + def read(self, name): + """Returns the content of the object, or raises CephError.""" + ioctx = self._connected(name) + try: + size = min(ioctx.stat(name)[0], MAX_OBJECT_SIZE) + return ioctx.read(name, size).decode("utf-8", "replace") if size else "" + except rados.Error: + raise CephError(name) + + def write(self, name, content): + """Writes the object, or raises CephError.""" + ioctx = self._connected(name) + try: + ioctx.write_full(name, content.encode("utf-8")) + except rados.Error: + raise CephError(name) + + def has_watcher(self, image): + """Returns whether the volume has a watcher, or raises WatchersError.""" + if self.ioctx is None: + raise WatchersError(image) + try: + with rbd.Image(self.ioctx, image, read_only=True) as volume: Review Comment: `read_only=True` is what keeps this script from registering a watcher of its own (librbd skips the watch registration for a read-only open). Worth a short comment, since dropping it would make every host look alive. ########## plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java: ########## @@ -330,37 +338,138 @@ public boolean isPoolSupportHA() { public String getHearthBeatPath() { if (StoragePoolType.NetworkFilesystem.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir); - } - return scriptPath; + return findKvmHaScript("kvmheartbeat.sh"); } else if (StoragePoolType.SharedMountPoint.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir); + return findKvmHaScript("kvmsmpheartbeat.sh"); + } else if (StoragePoolType.RBD.equals(type)) { + return findKvmHaScript("kvmheartbeat_rbd.py"); + } + return null; + } + + private String findKvmHaScript(String scriptName) { + String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + String scriptPath = Script.findScript(kvmScriptsDir, scriptName); + if (scriptPath == null) { + throw new CloudRuntimeException(String.format("Unable to find script '%s' in directory: %s", scriptName, kvmScriptsDir)); + } + return scriptPath; + } + + /** + * Returns the Ceph monitors as expected by "--mon-host": the comma-separated monitors of the pool, trimmed + * and without empty entries. If the pool has a monitor port, it is added to each monitor which has none yet. + */ + protected String getRbdMonitors() { Review Comment: This is now the fifth place that builds a monitor string from the host list and the port. `RbdEncryption.monSpec` produces the same librados `mon_host` syntax but just appends the port to the whole string, so with multiple monitors the port only lands on the last one. Consider making this a shared static helper and having `monSpec` use it. The rados-java `confSet("mon_host", ...)` calls could follow later. ########## scripts/vm/hypervisor/kvm/kvmha_rbd.py: ########## @@ -0,0 +1,149 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Access to Ceph for kvmheartbeat_rbd.py and kvmvmactivity_rbd.py. +# +# The objects of the heartbeat and of the activity state, and the watchers of the +# volumes, are read and written over one connection to the Ceph cluster with the +# Python bindings of librados and librbd (the packages python3-rados and python3-rbd). +# An operation which does not finish in TIMEOUT seconds fails, so that it does not +# block forever if the cluster can not be reached. +# +# The cephx key is not an argument of the scripts, as it would be visible in the +# process list: the scripts read it from the environment variable CEPH_KEY, and give +# it to the bindings in their configuration. +# +# Nothing is written to the standard error output: the agent merges it with the +# standard output and reads only the first line of the result, so a message of +# librados would hide the result of the script. + +import os + +try: + import rados +except ImportError: + rados = None + +try: + import rbd +except ImportError: + rbd = None + +KEY_ENV = "CEPH_KEY" +TIMEOUT = 30 +MAX_OBJECT_SIZE = 1024 * 1024 + + +def silence_standard_error(): + """Sends the standard error output, which librados may write to, to /dev/null.""" + try: + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, 2) + os.close(devnull) + except OSError: + pass + + +class BindingsError(Exception): + """The Python bindings of librados or librbd are not installed or can not be used.""" + + +class CephError(Exception): + """The cluster could not be connected, or an object could not be read, written or found out if it exists.""" + + +class WatchersError(Exception): + """The watchers of a volume could not be listed: it is not known whether the host is alive.""" + + +class Ceph: + def __init__(self, mon_hosts, pool, user, key, watchers=False): + """ + Connects to the cluster. Raises BindingsError if the bindings can not be used, in which case nothing can + be done. If the cluster can not be connected, the operations raise CephError. + + :param watchers: whether the watchers of volumes are listed, which needs librbd + """ + if rados is None: + raise BindingsError("the package python3-rados is not installed") + if watchers and (rbd is None or not hasattr(rbd.Image, "watchers_list")): + raise BindingsError("the package python3-rbd is not installed, or is too old to list the watchers of a volume") + self.pool = pool + self.cluster = None + self.ioctx = None + conf = {"mon_host": mon_hosts, "rados_osd_op_timeout": str(TIMEOUT), "rados_mon_op_timeout": str(TIMEOUT), + "client_mount_timeout": str(TIMEOUT), "log_to_stderr": "false", "err_to_stderr": "false"} + if user: + conf["key"] = key + cluster = rados.Rados(rados_id=user if user else None, conf=conf) + try: + cluster.connect(timeout=TIMEOUT) + self.ioctx = cluster.open_ioctx(pool) + self.cluster = cluster + except rados.Error: + cluster.shutdown() + + def close(self): + if self.ioctx is not None: + self.ioctx.close() + if self.cluster is not None: + self.cluster.shutdown() + self.ioctx = None + self.cluster = None + + def _connected(self, name): + if self.ioctx is None: + raise CephError(name) + return self.ioctx + + def exists(self, name): + """Returns whether the object exists, or raises CephError if it is not known.""" + ioctx = self._connected(name) + try: + ioctx.stat(name) + return True + except rados.ObjectNotFound: + return False + except rados.Error: + raise CephError(name) + + def read(self, name): + """Returns the content of the object, or raises CephError.""" + ioctx = self._connected(name) + try: + size = min(ioctx.stat(name)[0], MAX_OBJECT_SIZE) + return ioctx.read(name, size).decode("utf-8", "replace") if size else "" + except rados.Error: + raise CephError(name) + + def write(self, name, content): + """Writes the object, or raises CephError.""" + ioctx = self._connected(name) + try: + ioctx.write_full(name, content.encode("utf-8")) + except rados.Error: + raise CephError(name) + + def has_watcher(self, image): + """Returns whether the volume has a watcher, or raises WatchersError.""" + if self.ioctx is None: + raise WatchersError(image) + try: + with rbd.Image(self.ioctx, image, read_only=True) as volume: + return any(True for _ in volume.watchers_list()) + except (rados.Error, rbd.Error): Review Comment: `rbd.ImageNotFound` ends up here as well. A volume that was deleted but is still in the list from the management server then makes the whole check exit 2, even when another volume in the list has a live watcher. One stale entry blocks any verdict for that host. The NFS script ignores missing files. Suggest catching `rbd.ImageNotFound` first and returning False, so the check continues with the other volumes. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
