wido commented on code in PR #14320: URL: https://github.com/apache/cloudstack/pull/14320#discussion_r4254935225
########## scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh: ########## @@ -0,0 +1,167 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Ceph RBD flavor of kvmheartbeat.sh/kvmsmpheartbeat.sh. +# +# There is no shared POSIX mount point to write a heartbeat file to when the +# primary storage pool is Ceph RBD, so the heartbeat timestamp is instead +# stored as a small RADOS object (one object per host) in the same RBD pool. +# Any host with a working path to the Ceph cluster can write/read this object, +# which gives the same semantics as the NFS/SharedMountPoint heartbeat file. + +help() { + printf "Usage: $0 + -s ceph monitor host(s), comma separated + -o ceph/rbd pool name + -n cephx auth user (optional) + -k cephx auth key (optional, required if -n is set) + -h host + -r write/read hb log + -c cleanup + -t interval between read hb log\n" + exit 1 +} +#set -x +MonHosts= +PoolName= +CephUser= +CephKey= +HostIP= +interval= +rflag=0 +cflag=0 + +while getopts 's:o:n:k:h:t:rc' OPTION +do + case $OPTION in + s) + MonHosts="$OPTARG" + ;; + o) + PoolName="$OPTARG" + ;; + n) + CephUser="$OPTARG" + ;; + k) + CephKey="$OPTARG" + ;; + h) + HostIP="$OPTARG" + ;; + r) + rflag=1 + ;; + t) + interval="$OPTARG" + ;; + c) + cflag=1 + ;; + *) + help + ;; + esac +done + +if [ -z "$MonHosts" ] || [ -z "$PoolName" ] +then + exit 1 +fi + +# the host IP names the heartbeat object, so it is required except for the self-fencing (-c) +if [ "$cflag" != "1" ] && [ -z "$HostIP" ] +then + exit 1 +fi + +if [ -n "$CephUser" ] && [ -z "$CephKey" ] +then + exit 1 +fi + +# -t is the maximum age of the heartbeat, which is compared in a check +if [ "$rflag" == "1" ] && ! [[ "$interval" =~ ^[0-9]+$ ]] +then + exit 1 +fi + +RadosOpts=(--mon-host "$MonHosts") +if [ -n "$CephUser" ] && [ "$cflag" != "1" ] +then + # the key is given to rados in a file, to keep it out of the process list + KeyFile=$(mktemp) + trap 'rm -f "$KeyFile"' EXIT + printf '%s' "$CephKey" > "$KeyFile" + RadosOpts+=(--id "$CephUser" --keyfile "$KeyFile") +fi + +hbObject="KVMHA-hb-$HostIP" + +write_hbLog() { + tmpFile=$(mktemp) + date +%s > "$tmpFile" + rados -p "$PoolName" "${RadosOpts[@]}" put "$hbObject" "$tmpFile" &> /dev/null Review Comment: No need to write tmpfiles, rados can read from stdin. Saves writing files and cleaning them ########## plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java: ########## @@ -330,29 +332,94 @@ public boolean isPoolSupportHA() { public String getHearthBeatPath() { if (StoragePoolType.NetworkFilesystem.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir); - } - return scriptPath; + return findKvmHaScript("kvmheartbeat.sh"); } else if (StoragePoolType.SharedMountPoint.equals(type)) { - String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); - String scriptPath = Script.findScript(kvmScriptsDir, "kvmsmpheartbeat.sh"); - if (scriptPath == null) { - throw new CloudRuntimeException("Unable to find heartbeat script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir); - } - return scriptPath; + return findKvmHaScript("kvmsmpheartbeat.sh"); + } else if (StoragePoolType.RBD.equals(type)) { + return findKvmHaScript("kvmheartbeat_rbd.sh"); } return null; } + private String findKvmHaScript(String scriptName) { + String kvmScriptsDir = AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR); + String scriptPath = Script.findScript(kvmScriptsDir, scriptName); + if (scriptPath == null) { + throw new CloudRuntimeException(String.format("Unable to find script '%s' in directory: %s", scriptName, kvmScriptsDir)); + } + return scriptPath; + } + + /** + * Returns the Ceph monitors as expected by "--mon-host": the comma-separated monitors of the pool, trimmed + * and without empty entries. If the pool has a monitor port, it is added to each monitor which has none yet. + */ + protected String getRbdMonitors() { + List<String> monitors = new ArrayList<>(); + for (String monitor : sourceHost.split(",")) { + monitor = monitor.trim(); + if (monitor.isEmpty()) { + continue; + } + monitors.add(sourcePort > 0 ? addPortToRbdMonitor(monitor) : monitor); + } + return String.join(",", monitors); + } + + private String addPortToRbdMonitor(String monitor) { + if (monitor.startsWith("[")) { + // IPv6 address in square brackets, which has a port if followed by ":<port>" + return monitor.contains("]:") ? monitor : monitor + ":" + sourcePort; + } + int colons = StringUtils.countMatches(monitor, ":"); + if (colons == 0) { + return monitor + ":" + sourcePort; + } + if (colons == 1) { + // IPv4 address or host name, with a port + return monitor; + } + // IPv6 address without square brackets, so without a port + return "[" + monitor + "]:" + sourcePort; + } + + /** + * Adds the Ceph cluster connection details (monitors, pool and, if cephx is enabled, credentials) + * to a heartbeat/VM-activity check {@link Script} for a RBD storage pool. Mirrors the "mon_host"/"id"/"key" + * options that qemu itself uses to talk to RBD (see {@link KVMPhysicalDisk#RBDStringBuilder}). + */ + protected void addRbdConnectionArgs(Script cmd) { + boolean hasUser = StringUtils.isNotBlank(authUsername); + boolean hasSecret = StringUtils.isNotBlank(authSecret); + if (hasUser != hasSecret) { + throw new CloudRuntimeException(String.format("The Ceph user and key must be either both set or both unset for the storage pool %s", uuid)); + } + cmd.add("-s", getRbdMonitors()); + cmd.add("-o", sourceDir); + if (hasUser) { + cmd.add("-n", authUsername); + cmd.add("-k"); + cmd.addSensitive(authSecret); Review Comment: Do we want this? Can’t we mask this somehow? Now when you would do a ‘ps’ you will see the Ceph secret. Qemu now masks this. We could also provide this through an environment variable for the script and not as an argument ########## scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh: ########## @@ -0,0 +1,167 @@ +#!/bin/bash +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Ceph RBD flavor of kvmheartbeat.sh/kvmsmpheartbeat.sh. +# +# There is no shared POSIX mount point to write a heartbeat file to when the +# primary storage pool is Ceph RBD, so the heartbeat timestamp is instead +# stored as a small RADOS object (one object per host) in the same RBD pool. +# Any host with a working path to the Ceph cluster can write/read this object, +# which gives the same semantics as the NFS/SharedMountPoint heartbeat file. + +help() { + printf "Usage: $0 + -s ceph monitor host(s), comma separated + -o ceph/rbd pool name + -n cephx auth user (optional) + -k cephx auth key (optional, required if -n is set) + -h host + -r write/read hb log + -c cleanup + -t interval between read hb log\n" + exit 1 +} +#set -x +MonHosts= +PoolName= +CephUser= +CephKey= +HostIP= +interval= +rflag=0 +cflag=0 + +while getopts 's:o:n:k:h:t:rc' OPTION +do + case $OPTION in + s) + MonHosts="$OPTARG" + ;; + o) + PoolName="$OPTARG" + ;; + n) + CephUser="$OPTARG" + ;; + k) + CephKey="$OPTARG" + ;; + h) + HostIP="$OPTARG" + ;; + r) + rflag=1 + ;; + t) + interval="$OPTARG" + ;; + c) + cflag=1 + ;; + *) + help + ;; + esac +done + +if [ -z "$MonHosts" ] || [ -z "$PoolName" ] +then + exit 1 +fi + +# the host IP names the heartbeat object, so it is required except for the self-fencing (-c) +if [ "$cflag" != "1" ] && [ -z "$HostIP" ] +then + exit 1 +fi + +if [ -n "$CephUser" ] && [ -z "$CephKey" ] +then + exit 1 +fi + +# -t is the maximum age of the heartbeat, which is compared in a check +if [ "$rflag" == "1" ] && ! [[ "$interval" =~ ^[0-9]+$ ]] +then + exit 1 +fi + +RadosOpts=(--mon-host "$MonHosts") +if [ -n "$CephUser" ] && [ "$cflag" != "1" ] +then + # the key is given to rados in a file, to keep it out of the process list + KeyFile=$(mktemp) + trap 'rm -f "$KeyFile"' EXIT + printf '%s' "$CephKey" > "$KeyFile" Review Comment: Exactly my comment above! -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
