Copilot commented on code in PR #14320:
URL: https://github.com/apache/cloudstack/pull/14320#discussion_r4218622477
##########
plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePool.java:
##########
@@ -330,29 +332,83 @@ public boolean isPoolSupportHA() {
public String getHearthBeatPath() {
if (StoragePoolType.NetworkFilesystem.equals(type)) {
- String kvmScriptsDir =
AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR);
- String scriptPath = Script.findScript(kvmScriptsDir,
"kvmheartbeat.sh");
- if (scriptPath == null) {
- throw new CloudRuntimeException("Unable to find heartbeat
script 'kvmheartbeat.sh' in directory: " + kvmScriptsDir);
- }
- return scriptPath;
+ return findKvmHaScript("kvmheartbeat.sh");
} else if (StoragePoolType.SharedMountPoint.equals(type)) {
- String kvmScriptsDir =
AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR);
- String scriptPath = Script.findScript(kvmScriptsDir,
"kvmsmpheartbeat.sh");
- if (scriptPath == null) {
- throw new CloudRuntimeException("Unable to find heartbeat
script 'kvmsmpheartbeat.sh' in directory: " + kvmScriptsDir);
- }
- return scriptPath;
+ return findKvmHaScript("kvmsmpheartbeat.sh");
+ } else if (StoragePoolType.RBD.equals(type)) {
+ return findKvmHaScript("kvmheartbeat_rbd.sh");
}
return null;
}
+ private String findKvmHaScript(String scriptName) {
+ String kvmScriptsDir =
AgentPropertiesFileHandler.getPropertyValue(AgentProperties.KVM_SCRIPTS_DIR);
+ String scriptPath = Script.findScript(kvmScriptsDir, scriptName);
+ if (scriptPath == null) {
+ throw new CloudRuntimeException(String.format("Unable to find
script '%s' in directory: %s", scriptName, kvmScriptsDir));
+ }
+ return scriptPath;
+ }
+
+ /**
+ * Returns the Ceph monitors as expected by "--mon-host": the
comma-separated monitors of the pool, trimmed
+ * and without empty entries, each with the pool's monitor port if one is
set (IPv6 addresses are enclosed
+ * in square brackets then).
+ */
+ protected String getRbdMonitors() {
+ List<String> monitors = new ArrayList<>();
+ for (String monitor : sourceHost.split(",")) {
+ monitor = monitor.trim();
+ if (monitor.isEmpty()) {
+ continue;
+ }
+ if (sourcePort > 0) {
+ if (monitor.contains(":") && !monitor.startsWith("[")) {
+ monitor = "[" + monitor + "]";
+ }
+ monitor = monitor + ":" + sourcePort;
+ }
+ monitors.add(monitor);
+ }
+ return String.join(",", monitors);
+ }
Review Comment:
This logic treats any monitor containing `:` as an IPv6 literal and will
wrap it in brackets, which breaks valid `host:port` / `ipv4:port` inputs. It
also blindly appends `sourcePort`, producing invalid endpoints like
`10.0.0.1:3300:6789` if ports are already present. Consider detecting and
preserving already-port-suffixed entries (and only bracket true IPv6 literals)
before appending `sourcePort`.
##########
plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/storage/LibvirtStoragePoolTest.java:
##########
@@ -98,4 +100,126 @@ public void testExternalSnapshot() {
LibvirtStoragePool clvmPool = new LibvirtStoragePool(uuid, name,
StoragePoolType.CLVM, adapter, storage);
assertTrue(clvmPool.isExternalSnapshot());
}
+
+ @Test
+ public void testIsPoolSupportHA() {
+ String uuid = "0f7a58bd-1a85-4b1f-9f91-12f3d1ecf5a5";
+ String name = "myfirstpool";
+
+ StorageAdaptor adapter = Mockito.mock(LibvirtStorageAdaptor.class);
+ StoragePool storage = Mockito.mock(StoragePool.class);
+
+ // NetworkFilesystem, SharedMountPoint and RBD all support the KVM
Host-HA
+ // heartbeat/VM-activity check mechanism.
+ assertTrue(new LibvirtStoragePool(uuid, name,
StoragePoolType.NetworkFilesystem, adapter, storage).isPoolSupportHA());
+ assertTrue(new LibvirtStoragePool(uuid, name,
StoragePoolType.SharedMountPoint, adapter, storage).isPoolSupportHA());
+ assertTrue(new LibvirtStoragePool(uuid, name, StoragePoolType.RBD,
adapter, storage).isPoolSupportHA());
+
+ // Other pool types have no HA support.
+ assertFalse(new LibvirtStoragePool(uuid, name, StoragePoolType.CLVM,
adapter, storage).isPoolSupportHA());
+ assertFalse(new LibvirtStoragePool(uuid, name,
StoragePoolType.Filesystem, adapter, storage).isPoolSupportHA());
+ }
+
+ private String getRbdMonitors(String sourceHost, int sourcePort) {
+ LibvirtStoragePool pool = new
LibvirtStoragePool("0f7a58bd-1a85-4b1f-9f91-12f3d1ecf5a5", "myfirstpool",
StoragePoolType.RBD,
+ Mockito.mock(LibvirtStorageAdaptor.class),
Mockito.mock(StoragePool.class));
+ pool.setSourceHost(sourceHost);
+ pool.setSourcePort(sourcePort);
+ return pool.getRbdMonitors();
+ }
+
+ @Test
+ public void testRbdMonitorsWithoutPort() {
+ assertEquals("10.0.0.1", getRbdMonitors("10.0.0.1", 0));
+ assertEquals("10.0.0.1,10.0.0.2,10.0.0.3",
getRbdMonitors("10.0.0.1,10.0.0.2,10.0.0.3", 0));
+ assertEquals("fd00::1,fd00::2", getRbdMonitors("fd00::1,fd00::2", 0));
+ }
+
+ @Test
+ public void testRbdMonitorsIpv4WithPort() {
+ assertEquals("10.0.0.1:6789", getRbdMonitors("10.0.0.1", 6789));
+ assertEquals("10.0.0.1:3300,10.0.0.2:3300,10.0.0.3:3300",
getRbdMonitors("10.0.0.1,10.0.0.2,10.0.0.3", 3300));
+ }
+
+ @Test
Review Comment:
Given `getRbdMonitors()` currently branches on the presence of `:` (which
can also appear in `host:port`), add tests for inputs where `sourceHost`
entries already include ports (e.g., `10.0.0.1:6789`, `mon.example.com:6789`,
or `[fd00::1]:6789`) to lock in the intended behavior once the parsing is
corrected.
##########
scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh:
##########
@@ -0,0 +1,157 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Ceph RBD flavor of kvmheartbeat.sh/kvmsmpheartbeat.sh.
+#
+# There is no shared POSIX mount point to write a heartbeat file to when the
+# primary storage pool is Ceph RBD, so the heartbeat timestamp is instead
+# stored as a small RADOS object (one object per host) in the same RBD pool.
+# Any host with a working path to the Ceph cluster can write/read this object,
+# which gives the same semantics as the NFS/SharedMountPoint heartbeat file.
+
+help() {
+ printf "Usage: $0
+ -s ceph monitor host(s), comma separated
+ -o ceph/rbd pool name
+ -n cephx auth user (optional)
+ -k cephx auth key, base64 (optional, required if -n is set)
+ -h host
+ -r write/read hb log
+ -c cleanup
+ -t interval between read hb log\n"
+ exit 1
+}
+#set -x
+MonHosts=
+PoolName=
+CephUser=
+CephKey=
+HostIP=
+interval=
+rflag=0
+cflag=0
+
+while getopts 's:o:n:k:h:t:rc' OPTION
+do
+ case $OPTION in
+ s)
+ MonHosts="$OPTARG"
+ ;;
+ o)
+ PoolName="$OPTARG"
+ ;;
+ n)
+ CephUser="$OPTARG"
+ ;;
+ k)
+ CephKey="$OPTARG"
+ ;;
+ h)
+ HostIP="$OPTARG"
+ ;;
+ r)
+ rflag=1
+ ;;
+ t)
+ interval="$OPTARG"
+ ;;
+ c)
+ cflag=1
+ ;;
+ *)
+ help
+ ;;
+ esac
+done
+
+if [ -z "$MonHosts" ] || [ -z "$PoolName" ]
+then
+ exit 1
+fi
+
+# the host IP names the heartbeat object, so it is required except for the
self-fencing (-c)
+if [ "$cflag" != "1" ] && [ -z "$HostIP" ]
+then
+ exit 1
+fi
+
+if [ -n "$CephUser" ] && [ -z "$CephKey" ]
+then
+ exit 1
+fi
+
+RadosOpts=(--mon-host "$MonHosts")
+if [ -n "$CephUser" ]
+then
+ RadosOpts+=(--id "$CephUser" --key "$CephKey")
+fi
Review Comment:
Passing the Ceph key via `--key` exposes it in the process list (both for
this script’s args and the spawned `rados` process), which is a credential
leakage risk on multi-user hosts. Prefer using `--keyfile` (write to a
root-owned temp file with `chmod 600` + `trap` cleanup) or relying on a
pre-provisioned keyring/ceph.conf so secrets aren’t present on the command line.
##########
scripts/vm/hypervisor/kvm/kvmheartbeat_rbd.sh:
##########
@@ -0,0 +1,157 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Ceph RBD flavor of kvmheartbeat.sh/kvmsmpheartbeat.sh.
+#
+# There is no shared POSIX mount point to write a heartbeat file to when the
+# primary storage pool is Ceph RBD, so the heartbeat timestamp is instead
+# stored as a small RADOS object (one object per host) in the same RBD pool.
+# Any host with a working path to the Ceph cluster can write/read this object,
+# which gives the same semantics as the NFS/SharedMountPoint heartbeat file.
+
+help() {
+ printf "Usage: $0
+ -s ceph monitor host(s), comma separated
+ -o ceph/rbd pool name
+ -n cephx auth user (optional)
+ -k cephx auth key, base64 (optional, required if -n is set)
+ -h host
+ -r write/read hb log
+ -c cleanup
+ -t interval between read hb log\n"
+ exit 1
+}
+#set -x
+MonHosts=
+PoolName=
+CephUser=
+CephKey=
+HostIP=
+interval=
+rflag=0
+cflag=0
+
+while getopts 's:o:n:k:h:t:rc' OPTION
+do
+ case $OPTION in
+ s)
+ MonHosts="$OPTARG"
+ ;;
+ o)
+ PoolName="$OPTARG"
+ ;;
+ n)
+ CephUser="$OPTARG"
+ ;;
+ k)
+ CephKey="$OPTARG"
+ ;;
+ h)
+ HostIP="$OPTARG"
+ ;;
+ r)
+ rflag=1
+ ;;
+ t)
+ interval="$OPTARG"
+ ;;
+ c)
+ cflag=1
+ ;;
+ *)
+ help
+ ;;
+ esac
+done
+
+if [ -z "$MonHosts" ] || [ -z "$PoolName" ]
+then
+ exit 1
+fi
+
+# the host IP names the heartbeat object, so it is required except for the
self-fencing (-c)
+if [ "$cflag" != "1" ] && [ -z "$HostIP" ]
+then
+ exit 1
+fi
+
+if [ -n "$CephUser" ] && [ -z "$CephKey" ]
+then
+ exit 1
+fi
+
+RadosOpts=(--mon-host "$MonHosts")
+if [ -n "$CephUser" ]
+then
+ RadosOpts+=(--id "$CephUser" --key "$CephKey")
+fi
+
+hbObject="KVMHA-hb-$HostIP"
+
+write_hbLog() {
+ tmpFile=$(mktemp)
+ date +%s > "$tmpFile"
+ rados -p "$PoolName" "${RadosOpts[@]}" put "$hbObject" "$tmpFile" &>
/dev/null
+ rc=$?
+ rm -f "$tmpFile"
+ return $rc
+}
+
+check_hbLog() {
+ now=$(date +%s)
+ hb=$(rados -p "$PoolName" "${RadosOpts[@]}" get "$hbObject" - 2> /dev/null)
+ if ! [[ "$hb" =~ ^[0-9]+$ ]]
+ then
+ # Either the RADOS object doesn't exist yet (host never wrote a heartbeat)
+ # or the Ceph cluster can't be reached right now. Either way we can't
+ # confirm the host is alive, so fail safe and report it as DEAD.
+ hbAge=
+ return 1
+ fi
+ # the age is kept in a variable, not in the return status, as a status above
255 wraps around
+ hbAge=$(expr $now - $hb)
+ if [ $hbAge -gt $interval ]
+ then
+ return 1
+ fi
+ return 0
+}
Review Comment:
`interval` is used in a numeric comparison but is never validated when
running with `-r`. If `-r` is invoked without `-t` (or with a non-numeric
value), `[ $hbAge -gt $interval ]` will error and may produce incorrect
behavior. Make `-t` mandatory when `-r` is set (and validate it’s a positive
integer) before calling `check_hbLog()`.
##########
scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh:
##########
@@ -0,0 +1,194 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Ceph RBD flavor of kvmvmactivity.sh.
+#
+# On NFS/SharedMountPoint storage, VM disk activity is detected via the mtime
+# of the volume files on the shared mount point. RBD volumes aren't files on
+# a mount point, so instead activity is detected via RBD watchers: as long as
+# qemu has an RBD image open (i.e. a VM using that volume is running
+# somewhere), the image will have a live watcher. The most recent
+# suspect-time/watcher-state is persisted as a RADOS object (per host)
+# in place of the "ac-<host>" file used by the NFS/SMP scripts.
+
+help() {
+ printf "Usage: $0
+ -s ceph monitor host(s), comma separated
+ -o ceph/rbd pool name
+ -n cephx auth user (optional)
+ -k cephx auth key, base64 (optional, required if -n is set)
+ -h host
+ -u volume (rbd image) uuid list
+ -t current time in seconds (accepted for compatibility
with kvmvmactivity.sh, not used)
+ -d suspect time\n"
+ exit 1
+}
+
+#set -x
+
+MonHosts=
+PoolName=
+CephUser=
+CephKey=
+HostIP=
+UUIDList=
+SuspectTime=
+
+while getopts 's:o:n:k:h:u:t:d:' OPTION
+do
+ case $OPTION in
+ s)
+ MonHosts="$OPTARG"
+ ;;
+ o)
+ PoolName="$OPTARG"
+ ;;
+ n)
+ CephUser="$OPTARG"
+ ;;
+ k)
+ CephKey="$OPTARG"
+ ;;
+ h)
+ HostIP="$OPTARG"
+ ;;
+ u)
+ UUIDList="$OPTARG"
+ ;;
+ t)
+ # not used, see help
+ ;;
+ d)
+ SuspectTime="$OPTARG"
+ ;;
+ *)
+ help
+ ;;
+ esac
+done
+
+if [ -z "$MonHosts" ] || [ -z "$PoolName" ]
+then
+ exit 2
+fi
+
+if [ -z "$SuspectTime" ]
+then
+ exit 2
+fi
+
+# the host IP names the heartbeat and activity objects
+if [ -z "$HostIP" ]
+then
+ exit 2
+fi
+
+if [ -n "$CephUser" ] && [ -z "$CephKey" ]
+then
+ exit 2
+fi
+
+RadosOpts=(--mon-host "$MonHosts")
+RbdOpts=(--mon-host "$MonHosts")
+if [ -n "$CephUser" ]
+then
+ RadosOpts+=(--id "$CephUser" --key "$CephKey")
+ RbdOpts+=(--id "$CephUser" --key "$CephKey")
+fi
+
+hbObject="KVMHA-hb-$HostIP"
+acObject="KVMHA-ac-$HostIP"
+
+# First check: heartbeat object, same as kvmheartbeat_rbd.sh
+now=$(date +%s)
+hb=$(rados -p "$PoolName" "${RadosOpts[@]}" get "$hbObject" - 2> /dev/null)
+if [[ "$hb" =~ ^[0-9]+$ ]]
+then
+ diff=$(expr $now - $hb)
+ if [ $diff -lt 61 ]
+ then
+ echo "=====> ALIVE <====="
+ exit 0
+ fi
+fi
+
+if [ -z "$UUIDList" ]
+then
+ echo "=====> Considering host as DEAD due to empty UUIDList <======"
+ exit 0
+fi
+
+# Second check: RBD watcher based disk activity check.
+# If any of the host's volumes still has a live watcher, something (most
+# likely qemu on the host being checked) is actively using it right now.
+latestUpdateTime=0
+for image in ${UUIDList//,/ }
+do
+ watcherCount=$(rbd status "$PoolName/$image" "${RbdOpts[@]}" --format json
2> /dev/null | \
+ python3 -c 'import json,sys
+try:
+ print(len(json.load(sys.stdin).get("watchers", [])))
+except Exception:
+ print(0)' 2> /dev/null)
+ if [ -n "$watcherCount" ] && [ "$watcherCount" -gt 0 ] 2> /dev/null
+ then
+ latestUpdateTime=$now
+ break
+ fi
+done
+
+if [ ! -z "$(rados -p "$PoolName" "${RadosOpts[@]}" stat "$acObject" 2>
/dev/null)" ]
+then
+ acTime=$(rados -p "$PoolName" "${RadosOpts[@]}" get "$acObject" - 2>
/dev/null)
+else
+ acTime=
+fi
Review Comment:
Using `rados ... stat` output content to detect existence is brittle (output
formats can vary, and some `stat` info may go to stderr). Use the command’s
exit status instead (e.g., run `stat` with stdout/stderr redirected and branch
on `$?`) to reliably detect object existence.
##########
scripts/vm/hypervisor/kvm/kvmvmactivity_rbd.sh:
##########
@@ -0,0 +1,194 @@
+#!/bin/bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Ceph RBD flavor of kvmvmactivity.sh.
+#
+# On NFS/SharedMountPoint storage, VM disk activity is detected via the mtime
+# of the volume files on the shared mount point. RBD volumes aren't files on
+# a mount point, so instead activity is detected via RBD watchers: as long as
+# qemu has an RBD image open (i.e. a VM using that volume is running
+# somewhere), the image will have a live watcher. The most recent
+# suspect-time/watcher-state is persisted as a RADOS object (per host)
+# in place of the "ac-<host>" file used by the NFS/SMP scripts.
+
+help() {
+ printf "Usage: $0
+ -s ceph monitor host(s), comma separated
+ -o ceph/rbd pool name
+ -n cephx auth user (optional)
+ -k cephx auth key, base64 (optional, required if -n is set)
+ -h host
+ -u volume (rbd image) uuid list
+ -t current time in seconds (accepted for compatibility
with kvmvmactivity.sh, not used)
+ -d suspect time\n"
+ exit 1
+}
+
+#set -x
+
+MonHosts=
+PoolName=
+CephUser=
+CephKey=
+HostIP=
+UUIDList=
+SuspectTime=
+
+while getopts 's:o:n:k:h:u:t:d:' OPTION
+do
+ case $OPTION in
+ s)
+ MonHosts="$OPTARG"
+ ;;
+ o)
+ PoolName="$OPTARG"
+ ;;
+ n)
+ CephUser="$OPTARG"
+ ;;
+ k)
+ CephKey="$OPTARG"
+ ;;
+ h)
+ HostIP="$OPTARG"
+ ;;
+ u)
+ UUIDList="$OPTARG"
+ ;;
+ t)
+ # not used, see help
+ ;;
+ d)
+ SuspectTime="$OPTARG"
+ ;;
+ *)
+ help
+ ;;
+ esac
+done
+
+if [ -z "$MonHosts" ] || [ -z "$PoolName" ]
+then
+ exit 2
+fi
+
+if [ -z "$SuspectTime" ]
+then
+ exit 2
+fi
+
+# the host IP names the heartbeat and activity objects
+if [ -z "$HostIP" ]
+then
+ exit 2
+fi
+
+if [ -n "$CephUser" ] && [ -z "$CephKey" ]
+then
+ exit 2
+fi
+
+RadosOpts=(--mon-host "$MonHosts")
+RbdOpts=(--mon-host "$MonHosts")
+if [ -n "$CephUser" ]
+then
+ RadosOpts+=(--id "$CephUser" --key "$CephKey")
+ RbdOpts+=(--id "$CephUser" --key "$CephKey")
+fi
+
+hbObject="KVMHA-hb-$HostIP"
+acObject="KVMHA-ac-$HostIP"
+
+# First check: heartbeat object, same as kvmheartbeat_rbd.sh
+now=$(date +%s)
+hb=$(rados -p "$PoolName" "${RadosOpts[@]}" get "$hbObject" - 2> /dev/null)
+if [[ "$hb" =~ ^[0-9]+$ ]]
+then
+ diff=$(expr $now - $hb)
+ if [ $diff -lt 61 ]
+ then
+ echo "=====> ALIVE <====="
+ exit 0
+ fi
+fi
+
+if [ -z "$UUIDList" ]
+then
+ echo "=====> Considering host as DEAD due to empty UUIDList <======"
+ exit 0
+fi
+
+# Second check: RBD watcher based disk activity check.
+# If any of the host's volumes still has a live watcher, something (most
+# likely qemu on the host being checked) is actively using it right now.
+latestUpdateTime=0
+for image in ${UUIDList//,/ }
+do
Review Comment:
The unquoted `${UUIDList//,/ }` expansion relies on shell word-splitting and
can mis-handle whitespace (e.g., `uuid1, uuid2`) and can also trigger pathname
expansion. Parse the list using a controlled `IFS=',' read -ra` (or similar)
and ensure `image` is handled without glob expansion.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
