This is an automated email from the ASF dual-hosted git repository. weizhouapache pushed a commit to branch 4.20-kvm-reload-vnc-tls in repository https://gitbox.apache.org/repos/asf/cloudstack.git
commit dd4d1482f408cb43a3cdcb40e212abce31a0c8ca Author: Wei Zhou <[email protected]> AuthorDate: Fri Sep 11 12:49:00 2026 +0200 kvm: reload VNC TLS certificate on running VMs after cert renewal Fixes VM console access breaking after the VNC certificate expires and is renewed (issue apache/cloudstack#9718). On KVM, the VNC TLS certificate is the host's agent certificate, applied host-wide via libvirtd's vnc_tls_x509_cert_dir setting. Restarting libvirtd after a renewal does not affect VMs already running, since QEMU only loads that certificate once, at VM start. Extend the existing PostCertificateRenewalCommand handling to, after restarting libvirtd, reload the VNC TLS certificate live on every currently running VM via the QMP display-reload command (added in QEMU 6.0, commit 9cc07651655ee86eca41059f5ead8c4e5607c734), sent through libvirt's qemu-monitor-command passthrough for compatibility with older libvirt. Hosts running QEMU < 6.0 are detected and skipped with a warning, keeping the previous stop/start-or-migrate behavior. --- ...ibvirtPostCertificateRenewalCommandWrapper.java | 77 ++++++++++- ...rtPostCertificateRenewalCommandWrapperTest.java | 147 +++++++++++++++++++++ 2 files changed, 219 insertions(+), 5 deletions(-) diff --git a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java index 49e079348ef..60709c75100 100644 --- a/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java +++ b/plugins/hypervisors/kvm/src/main/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapper.java @@ -19,6 +19,9 @@ package com.cloud.hypervisor.kvm.resource.wrapper; import org.apache.cloudstack.ca.PostCertificateRenewalCommand; import org.apache.cloudstack.ca.SetupCertificateAnswer; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; import com.cloud.agent.api.Answer; import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; @@ -29,18 +32,82 @@ import com.cloud.utils.script.Script; @ResourceWrapper(handles = PostCertificateRenewalCommand.class) public final class LibvirtPostCertificateRenewalCommandWrapper extends CommandWrapper<PostCertificateRenewalCommand, Answer, LibvirtComputingResource> { + /** + * QMP {@code display-reload} command asking QEMU to reload the VNC display's TLS credentials off disk. + * {@code tls-certs: true} is required, otherwise QEMU reloads the display without touching the certificates. + * Added in QEMU 6.0 by commit 9cc07651655ee86eca41059f5ead8c4e5607c734 ("qmp: add new qmp display-reload", + * merged 2021-03-23) - https://github.com/qemu/qemu/commit/9cc07651655ee86eca41059f5ead8c4e5607c734 + */ + private static final String QEMU_MONITOR_DISPLAY_RELOAD_VNC_TLS_CERTS_COMMAND = + "{\"execute\":\"display-reload\",\"arguments\":{\"type\":\"vnc\",\"tls-certs\":true}}"; + + /** Minimum QEMU version supporting {@link #QEMU_MONITOR_DISPLAY_RELOAD_VNC_TLS_CERTS_COMMAND}. */ + private static final long MIN_QEMU_VERSION_FOR_VNC_TLS_CERT_RELOAD = 6000000L; @Override public Answer execute(final PostCertificateRenewalCommand command, final LibvirtComputingResource serverResource) { logger.info("Restarting libvirt after certificate provisioning/renewal"); if (command != null) { - final int timeout = 30000; - Script script = new Script(true, "service", timeout, logger); - script.add("libvirtd"); - script.add("restart"); - script.execute(); + restartLibvirtd(); + pushRenewedVncCertificateToRunningVms(serverResource); return new SetupCertificateAnswer(true); } return new SetupCertificateAnswer(false); } + + private void restartLibvirtd() { + final int timeout = 30000; + Script script = new Script(true, "service", timeout, logger); + script.add("libvirtd"); + script.add("restart"); + script.execute(); + } + + /** + * The VNC TLS certificate on KVM is the host's agent certificate, applied host-wide via libvirtd's + * {@code vnc_tls_x509_cert_dir} setting. Restarting libvirtd does not affect VMs already running, since QEMU + * only loads that certificate once, at VM start - so reload it live on every running VM here, instead of + * leaving them on the previous (possibly expired) certificate until stopped/started or migrated. + */ + private void pushRenewedVncCertificateToRunningVms(final LibvirtComputingResource serverResource) { + final long qemuVersion = serverResource.getHypervisorQemuVersion(); + if (qemuVersion < MIN_QEMU_VERSION_FOR_VNC_TLS_CERT_RELOAD) { + logger.warn("QEMU {} on this host does not support reloading the VNC TLS certificate of a running VM (QEMU >= {} required), " + + "running VMs will keep using the previous certificate until they are stopped/started or migrated", + qemuVersion, MIN_QEMU_VERSION_FOR_VNC_TLS_CERT_RELOAD); + return; + } + + final Connect conn; + final int[] domainIds; + try { + conn = serverResource.getLibvirtUtilitiesHelper().getConnection(); + domainIds = conn.listDomains(); + } catch (final LibvirtException e) { + logger.warn("Unable to list running VMs to reload their renewed VNC certificate", e); + return; + } + + for (final int domainId : domainIds) { + Domain vm = null; + String vmName = null; + try { + vm = conn.domainLookupByID(domainId); + vmName = vm.getName(); + vm.qemuMonitorCommand(QEMU_MONITOR_DISPLAY_RELOAD_VNC_TLS_CERTS_COMMAND, 0); + logger.debug("Reloaded VNC TLS certificate for VM [{}]", vmName); + } catch (final Exception e) { + logger.warn("Failed to reload the renewed VNC certificate for VM [{}], it will keep using the previous " + + "certificate until it is stopped/started or migrated", vmName, e); + } finally { + if (vm != null) { + try { + vm.free(); + } catch (final LibvirtException e) { + logger.trace("Ignoring libvirt error.", e); + } + } + } + } + } } diff --git a/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapperTest.java b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapperTest.java new file mode 100644 index 00000000000..9ee1fe75def --- /dev/null +++ b/plugins/hypervisors/kvm/src/test/java/com/cloud/hypervisor/kvm/resource/wrapper/LibvirtPostCertificateRenewalCommandWrapperTest.java @@ -0,0 +1,147 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. +package com.cloud.hypervisor.kvm.resource.wrapper; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.verifyNoInteractions; +import static org.mockito.Mockito.when; + +import org.apache.cloudstack.ca.PostCertificateRenewalCommand; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.libvirt.Connect; +import org.libvirt.Domain; +import org.libvirt.LibvirtException; +import org.mockito.ArgumentCaptor; +import org.mockito.Mock; +import org.mockito.Mockito; +import org.mockito.MockedConstruction; +import org.mockito.junit.MockitoJUnitRunner; + +import com.cloud.agent.api.Answer; +import com.cloud.hypervisor.kvm.resource.LibvirtComputingResource; +import com.cloud.utils.script.Script; + +@RunWith(MockitoJUnitRunner.class) +public class LibvirtPostCertificateRenewalCommandWrapperTest { + + private static final long SUPPORTED_QEMU_VERSION = 6000000L; + + @Mock + private LibvirtComputingResource libvirtComputingResource; + @Mock + private LibvirtUtilitiesHelper libvirtUtilitiesHelper; + @Mock + private Connect connect; + + private final LibvirtPostCertificateRenewalCommandWrapper wrapper = new LibvirtPostCertificateRenewalCommandWrapper(); + + @Before + public void setUp() { + when(libvirtComputingResource.getHypervisorQemuVersion()).thenReturn(SUPPORTED_QEMU_VERSION); + } + + private Answer executeWithScriptMocked() { + try (MockedConstruction<Script> ignored = Mockito.mockConstruction(Script.class)) { + return wrapper.execute(new PostCertificateRenewalCommand(), libvirtComputingResource); + } + } + + @Test + public void testExecuteReloadsVncTlsCertificateForEachRunningVm() throws Exception { + final Domain vm1 = mock(Domain.class); + when(vm1.getName()).thenReturn("i-2-3-VM"); + final Domain vm2 = mock(Domain.class); + when(vm2.getName()).thenReturn("i-4-5-VM"); + + when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect); + when(connect.listDomains()).thenReturn(new int[]{1, 2}); + when(connect.domainLookupByID(1)).thenReturn(vm1); + when(connect.domainLookupByID(2)).thenReturn(vm2); + + final Answer answer = executeWithScriptMocked(); + + assertTrue(answer.getResult()); + final ArgumentCaptor<String> monitorCommandCaptor = ArgumentCaptor.forClass(String.class); + verify(vm1, times(1)).qemuMonitorCommand(monitorCommandCaptor.capture(), Mockito.eq(0)); + verify(vm2, times(1)).qemuMonitorCommand(Mockito.anyString(), Mockito.eq(0)); + final String capturedCommand = monitorCommandCaptor.getValue(); + assertTrue(capturedCommand.contains("display-reload")); + assertTrue(capturedCommand.contains("\"type\":\"vnc\"")); + assertTrue(capturedCommand.contains("\"tls-certs\":true")); + verify(vm1, times(1)).free(); + verify(vm2, times(1)).free(); + } + + @Test + public void testExecuteContinuesWithOtherVmsWhenOneReloadFails() throws Exception { + final Domain failingVm = mock(Domain.class); + when(failingVm.getName()).thenReturn("i-2-3-VM"); + when(failingVm.qemuMonitorCommand(Mockito.anyString(), Mockito.eq(0))).thenThrow(mock(LibvirtException.class)); + final Domain workingVm = mock(Domain.class); + when(workingVm.getName()).thenReturn("i-4-5-VM"); + + when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect); + when(connect.listDomains()).thenReturn(new int[]{1, 2}); + when(connect.domainLookupByID(1)).thenReturn(failingVm); + when(connect.domainLookupByID(2)).thenReturn(workingVm); + + final Answer answer = executeWithScriptMocked(); + + assertTrue(answer.getResult()); + verify(workingVm, times(1)).qemuMonitorCommand(Mockito.anyString(), Mockito.eq(0)); + verify(failingVm, times(1)).free(); + verify(workingVm, times(1)).free(); + } + + @Test + public void testExecuteContinuesWhenNoRunningVms() throws Exception { + when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnection()).thenReturn(connect); + when(connect.listDomains()).thenReturn(new int[]{}); + + final Answer answer = executeWithScriptMocked(); + + assertTrue(answer.getResult()); + } + + @Test + public void testExecuteHandlesUnableToListRunningVms() throws Exception { + when(libvirtComputingResource.getLibvirtUtilitiesHelper()).thenReturn(libvirtUtilitiesHelper); + when(libvirtUtilitiesHelper.getConnection()).thenThrow(mock(LibvirtException.class)); + + final Answer answer = executeWithScriptMocked(); + + assertTrue(answer.getResult()); + } + + @Test + public void testExecuteSkipsVncCertReloadWhenQemuVersionTooOld() throws Exception { + when(libvirtComputingResource.getHypervisorQemuVersion()).thenReturn(5002000L); + + final Answer answer = executeWithScriptMocked(); + + assertTrue(answer.getResult()); + verifyNoInteractions(libvirtUtilitiesHelper); + } +}
