Some encoders can hang indefinitely (i.e. nvh264enc) if the pipeline is not stopped before it is destroyed (Observed on Debian bookworm).
Signed-off-by: Dietmar Maurer <diet...@proxmox.com> --- include/ui/console.h | 1 + system/runstate.c | 2 ++ ui/vnc-enc-h264.c | 18 ++++++++++++++++++ ui/vnc.c | 15 +++++++++++++++ 4 files changed, 36 insertions(+) diff --git a/include/ui/console.h b/include/ui/console.h index 46b3128185..ff46e9fe98 100644 --- a/include/ui/console.h +++ b/include/ui/console.h @@ -458,6 +458,7 @@ int vnc_display_password(const char *id, const char *password); int vnc_display_pw_expire(const char *id, time_t expires); void vnc_parse(const char *str); int vnc_init_func(void *opaque, QemuOpts *opts, Error **errp); +void vnc_cleanup(void); bool vnc_display_reload_certs(const char *id, Error **errp); bool vnc_display_update(DisplayUpdateOptionsVNC *arg, Error **errp); diff --git a/system/runstate.c b/system/runstate.c index 272801d307..4b2c6f3525 100644 --- a/system/runstate.c +++ b/system/runstate.c @@ -51,6 +51,7 @@ #include "qemu/thread.h" #include "qom/object.h" #include "qom/object_interfaces.h" +#include "ui/console.h" #include "system/cpus.h" #include "system/qtest.h" #include "system/replay.h" @@ -924,6 +925,7 @@ void qemu_cleanup(int status) job_cancel_sync_all(); bdrv_close_all(); + vnc_cleanup(); /* vhost-user must be cleaned up before chardevs. */ tpm_cleanup(); net_cleanup(); diff --git a/ui/vnc-enc-h264.c b/ui/vnc-enc-h264.c index 98055c095f..6618f156b4 100644 --- a/ui/vnc-enc-h264.c +++ b/ui/vnc-enc-h264.c @@ -95,6 +95,24 @@ static GstElement *create_encoder(const char *encoder_name) static void destroy_encoder_context(VncState *vs) { + GstStateChangeReturn state_change_ret; + + VNC_DEBUG("Destroy h264 context.\n"); + + /* + * Some encoders can hang indefinitely (i.e. nvh264enc) if + * the pipeline is not stopped before it is destroyed + * (Observed on Debian bookworm). + */ + if (vs->h264->pipeline != NULL) { + state_change_ret = gst_element_set_state( + vs->h264->pipeline, GST_STATE_NULL); + + if (state_change_ret == GST_STATE_CHANGE_FAILURE) { + VNC_DEBUG("Unable to stop the GST pipeline\n"); + } + } + gst_clear_object(&vs->h264->source); gst_clear_object(&vs->h264->convert); gst_clear_object(&vs->h264->gst_encoder); diff --git a/ui/vnc.c b/ui/vnc.c index 2d1e741705..062d6af0ab 100644 --- a/ui/vnc.c +++ b/ui/vnc.c @@ -4366,6 +4366,21 @@ int vnc_init_func(void *opaque, QemuOpts *opts, Error **errp) return 0; } +void vnc_cleanup(void) +{ + VncDisplay *vd; + VncState *vs; + + QTAILQ_FOREACH(vd, &vnc_displays, next) { + QTAILQ_FOREACH(vs, &vd->clients, next) { +#ifdef CONFIG_GSTREAMER + /* correctly close all h264 encoder pipelines */ + vnc_h264_clear(vs); +#endif + } + } +} + static void vnc_register_config(void) { qemu_add_opts(&qemu_vnc_opts); -- 2.39.5