[PATCH v4 8/8] h264: stop gstreamer pipeline before destroying, cleanup on exit

[Dietmar Maurer]

Some encoders can hang indefinitely (i.e. nvh264enc) if
the pipeline is not stopped before it is destroyed
(Observed on Debian bookworm).

Signed-off-by: Dietmar Maurer <diet...@proxmox.com>
---
 include/ui/console.h |  1 +
 system/runstate.c    |  2 ++
 ui/vnc-enc-h264.c    | 18 ++++++++++++++++++
 ui/vnc.c             | 15 +++++++++++++++
 4 files changed, 36 insertions(+)

diff --git a/include/ui/console.h b/include/ui/console.h
index 46b3128185..ff46e9fe98 100644
--- a/include/ui/console.h
+++ b/include/ui/console.h
@@ -458,6 +458,7 @@ int vnc_display_password(const char *id, const char 
*password);
 int vnc_display_pw_expire(const char *id, time_t expires);
 void vnc_parse(const char *str);
 int vnc_init_func(void *opaque, QemuOpts *opts, Error **errp);
+void vnc_cleanup(void);
 bool vnc_display_reload_certs(const char *id,  Error **errp);
 bool vnc_display_update(DisplayUpdateOptionsVNC *arg, Error **errp);
 
diff --git a/system/runstate.c b/system/runstate.c
index 272801d307..4b2c6f3525 100644
--- a/system/runstate.c
+++ b/system/runstate.c
@@ -51,6 +51,7 @@
 #include "qemu/thread.h"
 #include "qom/object.h"
 #include "qom/object_interfaces.h"
+#include "ui/console.h"
 #include "system/cpus.h"
 #include "system/qtest.h"
 #include "system/replay.h"
@@ -924,6 +925,7 @@ void qemu_cleanup(int status)
     job_cancel_sync_all();
     bdrv_close_all();
 
+    vnc_cleanup();
     /* vhost-user must be cleaned up before chardevs.  */
     tpm_cleanup();
     net_cleanup();
diff --git a/ui/vnc-enc-h264.c b/ui/vnc-enc-h264.c
index 98055c095f..6618f156b4 100644
--- a/ui/vnc-enc-h264.c
+++ b/ui/vnc-enc-h264.c
@@ -95,6 +95,24 @@ static GstElement *create_encoder(const char *encoder_name)
 
 static void destroy_encoder_context(VncState *vs)
 {
+    GstStateChangeReturn state_change_ret;
+
+    VNC_DEBUG("Destroy h264 context.\n");
+
+    /*
+     * Some encoders can hang indefinitely (i.e. nvh264enc) if
+     * the pipeline is not stopped before it is destroyed
+     * (Observed on Debian bookworm).
+     */
+    if (vs->h264->pipeline != NULL) {
+        state_change_ret = gst_element_set_state(
+            vs->h264->pipeline, GST_STATE_NULL);
+
+        if (state_change_ret == GST_STATE_CHANGE_FAILURE) {
+            VNC_DEBUG("Unable to stop the GST pipeline\n");
+        }
+    }
+
     gst_clear_object(&vs->h264->source);
     gst_clear_object(&vs->h264->convert);
     gst_clear_object(&vs->h264->gst_encoder);
diff --git a/ui/vnc.c b/ui/vnc.c
index 2d1e741705..062d6af0ab 100644
--- a/ui/vnc.c
+++ b/ui/vnc.c
@@ -4366,6 +4366,21 @@ int vnc_init_func(void *opaque, QemuOpts *opts, Error 
**errp)
     return 0;
 }
 
+void vnc_cleanup(void)
+{
+    VncDisplay *vd;
+    VncState *vs;
+
+    QTAILQ_FOREACH(vd, &vnc_displays, next) {
+        QTAILQ_FOREACH(vs, &vd->clients, next) {
+#ifdef CONFIG_GSTREAMER
+            /* correctly close all h264 encoder pipelines */
+            vnc_h264_clear(vs);
+#endif
+        }
+    }
+}
+
 static void vnc_register_config(void)
 {
     qemu_add_opts(&qemu_vnc_opts);
-- 
2.39.5