This is an automated email from the ASF dual-hosted git repository.
xiaoxiang781216 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/nuttx-apps.git
The following commit(s) were added to refs/heads/master by this push:
new 15a6f29e2 system/vncviewer: scan security types in chunks to prevent
stack overflow
15a6f29e2 is described below
commit 15a6f29e20f1ced34e88cf99db8b919df7a61c6c
Author: Swatantra Yadav <[email protected]>
AuthorDate: Sat Oct 31 02:23:21 2026 +0530
system/vncviewer: scan security types in chunks to prevent stack overflow
During the RFB security handshake, the server sends a 1-byte count of
supported security types (num_types, up to 255). rfb_negotiate_security()
previously passed num_types directly to rfb_recv_exact() into a fixed
16-byte stack array types[16], allowing a malicious server or network
attacker to overflow the stack buffer by up to 239 bytes before
authentication.
Rather than truncating the list or rejecting counts greater than 16,
receive and scan the security types in chunks of up to sizeof(types).
This inspects all server-offered security types without unbounded stack
growth. If RFB_SEC_NONE is found, any remaining unread types are drained
via rfb_discard_exact() to keep the stream synchronized.
Fixes #3942.
Assisted-by: Google Gemini
Signed-off-by: Swatantra Yadav <[email protected]>
---
system/vncviewer/rfb_protocol.c | 59 ++++++++++++++++++++++++++++-------------
1 file changed, 40 insertions(+), 19 deletions(-)
diff --git a/system/vncviewer/rfb_protocol.c b/system/vncviewer/rfb_protocol.c
index e2d384b49..acc5ef84c 100644
--- a/system/vncviewer/rfb_protocol.c
+++ b/system/vncviewer/rfb_protocol.c
@@ -520,12 +520,14 @@ static int rfb_negotiate_version(int sockfd)
static int rfb_negotiate_security(int sockfd, const char *password)
{
- uint8_t num_types;
- uint8_t types[16];
- uint8_t chosen;
+ uint32_t reason_len;
uint32_t result;
+ int chunk;
int ret;
int i;
+ uint8_t num_types;
+ uint8_t types[16];
+ uint8_t chosen = RFB_SEC_INVALID;
/* Receive number of security types */
@@ -539,35 +541,53 @@ static int rfb_negotiate_security(int sockfd, const char
*password)
{
/* Connection failed - read reason string */
- uint32_t reason_len;
rfb_recv_exact(sockfd, &reason_len, 4);
reason_len = ntohl(reason_len);
printf("vncviewer: connection refused by server\n");
return -ECONNREFUSED;
}
- /* Receive security type list */
+ /* Receive and examine security types in chunks to bound stack usage.
+ * Prefer None, fallback to VNC Auth.
+ */
- ret = rfb_recv_exact(sockfd, types, num_types);
- if (ret < 0)
+ while (num_types > 0)
{
- return ret;
- }
+ chunk = (num_types > (int)sizeof(types)) ?
+ (int)sizeof(types) : num_types;
- /* Choose security type: prefer None, fallback to VNC Auth */
+ ret = rfb_recv_exact(sockfd, types, chunk);
+ if (ret < 0)
+ {
+ return ret;
+ }
- chosen = RFB_SEC_INVALID;
- for (i = 0; i < num_types; i++)
- {
- if (types[i] == RFB_SEC_NONE)
+ num_types -= chunk;
+
+ for (i = 0; i < chunk; i++)
{
- chosen = RFB_SEC_NONE;
- break;
+ if (types[i] == RFB_SEC_NONE)
+ {
+ chosen = RFB_SEC_NONE;
+ break;
+ }
+ else if (types[i] == RFB_SEC_VNC_AUTH)
+ {
+ chosen = RFB_SEC_VNC_AUTH;
+ }
}
- if (types[i] == RFB_SEC_VNC_AUTH)
+ if (chosen == RFB_SEC_NONE && num_types > 0)
{
- chosen = RFB_SEC_VNC_AUTH;
+ /* None is the highest preference; drain remaining types */
+
+ ret = rfb_discard_exact(sockfd, num_types);
+ if (ret < 0)
+ {
+ return ret;
+ }
+
+ break;
}
}
@@ -946,6 +966,7 @@ int rfb_recv_update(struct rfb_conn_s *conn, rfb_rect_cb_t
rect_cb,
for (i = 0; i < num_rects; i++)
{
struct rfb_rect_hdr_s rect;
+ FAR uint8_t *rowbuf;
uint8_t rect_buf[12];
uint32_t row_bytes;
uint32_t row;
@@ -975,7 +996,7 @@ int rfb_recv_update(struct rfb_conn_s *conn, rfb_rect_cb_t
rect_cb,
row_bytes = rect.w * conn->bpp;
- FAR uint8_t *rowbuf = malloc(row_bytes);
+ rowbuf = malloc(row_bytes);
if (rowbuf == NULL)
{
return -ENOMEM;
