Returning VERR_* style vbox-runtime codes as positive values is an ugly hack and unlike any other ioctl call on Linux, breaking standard expectations of ioctls.
All Guest Additions user-space code calls the ioctl through the vbglR3DoIOCtl wrapper, which calls on RTErrConvertFromErrno on negative return values. The one exception is the src/VBox/GuestHost/OpenGL/util code, which gets fixed in this commit to deal with standard errno errors. All Guest Additions user-space code has been audited to check that any error codes requiring special handling survive being translated to an errno and back again by using RTErrConvertToErrno + RTErrConvertFromErrno. This means that we can return all errors as standard -errno values using RTErrConvertToErrno and vbglR3DoIOCtl will convert them back to VERR_* style vbox-runtime codes with any errors requiring special handling surviving as is. This commit removes the positive VERR_* style vbox-runtime codes and makes vgdrvLinuxIOCtl always return standard Linux errno values. Signed-off-by: Hans de Goede <hdego...@redhat.com> --- src/VBox/Additions/common/VBoxGuest/VBoxGuest-linux.c | 7 ++++++- src/VBox/Additions/common/VBoxGuestLib/VBoxGuestR3Lib.cpp | 4 +++- src/VBox/GuestHost/OpenGL/util/vboxhgcm.c | 2 ++ src/VBox/GuestHost/OpenGL/util/vboxhgsmi.c | 2 ++ 4 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/VBox/Additions/common/VBoxGuest/VBoxGuest-linux.c b/src/VBox/Additions/common/VBoxGuest/VBoxGuest-linux.c index 1ba3ba6..d48eb9e 100644 --- a/src/VBox/Additions/common/VBoxGuest/VBoxGuest-linux.c +++ b/src/VBox/Additions/common/VBoxGuest/VBoxGuest-linux.c @@ -818,7 +818,12 @@ static int vgdrvLinuxIOCtl(struct inode *pInode, struct file *pFilp, unsigned in else { Log(("vgdrvLinuxIOCtl: pFilp=%p uCmd=%#x ulArg=%p failed, rc=%d\n", pFilp, uCmd, (void *)ulArg, rc)); - rc = -rc; Assert(rc > 0); /* Positive returns == negated VBox error status codes. */ + /* Userspace code relies on VERR_* style vbox-runtime codes, + * vbglR3DoIOCtl will convert them back to VERR_* style + * vbox-runtime codes. This means that any VERR_* codes which + * require special handling must survive the to and from errno + * conversion as is. */ + rc = -RTErrConvertToErrno(rc); } } else diff --git a/src/VBox/Additions/common/VBoxGuestLib/VBoxGuestR3Lib.cpp b/src/VBox/Additions/common/VBoxGuestLib/VBoxGuestR3Lib.cpp index 28da83d..61b1c00 100644 --- a/src/VBox/Additions/common/VBoxGuestLib/VBoxGuestR3Lib.cpp +++ b/src/VBox/Additions/common/VBoxGuestLib/VBoxGuestR3Lib.cpp @@ -433,7 +433,9 @@ int vbglR3DoIOCtl(unsigned iFunction, void *pvData, size_t cbData) if (RT_LIKELY(rc == 0)) return VINF_SUCCESS; - /* Positive values are negated VBox error status codes. */ + /* Older kernel guest drivers use to return (negated) VBox error status + * codes as positive values, for now we still handle this case in case + * we end up running with an older guest driver (20170730) */ if (rc > 0) rc = -rc; else diff --git a/src/VBox/GuestHost/OpenGL/util/vboxhgcm.c b/src/VBox/GuestHost/OpenGL/util/vboxhgcm.c index fae2f22..813f3ec 100644 --- a/src/VBox/GuestHost/OpenGL/util/vboxhgcm.c +++ b/src/VBox/GuestHost/OpenGL/util/vboxhgcm.c @@ -650,6 +650,8 @@ static int crVBoxHGCMCall(CRConnection *conn, void *pvData, unsigned cbData) return VINF_SUCCESS; } # ifdef RT_OS_LINUX + if (rc < 0) + rc = -RTErrConvertFromErrno(errno); if (rc >= 0) /* positive values are negated VBox error status codes. */ { crWarning("vboxCall failed with VBox status code %d\n", -rc); diff --git a/src/VBox/GuestHost/OpenGL/util/vboxhgsmi.c b/src/VBox/GuestHost/OpenGL/util/vboxhgsmi.c index 7032381..d9c4818 100644 --- a/src/VBox/GuestHost/OpenGL/util/vboxhgsmi.c +++ b/src/VBox/GuestHost/OpenGL/util/vboxhgsmi.c @@ -277,6 +277,8 @@ static int crVBoxHGCMCall(void *pvData, unsigned cbData) return VINF_SUCCESS; } # ifdef RT_OS_LINUX + if (rc < 0) + rc = -RTErrConvertFromErrno(errno); if (rc >= 0) /* positive values are negated VBox error status codes. */ { crWarning("vboxCall failed with VBox status code %d\n", -rc); -- 2.9.4 _______________________________________________ vbox-dev mailing list vbox-dev@virtualbox.org https://www.virtualbox.org/mailman/listinfo/vbox-dev