Before, isatty() was an alias for WinAPI _isatty().
This resulted in wrong result for mintty.

Implement pipe name check in crt isatty().

Signed-off-by: Mihail Konev <k....@ya.ru>
Moved-from: https://github.com/Alexpux/mingw-w64/pull/3
Reference: https://cygwin.com/ml/cygwin-developers/2016-11/msg00002.html
---
 mingw-w64-crt/Makefile.am                      |   1 +
 mingw-w64-crt/def-include/msvcrt-common.def.in |   2 +-
 mingw-w64-crt/lib64/moldname-msvcrt.def        |   2 +-
 mingw-w64-crt/stdio/isatty.c                   | 111 +++++++++++++++++++++++++
 mingw-w64-headers/crt/io.h                     |   1 +
 5 files changed, 115 insertions(+), 2 deletions(-)
 create mode 100644 mingw-w64-crt/stdio/isatty.c

diff --git a/mingw-w64-crt/Makefile.am b/mingw-w64-crt/Makefile.am
index 581af0cc5811..49f7f58289c7 100644
--- a/mingw-w64-crt/Makefile.am
+++ b/mingw-w64-crt/Makefile.am
@@ -211,6 +211,7 @@ src_msvcrt=\
   secapi/vsprintf_s.c \
   secapi/wmemcpy_s.c \
   secapi/wmemmove_s.c \
+  stdio/isatty.c \
   stdio/mingw_lock.c
 
 src_msvcrt32=\
diff --git a/mingw-w64-crt/def-include/msvcrt-common.def.in 
b/mingw-w64-crt/def-include/msvcrt-common.def.in
index 76e1fa37e8b2..44ebccb49d46 100644
--- a/mingw-w64-crt/def-include/msvcrt-common.def.in
+++ b/mingw-w64-crt/def-include/msvcrt-common.def.in
@@ -47,7 +47,7 @@ ADD_UNDERSCORE(getcwd)
 ADD_UNDERSCORE(getpid)
 ADD_UNDERSCORE(getw)
 ADD_UNDERSCORE(heapwalk)
-ADD_UNDERSCORE(isatty)
+; isatty replaced for cygwin
 ADD_UNDERSCORE(itoa)
 ADD_UNDERSCORE(kbhit)
 ADD_UNDERSCORE(lfind)
diff --git a/mingw-w64-crt/lib64/moldname-msvcrt.def 
b/mingw-w64-crt/lib64/moldname-msvcrt.def
index 6f31518d5c82..30b45c3a71ca 100644
--- a/mingw-w64-crt/lib64/moldname-msvcrt.def
+++ b/mingw-w64-crt/lib64/moldname-msvcrt.def
@@ -64,7 +64,7 @@ getcwd
 getpid
 getw
 heapwalk
-isatty
+; isatty replaced for cygwin
 itoa
 kbhit
 lfind
diff --git a/mingw-w64-crt/stdio/isatty.c b/mingw-w64-crt/stdio/isatty.c
new file mode 100644
index 000000000000..bf940d6f225f
--- /dev/null
+++ b/mingw-w64-crt/stdio/isatty.c
@@ -0,0 +1,111 @@
+#include <wchar.h>
+#include <errno.h>
+#include <io.h>
+#include <winternl.h>
+#include <windows.h>
+
+int __cdecl (*__MINGW_IMP_SYMBOL(isatty))(int) = isatty;
+
+/* Cygwin-compatible isatty.
+ *
+ * Cygwin pty is a specially-named named pipe.
+ * Fetch [absolute] fd's NT object path (if any),
+ * and check it for the following pattern:
+ *
+ *   
\Device\NamedPipe\cygwin-[a-fA-F0-9]{16}-pty[0-9]{1,4}-(from-master|to-master|to-master-cyg)
+ *
+ * [a-fA-F0-9] is the cygwin installation key, 16 characters long.
+ * pty[0-9] is the pty name. Its index is of type int, but is safe to be 
limited to 4 characters.
+ *
+ * */
+
+int __cdecl isatty(int fd) {
+  wchar_t const expect_dev[] = L"\\Device\\NamedPipe\\cygwin-";
+  wchar_t const expect_pty[] = L"-pty";
+
+  typedef NTSTATUS (NTAPI proc_NtQueryObject) (HANDLE, 
OBJECT_INFORMATION_CLASS, PVOID, ULONG, PULONG);
+  proc_NtQueryObject *pNtQueryObject;
+
+  HANDLE h_fd, h_ntdll;
+
+  /* NtQueryObject needs space for OBJECT_NAME_INFORMATION.Name->Buffer also. 
*/
+  char ntfn_bytes[sizeof(OBJECT_NAME_INFORMATION) + MAX_PATH * sizeof(WCHAR)];
+
+  OBJECT_NAME_INFORMATION *ntfn = (OBJECT_NAME_INFORMATION*) ntfn_bytes;
+  NTSTATUS status;
+  ULONG ntfn_size = sizeof(ntfn_bytes);
+
+  wchar_t *s;
+  int l;
+
+  h_fd = (HANDLE) _get_osfhandle(fd);
+  if (!h_fd || h_fd == INVALID_HANDLE_VALUE) {
+    errno = EBADF;
+    return 0;
+  }
+
+  h_ntdll = GetModuleHandle("ntdll.dll");
+  if (!h_ntdll || h_ntdll == INVALID_HANDLE_VALUE) {
+    errno = EBADF;
+    return 0;
+  }
+
+  pNtQueryObject = (proc_NtQueryObject*) GetProcAddress(h_ntdll, 
"NtQueryObject");
+  if (!pNtQueryObject) {
+    goto no_tty;
+  }
+
+  memset(ntfn, 0, ntfn_size);
+  status = pNtQueryObject((HANDLE)h_fd, ObjectNameInformation,
+      ntfn, ntfn_size, &ntfn_size);
+
+  if (!NT_SUCCESS(status)) {
+    /* If it is not NUL (i.e. \Device\Null, which would succeed),
+     * then normal isatty() could be consulted.
+     * */
+    if (_isatty(fd)) {
+      return 1;
+    }
+    goto no_tty;
+  }
+
+  s = ntfn->Name.Buffer;
+  s[ntfn->Name.Length / sizeof(WCHAR)] = 0;
+
+  l = wcslen(expect_dev);
+  if (wcsncmp(s, expect_dev, l) != 0) {
+    goto no_tty;
+  }
+  s += l;
+
+  l = wcsspn(s, L"0123456789abcdefABCDEF");
+  if (l != 16) {
+    goto no_tty;
+  }
+  s += l;
+
+  l = wcslen(expect_pty);
+  if (wcsncmp(s, expect_pty, l) != 0) {
+    goto no_tty;
+  }
+  s += l;
+
+  l = wcsspn(s, L"0123456789");
+  if (l < 1 || l > 4) {
+    goto no_tty;
+  }
+  s += l;
+
+  if (wcscmp(s, L"-from-master") != 0 &&
+        wcscmp(s, L"-to-master") != 0 &&
+        wcscmp(s, L"-to-master-cyg") != 0)
+  {
+    goto no_tty;
+  }
+
+  return 1;
+
+no_tty:
+  errno = EINVAL;
+  return 0;
+}
diff --git a/mingw-w64-headers/crt/io.h b/mingw-w64-headers/crt/io.h
index c61e94ab8743..97b26762f831 100644
--- a/mingw-w64-headers/crt/io.h
+++ b/mingw-w64-headers/crt/io.h
@@ -191,6 +191,7 @@ _CRTIMP char* __cdecl _getcwd (char*, int);
   _CRTIMP int __cdecl _findnext32(intptr_t _FindHandle,struct _finddata32_t 
*_FindData);
   _CRTIMP int __cdecl _findclose(intptr_t _FindHandle);
   _CRTIMP int __cdecl _isatty(int _FileHandle);
+  _CRTIMP int __cdecl isatty(int _FileHandle);
   _CRTIMP int __cdecl _locking(int _FileHandle,int _LockMode,long _NumOfBytes);
   _CRTIMP long __cdecl _lseek(int _FileHandle,long _Offset,int _Origin);
   _off64_t lseek64(int fd,_off64_t offset, int whence);
-- 
2.9.2


------------------------------------------------------------------------------
_______________________________________________
Mingw-w64-public mailing list
Mingw-w64-public@lists.sourceforge.net
https://lists.sourceforge.net/lists/listinfo/mingw-w64-public

Reply via email to