From 9cb5307ab05b382df24f8412c2203b437d927f1e Mon Sep 17 00:00:00 2001
From: Johannes Schindelin <johannes.schindelin@gmx.de>
Date: Wed, 7 Feb 2018 13:50:03 +0100
Subject: [PATCH] mingw: work around incorrect standard handles
For some reason, when being called via TortoiseGit the standard handles,
or at least what is returned by _get_osfhandle(0) for standard input,
can take on the value (HANDLE)-2 (which is not a legal value, according
to the documentation).
Even if this value is not documented anywhere, CreateProcess() works
without complaints if hStdInput set to this value.
In contrast, when our new code to restrict which file handles get
inherited by spawned processes apparently uses such a value in its
handle list, it does *not* work, erroring out with
`ERROR_INVALID_PARAMETER`.
To help this, special-case the value (HANDLE)-2 returned by
_get_osfhandle() and replace it with INVALID_HANDLE_VALUE, which will
hopefully let the handle inheritance restriction work even when called
from TortoiseGit.
This addresses https://github.com/git-for-windows/git/issues/1481
Signed-off-by: Johannes Schindelin <johannes.schindelin@gmx.de>
---
compat/winansi.c | 12 +++++++++++-
1 file changed, 11 insertions(+), 1 deletion(-)
diff --git a/compat/winansi.c b/compat/winansi.c
index 691aac2a444621..98b3354d45df35 100644
--- a/compat/winansi.c
+++ b/compat/winansi.c
@@ -662,6 +662,8 @@ void winansi_init(void)
*/
HANDLE winansi_get_osfhandle(int fd)
{
+ HANDLE ret;
+
if (debug_1481)
warning("get_osf_handle for %d (fd_is_interactive %x %x): "
"%p (%p %p)",
@@ -672,5 +674,13 @@ HANDLE winansi_get_osfhandle(int fd)
if (fd == 2 && (fd_is_interactive[2] & FD_SWAPPED))
return hconsole2;
- return (HANDLE)_get_osfhandle(fd);
+ ret = (HANDLE)_get_osfhandle(fd);
+
+ /*
+ * There are obviously circumstances under which _get_osfhandle()
+ * returns (HANDLE)-2. This is not documented anywhere, but that is so
+ * clearly an invalid handle value that we can just work around this
+ * and return the correct value for invalid handles.
+ */
+ return ret == (HANDLE)-2 ? INVALID_HANDLE_VALUE : ret;
}