win32: don't pass std handles if they are attached to console

This is default behavior to attach to existing console, passing custom
handles is useful if we want to replace them, but in case they are
already attached to console we want to attach to console directly.

In theory, it should work out of the box because "console-like" handles
should be managed by Windows internally, which works for INPUT and
OUTPUT, but in certain cases, not for ERROR. This allows using VT
features in those cases for stderr too.
This commit is contained in:
Kacper Michajłow 2023-11-19 21:32:31 +01:00
parent dd88701aac
commit c3843d79de
1 changed files with 20 additions and 15 deletions

View File

@ -41,26 +41,31 @@ static void cr_perror(const wchar_t *prefix)
static int cr_runproc(wchar_t *name, wchar_t *cmdline)
{
STARTUPINFOW si;
STARTUPINFOW our_si;
PROCESS_INFORMATION pi;
DWORD retval = 1;
ZeroMemory(&si, sizeof(si));
si.cb = sizeof(si);
si.hStdInput = GetStdHandle(STD_INPUT_HANDLE);
si.hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE);
si.hStdError = GetStdHandle(STD_ERROR_HANDLE);
si.dwFlags |= STARTF_USESTDHANDLES;
// Copy the list of inherited CRT file descriptors to the new process
our_si.cb = sizeof(our_si);
STARTUPINFOW our_si = {sizeof(our_si)};
GetStartupInfoW(&our_si);
si.lpReserved2 = our_si.lpReserved2;
si.cbReserved2 = our_si.cbReserved2;
ZeroMemory(&pi, sizeof(pi));
// Don't redirect std streams if they are attached to a console. Let mpv
// attach to the console directly in this case. In theory, it should work
// out of the box because "console-like" handles should be managed by Windows
// internally, which works for INPUT and OUTPUT, but in certain cases,
// not for ERROR.
DWORD mode;
HANDLE hStdInput = GetStdHandle(STD_INPUT_HANDLE);
HANDLE hStdOutput = GetStdHandle(STD_OUTPUT_HANDLE);
HANDLE hStdError = GetStdHandle(STD_ERROR_HANDLE);
STARTUPINFOW si = {
.cb = sizeof(si),
.lpReserved2 = our_si.lpReserved2,
.cbReserved2 = our_si.cbReserved2,
.hStdInput = GetConsoleMode(hStdInput, &mode) ? NULL : hStdInput,
.hStdOutput = GetConsoleMode(hStdOutput, &mode) ? NULL : hStdOutput,
.hStdError = GetConsoleMode(hStdError, &mode) ? NULL : hStdError,
};
si.dwFlags = (si.hStdInput || si.hStdOutput || si.hStdError) ? STARTF_USESTDHANDLES : 0;
PROCESS_INFORMATION pi = {0};
if (!CreateProcessW(name, cmdline, NULL, NULL, TRUE, 0,
NULL, NULL, &si, &pi)) {