win32: only attach to the console from mpv.com

Previously, mpv.exe used the --terminal option to decide whether to
attach to the parent process's console, which made it impossible to tell
whether mpv would attach to the console before the config files were
parsed. Instead, make mpv always attach to the console when launched
from the console wrapper (mpv.com) and never attach otherwise. This will
be useful for the next commit, which will use the presence of the
console to decide whether to use the pseudo-gui profile.

This change should also be an improvement in behavior. The old code
would attach to the parent process's console, regardless of whether it
was mpv.com or some other program like cmd.exe. This could be confusing,
since mpv.exe is marked as a Windows GUI program and shouldn't write
text to its parent process's console when launched directly. (See #768.)

Visual Studio does something similar with its devenv.com wrapper.
devenv.exe only attaches to the console when launched from devenv.com.

osdep/terminal-win.c

@@ -250,24 +250,48 @@ void mp_write_console_ansi(HANDLE wstream, char *buf)
     }
 }
 
+static bool is_a_console(HANDLE h)
+{
+    return GetConsoleMode(h, &(DWORD){0});
+}
+
+static void reopen_console_handle(DWORD std, FILE *stream)
+{
+    HANDLE wstream = GetStdHandle(std);
+    if (is_a_console(wstream)) {
+        int fd = _open_osfhandle((intptr_t)wstream, _O_TEXT);
+        dup2(fd, fileno(stream));
+        setvbuf(stream, NULL, _IONBF, 0);
+    }
+}
+
+bool terminal_try_attach(void)
+{
+    // mpv.exe is a flagged as a GUI application, but it acts as a console
+    // application when started from the console wrapper (see
+    // osdep/win32-console-wrapper.c). The console wrapper sets
+    // _started_from_console=yes, so check that variable before trying to
+    // attach to the console.
+    wchar_t console_env[4] = { 0 };
+    if (!GetEnvironmentVariableW(L"_started_from_console", console_env, 4))
+        return false;
+    if (wcsncmp(console_env, L"yes", 4))
+        return false;
+    SetEnvironmentVariableW(L"_started_from_console", NULL);
+
+    if (!AttachConsole(ATTACH_PARENT_PROCESS))
+        return false;
+
+    // We have a console window. Redirect output streams to that console's
+    // low-level handles, so we can actually use WriteConsole later on.
+    reopen_console_handle(STD_OUTPUT_HANDLE, stdout);
+    reopen_console_handle(STD_ERROR_HANDLE, stderr);
+
+    return true;
+}
+
 int terminal_init(void)
 {
-    if (AttachConsole(ATTACH_PARENT_PROCESS)) {
-        // We have been started by something with a console window.
-        // Redirect output streams to that console's low-level handles,
-        // so we can actually use WriteConsole later on.
-
-        int hConHandle;
-
-        hConHandle = _open_osfhandle((intptr_t)hSTDOUT, _O_TEXT);
-        *stdout = *_fdopen(hConHandle, "w");
-        setvbuf(stdout, NULL, _IONBF, 0);
-
-        hConHandle = _open_osfhandle((intptr_t)hSTDERR, _O_TEXT);
-        *stderr = *_fdopen(hConHandle, "w");
-        setvbuf(stderr, NULL, _IONBF, 0);
-    }
-
     CONSOLE_SCREEN_BUFFER_INFO cinfo;
     DWORD cmode = 0;
     GetConsoleMode(hSTDOUT, &cmode);

osdep/terminal.h

@@ -47,4 +47,7 @@ void terminal_get_size(int *w, int *h);
 // Windows only.
 void mp_write_console_ansi(void *wstream, char *buf);
 
+/* Windows-only function to attach to the parent process's console */
+bool terminal_try_attach(void);
+
 #endif /* MPLAYER_GETCH2_H */

osdep/win32-console-wrapper.c

@@ -72,5 +72,9 @@ int wmain(int argc, wchar_t **argv, wchar_t **envp)
     GetModuleFileNameW(NULL, exe, MAX_PATH);
     wcscpy(wcsrchr(exe, '.') + 1, L"exe");
 
+    // Set an environment variable so the child process can tell whether it
+    // was started from this wrapper and attach to the console accordingly
+    SetEnvironmentVariableW(L"_started_from_console", L"yes");
+
     return cr_runproc(exe, cmd);
 }

player/main-fn-win.c

@@ -1,6 +1,7 @@
 #include "config.h"
 #include "core.h"
 #include "osdep/io.h"
+#include "osdep/terminal.h"
 
 int wmain(int argc, wchar_t *argv[]);
 
@@ -9,6 +10,10 @@ int _dowildcard = 0;
 
 int wmain(int argc, wchar_t *argv[])
 {
+    // If started from the console wrapper (see osdep/win32-console-wrapper.c),
+    // attach to the console and set up the standard IO handles
+    terminal_try_attach();
+
     char **argv_u8 = talloc_zero_array(NULL, char*, argc + 1);
     for (int i = 0; i < argc; i++)
         argv_u8[i] = mp_to_utf8(argv_u8, argv[i]);