issue #3741: attempt to fix terminal color output in Windows (#3753) (#3937)

* issue #3741: attempt to fix terminal color output in Windows (#3753)

* Fix for missing constant in older Windows versions (issue 3742)

CHANGELOG

@@ -1,6 +1,8 @@
 v3.3.rc7 (2017-12-07)
 ---------------------
 
+* fixed issue #3741: fix terminal color output in Windows 
+
 * UI: fixed issue #3822: disabled name input field for system collections
 
 * fixed issue #3640: limit in subquery

lib/ApplicationFeatures/ShellColorsFeature.cpp

@@ -24,6 +24,9 @@
 
 #ifdef _WIN32
 #include "Basics/win-utils.h"
+#ifndef ENABLE_VIRTUAL_TERMINAL_PROCESSING
+#define ENABLE_VIRTUAL_TERMINAL_PROCESSING 0x0004
+#endif
 #endif
 
 using namespace arangodb;
@@ -86,8 +89,32 @@ void ShellColorsFeature::prepare() {
 
 bool ShellColorsFeature::useColors() {
 #ifdef _WIN32
+  if (!prepareConsole()) {
+    return false;
+  }
   return terminalKnowsANSIColors();
 #else
   return true;
 #endif
 }
+
+bool ShellColorsFeature::prepareConsole() {
+#ifdef _WIN32
+  HANDLE hStdout = GetStdHandle(STD_OUTPUT_HANDLE);
+  if (hStdout == INVALID_HANDLE_VALUE) {
+    return false;
+  }
+
+  DWORD handleMode = 0;
+  if (!GetConsoleMode(hStdout, &handleMode)) {
+    return false;
+  }
+  handleMode |= ENABLE_VIRTUAL_TERMINAL_PROCESSING;
+  if (!SetConsoleMode(hStdout, handleMode)) {
+    return false;
+  }
+  return true;
+#else
+  return true;
+#endif
+}

lib/ApplicationFeatures/ShellColorsFeature.h

@@ -35,6 +35,7 @@ class ShellColorsFeature final : public application_features::ApplicationFeature
 
  private:
   bool useColors();
+  bool prepareConsole(); 
 
  public:
   static char const* SHELL_COLOR_RED;

lib/Basics/win-utils.cpp

@@ -31,6 +31,7 @@
 #include <string.h>
 #include <malloc.h>
 #include <crtdbg.h>
+#include <atlstr.h>
 #include <VersionHelpers.h>
 
 #include "Logger/Logger.h"
@@ -558,15 +559,14 @@ void ADB_WindowsExitFunction(int exitCode, void* data) {
 }
 
 // Detect cygwin ssh / terminals
-int _cyg_isatty(int fd)
+int _cyg_isatty(int fd) {
-{
   // detect standard windows ttys:
   if (_isatty (fd)) {
     return 1;
   }
 
   // stupid hack to allow forcing a tty..need to understand this better
-  // and create a thorugh fix..without this the logging stuff will not
+  // and create a thorough fix..without this the logging stuff will not
   // log to the foreground which is super annoying for debugging the
   // resilience tests
   char* forcetty = getenv("FORCE_WINDOWS_TTY");
@@ -660,8 +660,7 @@ int _is_cyg_tty(int fd)
   return 0;
 }
 
-bool terminalKnowsANSIColors()
+bool terminalKnowsANSIColors() {
-{
   if (_is_cyg_tty (STDOUT_FILENO)) {
     // Its a cygwin shell, expected to understand ANSI color codes.
     return true;
@@ -671,7 +670,6 @@ bool terminalKnowsANSIColors()
   return IsWindows8OrGreater();
 }
 
-#include <atlstr.h>
 std::string getFileNameFromHandle(HANDLE fileHandle) {
   char  buff[sizeof(FILE_NAME_INFO) + sizeof(WCHAR)*MAX_PATH];
   FILE_NAME_INFO *FileInformation = (FILE_NAME_INFO*) buff;