enable ANSI color (instead of ANSI color codes) in the Windows terminal #4393 (#4403)

* changed ie. -> i.e.(spelling error) on lines 667, 763 (issue scrapy#4332)

* updated all text files for issue #4332 (ie. -> i.e.)

* Apply ie. → i.e. in source comments

* ie → e.g.

* modified scrapy/utils/display.py to stop ANSI color sequences in the Windows terminal (issue #4393)

* modified scrapy/utils/display.py to stop ANSI color sequences in the Windows terminal (issue #4393)

* enabled virtual terminal processing (pr #4403)

* check for specific windows 10 version (pr #4403)

* fixing flake-8 test (pr #4403)

* added error handling for terminal info (pr #4403)

* corrected stderr (pr #4403)

* changed orientation, removed unwanted spaces (pr #4403)

* no need for style variable (pr #4403)

* fixing trailing whitespaces

* commenting windows check

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* small fixes

* Shifting _color_support_info() function

* enabled virtual terminal processing (pr #4403)

* check for specific windows 10 version (pr #4403)

* fixing flake-8 test (pr #4403)

* added error handling for terminal info (pr #4403)

* corrected stderr (pr #4403)

* changed orientation, removed unwanted spaces (pr #4403)

* no need for style variable (pr #4403)

* fixing trailing whitespaces

* commenting windows check

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* Update scrapy/utils/display.py

Co-Authored-By: Adrián Chaves <adrian@chaves.io>

* small fixes

* Shifting _color_support_info() function

* error handling

* error handlingy

* raise ValueError

* added in-built function for version comparison

* recommit changes

* changed check -> parse

* version comparison -> parse_version

* added scrapy/utils/display.py in pytest.ini

* Trigger

* Add simple test for scrapy.utils.display._colorize

* Flake8: E501 for tests/test_utils_display.py

* assertEquals -> assertEqual

* Normal formatter for all platforms

* separate test for windows

* all curses under try block

* added global TestStr

* more test added

* small fix

* covering exceptions

* windows test failing

* Refactor output color handling

* Fix pprint test

* fix flake8

Co-authored-by: Adrián Chaves <adrian@chaves.io>
Co-authored-by: Eugenio Lacuesta <eugenio.lacuesta@gmail.com>

scrapy/utils/display.py

@@ -2,20 +2,42 @@
 pprint and pformat wrappers with colorization support
 """
 
+import ctypes
+import platform
 import sys
+from distutils.version import LooseVersion as parse_version
 from pprint import pformat as pformat_
 
 
+def _enable_windows_terminal_processing():
+    # https://stackoverflow.com/a/36760881
+    kernel32 = ctypes.windll.kernel32
+    return bool(kernel32.SetConsoleMode(kernel32.GetStdHandle(-11), 7))
+
+
+def _tty_supports_color():
+    if sys.platform != "win32":
+        return True
+
+    if parse_version(platform.version()) < parse_version("10.0.14393"):
+        return True
+
+    # Windows >= 10.0.14393 interprets ANSI escape sequences providing terminal
+    # processing is enabled.
+    return _enable_windows_terminal_processing()
+
+
 def _colorize(text, colorize=True):
-    if not colorize or not sys.stdout.isatty():
+    if not colorize or not sys.stdout.isatty() or not _tty_supports_color():
         return text
     try:
         from pygments import highlight
+    except ImportError:
+        return text
+    else:
         from pygments.formatters import TerminalFormatter
         from pygments.lexers import PythonLexer
         return highlight(text, PythonLexer(), TerminalFormatter())
-    except ImportError:
-        return text
 
 
 def pformat(obj, *args, **kwargs):

tests/test_utils_display.py

@@ -0,0 +1,78 @@
+from io import StringIO
+
+from unittest import mock, TestCase
+
+from scrapy.utils.display import pformat, pprint
+
+
+class TestDisplay(TestCase):
+    object = {'a': 1}
+    colorized_string = (
+        "{\x1b[33m'\x1b[39;49;00m\x1b[33ma\x1b[39;49;00m\x1b[33m'"
+        "\x1b[39;49;00m: \x1b[34m1\x1b[39;49;00m}\n"
+    )
+    plain_string = "{'a': 1}"
+
+    @mock.patch('sys.platform', 'linux')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat(self, isatty):
+        isatty.return_value = True
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_dont_colorize(self, isatty):
+        isatty.return_value = True
+        self.assertEqual(pformat(self.object, colorize=False), self.plain_string)
+
+    def test_pformat_not_tty(self):
+        self.assertEqual(pformat(self.object), self.plain_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_old_windows(self, isatty, version):
+        isatty.return_value = True
+        version.return_value = '10.0.14392'
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_windows_no_terminal_processing(self, isatty, version, terminal_processing):
+        isatty.return_value = True
+        version.return_value = '10.0.14393'
+        terminal_processing.return_value = False
+        self.assertEqual(pformat(self.object), self.plain_string)
+
+    @mock.patch('sys.platform', 'win32')
+    @mock.patch('scrapy.utils.display._enable_windows_terminal_processing')
+    @mock.patch('platform.version')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_windows(self, isatty, version, terminal_processing):
+        isatty.return_value = True
+        version.return_value = '10.0.14393'
+        terminal_processing.return_value = True
+        self.assertEqual(pformat(self.object), self.colorized_string)
+
+    @mock.patch('sys.platform', 'linux')
+    @mock.patch("sys.stdout.isatty")
+    def test_pformat_no_pygments(self, isatty):
+        isatty.return_value = True
+
+        import builtins
+        real_import = builtins.__import__
+
+        def mock_import(name, globals, locals, fromlist, level):
+            if 'pygments' in name:
+                raise ImportError
+            return real_import(name, globals, locals, fromlist, level)
+
+        builtins.__import__ = mock_import
+        self.assertEqual(pformat(self.object), self.plain_string)
+        builtins.__import__ = real_import
+
+    def test_pprint(self):
+        with mock.patch('sys.stdout', new=StringIO()) as mock_out:
+            pprint(self.object)
+            self.assertEqual(mock_out.getvalue(), "{'a': 1}\n")