Update CI to support Twisted 21.2.0 (#5027)

scrapy/pipelines/images.py

@@ -9,9 +9,8 @@ from contextlib import suppress
 from io import BytesIO
 
 from itemadapter import ItemAdapter
-from PIL import Image
 
-from scrapy.exceptions import DropItem
+from scrapy.exceptions import DropItem, NotConfigured
 from scrapy.http import Request
 from scrapy.pipelines.files import FileException, FilesPipeline
 # TODO: from scrapy.pipelines.media import MediaPipeline
@@ -45,6 +44,14 @@ class ImagesPipeline(FilesPipeline):
     DEFAULT_IMAGES_RESULT_FIELD = 'images'
 
     def __init__(self, store_uri, download_func=None, settings=None):
+        try:
+            from PIL import Image
+            self._Image = Image
+        except ImportError:
+            raise NotConfigured(
+                'ImagesPipeline requires installing Pillow 4.0.0 or later'
+            )
+
         super().__init__(store_uri, settings=settings, download_func=download_func)
 
         if isinstance(settings, dict) or settings is None:
@@ -121,7 +128,7 @@ class ImagesPipeline(FilesPipeline):
 
     def get_images(self, response, request, info, *, item=None):
         path = self.file_path(request, response=response, info=info, item=item)
-        orig_image = Image.open(BytesIO(response.body))
+        orig_image = self._Image.open(BytesIO(response.body))
 
         width, height = orig_image.size
         if width < self.min_width or height < self.min_height:
@@ -139,12 +146,12 @@ class ImagesPipeline(FilesPipeline):
 
     def convert_image(self, image, size=None):
         if image.format == 'PNG' and image.mode == 'RGBA':
-            background = Image.new('RGBA', image.size, (255, 255, 255))
+            background = self._Image.new('RGBA', image.size, (255, 255, 255))
             background.paste(image, image)
             image = background.convert('RGB')
         elif image.mode == 'P':
             image = image.convert("RGBA")
-            background = Image.new('RGBA', image.size, (255, 255, 255))
+            background = self._Image.new('RGBA', image.size, (255, 255, 255))
             background.paste(image, image)
             image = background.convert('RGB')
         elif image.mode != 'RGB':
@@ -152,7 +159,7 @@ class ImagesPipeline(FilesPipeline):
 
         if size:
             image = image.copy()
-            image.thumbnail(size, Image.ANTIALIAS)
+            image.thumbnail(size, self._Image.ANTIALIAS)
 
         buf = BytesIO()
         image.save(buf, 'JPEG')

tests/test_commands.py

@@ -17,6 +17,8 @@ from threading import Timer
 from unittest import skipIf
 
 from pytest import mark
+from twisted import version as twisted_version
+from twisted.python.versions import Version
 from twisted.trial import unittest
 
 import scrapy
@@ -630,6 +632,7 @@ class MySpider(scrapy.Spider):
 
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
     @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
     def test_custom_asyncio_loop_enabled_true(self):
         log = self.get_log(self.debug_log_spider, args=[
             '-s',

tests/test_crawler.py

@@ -8,7 +8,9 @@ from unittest import skipIf
 
 from pytest import raises, mark
 from testfixtures import LogCapture
+from twisted import version as twisted_version
 from twisted.internet import defer
+from twisted.python.versions import Version
 from twisted.trial import unittest
 
 import scrapy
@@ -358,6 +360,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
 
     @mark.skipif(sys.implementation.name == 'pypy', reason='uvloop does not support pypy properly')
     @mark.skipif(platform.system() == 'Windows', reason='uvloop does not support Windows')
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
     def test_custom_loop_asyncio(self):
         log = self.run_script("asyncio_custom_loop.py")
         self.assertIn("Spider closed (finished)", log)
@@ -366,6 +369,7 @@ class CrawlerProcessSubprocess(ScriptRunnerMixin, unittest.TestCase):
 
     @mark.skipif(sys.implementation.name == "pypy", reason="uvloop does not support pypy properly")
     @mark.skipif(platform.system() == "Windows", reason="uvloop does not support Windows")
+    @mark.skipif(twisted_version == Version('twisted', 21, 2, 0), reason='https://twistedmatrix.com/trac/ticket/10106')
     def test_custom_loop_asyncio_deferred_signal(self):
         log = self.run_script("asyncio_deferred_signal.py", "uvloop.Loop")
         self.assertIn("Spider closed (finished)", log)

tests/test_pipeline_crawl.py

@@ -180,7 +180,18 @@ class FileDownloadCrawlTestCase(TestCase):
         self.assertEqual(crawler.stats.get_value('downloader/response_status_count/302'), 3)
 
 
+try:
+    from PIL import Image  # noqa: imported just to check for the import error
+except ImportError:
+    skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+else:
+    skip_pillow = None
+
+
 class ImageDownloadCrawlTestCase(FileDownloadCrawlTestCase):
+
+    skip = skip_pillow
+
     pipeline_class = 'scrapy.pipelines.images.ImagesPipeline'
     store_setting_key = 'IMAGES_STORE'
     media_key = 'images'

tests/test_pipeline_images.py

@@ -23,15 +23,16 @@ except ImportError:
     dataclass_field = None
 
 
-skip = False
 try:
     from PIL import Image
 except ImportError:
-    skip = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+    skip_pillow = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
 else:
     encoders = {'jpeg_encoder', 'jpeg_decoder'}
     if not encoders.issubset(set(Image.core.__dict__)):
-        skip = 'Missing JPEG encoders'
+        skip_pillow = 'Missing JPEG encoders'
+    else:
+        skip_pillow = None
 
 
 def _mocked_download_func(request, info):
@@ -41,7 +42,7 @@ def _mocked_download_func(request, info):
 
 class ImagesPipelineTestCase(unittest.TestCase):
 
-    skip = skip
+    skip = skip_pillow
 
     def setUp(self):
         self.tempdir = mkdtemp()
@@ -137,6 +138,8 @@ class DeprecatedImagesPipeline(ImagesPipeline):
 
 class ImagesPipelineTestCaseFieldsMixin:
 
+    skip = skip_pillow
+
     def test_item_fields_default(self):
         url = 'http://www.example.com/images/1.jpg'
         item = self.item_class(name='item1', image_urls=[url])
@@ -221,6 +224,9 @@ class ImagesPipelineTestCaseFieldsAttrsItem(ImagesPipelineTestCaseFieldsMixin, u
 
 
 class ImagesPipelineTestCaseCustomSettings(unittest.TestCase):
+
+    skip = skip_pillow
+
     img_cls_attribute_names = [
         # Pipeline attribute names with corresponding setting names.
         ("EXPIRES", "IMAGES_EXPIRES"),

tests/test_pipeline_media.py

@@ -1,3 +1,5 @@
+from typing import Optional
+
 from testfixtures import LogCapture
 from twisted.trial import unittest
 from twisted.python.failure import Failure
@@ -17,6 +19,14 @@ from scrapy.utils.signal import disconnect_all
 from scrapy import signals
 
 
+try:
+    from PIL import Image  # noqa: imported just to check for the import error
+except ImportError:
+    skip_pillow: Optional[str] = 'Missing Python Imaging Library, install https://pypi.python.org/pypi/Pillow'
+else:
+    skip_pillow = None
+
+
 def _mocked_download_func(request, info):
     response = request.meta.get('response')
     return response() if callable(response) else response
@@ -379,6 +389,7 @@ class MockedMediaPipelineDeprecatedMethods(ImagesPipeline):
 
 
 class MediaPipelineDeprecatedMethodsTestCase(unittest.TestCase):
+    skip = skip_pillow
 
     def setUp(self):
         self.pipe = MockedMediaPipelineDeprecatedMethods(store_uri='store-uri', download_func=_mocked_download_func)

tox.ini

@@ -19,9 +19,6 @@ deps =
     mitmproxy >= 4.0.4, < 5; python_version >= '3.6' and python_version < '3.7' and platform_system != 'Windows' and implementation_name != 'pypy'
     # Extras
     botocore>=1.4.87
-    Pillow>=4.0.0
-    # Twisted 21+ causes issues in tests that use skipIf
-    Twisted[http2]>=17.9.0,<21
 passenv =
     S3_TEST_FILE_URI
     AWS_ACCESS_KEY_ID
@@ -124,6 +121,7 @@ deps =
     {[testenv]deps}
     reppy
     robotexclusionrulesparser
+    Pillow>=4.0.0
 
 [testenv:asyncio]
 commands =