BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-21 04:33:24 +00:00
Code Releases Wiki Activity

Add async def support to downloader middlewares.

Browse Source
This commit is contained in:
Andrey Rakhmatullin 2019-07-30 19:46:18 +05:00
parent 14d4428e70
commit 21f50c795a
2 changed files with 28 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/core/downloader/middleware.py
View File

@ -8,7 +8,7 @@ from twisted.internet import defer
from scrapy.exceptions import _InvalidOutput
from scrapy.http import Request, Response
from scrapy.middleware import MiddlewareManager
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.defer import mustbe_deferred, deferred_from_coro
from scrapy.utils.conf import build_component_list
@ -33,7 +33,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
@defer.inlineCallbacks
def process_request(request):
for method in self.methods['process_request']:
response = yield method(request=request, spider=spider)
response = yield deferred_from_coro(method(request=request, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_request must return None, Response or Request, got %s' % \
(method.__self__.__class__.__name__, response.__class__.__name__))
@ -48,7 +48,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
defer.returnValue(response)
for method in self.methods['process_response']:
response = yield method(request=request, response=response, spider=spider)
response = yield deferred_from_coro(method(request=request, response=response, spider=spider))
if not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_response must return Response or Request, got %s' % \
(method.__self__.__class__.__name__, type(response)))
@ -60,7 +60,7 @@ class DownloaderMiddlewareManager(MiddlewareManager):
def process_exception(_failure):
exception = _failure.value
for method in self.methods['process_exception']:
response = yield method(request=request, exception=exception, spider=spider)
response = yield deferred_from_coro(method(request=request, exception=exception, spider=spider))
if response is not None and not isinstance(response, (Response, Request)):
raise _InvalidOutput('Middleware %s.process_exception must return None, Response or Request, got %s' % \
(method.__self__.__class__.__name__, type(response)))

24
tests/test_downloadermiddleware.py
View File

@ -1,3 +1,4 @@
import asyncio
from unittest import mock
from twisted.internet.defer import Deferred
@ -206,3 +207,26 @@ class MiddlewareUsingDeferreds(ManagerTestCase):
self.assertIs(results[0], resp)
self.assertFalse(download_func.called)
class MiddlewareUsingCoro(ManagerTestCase):
"""Middlewares using asyncio coroutines should work"""
def test_asyncdef(self):
resp = Response('http://example.com/index.html')
class CoroMiddleware:
async def process_request(self, request, spider):
await asyncio.sleep(0.1)
return resp
self.mwman._add_middleware(CoroMiddleware())
req = Request('http://example.com/index.html')
download_func = mock.MagicMock()
dfd = self.mwman.download(download_func, req, self.spider)
results = []
dfd.addBoth(results.append)
self._wait(dfd)
self.assertIs(results[0], resp)
self.assertFalse(download_func.called)