BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-25 11:03:58 +00:00
Code Releases Wiki Activity

restore downloader enqueing after middleware

Browse Source
This commit is contained in:
Daniel Grana 2009-06-21 03:03:14 -03:00
parent cd1ad337b1
commit 5944f6590a
2 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/core/downloader/manager.py
View File

@ -9,6 +9,7 @@ from twisted.internet import reactor, defer
from scrapy.core.exceptions import IgnoreRequest
from scrapy.spider import spiders
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.core.downloader.handlers import download_any
from scrapy.conf import settings
from scrapy.utils.defer import chain_deferred, mustbe_deferred
@ -74,7 +75,7 @@ class Downloader(object):
site.active.remove(request)
return _
return self.middleware.download(request, spider).addBoth(_deactivate)
return self.middleware.download(self.enqueue, request, spider).addBoth(_deactivate)
def enqueue(self, request, spider):
"""Enqueue a Request for a effective download from site"""
@ -116,7 +117,7 @@ class Downloader(object):
site.downloading.remove(request)
self.process_queue(spider)
dwld = mustbe_deferred(request, spider)
dwld = mustbe_deferred(download_any, request, spider)
chain_deferred(dwld, deferred)
return dwld.addBoth(_finish)

5
scrapy/core/downloader/middleware.py
View File

@ -13,7 +13,6 @@ from scrapy.core.exceptions import NotConfigured
from scrapy.utils.misc import load_object
from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.middleware import build_middleware_list
from scrapy.core.downloader.handlers import download_any
from scrapy.conf import settings
class DownloaderMiddlewareManager(object):
@ -51,7 +50,7 @@ class DownloaderMiddlewareManager(object):
log.msg("Enabled downloader middlewares: %s" % ", ".join([type(m).__name__ for m in mws]))
self.loaded = True
def download(self, request, spider):
def download(self, download_func, request, spider):
def process_request(request):
for method in self.request_middleware:
response = method(request=request, spider=spider)
@ -60,7 +59,7 @@ class DownloaderMiddlewareManager(object):
(method.im_self.__class__.__name__, response.__class__.__name__)
if response:
return response
return download_any(request=request, spider=spider)
return download_func(request=request, spider=spider)
def process_response(response):
assert response is not None, 'Received None in process_response'