BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-25 14:03:41 +00:00
Code Releases Wiki Activity

restore downloader enqueing after middleware

Browse Source
This commit is contained in:
Daniel Grana 2009-06-21 03:03:14 -03:00
parent cd1ad337b1
commit 5944f6590a
2 changed files with 5 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
scrapy/core/downloader/manager.py
View File

@ -9,6 +9,7 @@ from twisted.internet import reactor, defer
from scrapy.core.exceptions import IgnoreRequest from scrapy.core.exceptions import IgnoreRequest
from scrapy.spider import spiders from scrapy.spider import spiders
from scrapy.core.downloader.middleware import DownloaderMiddlewareManager from scrapy.core.downloader.middleware import DownloaderMiddlewareManager
from scrapy.core.downloader.handlers import download_any
from scrapy.conf import settings from scrapy.conf import settings
from scrapy.utils.defer import chain_deferred, mustbe_deferred from scrapy.utils.defer import chain_deferred, mustbe_deferred
@ -74,7 +75,7 @@ class Downloader(object):
site.active.remove(request) site.active.remove(request)
return _ return _
return self.middleware.download(request, spider).addBoth(_deactivate) return self.middleware.download(self.enqueue, request, spider).addBoth(_deactivate)
def enqueue(self, request, spider): def enqueue(self, request, spider):
"""Enqueue a Request for a effective download from site""" """Enqueue a Request for a effective download from site"""
@ -116,7 +117,7 @@ class Downloader(object):
site.downloading.remove(request) site.downloading.remove(request)
self.process_queue(spider) self.process_queue(spider)
dwld = mustbe_deferred(request, spider) dwld = mustbe_deferred(download_any, request, spider)
chain_deferred(dwld, deferred) chain_deferred(dwld, deferred)
return dwld.addBoth(_finish) return dwld.addBoth(_finish)

5
scrapy/core/downloader/middleware.py
View File

@ -13,7 +13,6 @@ from scrapy.core.exceptions import NotConfigured
from scrapy.utils.misc import load_object from scrapy.utils.misc import load_object
from scrapy.utils.defer import mustbe_deferred from scrapy.utils.defer import mustbe_deferred
from scrapy.utils.middleware import build_middleware_list from scrapy.utils.middleware import build_middleware_list
from scrapy.core.downloader.handlers import download_any
from scrapy.conf import settings from scrapy.conf import settings
class DownloaderMiddlewareManager(object): class DownloaderMiddlewareManager(object):
@ -51,7 +50,7 @@ class DownloaderMiddlewareManager(object):
log.msg("Enabled downloader middlewares: %s" % ", ".join([type(m).__name__ for m in mws])) log.msg("Enabled downloader middlewares: %s" % ", ".join([type(m).__name__ for m in mws]))
self.loaded = True self.loaded = True
def download(self, request, spider): def download(self, download_func, request, spider):
def process_request(request): def process_request(request):
for method in self.request_middleware: for method in self.request_middleware:
response = method(request=request, spider=spider) response = method(request=request, spider=spider)
@ -60,7 +59,7 @@ class DownloaderMiddlewareManager(object):
(method.im_self.__class__.__name__, response.__class__.__name__) (method.im_self.__class__.__name__, response.__class__.__name__)
if response: if response:
return response return response
return download_any(request=request, spider=spider) return download_func(request=request, spider=spider)
def process_response(response): def process_response(response):
assert response is not None, 'Received None in process_response' assert response is not None, 'Received None in process_response'