BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-25 23:23:43 +00:00
Code Releases Wiki Activity

fixed unhandled error in deferred (RobotsTxtMiddleware)

Browse Source
This commit is contained in:
Victoria Terenina (torymur) 2015-04-02 17:48:38 +03:00
parent 2183984832
commit 42f7674619
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/contrib/downloadermiddleware/robotstxt.py
View File

@ -49,8 +49,14 @@ class RobotsTxtMiddleware(object):
) )
dfd = self.crawler.engine.download(robotsreq, spider) dfd = self.crawler.engine.download(robotsreq, spider)
dfd.addCallback(self._parse_robots) dfd.addCallback(self._parse_robots)
dfd.addErrback(self._logerror, robotsreq, spider)
return self._parsers[netloc] return self._parsers[netloc]
def _logerror(self, failure, request, spider):
if failure.type is not IgnoreRequest:
log.msg(format="Error downloading %%(request)s: %s" % failure.value,
level=log.ERROR, request=request, spider=spider)
def _parse_robots(self, response): def _parse_robots(self, response):
rp = robotparser.RobotFileParser(response.url) rp = robotparser.RobotFileParser(response.url)
rp.parse(response.body.splitlines()) rp.parse(response.body.splitlines())