BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-25 20:04:28 +00:00
Code Releases Wiki Activity

Merge pull request #1131 from torymur/master

Browse Source 
Fix #1130
This commit is contained in:
Mikhail Korobov 2015-04-15 03:52:07 +05:00
commit f40587c661
1 changed files with 6 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/contrib/downloadermiddleware/robotstxt.py
View File

@ -49,8 +49,14 @@ class RobotsTxtMiddleware(object):
)
dfd = self.crawler.engine.download(robotsreq, spider)
dfd.addCallback(self._parse_robots)
dfd.addErrback(self._logerror, robotsreq, spider)
return self._parsers[netloc]
def _logerror(self, failure, request, spider):
if failure.type is not IgnoreRequest:
log.msg(format="Error downloading %%(request)s: %s" % failure.value,
level=log.ERROR, request=request, spider=spider)
def _parse_robots(self, response):
rp = robotparser.RobotFileParser(response.url)
rp.parse(response.body.splitlines())