diff --git a/scrapy/contrib/downloadermiddleware/robotstxt.py b/scrapy/contrib/downloadermiddleware/robotstxt.py index 0ea402719..a58ecca8e 100644 --- a/scrapy/contrib/downloadermiddleware/robotstxt.py +++ b/scrapy/contrib/downloadermiddleware/robotstxt.py @@ -49,8 +49,14 @@ class RobotsTxtMiddleware(object): ) dfd = self.crawler.engine.download(robotsreq, spider) dfd.addCallback(self._parse_robots) + dfd.addErrback(self._logerror, robotsreq, spider) return self._parsers[netloc] + def _logerror(self, failure, request, spider): + if failure.type is not IgnoreRequest: + log.msg(format="Error downloading %%(request)s: %s" % failure.value, + level=log.ERROR, request=request, spider=spider) + def _parse_robots(self, response): rp = robotparser.RobotFileParser(response.url) rp.parse(response.body.splitlines())