BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-26 23:04:29 +00:00
Code Releases Wiki Activity

added spider_exceptions to scrapy stats

Browse Source 
--HG--
extra : convert_revision : svn%3Ab85faa78-f9eb-468e-a121-7cced6da292c%40556
This commit is contained in:
Pablo Hoffman 2008-12-27 17:37:05 +00:00
parent 6fa33f4463
commit f932a3d842
1 changed files with 2 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/trunk/scrapy/core/engine.py
View File

@ -9,6 +9,7 @@ from twisted.python.failure import Failure
from pydispatch import dispatcher
from scrapy import log
from scrapy.stats import stats
from scrapy.conf import settings
from scrapy.core import signals
from scrapy.core.scheduler import Scheduler
@ -259,6 +260,7 @@ class ExecutionEngine(object):
if not isinstance(_failure.value, IgnoreRequest):
referer = None if not isinstance(response, Response) else response.request.headers.get('Referer', None)
log.msg("Error while spider was processing <%s> from <%s>: %s" % (request.url, referer, _failure), log.ERROR, domain=domain)
stats.incpath("%s/spider_exceptions/%s" % (domain, _failure.value.__class__.__name__))
def _bugtrap(_failure):
log.msg('FRAMEWORK BUG processing %s: %s' % (request, _failure), log.ERROR, domain=domain)