BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-25 10:24:14 +00:00
Code Releases Wiki Activity

Merge pull request #214 from tonal/log-level-dropped-item

Browse Source 
Make LogFormatter return the log level (and require it)
This commit is contained in:
Pablo Hoffman 2012-12-29 19:21:54 -08:00
commit 1aa25cdff2
3 changed files with 8 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
scrapy/core/engine.py
View File

@ -195,7 +195,7 @@ class ExecutionEngine(object):
if isinstance(response, Response):
response.request = request # tie request to response received
logkws = self.logformatter.crawled(request, response, spider)
log.msg(level=log.DEBUG, spider=spider, **logkws)
log.msg(spider=spider, **logkws)
self.signals.send_catch_log(signal=signals.response_received, \
response=response, request=request, spider=spider)
return response

4
scrapy/core/scraper.py
View File

@ -201,14 +201,14 @@ class Scraper(object):
ex = output.value
if isinstance(ex, DropItem):
logkws = self.logformatter.dropped(item, ex, response, spider)
log.msg(level=log.WARNING, spider=spider, **logkws)
log.msg(spider=spider, **logkws)
return self.signals.send_catch_log_deferred(signal=signals.item_dropped, \
item=item, spider=spider, exception=output.value)
else:
log.err(output, 'Error processing %(item)s', item=item, spider=spider)
else:
logkws = self.logformatter.scraped(output, response, spider)
log.msg(level=log.DEBUG, spider=spider, **logkws)
log.msg(spider=spider, **logkws)
return self.signals.send_catch_log_deferred(signal=signals.item_scraped, \
item=output, response=response, spider=spider)

5
scrapy/logformatter.py
View File

@ -2,6 +2,8 @@ import os
from twisted.python.failure import Failure
from scrapy import log
SCRAPEDFMT = u"Scraped from %(src)s" + os.linesep + "%(item)s"
DROPPEDFMT = u"Dropped: %(exception)s" + os.linesep + "%(item)s"
@ -16,6 +18,7 @@ class LogFormatter(object):
def crawled(self, request, response, spider):
flags = ' %s' % str(response.flags) if response.flags else ''
return {
'level': log.DEBUG,
'format': CRAWLEDFMT,
'status': response.status,
'request': request,
@ -26,6 +29,7 @@ class LogFormatter(object):
def scraped(self, item, response, spider):
src = response.getErrorMessage() if isinstance(response, Failure) else response
return {
'level': log.DEBUG,
'format': SCRAPEDFMT,
'src': src,
'item': item,
@ -33,6 +37,7 @@ class LogFormatter(object):
def dropped(self, item, exception, response, spider):
return {
'level': log.WARNING,
'format': DROPPEDFMT,
'exception': exception,
'item': item,