BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-25 07:03:56 +00:00
Code Releases Wiki Activity

Added support for logging twisted errors generated outside of Scrapy - refs #188

Browse Source
This commit is contained in:
Pablo Hoffman 2010-08-05 20:46:54 -03:00
parent c82260f5ec
commit 4d66f4c6f8
2 changed files with 37 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
scrapy/log.py
View File

@ -38,24 +38,30 @@ class ScrapyFileLogObserver(log.FileLogObserver):
log.FileLogObserver.__init__(self, f)
def emit(self, eventDict):
if eventDict.get('system') != 'scrapy':
ev = eventDict.copy()
if ev['isError']:
ev.setdefault('logLevel', ERROR)
# ignore non-error messages from outside scrapy
if ev.get('system') != 'scrapy' and not ev['isError']:
return
level = eventDict.get('logLevel')
level = ev.get('logLevel')
if level < self.level:
return
spider = eventDict.get('spider')
message = eventDict.get('message')
spider = ev.get('spider')
if spider:
ev['system'] = spider.name
message = ev.get('message')
lvlname = level_names.get(level, 'NOLEVEL')
if message:
message = [unicode_to_str(x, self.encoding) for x in message]
message[0] = "%s: %s" % (lvlname, message[0])
why = eventDict.get('why')
ev['message'] = message
why = ev.get('why')
if why:
why = "%s: %s" % (lvlname, unicode_to_str(why, self.encoding))
eventDict['message'] = message
eventDict['why'] = why
eventDict['system'] = spider.name if spider else '-'
log.FileLogObserver.emit(self, eventDict)
ev['why'] = why
log.FileLogObserver.emit(self, ev)
def _get_log_level(level_name_or_id=None):
if level_name_or_id is None:

35
scrapy/tests/test_log.py
View File

@ -39,7 +39,7 @@ class ScrapyFileLogObserverTest(unittest.TestCase):
def test_msg_basic(self):
log.msg("Hello")
self.assertEqual(self.logged(), "[-] INFO: Hello")
self.assertEqual(self.logged(), "[scrapy] INFO: Hello")
def test_msg_spider(self):
spider = BaseSpider("myspider")
@ -48,15 +48,15 @@ class ScrapyFileLogObserverTest(unittest.TestCase):
def test_msg_level1(self):
log.msg("Hello", level=log.WARNING)
self.assertEqual(self.logged(), "[-] WARNING: Hello")
self.assertEqual(self.logged(), "[scrapy] WARNING: Hello")
def test_msg_level2(self):
log.msg("Hello", log.WARNING)
self.assertEqual(self.logged(), "[-] WARNING: Hello")
self.assertEqual(self.logged(), "[scrapy] WARNING: Hello")
def test_msg_wrong_level(self):
log.msg("Hello", level=9999)
self.assertEqual(self.logged(), "[-] NOLEVEL: Hello")
self.assertEqual(self.logged(), "[scrapy] NOLEVEL: Hello")
def test_msg_level_spider(self):
spider = BaseSpider("myspider")
@ -65,12 +65,12 @@ class ScrapyFileLogObserverTest(unittest.TestCase):
def test_msg_encoding(self):
log.msg(u"Price: \xa3100")
self.assertEqual(self.logged(), "[-] INFO: Price: \xc2\xa3100")
self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xc2\xa3100")
def test_msg_ignore_level(self):
log.msg("Hello", level=log.DEBUG)
log.msg("World", level=log.INFO)
self.assertEqual(self.logged(), "[-] INFO: World")
self.assertEqual(self.logged(), "[scrapy] INFO: World")
def test_msg_ignore_system(self):
txlog.msg("Hello")
@ -90,13 +90,21 @@ class ScrapyFileLogObserverTest(unittest.TestCase):
def test_err_why(self):
log.err(TypeError("bad type"), "Wrong type")
self.assertEqual(self.first_log_line(), "[scrapy] ERROR: Wrong type")
self.failUnless('TypeError' in self.logged())
self.failUnless('bad type' in self.logged())
def test_error_outside_scrapy(self):
"""Scrapy logger should still print outside errors"""
txlog.err(TypeError("bad type"), "Wrong type")
self.assertEqual(self.first_log_line(), "[-] ERROR: Wrong type")
self.failUnless('TypeError' in self.logged())
self.failUnless('bad type' in self.logged())
def test_err_why_encoding(self):
log.err(TypeError("bad type"), u"\xa3")
self.assertEqual(self.first_log_line(), "[-] ERROR: \xc2\xa3")
# this test fails in twisted trial observer, not in scrapy observer
# def test_err_why_encoding(self):
# log.err(TypeError("bad type"), u"\xa3")
# self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xc2\xa3")
def test_err_exc(self):
log.err(TypeError("bad type"))
@ -118,11 +126,12 @@ class Latin1ScrapyFileLogObserverTest(ScrapyFileLogObserverTest):
def test_msg_encoding(self):
log.msg(u"Price: \xa3100")
logged = self.f.getvalue().strip()[25:]
self.assertEqual(self.logged(), "[-] INFO: Price: \xa3100")
self.assertEqual(self.logged(), "[scrapy] INFO: Price: \xa3100")
def test_err_why_encoding(self):
log.err(TypeError("bad type"), u"\xa3")
self.assertEqual(self.first_log_line(), "[-] ERROR: \xa3")
# this test fails in twisted trial observer, not in scrapy observer
# def test_err_why_encoding(self):
# log.err(TypeError("bad type"), u"\xa3")
# self.assertEqual(self.first_log_line(), "[scrapy] ERROR: \xa3")
if __name__ == "__main__":