mirror of
https://github.com/scrapy/scrapy.git
synced 2025-02-22 05:53:39 +00:00
94 lines
3.1 KiB
Python
94 lines
3.1 KiB
Python
import unittest
|
|
import six
|
|
|
|
from scrapy.spiders import Spider
|
|
from scrapy.http import Request, Response
|
|
from scrapy.item import Item, Field
|
|
from scrapy.logformatter import LogFormatter
|
|
|
|
|
|
class CustomItem(Item):
|
|
|
|
name = Field()
|
|
|
|
def __str__(self):
|
|
return "name: %s" % self['name']
|
|
|
|
|
|
class LoggingContribTest(unittest.TestCase):
|
|
|
|
def setUp(self):
|
|
self.formatter = LogFormatter()
|
|
self.spider = Spider('default')
|
|
|
|
def test_crawled(self):
|
|
req = Request("http://www.example.com")
|
|
res = Response("http://www.example.com")
|
|
logkws = self.formatter.crawled(req, res, self.spider)
|
|
logline = logkws['msg'] % logkws['args']
|
|
self.assertEqual(logline,
|
|
"Crawled (200) <GET http://www.example.com> (referer: None)")
|
|
|
|
req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
|
|
res = Response("http://www.example.com", flags=['cached'])
|
|
logkws = self.formatter.crawled(req, res, self.spider)
|
|
logline = logkws['msg'] % logkws['args']
|
|
self.assertEqual(logline,
|
|
"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
|
|
|
|
def test_flags_in_request(self):
|
|
req = Request("http://www.example.com", flags=['test','flag'])
|
|
res = Response("http://www.example.com")
|
|
logkws = self.formatter.crawled(req, res, self.spider)
|
|
logline = logkws['msg'] % logkws['args']
|
|
self.assertEqual(logline,
|
|
"Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
|
|
|
|
def test_dropped(self):
|
|
item = {}
|
|
exception = Exception(u"\u2018")
|
|
response = Response("http://www.example.com")
|
|
logkws = self.formatter.dropped(item, exception, response, self.spider)
|
|
logline = logkws['msg'] % logkws['args']
|
|
lines = logline.splitlines()
|
|
assert all(isinstance(x, six.text_type) for x in lines)
|
|
self.assertEqual(lines, [u"Dropped: \u2018", '{}'])
|
|
|
|
def test_scraped(self):
|
|
item = CustomItem()
|
|
item['name'] = u'\xa3'
|
|
response = Response("http://www.example.com")
|
|
logkws = self.formatter.scraped(item, response, self.spider)
|
|
logline = logkws['msg'] % logkws['args']
|
|
lines = logline.splitlines()
|
|
assert all(isinstance(x, six.text_type) for x in lines)
|
|
self.assertEqual(lines, [u"Scraped from <200 http://www.example.com>", u'name: \xa3'])
|
|
|
|
|
|
class LogFormatterSubclass(LogFormatter):
|
|
def crawled(self, request, response, spider):
|
|
kwargs = super(LogFormatterSubclass, self).crawled(
|
|
request, response, spider)
|
|
CRAWLEDMSG = (
|
|
u"Crawled (%(status)s) %(request)s (referer: "
|
|
u"%(referer)s)%(flags)s"
|
|
)
|
|
return {
|
|
'level': kwargs['level'],
|
|
'msg': CRAWLEDMSG,
|
|
'args': kwargs['args']
|
|
}
|
|
|
|
|
|
class LogformatterSubclassTest(LoggingContribTest):
|
|
def setUp(self):
|
|
self.formatter = LogFormatterSubclass()
|
|
self.spider = Spider('default')
|
|
|
|
def test_flags_in_request(self):
|
|
pass
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|