2019-10-01 11:54:15 -03:00
|
|
|
from datetime import datetime
|
2013-12-28 00:47:32 +06:00
|
|
|
import unittest
|
2019-08-20 21:27:52 +05:00
|
|
|
from unittest import mock
|
2019-10-01 11:54:15 -03:00
|
|
|
|
|
|
|
from scrapy.extensions.corestats import CoreStats
|
2015-05-09 04:20:09 -03:00
|
|
|
from scrapy.spiders import Spider
|
2015-04-23 13:07:48 -03:00
|
|
|
from scrapy.statscollectors import StatsCollector, DummyStatsCollector
|
2012-03-06 08:59:20 -02:00
|
|
|
from scrapy.utils.test import get_crawler
|
2008-10-22 13:44:02 +00:00
|
|
|
|
2015-08-26 01:58:59 +05:00
|
|
|
|
2019-10-01 11:54:15 -03:00
|
|
|
class CoreStatsExtensionTest(unittest.TestCase):
|
|
|
|
|
|
|
|
def setUp(self):
|
|
|
|
self.crawler = get_crawler(Spider)
|
|
|
|
self.spider = self.crawler._create_spider('foo')
|
|
|
|
|
|
|
|
@mock.patch('scrapy.extensions.corestats.datetime')
|
|
|
|
def test_core_stats_default_stats_collector(self, mock_datetime):
|
|
|
|
fixed_datetime = datetime(2019, 12, 1, 11, 38)
|
|
|
|
mock_datetime.utcnow = mock.Mock(return_value=fixed_datetime)
|
|
|
|
self.crawler.stats = StatsCollector(self.crawler)
|
|
|
|
ext = CoreStats.from_crawler(self.crawler)
|
|
|
|
ext.spider_opened(self.spider)
|
|
|
|
ext.item_scraped({}, self.spider)
|
|
|
|
ext.response_received(self.spider)
|
|
|
|
ext.item_dropped({}, self.spider, ZeroDivisionError())
|
|
|
|
ext.spider_closed(self.spider, 'finished')
|
|
|
|
self.assertEqual(
|
|
|
|
ext.stats._stats,
|
|
|
|
{
|
|
|
|
'start_time': fixed_datetime,
|
|
|
|
'finish_time': fixed_datetime,
|
|
|
|
'item_scraped_count': 1,
|
|
|
|
'response_received_count': 1,
|
|
|
|
'item_dropped_count': 1,
|
|
|
|
'item_dropped_reasons_count/ZeroDivisionError': 1,
|
|
|
|
'finish_reason': 'finished',
|
|
|
|
'elapsed_time_seconds': 0.0,
|
|
|
|
}
|
|
|
|
)
|
|
|
|
|
|
|
|
def test_core_stats_dummy_stats_collector(self):
|
|
|
|
self.crawler.stats = DummyStatsCollector(self.crawler)
|
|
|
|
ext = CoreStats.from_crawler(self.crawler)
|
|
|
|
ext.spider_opened(self.spider)
|
|
|
|
ext.item_scraped({}, self.spider)
|
|
|
|
ext.response_received(self.spider)
|
|
|
|
ext.item_dropped({}, self.spider, ZeroDivisionError())
|
|
|
|
ext.spider_closed(self.spider, 'finished')
|
|
|
|
self.assertEqual(ext.stats._stats, {})
|
|
|
|
|
|
|
|
|
2009-08-11 17:10:37 -03:00
|
|
|
class StatsCollectorTest(unittest.TestCase):
|
2008-10-22 13:44:02 +00:00
|
|
|
|
2009-11-14 20:28:59 -02:00
|
|
|
def setUp(self):
|
2014-07-31 04:12:12 -03:00
|
|
|
self.crawler = get_crawler(Spider)
|
|
|
|
self.spider = self.crawler._create_spider('foo')
|
2009-11-14 20:28:59 -02:00
|
|
|
|
2009-08-11 17:10:37 -03:00
|
|
|
def test_collector(self):
|
2012-03-06 08:59:20 -02:00
|
|
|
stats = StatsCollector(self.crawler)
|
2009-08-11 17:10:37 -03:00
|
|
|
self.assertEqual(stats.get_stats(), {})
|
|
|
|
self.assertEqual(stats.get_value('anything'), None)
|
|
|
|
self.assertEqual(stats.get_value('anything', 'default'), 'default')
|
|
|
|
stats.set_value('test', 'value')
|
|
|
|
self.assertEqual(stats.get_stats(), {'test': 'value'})
|
|
|
|
stats.set_value('test2', 23)
|
|
|
|
self.assertEqual(stats.get_stats(), {'test': 'value', 'test2': 23})
|
|
|
|
self.assertEqual(stats.get_value('test2'), 23)
|
|
|
|
stats.inc_value('test2')
|
|
|
|
self.assertEqual(stats.get_value('test2'), 24)
|
|
|
|
stats.inc_value('test2', 6)
|
|
|
|
self.assertEqual(stats.get_value('test2'), 30)
|
|
|
|
stats.max_value('test2', 6)
|
|
|
|
self.assertEqual(stats.get_value('test2'), 30)
|
|
|
|
stats.max_value('test2', 40)
|
|
|
|
self.assertEqual(stats.get_value('test2'), 40)
|
|
|
|
stats.max_value('test3', 1)
|
|
|
|
self.assertEqual(stats.get_value('test3'), 1)
|
|
|
|
stats.min_value('test2', 60)
|
|
|
|
self.assertEqual(stats.get_value('test2'), 40)
|
|
|
|
stats.min_value('test2', 35)
|
|
|
|
self.assertEqual(stats.get_value('test2'), 35)
|
|
|
|
stats.min_value('test4', 7)
|
|
|
|
self.assertEqual(stats.get_value('test4'), 7)
|
2008-10-22 13:44:02 +00:00
|
|
|
|
2009-08-11 17:10:37 -03:00
|
|
|
def test_dummy_collector(self):
|
2012-03-06 08:59:20 -02:00
|
|
|
stats = DummyStatsCollector(self.crawler)
|
2009-08-11 17:10:37 -03:00
|
|
|
self.assertEqual(stats.get_stats(), {})
|
|
|
|
self.assertEqual(stats.get_value('anything'), None)
|
|
|
|
self.assertEqual(stats.get_value('anything', 'default'), 'default')
|
|
|
|
stats.set_value('test', 'value')
|
|
|
|
stats.inc_value('v1')
|
|
|
|
stats.max_value('v2', 100)
|
|
|
|
stats.min_value('v3', 100)
|
2009-11-14 20:28:59 -02:00
|
|
|
stats.open_spider('a')
|
|
|
|
stats.set_value('test', 'value', spider=self.spider)
|
2009-08-11 17:10:37 -03:00
|
|
|
self.assertEqual(stats.get_stats(), {})
|
|
|
|
self.assertEqual(stats.get_stats('a'), {})
|