BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-26 06:43:44 +00:00
Code Releases Wiki Activity

Merge pull request #566 from redapple/offsite-stats

Browse Source 
OffsiteMiddleware: add 2 stats counters
This commit is contained in:
Daniel Graña 2014-01-31 10:05:35 -08:00
commit e1c6d3ffdf
2 changed files with 9 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
scrapy/contrib/spidermiddleware/offsite.py
View File

@ -13,9 +13,12 @@ from scrapy import log
class OffsiteMiddleware(object):
def __init__(self, stats):
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
o = cls()
o = cls(crawler.stats)
crawler.signals.connect(o.spider_opened, signal=signals.spider_opened)
return o
@ -30,6 +33,8 @@ class OffsiteMiddleware(object):
self.domains_seen.add(domain)
log.msg(format="Filtered offsite request to %(domain)r: %(request)s",
level=log.DEBUG, spider=spider, domain=domain, request=x)
self.stats.inc_value('offsite/domains', spider=spider)
self.stats.inc_value('offsite/filtered', spider=spider)
else:
yield x

4
scrapy/tests/test_spidermiddleware_offsite.py
View File

@ -3,13 +3,15 @@ from unittest import TestCase
from scrapy.http import Response, Request
from scrapy.spider import Spider
from scrapy.contrib.spidermiddleware.offsite import OffsiteMiddleware
from scrapy.utils.test import get_crawler
class TestOffsiteMiddleware(TestCase):
def setUp(self):
self.spider = self._get_spider()
self.mw = OffsiteMiddleware()
crawler = get_crawler()
self.mw = OffsiteMiddleware.from_crawler(crawler)
self.mw.spider_opened(self.spider)
def _get_spider(self):