BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-28 22:58:06 +00:00
Code Releases Wiki Activity

Merge pull request #330 from nramirezuy/spider-from_crawler

Browse Source 
use from_crawler method if available in spiders
This commit is contained in:
Daniel Graña 2013-07-16 12:42:31 -07:00
commit 3eb5d5e931
3 changed files with 20 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
scrapy/spidermanager.py
View File

@ -33,6 +33,7 @@ class SpiderManager(object):
@classmethod @classmethod
def from_crawler(cls, crawler): def from_crawler(cls, crawler):
sm = cls.from_settings(crawler.settings) sm = cls.from_settings(crawler.settings)
sm.crawler = crawler
crawler.signals.connect(sm.close_spider, signals.spider_closed) crawler.signals.connect(sm.close_spider, signals.spider_closed)
return sm return sm
@ -41,7 +42,10 @@ class SpiderManager(object):
spcls = self._spiders[spider_name] spcls = self._spiders[spider_name]
except KeyError: except KeyError:
raise KeyError("Spider not found: %s" % spider_name) raise KeyError("Spider not found: %s" % spider_name)
return spcls(**spider_kwargs) if hasattr(self, 'crawler') and hasattr(spcls, 'from_crawler'):
return spcls.from_crawler(self.crawler, **spider_kwargs)
else:
return spcls(**spider_kwargs)
def find_by_request(self, request): def find_by_request(self, request):
return [name for name, cls in self._spiders.iteritems() return [name for name, cls in self._spiders.iteritems()

8
scrapy/tests/test_spidermanager/__init__.py
View File

@ -1,6 +1,5 @@
import sys import sys
import os import os
import weakref
import shutil import shutil
from zope.interface.verify import verifyObject from zope.interface.verify import verifyObject
@ -9,7 +8,6 @@ from twisted.trial import unittest
# ugly hack to avoid cyclic imports of scrapy.spider when running this test # ugly hack to avoid cyclic imports of scrapy.spider when running this test
# alone # alone
import scrapy.spider
from scrapy.interfaces import ISpiderManager from scrapy.interfaces import ISpiderManager
from scrapy.spidermanager import SpiderManager from scrapy.spidermanager import SpiderManager
from scrapy.http import Request from scrapy.http import Request
@ -36,7 +34,7 @@ class SpiderManagerTest(unittest.TestCase):
def test_list(self): def test_list(self):
self.assertEqual(set(self.spiderman.list()), self.assertEqual(set(self.spiderman.list()),
set(['spider1', 'spider2', 'spider3'])) set(['spider1', 'spider2', 'spider3', 'spider4']))
def test_create(self): def test_create(self):
spider1 = self.spiderman.create("spider1") spider1 = self.spiderman.create("spider1")
@ -66,3 +64,7 @@ class SpiderManagerTest(unittest.TestCase):
def test_load_base_spider(self): def test_load_base_spider(self):
self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider0']) self.spiderman = SpiderManager(['scrapy.tests.test_spidermanager.test_spiders.spider0'])
assert len(self.spiderman._spiders) == 0 assert len(self.spiderman._spiders) == 0
def test_load_from_crawler(self):
spider = self.spiderman.create('spider4', a='OK')
self.assertEqual(spider.a, 'OK')

10
scrapy/tests/test_spidermanager/test_spiders/spider4.py Normal file
View File

@ -0,0 +1,10 @@
from scrapy.spider import BaseSpider
class Spider4(BaseSpider):
name = "spider4"
@classmethod
def from_crawler(cls, crawler, **kwargs):
o = cls(**kwargs)
o.crawler = crawler
return o