diff --git a/bin/runtests.bat b/bin/runtests.bat index 8a0869938..8e4247681 100644 --- a/bin/runtests.bat +++ b/bin/runtests.bat @@ -1,6 +1,6 @@ @ECHO off -SET test="scrapy" +SET test="scrapy scrapyd" IF NOT "%1" == "" SET test="%1" IF EXIST c:\python26\scripts\trial.py GOTO py26 diff --git a/bin/runtests.sh b/bin/runtests.sh index 33ddc0565..a714a208d 100755 --- a/bin/runtests.sh +++ b/bin/runtests.sh @@ -40,7 +40,7 @@ fi find -name '*.py[co]' -delete if [ $# -eq 0 ]; then - $trial scrapy + $trial scrapy scrapyd else $trial "$@" fi diff --git a/scrapyd/config.py b/scrapyd/config.py index d91efdad1..a01296fee 100644 --- a/scrapyd/config.py +++ b/scrapyd/config.py @@ -9,12 +9,16 @@ class Config(object): SECTION = 'scrapyd' - def __init__(self): - sources = self._getsources() - default_config = pkgutil.get_data(__package__, 'default_scrapyd.conf') - self.cp = SafeConfigParser() - self.cp.readfp(StringIO(default_config)) - self.cp.read(sources) + def __init__(self, values=None): + if values is None: + sources = self._getsources() + default_config = pkgutil.get_data(__package__, 'default_scrapyd.conf') + self.cp = SafeConfigParser() + self.cp.readfp(StringIO(default_config)) + self.cp.read(sources) + else: + self.cp = SafeConfigParser(values) + self.cp.add_section(self.SECTION) def _getsources(self): sources = ['/etc/scrapyd/scrapyd.conf', r'c:\scrapyd\scrapyd.conf'] diff --git a/scrapyd/eggstorage.py b/scrapyd/eggstorage.py index 956b02409..f46da4750 100644 --- a/scrapyd/eggstorage.py +++ b/scrapyd/eggstorage.py @@ -6,11 +6,10 @@ from shutil import copyfileobj, rmtree from distutils.version import LooseVersion from zope.interface import implements -from twisted.application.service import Service from .interfaces import IEggStorage -class FilesystemEggStorage(Service): +class FilesystemEggStorage(object): implements(IEggStorage) diff --git a/scrapyd/interfaces.py b/scrapyd/interfaces.py index ae33ec75f..e8a1f2cf2 100644 --- a/scrapyd/interfaces.py +++ b/scrapyd/interfaces.py @@ -12,11 +12,11 @@ class IEggStorage(Interface): project and version. If version is None, the latest version is returned.""" - def list(self, project): + def list(project): """Return the list of versions which have eggs stored (for the given project) in order (the latest version is the currently used).""" - def delete(self, project, version=None): + def delete(project, version=None): """Delete the egg stored for the given project and version. If should also delete the project if no versions are left""" diff --git a/scrapyd/tests/__init__.py b/scrapyd/tests/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/scrapyd/tests/test_eggstorage.py b/scrapyd/tests/test_eggstorage.py new file mode 100644 index 000000000..9254f0894 --- /dev/null +++ b/scrapyd/tests/test_eggstorage.py @@ -0,0 +1,41 @@ +from cStringIO import StringIO + +from twisted.trial import unittest + +from zope.interface.verify import verifyObject + +from scrapyd.interfaces import IEggStorage +from scrapyd.config import Config +from scrapyd.eggstorage import FilesystemEggStorage + +class EggStorageTest(unittest.TestCase): + + def setUp(self): + d = self.mktemp() + config = Config(values={'eggs_dir': d}) + self.eggst = FilesystemEggStorage(config) + + def test_interface(self): + verifyObject(IEggStorage, self.eggst) + + def test_put_get_list_delete(self): + self.eggst.put(StringIO("egg01"), 'mybot', '01') + self.eggst.put(StringIO("egg03"), 'mybot', '03') + self.eggst.put(StringIO("egg02"), 'mybot', '02') + + self.assertEqual(self.eggst.list('mybot'), ['01', '02', '03']) + self.assertEqual(self.eggst.list('mybot2'), []) + + v, f = self.eggst.get('mybot') + self.assertEqual(v, "03") + self.assertEqual(f.read(), "egg03") + + v, f = self.eggst.get('mybot', '02') + self.assertEqual(v, "02") + self.assertEqual(f.read(), "egg02") + + self.eggst.delete('mybot', '02') + self.assertEqual(self.eggst.list('mybot'), ['01', '03']) + + self.eggst.delete('mybot') + self.assertEqual(self.eggst.list('mybot'), []) diff --git a/scrapyd/tests/test_envion.py b/scrapyd/tests/test_envion.py new file mode 100644 index 000000000..0d1c657ee --- /dev/null +++ b/scrapyd/tests/test_envion.py @@ -0,0 +1,28 @@ +import os + +from twisted.trial import unittest + +from zope.interface.verify import verifyObject + +from scrapyd.interfaces import IEnvironment +from scrapyd.config import Config +from scrapyd.environ import Environment + +class EggStorageTest(unittest.TestCase): + + def setUp(self): + d = self.mktemp() + os.mkdir(d) + config = Config(values={'eggs_dir': d, 'logs_dir': d}) + self.environ = Environment(config) + + def test_interface(self): + verifyObject(IEnvironment, self.environ) + + def test_get_environment(self): + msg = {'project': 'mybot'} + slot = 3 + env = self.environ.get_environment(msg, slot) + self.assertEqual(env['SCRAPY_PROJECT'], 'mybot') + self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db')) + self.assert_(env['SCRAPY_LOG_FILE'].endswith('slot3.log')) diff --git a/scrapyd/tests/test_poller.py b/scrapyd/tests/test_poller.py new file mode 100644 index 000000000..d8dae49ac --- /dev/null +++ b/scrapyd/tests/test_poller.py @@ -0,0 +1,41 @@ +import os + +from twisted.trial import unittest +from twisted.internet.defer import Deferred + +from zope.interface.verify import verifyObject + +from scrapyd.interfaces import IPoller +from scrapyd.config import Config +from scrapyd.poller import QueuePoller +from scrapyd.utils import get_spider_queues + +class QueuePollerTest(unittest.TestCase): + + def setUp(self): + d = self.mktemp() + eggs_dir = os.path.join(d, 'eggs') + dbs_dir = os.path.join(d, 'dbs') + os.makedirs(eggs_dir) + os.makedirs(dbs_dir) + os.makedirs(os.path.join(eggs_dir, 'mybot1')) + os.makedirs(os.path.join(eggs_dir, 'mybot2')) + config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir}) + self.queues = get_spider_queues(eggs_dir, dbs_dir) + self.poller = QueuePoller(config) + + def test_interface(self): + verifyObject(IPoller, self.poller) + + def test_poll_next(self): + self.queues['mybot1'].add('spider1') + self.queues['mybot2'].add('spider2') + d1 = self.poller.next() + d2 = self.poller.next() + self.failUnless(isinstance(d1, Deferred)) + self.failIf(hasattr(d1, 'result')) + self.poller.poll() + self.queues['mybot1'].pop() + self.poller.poll() + self.failUnlessEqual(d1.result, {'project': 'mybot1'}) + self.failUnlessEqual(d2.result, {'project': 'mybot2'}) diff --git a/scrapyd/tests/test_scheduler.py b/scrapyd/tests/test_scheduler.py new file mode 100644 index 000000000..a9fb7e6f0 --- /dev/null +++ b/scrapyd/tests/test_scheduler.py @@ -0,0 +1,44 @@ +import os + +from twisted.trial import unittest + +from zope.interface.verify import verifyObject + +from scrapyd.interfaces import ISpiderScheduler +from scrapyd.config import Config +from scrapyd.scheduler import SpiderScheduler +from scrapyd.utils import get_spider_queues + +class SpiderSchedulerTest(unittest.TestCase): + + def setUp(self): + d = self.mktemp() + eggs_dir = self.eggs_dir = os.path.join(d, 'eggs') + dbs_dir = os.path.join(d, 'dbs') + os.mkdir(d) + os.makedirs(eggs_dir) + os.makedirs(dbs_dir) + os.makedirs(os.path.join(eggs_dir, 'mybot1')) + os.makedirs(os.path.join(eggs_dir, 'mybot2')) + config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir}) + self.queues = get_spider_queues(eggs_dir, dbs_dir) + self.sched = SpiderScheduler(config) + + def test_interface(self): + verifyObject(ISpiderScheduler, self.sched) + + def test_list_update_projects(self): + self.assertEqual(sorted(self.sched.list_projects()), sorted(['mybot1', 'mybot2'])) + os.makedirs(os.path.join(self.eggs_dir, 'mybot3')) + self.sched.update_projects() + self.assertEqual(sorted(self.sched.list_projects()), sorted(['mybot1', 'mybot2', 'mybot3'])) + + def test_schedule(self): + q = self.queues['mybot1'] + self.failIf(q.count()) + self.sched.schedule('mybot1', 'myspider1', a='b') + self.sched.schedule('mybot2', 'myspider2', c='d') + self.assertEqual(q.pop(), {'name': 'myspider1', 'a': 'b'}) + q = self.queues['mybot2'] + self.assertEqual(q.pop(), {'name': 'myspider2', 'c': 'd'}) +