Added tests for Scrapyd components. Closes #237

bin/runtests.bat

@@ -1,6 +1,6 @@
 @ECHO off
 
-SET test="scrapy"
+SET test="scrapy scrapyd"
 IF NOT "%1" == "" SET test="%1"
 
 IF EXIST c:\python26\scripts\trial.py GOTO py26

bin/runtests.sh

@@ -40,7 +40,7 @@ fi
 
 find -name '*.py[co]' -delete
 if [ $# -eq 0 ]; then
-    $trial scrapy
+    $trial scrapy scrapyd
 else
     $trial "$@"
 fi

scrapyd/config.py

@@ -9,12 +9,16 @@ class Config(object):
 
     SECTION = 'scrapyd'
 
-    def __init__(self):
-        sources = self._getsources()
-        default_config = pkgutil.get_data(__package__, 'default_scrapyd.conf')
-        self.cp = SafeConfigParser()
-        self.cp.readfp(StringIO(default_config))
-        self.cp.read(sources)
+    def __init__(self, values=None):
+        if values is None:
+            sources = self._getsources()
+            default_config = pkgutil.get_data(__package__, 'default_scrapyd.conf')
+            self.cp = SafeConfigParser()
+            self.cp.readfp(StringIO(default_config))
+            self.cp.read(sources)
+        else:
+            self.cp = SafeConfigParser(values)
+            self.cp.add_section(self.SECTION)
 
     def _getsources(self):
         sources = ['/etc/scrapyd/scrapyd.conf', r'c:\scrapyd\scrapyd.conf']

scrapyd/eggstorage.py

@@ -6,11 +6,10 @@ from shutil import copyfileobj, rmtree
 from distutils.version import LooseVersion
 
 from zope.interface import implements
-from twisted.application.service import Service
 
 from .interfaces import IEggStorage
 
-class FilesystemEggStorage(Service):
+class FilesystemEggStorage(object):
 
     implements(IEggStorage)
 

scrapyd/interfaces.py

@@ -12,11 +12,11 @@ class IEggStorage(Interface):
         project and version. If version is None, the latest version is
         returned."""
 
-    def list(self, project):
+    def list(project):
         """Return the list of versions which have eggs stored (for the given
         project) in order (the latest version is the currently used)."""
 
-    def delete(self, project, version=None):
+    def delete(project, version=None):
         """Delete the egg stored for the given project and version. If should
         also delete the project if no versions are left"""
 

scrapyd/tests/test_eggstorage.py

@@ -0,0 +1,41 @@
+from cStringIO import StringIO
+
+from twisted.trial import unittest
+
+from zope.interface.verify import verifyObject
+
+from scrapyd.interfaces import IEggStorage
+from scrapyd.config import Config
+from scrapyd.eggstorage import FilesystemEggStorage
+
+class EggStorageTest(unittest.TestCase):
+
+    def setUp(self):
+        d = self.mktemp()
+        config = Config(values={'eggs_dir': d})
+        self.eggst = FilesystemEggStorage(config)
+
+    def test_interface(self):
+        verifyObject(IEggStorage, self.eggst)
+
+    def test_put_get_list_delete(self):
+        self.eggst.put(StringIO("egg01"), 'mybot', '01')
+        self.eggst.put(StringIO("egg03"), 'mybot', '03')
+        self.eggst.put(StringIO("egg02"), 'mybot', '02')
+
+        self.assertEqual(self.eggst.list('mybot'), ['01', '02', '03'])
+        self.assertEqual(self.eggst.list('mybot2'), [])
+
+        v, f = self.eggst.get('mybot')
+        self.assertEqual(v, "03")
+        self.assertEqual(f.read(), "egg03")
+
+        v, f = self.eggst.get('mybot', '02')
+        self.assertEqual(v, "02")
+        self.assertEqual(f.read(), "egg02")
+
+        self.eggst.delete('mybot', '02')
+        self.assertEqual(self.eggst.list('mybot'), ['01', '03'])
+
+        self.eggst.delete('mybot')
+        self.assertEqual(self.eggst.list('mybot'), [])

scrapyd/tests/test_envion.py

@@ -0,0 +1,28 @@
+import os
+
+from twisted.trial import unittest
+
+from zope.interface.verify import verifyObject
+
+from scrapyd.interfaces import IEnvironment
+from scrapyd.config import Config
+from scrapyd.environ import Environment
+
+class EggStorageTest(unittest.TestCase):
+
+    def setUp(self):
+        d = self.mktemp()
+        os.mkdir(d)
+        config = Config(values={'eggs_dir': d, 'logs_dir': d})
+        self.environ = Environment(config)
+
+    def test_interface(self):
+        verifyObject(IEnvironment, self.environ)
+
+    def test_get_environment(self):
+        msg = {'project': 'mybot'}
+        slot = 3
+        env = self.environ.get_environment(msg, slot)
+        self.assertEqual(env['SCRAPY_PROJECT'], 'mybot')
+        self.assert_(env['SCRAPY_SQLITE_DB'].endswith('mybot.db'))
+        self.assert_(env['SCRAPY_LOG_FILE'].endswith('slot3.log'))

scrapyd/tests/test_poller.py

@@ -0,0 +1,41 @@
+import os
+
+from twisted.trial import unittest
+from twisted.internet.defer import Deferred
+
+from zope.interface.verify import verifyObject
+
+from scrapyd.interfaces import IPoller
+from scrapyd.config import Config
+from scrapyd.poller import QueuePoller
+from scrapyd.utils import get_spider_queues
+
+class QueuePollerTest(unittest.TestCase):
+
+    def setUp(self):
+        d = self.mktemp()
+        eggs_dir = os.path.join(d, 'eggs')
+        dbs_dir = os.path.join(d, 'dbs')
+        os.makedirs(eggs_dir)
+        os.makedirs(dbs_dir)
+        os.makedirs(os.path.join(eggs_dir, 'mybot1'))
+        os.makedirs(os.path.join(eggs_dir, 'mybot2'))
+        config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
+        self.queues = get_spider_queues(eggs_dir, dbs_dir)
+        self.poller = QueuePoller(config)
+
+    def test_interface(self):
+        verifyObject(IPoller, self.poller)
+
+    def test_poll_next(self):
+        self.queues['mybot1'].add('spider1')
+        self.queues['mybot2'].add('spider2')
+        d1 = self.poller.next()
+        d2 = self.poller.next()
+        self.failUnless(isinstance(d1, Deferred))
+        self.failIf(hasattr(d1, 'result'))
+        self.poller.poll()
+        self.queues['mybot1'].pop()
+        self.poller.poll()
+        self.failUnlessEqual(d1.result, {'project': 'mybot1'})
+        self.failUnlessEqual(d2.result, {'project': 'mybot2'})

scrapyd/tests/test_scheduler.py

@@ -0,0 +1,44 @@
+import os
+
+from twisted.trial import unittest
+
+from zope.interface.verify import verifyObject
+
+from scrapyd.interfaces import ISpiderScheduler
+from scrapyd.config import Config
+from scrapyd.scheduler import SpiderScheduler
+from scrapyd.utils import get_spider_queues
+
+class SpiderSchedulerTest(unittest.TestCase):
+
+    def setUp(self):
+        d = self.mktemp()
+        eggs_dir = self.eggs_dir = os.path.join(d, 'eggs')
+        dbs_dir = os.path.join(d, 'dbs')
+        os.mkdir(d)
+        os.makedirs(eggs_dir)
+        os.makedirs(dbs_dir)
+        os.makedirs(os.path.join(eggs_dir, 'mybot1'))
+        os.makedirs(os.path.join(eggs_dir, 'mybot2'))
+        config = Config(values={'eggs_dir': eggs_dir, 'dbs_dir': dbs_dir})
+        self.queues = get_spider_queues(eggs_dir, dbs_dir)
+        self.sched = SpiderScheduler(config)
+
+    def test_interface(self):
+        verifyObject(ISpiderScheduler, self.sched)
+
+    def test_list_update_projects(self):
+        self.assertEqual(sorted(self.sched.list_projects()), sorted(['mybot1', 'mybot2']))
+        os.makedirs(os.path.join(self.eggs_dir, 'mybot3'))
+        self.sched.update_projects()
+        self.assertEqual(sorted(self.sched.list_projects()), sorted(['mybot1', 'mybot2', 'mybot3']))
+
+    def test_schedule(self):
+        q = self.queues['mybot1']
+        self.failIf(q.count())
+        self.sched.schedule('mybot1', 'myspider1', a='b')
+        self.sched.schedule('mybot2', 'myspider2', c='d')
+        self.assertEqual(q.pop(), {'name': 'myspider1', 'a': 'b'})
+        q = self.queues['mybot2']
+        self.assertEqual(q.pop(), {'name': 'myspider2', 'c': 'd'})
+