make SpiderState extension always available, regardless of whether there is a job dir, and make sure it always set spider.state attribute, for consistency between in-memory and on-disk runs

scrapy/contrib/spiderstate.py

@@ -9,26 +9,28 @@ from scrapy.xlib.pydispatch import dispatcher
 class SpiderState(object):
     """Store and load spider state during a scraping job"""
 
-    def __init__(self, jobdir):
-        self.statefn = os.path.join(jobdir, 'spider.state')
+    def __init__(self, jobdir=None):
+        self.jobdir = jobdir
 
     @classmethod
     def from_crawler(cls, crawler):
-        jobdir = crawler.settings.get('JOBDIR')
-        if not jobdir:
-            raise NotConfigured
-        obj = cls(jobdir)
+        obj = cls(crawler.settings.get('JOBDIR'))
         dispatcher.connect(obj.spider_closed, signal=signals.spider_closed)
         dispatcher.connect(obj.spider_opened, signal=signals.spider_opened)
         return obj
 
     def spider_closed(self, spider):
-        with open(self.statefn, 'wb') as f:
-            pickle.dump(spider.state, f, protocol=2)
+        if self.jobdir:
+            with open(self.statefn, 'wb') as f:
+                pickle.dump(spider.state, f, protocol=2)
 
     def spider_opened(self, spider):
-        if os.path.exists(self.statefn):
+        if self.jobdir and os.path.exists(self.statefn):
             with open(self.statefn) as f:
                 spider.state = pickle.load(f)
         else:
             spider.state = {}
+
+    @property
+    def statefn(self):
+        return os.path.join(self.jobdir, 'spider.state')

scrapy/tests/test_contrib_spiderstate.py

@@ -25,3 +25,12 @@ class SpiderStateTest(unittest.TestCase):
         ss2.spider_opened(spider2)
         self.assertEqual(spider.state, {'one': 1, 'dt': dt})
         ss2.spider_closed(spider2)
+
+    def test_state_attribute(self):
+        # state attribute must be present if jobdir is not set, to provide a
+        # consistent interface 
+        spider = BaseSpider(name='default')
+        ss = SpiderState()
+        ss.spider_opened(spider)
+        self.assertEqual(spider.state, {})
+        ss.spider_closed(spider)