added SpiderState extension

docs/topics/settings.rst

@@ -599,6 +599,7 @@ Default::
         'scrapy.contrib.feedexport.FeedExporter': 0,
         'scrapy.contrib.spidercontext.SpiderContext': 0,
         'scrapy.contrib.logstats.LogStats': 0,
+        'scrapy.contrib.spiderstate.SpiderState': 0,
     }
 
 The list of available extensions. Keep in mind that some of them need to

scrapy/contrib/spiderstate.py

+from __future__ import with_statement
+
+import os, cPickle as pickle
+
+from scrapy import signals
+from scrapy.exceptions import NotConfigured
+from scrapy.xlib.pydispatch import dispatcher
+
+class SpiderState(object):
+    """Stores and loads spider state"""
+
+    def __init__(self, jobdir):
+        self.statefn = os.path.join(jobdir, 'spider.state')
+        dispatcher.connect(self.spider_closed, signal=signals.spider_closed)
+        dispatcher.connect(self.spider_opened, signal=signals.spider_opened)
+
+    @classmethod
+    def from_crawler(cls, crawler):
+        jobdir = crawler.settings.get('JOBDIR')
+        if not jobdir:
+            raise NotConfigured
+        return cls(jobdir)
+
+    def spider_closed(self, spider):
+        with open(self.statefn, 'wb') as f:
+            pickle.dump(spider.state, f, protocol=2)
+
+    def spider_opened(self, spider):
+        if os.path.exists(self.statefn):
+            with open(self.statefn) as f:
+                spider.state = pickle.load(f)
+        else:
+            spider.state = {}

scrapy/settings/default_settings.py

@@ -144,6 +144,7 @@ EXTENSIONS_BASE = {
     'scrapy.contrib.closespider.CloseSpider': 0,
     'scrapy.contrib.feedexport.FeedExporter': 0,
     'scrapy.contrib.logstats.LogStats': 0,
+    'scrapy.contrib.spiderstate.SpiderState': 0,
 }
 
 FEED_URI = None