utils.serialize: added support for encoding Deferreds, and to refer spiders by...

utils.serialize: added support for encoding Deferreds, and to refer spiders by name using 'spider::name'

scrapy/tests/test_utils_serialize.py

@@ -2,6 +2,8 @@ import unittest
 import datetime
 from decimal import Decimal
 
+from twisted.internet import defer
+
 from scrapy.utils.serialize import SpiderReferencer, ScrapyJSONEncoder, ScrapyJSONDecoder
 from scrapy.utils.py26 import json
 from scrapy.spider import BaseSpider
@@ -45,6 +47,10 @@ class SpiderReferencerTestCase(BaseTestCase):
         assert sp1 is not sp2
         assert sp1 is sp1_
 
+        # referring to spiders by name
+        assert sp1 is self.spref.get_spider_from_reference('spider::name1')
+        assert sp2 is self.spref.get_spider_from_reference('spider::name2')
+
         # must return string as-is if spider id not found
         assert 'lala' == self.spref.get_spider_from_reference('lala')
         # must raise RuntimeError if spider id is not found and spider is not running
@@ -107,6 +113,8 @@ class JsonEncoderTestCase(BaseTestCase):
         for spiders, refs in examples_encode_only:
             self.assertEqual(self.encoder.encode(spiders), json.dumps(refs))
 
+        assert 'Deferred' in self.encoder.encode(defer.Deferred())
+
     def test_encode_request(self):
         r = Request("http://www.example.com/lala")
         rs = self.encoder.encode(r)

scrapy/utils/serialize.py

@@ -2,6 +2,8 @@ import re
 import datetime
 import decimal
 
+from twisted.internet import defer
+
 from scrapy.core.manager import scrapymanager
 from scrapy.spider import BaseSpider
 from scrapy.http import Request, Response
@@ -16,7 +18,7 @@ class SpiderReferencer(object):
     ScrapyJSONEncoder.default() with a custom encoding mechanism.
     """
 
-    spider_ref_re = re.compile('^spider:([0-9a-f]+)(:.*)?$')
+    spider_ref_re = re.compile('^spider:([0-9a-f]+)?:?(.+)?$')
 
     def __init__(self, manager=None):
         self.manager = manager or scrapymanager
@@ -31,9 +33,9 @@ class SpiderReferencer(object):
         """
         m = self.spider_ref_re.search(ref)
         if m:
-            spid = int(m.group(1), 16)
+            spid, spname = m.groups()
             for spider in self.manager.engine.open_spiders:
-                if id(spider) == spid:
+                if "%x" % id(spider) == spid or spider.name == spname:
                     return spider
             raise RuntimeError("Spider not running: %s" % ref)
         return ref
@@ -94,6 +96,8 @@ class ScrapyJSONEncoder(json.JSONEncoder):
             return o.strftime(self.TIME_FORMAT)
         elif isinstance(o, decimal.Decimal):
             return str(o)
+        elif isinstance(o, defer.Deferred):
+            return str(o)
         elif isinstance(o, Request):
             return "<%s %s %s>" % (type(o).__name__, o.method, o.url)
         elif isinstance(o, Response):