diff --git a/scrapy/tests/test_utils_serialize.py b/scrapy/tests/test_utils_serialize.py index 01b5fab2ae4a8513b5647ff8cb077b987045e32c..0d3ee297dc1727d9687b902c44d8e3788614451f 100644 --- a/scrapy/tests/test_utils_serialize.py +++ b/scrapy/tests/test_utils_serialize.py @@ -2,6 +2,8 @@ import unittest import datetime from decimal import Decimal +from twisted.internet import defer + from scrapy.utils.serialize import SpiderReferencer, ScrapyJSONEncoder, ScrapyJSONDecoder from scrapy.utils.py26 import json from scrapy.spider import BaseSpider @@ -45,6 +47,10 @@ class SpiderReferencerTestCase(BaseTestCase): assert sp1 is not sp2 assert sp1 is sp1_ + # referring to spiders by name + assert sp1 is self.spref.get_spider_from_reference('spider::name1') + assert sp2 is self.spref.get_spider_from_reference('spider::name2') + # must return string as-is if spider id not found assert 'lala' == self.spref.get_spider_from_reference('lala') # must raise RuntimeError if spider id is not found and spider is not running @@ -107,6 +113,8 @@ class JsonEncoderTestCase(BaseTestCase): for spiders, refs in examples_encode_only: self.assertEqual(self.encoder.encode(spiders), json.dumps(refs)) + assert 'Deferred' in self.encoder.encode(defer.Deferred()) + def test_encode_request(self): r = Request("http://www.example.com/lala") rs = self.encoder.encode(r) diff --git a/scrapy/utils/serialize.py b/scrapy/utils/serialize.py index aa07c786528442aef3f196ea0cb9a2ab2e8dcc2c..b0bec94b745eb1d4e8307b8144b48306bcb6ed14 100644 --- a/scrapy/utils/serialize.py +++ b/scrapy/utils/serialize.py @@ -2,6 +2,8 @@ import re import datetime import decimal +from twisted.internet import defer + from scrapy.core.manager import scrapymanager from scrapy.spider import BaseSpider from scrapy.http import Request, Response @@ -16,7 +18,7 @@ class SpiderReferencer(object): ScrapyJSONEncoder.default() with a custom encoding mechanism. """ - spider_ref_re = re.compile('^spider:([0-9a-f]+)(:.*)?$') + spider_ref_re = re.compile('^spider:([0-9a-f]+)?:?(.+)?$') def __init__(self, manager=None): self.manager = manager or scrapymanager @@ -31,9 +33,9 @@ class SpiderReferencer(object): """ m = self.spider_ref_re.search(ref) if m: - spid = int(m.group(1), 16) + spid, spname = m.groups() for spider in self.manager.engine.open_spiders: - if id(spider) == spid: + if "%x" % id(spider) == spid or spider.name == spname: return spider raise RuntimeError("Spider not running: %s" % ref) return ref @@ -94,6 +96,8 @@ class ScrapyJSONEncoder(json.JSONEncoder): return o.strftime(self.TIME_FORMAT) elif isinstance(o, decimal.Decimal): return str(o) + elif isinstance(o, defer.Deferred): + return str(o) elif isinstance(o, Request): return "<%s %s %s>" % (type(o).__name__, o.method, o.url) elif isinstance(o, Response):