提交 d4b2b612 编写于 作者: D Dmitriy Pomazunovskiy

Use from_settings for backward compatibility

上级 6e5ea792
......@@ -14,17 +14,15 @@ logger = logging.getLogger(__name__)
class UrlLengthMiddleware:
def __init__(self, maxlength, stats):
def __init__(self, maxlength):
self.maxlength = maxlength
self.stats = stats
@classmethod
def from_crawler(cls, crawler):
settings = crawler.settings
def from_settings(cls, settings):
maxlength = settings.getint('URLLENGTH_LIMIT')
if not maxlength:
raise NotConfigured
return cls(maxlength, crawler.stats)
return cls(maxlength)
def process_spider_output(self, response, result, spider):
def _filter(request):
......@@ -34,7 +32,7 @@ class UrlLengthMiddleware:
{'maxlength': self.maxlength, 'url': request.url},
extra={'spider': spider}
)
self.stats.inc_value('urllength/request_ignored_count', spider=spider)
spider.crawler.stats.inc_value('urllength/request_ignored_count', spider=spider)
return False
else:
return True
......
......@@ -5,7 +5,6 @@ from testfixtures import LogCapture
from scrapy.spidermiddlewares.urllength import UrlLengthMiddleware
from scrapy.http import Response, Request
from scrapy.spiders import Spider
from scrapy.statscollectors import StatsCollector
from scrapy.utils.test import get_crawler
......@@ -14,21 +13,16 @@ class TestUrlLengthMiddleware(TestCase):
def setUp(self):
crawler = get_crawler(Spider)
self.spider = crawler._create_spider('foo')
self.stats = StatsCollector(crawler)
self.stats.open_spider(self.spider)
self.stats = self.spider.crawler.stats
self.maxlength = 25
self.mw = UrlLengthMiddleware(maxlength=self.maxlength, stats=self.stats)
self.mw = UrlLengthMiddleware(maxlength=self.maxlength)
self.response = Response('http://scrapytest.org')
self.short_url_req = Request('http://scrapytest.org/')
self.long_url_req = Request('http://scrapytest.org/this_is_a_long_url')
self.reqs = [self.short_url_req, self.long_url_req]
def tearDown(self):
self.stats.close_spider(self.spider, '')
def process_spider_output(self):
return list(self.mw.process_spider_output(self.response, self.reqs, self.spider))
......
Markdown is supported
0% .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑!
想要评论请 注册