Skip to content

S
scrapy

looyolo / scrapy
与 Fork 源项目一致

从无法访问的项目Fork

通知 2
Star 0
Fork 0
S
scrapy

前往新版Gitcode，体验更适合开发者的 AI 搜索 >>

未验证 提交 bf56517a 编写于 作者: E Eugenio Lacuesta
浏览文件
操作

Flake8: Remove E128 (tests, part 2)

上级 9c6d1307
隐藏空白更改
内联 并排
Showing with 94 addition and 97 deletion
pytest.ini
浏览文件 @ bf56517a
......@@ -200,19 +200,19 @@ flake8-ignore =
tests/test_feedexport.py E501 F841
tests/test_http_cookies.py E501
tests/test_http_headers.py E501
tests/test_item.py E128 F841
tests/test_http_request.py E402 E501
tests/test_http_response.py E501
tests/test_item.py E501 F841
tests/test_link.py E501
tests/test_linkextractors.py E501 E128
tests/test_loader.py E501 E741 E128 E117
tests/test_logformatter.py E128 E501
tests/test_mail.py E128 E501
tests/test_middleware.py E501 E128
tests/test_pipeline_crawl.py E501 E128
tests/test_linkextractors.py E501
tests/test_loader.py E501 E741 E117
tests/test_logformatter.py E501
tests/test_mail.py E501
tests/test_middleware.py E501
tests/test_pipeline_crawl.py E501
tests/test_pipeline_files.py E501
tests/test_pipeline_images.py F841 E501
tests/test_pipeline_media.py E501 E741 E128
tests/test_pipeline_media.py E501 E741
tests/test_proxy_connect.py E501 E741
tests/test_request_cb_kwargs.py E501
tests/test_responsetypes.py E501
......@@ -221,8 +221,8 @@ flake8-ignore =
tests/test_selector.py E501
tests/test_spider.py E501
tests/test_spidermiddleware.py E501
tests/test_spidermiddleware_httperror.py E128 E501 E121
tests/test_spidermiddleware_offsite.py E501 E128 E111
tests/test_spidermiddleware_httperror.py E501 E121
tests/test_spidermiddleware_offsite.py E501 E111
tests/test_spidermiddleware_output_chain.py E501
tests/test_spidermiddleware_referer.py E501 F841 E501 E121
tests/test_squeues.py E501 E741
......
tests/test_item.py
浏览文件 @ bf56517a
......@@ -162,8 +162,7 @@ class ItemTest(unittest.TestCase):
item = D(save='X', load='Y')
self.assertEqual(item['save'], 'X')
self.assertEqual(item['load'], 'Y')
self.assertEqual(D.fields, {'load': {'default': 'A'},
'save': {'default': 'A'}})
self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}})
# D class inverted
class E(C, B):
......@@ -171,8 +170,7 @@ class ItemTest(unittest.TestCase):
self.assertEqual(E(save='X')['save'], 'X')
self.assertEqual(E(load='X')['load'], 'X')
self.assertEqual(E.fields, {'load': {'default': 'C'},
'save': {'default': 'C'}})
self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}})
def test_metaclass_multiple_inheritance_diamond(self):
class A(Item):
......@@ -193,8 +191,9 @@ class ItemTest(unittest.TestCase):
self.assertEqual(D(save='X')['save'], 'X')
self.assertEqual(D(load='X')['load'], 'X')
self.assertEqual(D.fields, {'save': {'default': 'C'},
'load': {'default': 'D'}, 'update': {'default': 'D'}})
self.assertEqual(
D.fields,
{'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}})
# D class inverted
class E(C, B):
......@@ -202,8 +201,9 @@ class ItemTest(unittest.TestCase):
self.assertEqual(E(save='X')['save'], 'X')
self.assertEqual(E(load='X')['load'], 'X')
self.assertEqual(E.fields, {'save': {'default': 'C'},
'load': {'default': 'E'}, 'update': {'default': 'C'}})
self.assertEqual(
E.fields,
{'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}})
def test_metaclass_multiple_inheritance_without_metaclass(self):
class A(Item):
......@@ -223,8 +223,7 @@ class ItemTest(unittest.TestCase):
self.assertRaises(KeyError, D, not_allowed='value')
self.assertEqual(D(save='X')['save'], 'X')
self.assertEqual(D.fields, {'save': {'default': 'A'},
'load': {'default': 'A'}})
self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
# D class inverted
class E(C, B):
......@@ -232,8 +231,7 @@ class ItemTest(unittest.TestCase):
self.assertRaises(KeyError, E, not_allowed='value')
self.assertEqual(E(save='X')['save'], 'X')
self.assertEqual(E.fields, {'save': {'default': 'A'},
'load': {'default': 'A'}})
self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}})
def test_to_dict(self):
class TestItem(Item):
......
tests/test_linkextractors.py
浏览文件 @ bf56517a
......@@ -171,9 +171,9 @@ class Base:
self.assertEqual(lx.matches(url1), False)
self.assertEqual(lx.matches(url2), True)
lx = self.extractor_cls(allow=('blah1',), deny=('blah2',),
allow_domains=('blah1.com',),
deny_domains=('blah2.com',))
lx = self.extractor_cls(allow=['blah1'], deny=['blah2'],
allow_domains=['blah1.com'],
deny_domains=['blah2.com'])
self.assertEqual(lx.matches('http://blah1.com/blah1'), True)
self.assertEqual(lx.matches('http://blah1.com/blah2'), False)
self.assertEqual(lx.matches('http://blah2.com/blah1'), False)
......
tests/test_logformatter.py
浏览文件 @ bf56517a
......@@ -34,15 +34,15 @@ class LogFormatterTestCase(unittest.TestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
"Crawled (200) <GET http://www.example.com> (referer: None)")
self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None)")
def test_crawled_without_referer(self):
req = Request("http://www.example.com", headers={'referer': 'http://example.com'})
res = Response("http://www.example.com", flags=['cached'])
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
def test_flags_in_request(self):
......@@ -50,8 +50,9 @@ class LogFormatterTestCase(unittest.TestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
"Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> ['test', 'flag'] (referer: None)")
def test_dropped(self):
item = {}
......@@ -140,7 +141,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: None) []")
def test_crawled_without_referer(self):
......@@ -148,7 +150,8 @@ class LogformatterSubclassTest(LogFormatterTestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline,
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: http://example.com) ['cached']")
def test_flags_in_request(self):
......@@ -156,7 +159,9 @@ class LogformatterSubclassTest(LogFormatterTestCase):
res = Response("http://www.example.com")
logkws = self.formatter.crawled(req, res, self.spider)
logline = logkws['msg'] % logkws['args']
self.assertEqual(logline, "Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
self.assertEqual(
logline,
"Crawled (200) <GET http://www.example.com> (referer: None) ['test', 'flag']")
class SkipMessagesLogFormatter(LogFormatter):
......
tests/test_mail.py
浏览文件 @ bf56517a
......@@ -49,7 +49,7 @@ class MailSenderTest(unittest.TestCase):
mailsender = MailSender(debug=True)
mailsender.send(to=['test@scrapy.org'], subject='subject', body='body',
attachs=attachs, _callback=self._catch_mail_sent)
attachs=attachs, _callback=self._catch_mail_sent)
assert self.catched_msg
self.assertEqual(self.catched_msg['to'], ['test@scrapy.org'])
......
tests/test_middleware.py
浏览文件 @ bf56517a
......@@ -69,11 +69,14 @@ class MiddlewareManagerTest(unittest.TestCase):
def test_methods(self):
mwman = TestMiddlewareManager(M1(), M2(), M3())
self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']],
self.assertEqual(
[x.__self__.__class__ for x in mwman.methods['open_spider']],
[M1, M2])
self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']],
self.assertEqual(
[x.__self__.__class__ for x in mwman.methods['close_spider']],
[M2, M1])
self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']],
self.assertEqual(
[x.__self__.__class__ for x in mwman.methods['process']],
[M1, M3])
def test_enabled(self):
......
tests/test_pipeline_crawl.py
浏览文件 @ bf56517a
......@@ -132,7 +132,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media(self):
crawler = self._create_crawler(MediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key)
self._assert_files_downloaded(self.items, str(log))
......@@ -141,7 +142,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media_wrong_urls(self):
crawler = self._create_crawler(BrokenLinksMediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key)
self._assert_files_download_failure(crawler, self.items, 404, str(log))
......@@ -150,7 +152,8 @@ class FileDownloadCrawlTestCase(TestCase):
def test_download_media_redirected_default_failure(self):
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key,
mockserver=self.mockserver)
......@@ -164,7 +167,8 @@ class FileDownloadCrawlTestCase(TestCase):
crawler = self._create_crawler(RedirectedMediaDownloadSpider)
with LogCapture() as log:
yield crawler.crawl(self.mockserver.url("/files/images/"),
yield crawler.crawl(
self.mockserver.url("/files/images/"),
media_key=self.media_key,
media_urls_key=self.media_urls_key,
mockserver=self.mockserver)
......
tests/test_pipeline_media.py
浏览文件 @ bf56517a
......@@ -214,9 +214,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(True, rsp)])
self.assertEqual(self.pipe._mockcalled,
['get_media_requests', 'media_to_download',
'media_downloaded', 'request_callback', 'item_completed'])
self.assertEqual(
self.pipe._mockcalled,
['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed'])
@inlineCallbacks
def test_result_failure(self):
......@@ -227,9 +227,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(False, fail)])
self.assertEqual(self.pipe._mockcalled,
['get_media_requests', 'media_to_download',
'media_failed', 'request_errback', 'item_completed'])
self.assertEqual(
self.pipe._mockcalled,
['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed'])
@inlineCallbacks
def test_mix_of_success_and_failure(self):
......
tests/test_spidermiddleware_httperror.py
浏览文件 @ bf56517a
......@@ -68,29 +68,23 @@ class TestHttpErrorMiddleware(TestCase):
self.res200, self.res404 = _responses(self.req, [200, 404])
def test_process_spider_input(self):
self.assertEqual(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res404, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
def test_process_spider_exception(self):
self.assertEqual([],
self.mw.process_spider_exception(self.res404,
HttpError(self.res404), self.spider))
self.assertEqual(None,
self.mw.process_spider_exception(self.res404,
Exception(), self.spider))
self.assertEqual(
[],
self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider))
self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider))
def test_handle_httpstatus_list(self):
res = self.res404.copy()
res.request = Request('http://scrapytest.org',
meta={'handle_httpstatus_list': [404]})
self.assertEqual(None,
self.mw.process_spider_input(res, self.spider))
self.assertIsNone(self.mw.process_spider_input(res, self.spider))
self.spider.handle_httpstatus_list = [404]
self.assertEqual(None,
self.mw.process_spider_input(self.res404, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
class TestHttpErrorMiddlewareSettings(TestCase):
......@@ -103,12 +97,9 @@ class TestHttpErrorMiddlewareSettings(TestCase):
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def test_process_spider_input(self):
self.assertEqual(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res404, self.spider)
self.assertEqual(None,
self.mw.process_spider_input(self.res402, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider))
def test_meta_overrides_settings(self):
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
......@@ -117,17 +108,13 @@ class TestHttpErrorMiddlewareSettings(TestCase):
res402 = self.res402.copy()
res402.request = request
self.assertEqual(None,
self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, res402, self.spider)
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
def test_spider_override_settings(self):
self.spider.handle_httpstatus_list = [404]
self.assertEqual(None,
self.mw.process_spider_input(self.res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, self.res402, self.spider)
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider)
class TestHttpErrorMiddlewareHandleAll(TestCase):
......@@ -139,10 +126,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402])
def test_process_spider_input(self):
self.assertEqual(None,
self.mw.process_spider_input(self.res200, self.spider))
self.assertEqual(None,
self.mw.process_spider_input(self.res404, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider))
self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider))
def test_meta_overrides_settings(self):
request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]})
......@@ -151,10 +136,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
res402 = self.res402.copy()
res402.request = request
self.assertEqual(None,
self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError,
self.mw.process_spider_input, res402, self.spider)
self.assertIsNone(self.mw.process_spider_input(res404, self.spider))
self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider)
class TestHttpErrorMiddlewareIntegrational(TrialTestCase):
......
tests/test_spidermiddleware_offsite.py
浏览文件 @ bf56517a
......@@ -22,20 +22,24 @@ class TestOffsiteMiddleware(TestCase):
def test_process_spider_output(self):
res = Response('http://scrapytest.org')
onsite_reqs = [Request('http://scrapytest.org/1'),
Request('http://scrapy.org/1'),
Request('http://sub.scrapy.org/1'),
Request('http://offsite.tld/letmepass', dont_filter=True),
Request('http://scrapy.test.org/'),
Request('http://scrapy.test.org:8000/')]
offsite_reqs = [Request('http://scrapy2.org'),
Request('http://offsite.tld/'),
Request('http://offsite.tld/scrapytest.org'),
Request('http://offsite.tld/rogue.scrapytest.org'),
Request('http://rogue.scrapytest.org.haha.com'),
Request('http://roguescrapytest.org'),
Request('http://test.org/'),
Request('http://notscrapy.test.org/')]
onsite_reqs = [
Request('http://scrapytest.org/1'),
Request('http://scrapy.org/1'),
Request('http://sub.scrapy.org/1'),
Request('http://offsite.tld/letmepass', dont_filter=True),
Request('http://scrapy.test.org/'),
Request('http://scrapy.test.org:8000/'),
]
offsite_reqs = [
Request('http://scrapy2.org'),
Request('http://offsite.tld/'),
Request('http://offsite.tld/scrapytest.org'),
Request('http://offsite.tld/rogue.scrapytest.org'),
Request('http://rogue.scrapytest.org.haha.com'),
Request('http://roguescrapytest.org'),
Request('http://test.org/'),
Request('http://notscrapy.test.org/'),
]
reqs = onsite_reqs + offsite_reqs
out = list(self.mw.process_spider_output(res, reqs, self.spider))
......
Markdown is supported
0% 或 .
You are about to add 0 people to the discussion. Proceed with caution.
先完成此消息的编辑！
想要评论请 注册 或