From bf56517abfc3d2287c08e1c88f07de7d3b1c499c Mon Sep 17 00:00:00 2001 From: Eugenio Lacuesta Date: Wed, 6 May 2020 19:18:29 -0300 Subject: [PATCH] Flake8: Remove E128 (tests, part 2) --- pytest.ini | 20 ++++----- tests/test_item.py | 22 +++++----- tests/test_linkextractors.py | 6 +-- tests/test_logformatter.py | 21 +++++---- tests/test_mail.py | 2 +- tests/test_middleware.py | 9 ++-- tests/test_pipeline_crawl.py | 12 ++++-- tests/test_pipeline_media.py | 12 +++--- tests/test_spidermiddleware_httperror.py | 55 ++++++++---------------- tests/test_spidermiddleware_offsite.py | 32 ++++++++------ 10 files changed, 94 insertions(+), 97 deletions(-) diff --git a/pytest.ini b/pytest.ini index d8eefea7e..0b2766031 100644 --- a/pytest.ini +++ b/pytest.ini @@ -200,19 +200,19 @@ flake8-ignore = tests/test_feedexport.py E501 F841 tests/test_http_cookies.py E501 tests/test_http_headers.py E501 - tests/test_item.py E128 F841 tests/test_http_request.py E402 E501 tests/test_http_response.py E501 + tests/test_item.py E501 F841 tests/test_link.py E501 - tests/test_linkextractors.py E501 E128 - tests/test_loader.py E501 E741 E128 E117 - tests/test_logformatter.py E128 E501 - tests/test_mail.py E128 E501 - tests/test_middleware.py E501 E128 - tests/test_pipeline_crawl.py E501 E128 + tests/test_linkextractors.py E501 + tests/test_loader.py E501 E741 E117 + tests/test_logformatter.py E501 + tests/test_mail.py E501 + tests/test_middleware.py E501 + tests/test_pipeline_crawl.py E501 tests/test_pipeline_files.py E501 tests/test_pipeline_images.py F841 E501 - tests/test_pipeline_media.py E501 E741 E128 + tests/test_pipeline_media.py E501 E741 tests/test_proxy_connect.py E501 E741 tests/test_request_cb_kwargs.py E501 tests/test_responsetypes.py E501 @@ -221,8 +221,8 @@ flake8-ignore = tests/test_selector.py E501 tests/test_spider.py E501 tests/test_spidermiddleware.py E501 - tests/test_spidermiddleware_httperror.py E128 E501 E121 - tests/test_spidermiddleware_offsite.py E501 E128 E111 + tests/test_spidermiddleware_httperror.py E501 E121 + tests/test_spidermiddleware_offsite.py E501 E111 tests/test_spidermiddleware_output_chain.py E501 tests/test_spidermiddleware_referer.py E501 F841 E501 E121 tests/test_squeues.py E501 E741 diff --git a/tests/test_item.py b/tests/test_item.py index 4017f6e84..4fe247cda 100644 --- a/tests/test_item.py +++ b/tests/test_item.py @@ -162,8 +162,7 @@ class ItemTest(unittest.TestCase): item = D(save='X', load='Y') self.assertEqual(item['save'], 'X') self.assertEqual(item['load'], 'Y') - self.assertEqual(D.fields, {'load': {'default': 'A'}, - 'save': {'default': 'A'}}) + self.assertEqual(D.fields, {'load': {'default': 'A'}, 'save': {'default': 'A'}}) # D class inverted class E(C, B): @@ -171,8 +170,7 @@ class ItemTest(unittest.TestCase): self.assertEqual(E(save='X')['save'], 'X') self.assertEqual(E(load='X')['load'], 'X') - self.assertEqual(E.fields, {'load': {'default': 'C'}, - 'save': {'default': 'C'}}) + self.assertEqual(E.fields, {'load': {'default': 'C'}, 'save': {'default': 'C'}}) def test_metaclass_multiple_inheritance_diamond(self): class A(Item): @@ -193,8 +191,9 @@ class ItemTest(unittest.TestCase): self.assertEqual(D(save='X')['save'], 'X') self.assertEqual(D(load='X')['load'], 'X') - self.assertEqual(D.fields, {'save': {'default': 'C'}, - 'load': {'default': 'D'}, 'update': {'default': 'D'}}) + self.assertEqual( + D.fields, + {'save': {'default': 'C'}, 'load': {'default': 'D'}, 'update': {'default': 'D'}}) # D class inverted class E(C, B): @@ -202,8 +201,9 @@ class ItemTest(unittest.TestCase): self.assertEqual(E(save='X')['save'], 'X') self.assertEqual(E(load='X')['load'], 'X') - self.assertEqual(E.fields, {'save': {'default': 'C'}, - 'load': {'default': 'E'}, 'update': {'default': 'C'}}) + self.assertEqual( + E.fields, + {'save': {'default': 'C'}, 'load': {'default': 'E'}, 'update': {'default': 'C'}}) def test_metaclass_multiple_inheritance_without_metaclass(self): class A(Item): @@ -223,8 +223,7 @@ class ItemTest(unittest.TestCase): self.assertRaises(KeyError, D, not_allowed='value') self.assertEqual(D(save='X')['save'], 'X') - self.assertEqual(D.fields, {'save': {'default': 'A'}, - 'load': {'default': 'A'}}) + self.assertEqual(D.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}}) # D class inverted class E(C, B): @@ -232,8 +231,7 @@ class ItemTest(unittest.TestCase): self.assertRaises(KeyError, E, not_allowed='value') self.assertEqual(E(save='X')['save'], 'X') - self.assertEqual(E.fields, {'save': {'default': 'A'}, - 'load': {'default': 'A'}}) + self.assertEqual(E.fields, {'save': {'default': 'A'}, 'load': {'default': 'A'}}) def test_to_dict(self): class TestItem(Item): diff --git a/tests/test_linkextractors.py b/tests/test_linkextractors.py index 68e8514ba..7e1b62b7f 100644 --- a/tests/test_linkextractors.py +++ b/tests/test_linkextractors.py @@ -171,9 +171,9 @@ class Base: self.assertEqual(lx.matches(url1), False) self.assertEqual(lx.matches(url2), True) - lx = self.extractor_cls(allow=('blah1',), deny=('blah2',), - allow_domains=('blah1.com',), - deny_domains=('blah2.com',)) + lx = self.extractor_cls(allow=['blah1'], deny=['blah2'], + allow_domains=['blah1.com'], + deny_domains=['blah2.com']) self.assertEqual(lx.matches('http://blah1.com/blah1'), True) self.assertEqual(lx.matches('http://blah1.com/blah2'), False) self.assertEqual(lx.matches('http://blah2.com/blah1'), False) diff --git a/tests/test_logformatter.py b/tests/test_logformatter.py index cd6cb8016..7064337ad 100644 --- a/tests/test_logformatter.py +++ b/tests/test_logformatter.py @@ -34,15 +34,15 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, - "Crawled (200) (referer: None)") + self.assertEqual(logline, "Crawled (200) (referer: None)") def test_crawled_without_referer(self): req = Request("http://www.example.com", headers={'referer': 'http://example.com'}) res = Response("http://www.example.com", flags=['cached']) logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: http://example.com) ['cached']") def test_flags_in_request(self): @@ -50,8 +50,9 @@ class LogFormatterTestCase(unittest.TestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, - "Crawled (200) ['test', 'flag'] (referer: None)") + self.assertEqual( + logline, + "Crawled (200) ['test', 'flag'] (referer: None)") def test_dropped(self): item = {} @@ -140,7 +141,8 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: None) []") def test_crawled_without_referer(self): @@ -148,7 +150,8 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, + self.assertEqual( + logline, "Crawled (200) (referer: http://example.com) ['cached']") def test_flags_in_request(self): @@ -156,7 +159,9 @@ class LogformatterSubclassTest(LogFormatterTestCase): res = Response("http://www.example.com") logkws = self.formatter.crawled(req, res, self.spider) logline = logkws['msg'] % logkws['args'] - self.assertEqual(logline, "Crawled (200) (referer: None) ['test', 'flag']") + self.assertEqual( + logline, + "Crawled (200) (referer: None) ['test', 'flag']") class SkipMessagesLogFormatter(LogFormatter): diff --git a/tests/test_mail.py b/tests/test_mail.py index f5cb81a8b..53dbc0686 100644 --- a/tests/test_mail.py +++ b/tests/test_mail.py @@ -49,7 +49,7 @@ class MailSenderTest(unittest.TestCase): mailsender = MailSender(debug=True) mailsender.send(to=['test@scrapy.org'], subject='subject', body='body', - attachs=attachs, _callback=self._catch_mail_sent) + attachs=attachs, _callback=self._catch_mail_sent) assert self.catched_msg self.assertEqual(self.catched_msg['to'], ['test@scrapy.org']) diff --git a/tests/test_middleware.py b/tests/test_middleware.py index 3af514bb0..3364d2258 100644 --- a/tests/test_middleware.py +++ b/tests/test_middleware.py @@ -69,11 +69,14 @@ class MiddlewareManagerTest(unittest.TestCase): def test_methods(self): mwman = TestMiddlewareManager(M1(), M2(), M3()) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['open_spider']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['open_spider']], [M1, M2]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['close_spider']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['close_spider']], [M2, M1]) - self.assertEqual([x.__self__.__class__ for x in mwman.methods['process']], + self.assertEqual( + [x.__self__.__class__ for x in mwman.methods['process']], [M1, M3]) def test_enabled(self): diff --git a/tests/test_pipeline_crawl.py b/tests/test_pipeline_crawl.py index 24c516473..188c5c3cf 100644 --- a/tests/test_pipeline_crawl.py +++ b/tests/test_pipeline_crawl.py @@ -132,7 +132,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media(self): crawler = self._create_crawler(MediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_downloaded(self.items, str(log)) @@ -141,7 +142,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_wrong_urls(self): crawler = self._create_crawler(BrokenLinksMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key) self._assert_files_download_failure(crawler, self.items, 404, str(log)) @@ -150,7 +152,8 @@ class FileDownloadCrawlTestCase(TestCase): def test_download_media_redirected_default_failure(self): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver) @@ -164,7 +167,8 @@ class FileDownloadCrawlTestCase(TestCase): crawler = self._create_crawler(RedirectedMediaDownloadSpider) with LogCapture() as log: - yield crawler.crawl(self.mockserver.url("/files/images/"), + yield crawler.crawl( + self.mockserver.url("/files/images/"), media_key=self.media_key, media_urls_key=self.media_urls_key, mockserver=self.mockserver) diff --git a/tests/test_pipeline_media.py b/tests/test_pipeline_media.py index 949f0dea1..b51f82422 100644 --- a/tests/test_pipeline_media.py +++ b/tests/test_pipeline_media.py @@ -214,9 +214,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item['results'], [(True, rsp)]) - self.assertEqual(self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', - 'media_downloaded', 'request_callback', 'item_completed']) + self.assertEqual( + self.pipe._mockcalled, + ['get_media_requests', 'media_to_download', 'media_downloaded', 'request_callback', 'item_completed']) @inlineCallbacks def test_result_failure(self): @@ -227,9 +227,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase): item = dict(requests=req) new_item = yield self.pipe.process_item(item, self.spider) self.assertEqual(new_item['results'], [(False, fail)]) - self.assertEqual(self.pipe._mockcalled, - ['get_media_requests', 'media_to_download', - 'media_failed', 'request_errback', 'item_completed']) + self.assertEqual( + self.pipe._mockcalled, + ['get_media_requests', 'media_to_download', 'media_failed', 'request_errback', 'item_completed']) @inlineCallbacks def test_mix_of_success_and_failure(self): diff --git a/tests/test_spidermiddleware_httperror.py b/tests/test_spidermiddleware_httperror.py index 6b61df56f..714279ae0 100644 --- a/tests/test_spidermiddleware_httperror.py +++ b/tests/test_spidermiddleware_httperror.py @@ -68,29 +68,23 @@ class TestHttpErrorMiddleware(TestCase): self.res200, self.res404 = _responses(self.req, [200, 404]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res404, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) def test_process_spider_exception(self): - self.assertEqual([], - self.mw.process_spider_exception(self.res404, - HttpError(self.res404), self.spider)) - self.assertEqual(None, - self.mw.process_spider_exception(self.res404, - Exception(), self.spider)) + self.assertEqual( + [], + self.mw.process_spider_exception(self.res404, HttpError(self.res404), self.spider)) + self.assertIsNone(self.mw.process_spider_exception(self.res404, Exception(), self.spider)) def test_handle_httpstatus_list(self): res = self.res404.copy() res.request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) - self.assertEqual(None, - self.mw.process_spider_input(res, self.spider)) + self.assertIsNone(self.mw.process_spider_input(res, self.spider)) self.spider.handle_httpstatus_list = [404] - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) class TestHttpErrorMiddlewareSettings(TestCase): @@ -103,12 +97,9 @@ class TestHttpErrorMiddlewareSettings(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res404, self.spider) - self.assertEqual(None, - self.mw.process_spider_input(self.res402, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res404, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res402, self.spider)) def test_meta_overrides_settings(self): request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) @@ -117,17 +108,13 @@ class TestHttpErrorMiddlewareSettings(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEqual(None, - self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) def test_spider_override_settings(self): self.spider.handle_httpstatus_list = [404] - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, self.res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, self.res402, self.spider) class TestHttpErrorMiddlewareHandleAll(TestCase): @@ -139,10 +126,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): self.res200, self.res404, self.res402 = _responses(self.req, [200, 404, 402]) def test_process_spider_input(self): - self.assertEqual(None, - self.mw.process_spider_input(self.res200, self.spider)) - self.assertEqual(None, - self.mw.process_spider_input(self.res404, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res200, self.spider)) + self.assertIsNone(self.mw.process_spider_input(self.res404, self.spider)) def test_meta_overrides_settings(self): request = Request('http://scrapytest.org', meta={'handle_httpstatus_list': [404]}) @@ -151,10 +136,8 @@ class TestHttpErrorMiddlewareHandleAll(TestCase): res402 = self.res402.copy() res402.request = request - self.assertEqual(None, - self.mw.process_spider_input(res404, self.spider)) - self.assertRaises(HttpError, - self.mw.process_spider_input, res402, self.spider) + self.assertIsNone(self.mw.process_spider_input(res404, self.spider)) + self.assertRaises(HttpError, self.mw.process_spider_input, res402, self.spider) class TestHttpErrorMiddlewareIntegrational(TrialTestCase): diff --git a/tests/test_spidermiddleware_offsite.py b/tests/test_spidermiddleware_offsite.py index b96807bc2..0f4b98a07 100644 --- a/tests/test_spidermiddleware_offsite.py +++ b/tests/test_spidermiddleware_offsite.py @@ -22,20 +22,24 @@ class TestOffsiteMiddleware(TestCase): def test_process_spider_output(self): res = Response('http://scrapytest.org') - onsite_reqs = [Request('http://scrapytest.org/1'), - Request('http://scrapy.org/1'), - Request('http://sub.scrapy.org/1'), - Request('http://offsite.tld/letmepass', dont_filter=True), - Request('http://scrapy.test.org/'), - Request('http://scrapy.test.org:8000/')] - offsite_reqs = [Request('http://scrapy2.org'), - Request('http://offsite.tld/'), - Request('http://offsite.tld/scrapytest.org'), - Request('http://offsite.tld/rogue.scrapytest.org'), - Request('http://rogue.scrapytest.org.haha.com'), - Request('http://roguescrapytest.org'), - Request('http://test.org/'), - Request('http://notscrapy.test.org/')] + onsite_reqs = [ + Request('http://scrapytest.org/1'), + Request('http://scrapy.org/1'), + Request('http://sub.scrapy.org/1'), + Request('http://offsite.tld/letmepass', dont_filter=True), + Request('http://scrapy.test.org/'), + Request('http://scrapy.test.org:8000/'), + ] + offsite_reqs = [ + Request('http://scrapy2.org'), + Request('http://offsite.tld/'), + Request('http://offsite.tld/scrapytest.org'), + Request('http://offsite.tld/rogue.scrapytest.org'), + Request('http://rogue.scrapytest.org.haha.com'), + Request('http://roguescrapytest.org'), + Request('http://test.org/'), + Request('http://notscrapy.test.org/'), + ] reqs = onsite_reqs + offsite_reqs out = list(self.mw.process_spider_output(res, reqs, self.spider)) -- GitLab