Fix pycodestyle E2XX (whitespace) (#4468)

2025-03-13 11:14:09 +00:00 · 2020-04-15 09:11:37 -03:00 · 2020-04-15 09:11:37 -03:00 · 94d7ad76cb
commit 94d7ad76cb
parent 39b01b6892
13 changed files with 268 additions and 241 deletions
--- a/pytest.ini
+++ b/pytest.ini
@ -54,7 +54,7 @@ flake8-ignore =
    scrapy/core/downloader/__init__.py E501
    scrapy/core/downloader/contextfactory.py E501 E128 E126
    scrapy/core/downloader/middleware.py E501
-    scrapy/core/downloader/tls.py E501 E241
+    scrapy/core/downloader/tls.py E501
    scrapy/core/downloader/webclient.py E731 E501 E128 E126
    scrapy/core/downloader/handlers/__init__.py E501
    scrapy/core/downloader/handlers/ftp.py E501 E128 E127
@ -97,9 +97,9 @@ flake8-ignore =
    scrapy/loader/processors.py E501
    # scrapy/pipelines
    scrapy/pipelines/__init__.py E501
-    scrapy/pipelines/files.py E116 E501 E266
-    scrapy/pipelines/images.py E265 E501
-    scrapy/pipelines/media.py E125 E501 E266
+    scrapy/pipelines/files.py E116 E501
+    scrapy/pipelines/images.py E501
+    scrapy/pipelines/media.py E125 E501
    # scrapy/selector
    scrapy/selector/__init__.py F403
    scrapy/selector/unified.py E501 E111
@ -149,7 +149,7 @@ flake8-ignore =
    scrapy/__init__.py E402 E501
    scrapy/cmdline.py E501
    scrapy/crawler.py E501
-    scrapy/dupefilters.py E501 E202
+    scrapy/dupefilters.py E501
    scrapy/exceptions.py E501
    scrapy/exporters.py E501
    scrapy/interfaces.py E501
@ -178,13 +178,13 @@ flake8-ignore =
    tests/test_command_shell.py E501 E128
    tests/test_commands.py E128 E501
    tests/test_contracts.py E501 E128
-    tests/test_crawl.py E501 E741 E265
+    tests/test_crawl.py E501 E741
    tests/test_crawler.py F841 E501
    tests/test_dependencies.py F841 E501
-    tests/test_downloader_handlers.py E124 E127 E128 E265 E501 E126 E123
+    tests/test_downloader_handlers.py E124 E127 E128 E501 E126 E123
    tests/test_downloadermiddleware.py E501
    tests/test_downloadermiddleware_ajaxcrawlable.py E501
-    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E265 E126
+    tests/test_downloadermiddleware_cookies.py E731 E741 E501 E128 E126
    tests/test_downloadermiddleware_decompression.py E127
    tests/test_downloadermiddleware_defaultheaders.py E501
    tests/test_downloadermiddleware_downloadtimeout.py E501
@ -199,15 +199,15 @@ flake8-ignore =
    tests/test_engine.py E401 E501 E128
    tests/test_exporters.py E501 E731 E128 E124
    tests/test_extension_telnet.py F841
-    tests/test_feedexport.py E501 F841 E241
+    tests/test_feedexport.py E501 F841
    tests/test_http_cookies.py E501
    tests/test_http_headers.py E501
    tests/test_http_request.py E402 E501 E127 E128 E128 E126 E123
-    tests/test_http_response.py E501 E128 E265
+    tests/test_http_response.py E501 E128
    tests/test_item.py E128 F841
    tests/test_link.py E501
    tests/test_linkextractors.py E501 E128 E124
-    tests/test_loader.py E501 E731 E741 E128 E117 E241
+    tests/test_loader.py E501 E731 E741 E128 E117
    tests/test_logformatter.py E128 E501 E122
    tests/test_mail.py E128 E501
    tests/test_middleware.py E501 E128
@ -226,7 +226,7 @@ flake8-ignore =
    tests/test_spidermiddleware_httperror.py E128 E501 E127 E121
    tests/test_spidermiddleware_offsite.py E501 E128 E111
    tests/test_spidermiddleware_output_chain.py E501
-    tests/test_spidermiddleware_referer.py E501 F841 E125 E201 E124 E501 E241 E121
+    tests/test_spidermiddleware_referer.py E501 F841 E125 E124 E501 E121
    tests/test_squeues.py E501 E741
    tests/test_utils_asyncio.py E501
    tests/test_utils_conf.py E501 E128
@ -235,7 +235,7 @@ flake8-ignore =
    tests/test_utils_defer.py E501 F841
    tests/test_utils_deprecate.py F841 E501
    tests/test_utils_http.py E501 E128 W504
-    tests/test_utils_iterators.py E501 E128 E129 E241
+    tests/test_utils_iterators.py E501 E128 E129
    tests/test_utils_log.py E741
    tests/test_utils_python.py E501 E731
    tests/test_utils_reqser.py E501 E128
@ -243,8 +243,8 @@ flake8-ignore =
    tests/test_utils_response.py E501
    tests/test_utils_signal.py E741 F841 E731
    tests/test_utils_sitemap.py E128 E501 E124
-    tests/test_utils_url.py E501 E127 E125 E501 E241 E126 E123
-    tests/test_webclient.py E501 E128 E122 E402 E241 E123 E126
+    tests/test_utils_url.py E501 E127 E125 E501 E126 E123
+    tests/test_webclient.py E501 E128 E122 E402 E123 E126
    tests/test_cmdline/__init__.py E501
    tests/test_settings/__init__.py E501 E128
    tests/test_spiderloader/__init__.py E128 E501
--- a/scrapy/dupefilters.py
+++ b/scrapy/dupefilters.py
@ -61,7 +61,7 @@ class RFPDupeFilter(BaseDupeFilter):
    def log(self, request, spider):
        if self.debug:
            msg = "Filtered duplicate request: %(request)s (referer: %(referer)s)"
-            args = {'request': request, 'referer': referer_str(request) }
+            args = {'request': request, 'referer': referer_str(request)}
            self.logger.debug(msg, args, extra={'spider': spider})
        elif self.logdupes:
            msg = ("Filtered duplicate request: %(request)s"
--- a/scrapy/pipelines/files.py
+++ b/scrapy/pipelines/files.py
@ -500,7 +500,7 @@ class FilesPipeline(MediaPipeline):
        spider.crawler.stats.inc_value('file_count', spider=spider)
        spider.crawler.stats.inc_value('file_status_count/%s' % status, spider=spider)

-    ### Overridable Interface
+    # Overridable Interface
    def get_media_requests(self, item, info):
        return [Request(x) for x in item.get(self.files_urls_field, [])]

--- a/scrapy/pipelines/images.py
+++ b/scrapy/pipelines/images.py
@ -14,7 +14,7 @@ from scrapy.utils.python import to_bytes
 from scrapy.http import Request
 from scrapy.settings import Settings
 from scrapy.exceptions import DropItem
-#TODO: from scrapy.pipelines.media import MediaPipeline
+# TODO: from scrapy.pipelines.media import MediaPipeline
 from scrapy.pipelines.files import FileException, FilesPipeline


--- a/scrapy/pipelines/media.py
+++ b/scrapy/pipelines/media.py
@ -166,7 +166,7 @@ class MediaPipeline:
        for wad in info.waiting.pop(fp):
            defer_result(result).chainDeferred(wad)

-    ### Overridable Interface
+    # Overridable Interface
    def media_to_download(self, request, info):
        """Check request before starting download"""
        pass
--- a/tests/test_crawl.py
+++ b/tests/test_crawl.py
@ -147,9 +147,9 @@ class CrawlTestCase(TestCase):
        settings = {"CONCURRENT_REQUESTS": 1}
        crawler = CrawlerRunner(settings).create_crawler(BrokenStartRequestsSpider)
        yield crawler.crawl(mockserver=self.mockserver)
-        #self.assertTrue(False, crawler.spider.seedsseen)
-        #self.assertTrue(crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
-        #                crawler.spider.seedsseen)
+        self.assertTrue(
+            crawler.spider.seedsseen.index(None) < crawler.spider.seedsseen.index(99),
+            crawler.spider.seedsseen)

    @defer.inlineCallbacks
    def test_start_requests_dupes(self):
--- a/tests/test_downloadermiddleware_cookies.py
+++ b/tests/test_downloadermiddleware_cookies.py
@ -202,7 +202,7 @@ class CookiesMiddlewareTest(TestCase):
        assert self.mw.process_request(req4, self.spider) is None
        self.assertCookieValEqual(req4.headers.get('Cookie'), b'C2=value2; galleta=dulce')

-        #cookies from hosts with port
+        # cookies from hosts with port
        req5_1 = Request('http://scrapytest.org:1104/')
        assert self.mw.process_request(req5_1, self.spider) is None

@ -218,7 +218,7 @@ class CookiesMiddlewareTest(TestCase):
        assert self.mw.process_request(req5_3, self.spider) is None
        self.assertEqual(req5_3.headers.get('Cookie'), b'C1=value1')

-        #skip cookie retrieval for not http request
+        # skip cookie retrieval for not http request
        req6 = Request('file:///scrapy/sometempfile')
        assert self.mw.process_request(req6, self.spider) is None
        self.assertEqual(req6.headers.get('Cookie'), None)
--- a/tests/test_http_response.py
+++ b/tests/test_http_response.py
@ -438,8 +438,8 @@ class TextResponseTest(BaseResponseTest):
        assert u'<span>value</span>' in r.text, repr(r.text)

        # FIXME: This test should pass once we stop using BeautifulSoup's UnicodeDammit in TextResponse
-        #r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
-        #assert u'\ufffd' in r.text, repr(r.text)
+        # r = self.response_class("http://www.example.com", body=b'PREFIX\xe3\xabSUFFIX')
+        # assert u'\ufffd' in r.text, repr(r.text)

    def test_selector(self):
        body = b"<html><head><title>Some page</title><body></body></html>"
--- a/tests/test_spidermiddleware_referer.py
+++ b/tests/test_spidermiddleware_referer.py
@ -541,7 +541,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):

    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
    scenarii = [
-        (   'http://scrapytest.org/1',      # parent
+        (
+            'http://scrapytest.org/1',      # parent
            'http://scrapytest.org/2',      # target
            (
                # redirections: code, URL
@ -551,7 +552,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
            b'http://scrapytest.org/1',  # expected initial referer
            b'http://scrapytest.org/1',  # expected referer for the redirection request
        ),
-        (   'https://scrapytest.org/1',
+        (
+            'https://scrapytest.org/1',
            'https://scrapytest.org/2',
            (
                # redirecting to non-secure URL
@ -560,7 +562,8 @@ class TestReferrerOnRedirect(TestRefererMiddleware):
            b'https://scrapytest.org/1',
            b'https://scrapytest.org/1',
        ),
-        (   'https://scrapytest.org/1',
+        (
+            'https://scrapytest.org/1',
            'https://scrapytest.com/2',
            (
                # redirecting to non-secure URL: different origin
@ -602,7 +605,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
    """
    settings = {'REFERRER_POLICY': 'no-referrer'}
    scenarii = [
-        (   'http://scrapytest.org/1',      # parent
+        (
+            'http://scrapytest.org/1',      # parent
            'http://scrapytest.org/2',      # target
            (
                # redirections: code, URL
@ -612,7 +616,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
            None,  # expected initial "Referer"
            None,  # expected "Referer" for the redirection request
        ),
-        (   'https://scrapytest.org/1',
+        (
+            'https://scrapytest.org/1',
            'https://scrapytest.org/2',
            (
                (301, 'http://scrapytest.org/3'),
@ -620,7 +625,8 @@ class TestReferrerOnRedirectNoReferrer(TestReferrerOnRedirect):
            None,
            None,
        ),
-        (   'https://scrapytest.org/1',
+        (
+            'https://scrapytest.org/1',
            'https://example.com/2',    # different origin
            (
                (301, 'http://scrapytest.com/3'),
@ -641,7 +647,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
    """
    settings = {'REFERRER_POLICY': 'same-origin'}
    scenarii = [
-        (   'http://scrapytest.org/101',      # origin
+        (
+            'http://scrapytest.org/101',      # origin
            'http://scrapytest.org/102',      # target
            (
                # redirections: code, URL
@ -651,7 +658,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
            b'http://scrapytest.org/101',  # expected initial "Referer"
            b'http://scrapytest.org/101',  # expected referer for the redirection request
        ),
-        (   'https://scrapytest.org/201',
+        (
+            'https://scrapytest.org/201',
            'https://scrapytest.org/202',
            (
                # redirecting from secure to non-secure URL == different origin
@ -660,7 +668,8 @@ class TestReferrerOnRedirectSameOrigin(TestReferrerOnRedirect):
            b'https://scrapytest.org/201',
            None,
        ),
-        (   'https://scrapytest.org/301',
+        (
+            'https://scrapytest.org/301',
            'https://scrapytest.org/302',
            (
                # different domain == different origin
@ -683,7 +692,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
    """
    settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN}
    scenarii = [
-        (   'http://scrapytest.org/101',
+        (
+            'http://scrapytest.org/101',
            'http://scrapytest.org/102',
            (
                (301, 'http://scrapytest.org/103'),
@ -692,7 +702,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
            b'http://scrapytest.org/',  # send origin
            b'http://scrapytest.org/',  # redirects to same origin: send origin
        ),
-        (   'https://scrapytest.org/201',
+        (
+            'https://scrapytest.org/201',
            'https://scrapytest.org/202',
            (
                # redirecting to non-secure URL: no referrer
@ -701,7 +712,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
            b'https://scrapytest.org/',
            None,
        ),
-        (   'https://scrapytest.org/301',
+        (
+            'https://scrapytest.org/301',
            'https://scrapytest.org/302',
            (
                # redirecting to non-secure URL (different domain): no referrer
@ -710,7 +722,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
            b'https://scrapytest.org/',
            None,
        ),
-        (   'http://scrapy.org/401',
+        (
+            'http://scrapy.org/401',
            'http://example.com/402',
            (
                (301, 'http://scrapytest.org/403'),
@ -718,7 +731,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
            b'http://scrapy.org/',
            b'http://scrapy.org/',
        ),
-        (   'https://scrapy.org/501',
+        (
+            'https://scrapy.org/501',
            'https://example.com/502',
            (
                # HTTPS all along, so origin referrer is kept as-is
@ -728,7 +742,8 @@ class TestReferrerOnRedirectStrictOrigin(TestReferrerOnRedirect):
            b'https://scrapy.org/',
            b'https://scrapy.org/',
        ),
-        (   'https://scrapytest.org/601',
+        (
+            'https://scrapytest.org/601',
            'http://scrapytest.org/602',                # TLS to non-TLS: no referrer
            (
                (301, 'https://scrapytest.org/603'),    # TLS URL again: (still) no referrer
@ -750,7 +765,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
    """
    settings = {'REFERRER_POLICY': POLICY_ORIGIN_WHEN_CROSS_ORIGIN}
    scenarii = [
-        (   'http://scrapytest.org/101',      # origin
+        (
+            'http://scrapytest.org/101',      # origin
            'http://scrapytest.org/102',      # target + redirection
            (
                # redirections: code, URL
@ -760,7 +776,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
            b'http://scrapytest.org/101',  # expected initial referer
            b'http://scrapytest.org/101',  # expected referer for the redirection request
        ),
-        (   'https://scrapytest.org/201',
+        (
+            'https://scrapytest.org/201',
            'https://scrapytest.org/202',
            (
                # redirecting to non-secure URL: send origin
@ -769,7 +786,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
            b'https://scrapytest.org/201',
            b'https://scrapytest.org/',
        ),
-        (   'https://scrapytest.org/301',
+        (
+            'https://scrapytest.org/301',
            'https://scrapytest.org/302',
            (
                # redirecting to non-secure URL (different domain): send origin
@ -778,7 +796,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
            b'https://scrapytest.org/301',
            b'https://scrapytest.org/',
        ),
-        (   'http://scrapy.org/401',
+        (
+            'http://scrapy.org/401',
            'http://example.com/402',
            (
                (301, 'http://scrapytest.org/403'),
@ -786,7 +805,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
            b'http://scrapy.org/',
            b'http://scrapy.org/',
        ),
-        (   'https://scrapy.org/501',
+        (
+            'https://scrapy.org/501',
            'https://example.com/502',
            (
                # all different domains: send origin
@ -796,7 +816,8 @@ class TestReferrerOnRedirectOriginWhenCrossOrigin(TestReferrerOnRedirect):
            b'https://scrapy.org/',
            b'https://scrapy.org/',
        ),
-        (   'https://scrapytest.org/301',
+        (
+            'https://scrapytest.org/301',
            'http://scrapytest.org/302',                # TLS to non-TLS: send origin
            (
                (301, 'https://scrapytest.org/303'),    # TLS URL again: send origin (also)
@ -820,7 +841,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
    """
    settings = {'REFERRER_POLICY': POLICY_STRICT_ORIGIN_WHEN_CROSS_ORIGIN}
    scenarii = [
-        (   'http://scrapytest.org/101',      # origin
+        (
+            'http://scrapytest.org/101',      # origin
            'http://scrapytest.org/102',      # target + redirection
            (
                # redirections: code, URL
@ -830,7 +852,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
            b'http://scrapytest.org/101',  # expected initial referer
            b'http://scrapytest.org/101',  # expected referer for the redirection request
        ),
-        (   'https://scrapytest.org/201',
+        (
+            'https://scrapytest.org/201',
            'https://scrapytest.org/202',
            (
                # redirecting to non-secure URL: do not send the "Referer" header
@ -839,7 +862,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
            b'https://scrapytest.org/201',
            None,
        ),
-        (   'https://scrapytest.org/301',
+        (
+            'https://scrapytest.org/301',
            'https://scrapytest.org/302',
            (
                # redirecting to non-secure URL (different domain): send origin
@ -848,7 +872,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
            b'https://scrapytest.org/301',
            None,
        ),
-        (   'http://scrapy.org/401',
+        (
+            'http://scrapy.org/401',
            'http://example.com/402',
            (
                (301, 'http://scrapytest.org/403'),
@ -856,7 +881,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
            b'http://scrapy.org/',
            b'http://scrapy.org/',
        ),
-        (   'https://scrapy.org/501',
+        (
+            'https://scrapy.org/501',
            'https://example.com/502',
            (
                # all different domains: send origin
@ -866,7 +892,8 @@ class TestReferrerOnRedirectStrictOriginWhenCrossOrigin(TestReferrerOnRedirect):
            b'https://scrapy.org/',
            b'https://scrapy.org/',
        ),
-        (   'https://scrapytest.org/601',
+        (
+            'https://scrapytest.org/601',
            'http://scrapytest.org/602',                # TLS to non-TLS: do not send "Referer"
            (
                (301, 'https://scrapytest.org/603'),    # TLS URL again: (still) send nothing