Remove six.iter* occurrences

scrapy/core/downloader/__init__.py

@@ -4,7 +4,6 @@ from time import time
 from datetime import datetime
 from collections import deque
 
-import six
 from twisted.internet import reactor, defer, task
 
 from scrapy.utils.defer import mustbe_deferred
@@ -189,7 +188,7 @@ class Downloader(object):
 
     def close(self):
         self._slot_gc_loop.stop()
-        for slot in six.itervalues(self.slots):
+        for slot in self.slots.values():
             slot.close()
 
     def _slot_gc(self, age=60):

scrapy/core/downloader/handlers/__init__.py

@@ -1,8 +1,9 @@
 """Download handlers for different schemes"""
 
 import logging
+
 from twisted.internet import defer
-import six
+
 from scrapy.exceptions import NotSupported, NotConfigured
 from scrapy.utils.httpobj import urlparse_cached
 from scrapy.utils.misc import load_object
@@ -22,7 +23,7 @@ class DownloadHandlers(object):
         self._notconfigured = {}  # remembers failed handlers
         handlers = without_none_values(
             crawler.settings.getwithbase('DOWNLOAD_HANDLERS'))
-        for scheme, clspath in six.iteritems(handlers):
+        for scheme, clspath in handlers.items():
             self._schemes[scheme] = clspath
             self._load_handler(scheme, skip_lazy=True)
 

scrapy/downloadermiddlewares/cookies.py

@@ -1,5 +1,4 @@
 import os
-import six
 import logging
 from collections import defaultdict
 
@@ -8,6 +7,7 @@ from scrapy.http import Response
 from scrapy.http.cookies import CookieJar
 from scrapy.utils.python import to_unicode
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -82,8 +82,10 @@ class CookiesMiddleware(object):
 
     def _get_request_cookies(self, jar, request):
         if isinstance(request.cookies, dict):
-            cookie_list = [{'name': k, 'value': v} for k, v in \
-                    six.iteritems(request.cookies)]
+            cookie_list = [
+                {'name': k, 'value': v}
+                for k, v in request.cookies.items()
+            ]
         else:
             cookie_list = request.cookies
 

scrapy/downloadermiddlewares/decompression.py

@@ -4,16 +4,15 @@ and extract the potentially compressed responses that may arrive.
 
 import bz2
 import gzip
-from io import BytesIO
 import zipfile
 import tarfile
 import logging
+from io import BytesIO
 from tempfile import mktemp
 
-import six
-
 from scrapy.responsetypes import responsetypes
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -75,7 +74,7 @@ class DecompressionMiddleware(object):
         if not response.body:
             return response
 
-        for fmt, func in six.iteritems(self._formats):
+        for fmt, func in self._formats.items():
             new_response = func(response)
             if new_response:
                 logger.debug('Decompressed response with format: %(responsefmt)s',

scrapy/exporters.py

@@ -62,9 +62,9 @@ class BaseItemExporter(object):
             include_empty = self.export_empty_fields
         if self.fields_to_export is None:
             if include_empty and not isinstance(item, dict):
-                field_iter = six.iterkeys(item.fields)
+                field_iter = item.fields.keys()
             else:
-                field_iter = six.iterkeys(item)
+                field_iter = item.keys()
         else:
             if include_empty:
                 field_iter = self.fields_to_export
@@ -326,7 +326,7 @@ class PythonItemExporter(BaseItemExporter):
         return value
 
     def _serialize_dict(self, value):
-        for key, val in six.iteritems(value):
+        for key, val in value.items():
             key = to_bytes(key) if self.binary else key
             yield key, self._serialize_value(val)
 

scrapy/extensions/memdebug.py

@@ -5,7 +5,6 @@ See documentation in docs/topics/extensions.rst
 """
 
 import gc
-import six
 
 from scrapy import signals
 from scrapy.exceptions import NotConfigured
@@ -28,7 +27,7 @@ class MemoryDebugger(object):
     def spider_closed(self, spider, reason):
         gc.collect()
         self.stats.set_value('memdebug/gc_garbage_count', len(gc.garbage), spider=spider)
-        for cls, wdict in six.iteritems(live_refs):
+        for cls, wdict in live_refs.items():
             if not wdict:
                 continue
             self.stats.set_value('memdebug/live_refs/%s' % cls.__name__, len(wdict), spider=spider)

scrapy/http/request/form.py

@@ -8,7 +8,6 @@ See documentation in docs/topics/request-response.rst
 from urllib.parse import urljoin, urlencode
 
 import lxml.html
-import six
 from parsel.selector import create_root_node
 from w3lib.html import strip_html5_whitespace
 
@@ -208,7 +207,7 @@ def _get_clickable(clickdata, form):
     # We didn't find it, so now we build an XPath expression out of the other
     # arguments, because they can be used as such
     xpath = u'.//*' + \
-            u''.join(u'[@%s="%s"]' % c for c in six.iteritems(clickdata))
+            u''.join(u'[@%s="%s"]' % c for c in clickdata.items())
     el = form.xpath(xpath)
     if len(el) == 1:
         return (el[0].get('name'), el[0].get('value') or '')

scrapy/item.py

@@ -78,7 +78,7 @@ class DictItem(MutableMapping, BaseItem):
     def __init__(self, *args, **kwargs):
         self._values = {}
         if args or kwargs:  # avoid creating dict for most common case
-            for k, v in six.iteritems(dict(*args, **kwargs)):
+            for k, v in dict(*args, **kwargs).items():
                 self[k] = v
 
     def __getitem__(self, key):

scrapy/loader/__init__.py

@@ -5,8 +5,6 @@ See documentation in docs/topics/loaders.rst
 """
 from collections import defaultdict
 
-import six
-
 from scrapy.item import Item
 from scrapy.loader.common import wrap_loader_context
 from scrapy.loader.processors import Identity
@@ -72,7 +70,7 @@ class ItemLoader(object):
         if value is None:
             return
         if not field_name:
-            for k, v in six.iteritems(value):
+            for k, v in value.items():
                 self._add_value(k, v)
         else:
             self._add_value(field_name, value)
@@ -82,7 +80,7 @@ class ItemLoader(object):
         if value is None:
             return
         if not field_name:
-            for k, v in six.iteritems(value):
+            for k, v in value.items():
                 self._replace_value(k, v)
         else:
             self._replace_value(field_name, value)

scrapy/pipelines/files.py

@@ -8,14 +8,12 @@ import hashlib
 import logging
 import mimetypes
 import os
-import os.path
 import time
 from collections import defaultdict
 from email.utils import parsedate_tz, mktime_tz
 from io import BytesIO
 from urllib.parse import urlparse
 
-import six
 from twisted.internet import defer, threads
 
 from scrapy.pipelines.media import MediaPipeline
@@ -29,6 +27,7 @@ from scrapy.utils.request import referer_str
 from scrapy.utils.boto import is_botocore
 from scrapy.utils.datatypes import CaselessDict
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -153,14 +152,14 @@ class S3FilesStore(object):
                 Bucket=self.bucket,
                 Key=key_name,
                 Body=buf,
-                Metadata={k: str(v) for k, v in six.iteritems(meta or {})},
+                Metadata={k: str(v) for k, v in (meta or {}).items()},
                 ACL=self.POLICY,
                 **extra)
         else:
             b = self._get_boto_bucket()
             k = b.new_key(key_name)
             if meta:
-                for metakey, metavalue in six.iteritems(meta):
+                for metakey, metavalue in meta.items():
                     k.set_metadata(metakey, str(metavalue))
             h = self.HEADERS.copy()
             if headers:
@@ -201,7 +200,7 @@ class S3FilesStore(object):
             'X-Amz-Website-Redirect-Location': 'WebsiteRedirectLocation',
         })
         extra = {}
-        for key, value in six.iteritems(headers):
+        for key, value in headers.items():
             try:
                 kwarg = mapping[key]
             except KeyError:
@@ -249,7 +248,7 @@ class GCSFilesStore(object):
     def persist_file(self, path, buf, info, meta=None, headers=None):
         blob = self.bucket.blob(self.prefix + path)
         blob.cache_control = self.CACHE_CONTROL
-        blob.metadata = {k: str(v) for k, v in six.iteritems(meta or {})}
+        blob.metadata = {k: str(v) for k, v in (meta or {}).items()}
         return threads.deferToThread(
             blob.upload_from_string,
             data=buf.getvalue(),

scrapy/pipelines/images.py

@@ -6,7 +6,6 @@ See documentation in topics/media-pipeline.rst
 import functools
 import hashlib
 from io import BytesIO
-import six
 
 from PIL import Image
 
@@ -126,7 +125,7 @@ class ImagesPipeline(FilesPipeline):
         image, buf = self.convert_image(orig_image)
         yield path, image, buf
 
-        for thumb_id, size in six.iteritems(self.thumbs):
+        for thumb_id, size in self.thumbs.items():
             thumb_path = self.thumb_path(request, thumb_id, response=response, info=info)
             thumb_image, thumb_buf = self.convert_image(image, size)
             yield thumb_path, thumb_image, thumb_buf

scrapy/responsetypes.py

@@ -5,7 +5,6 @@ based on different criteria.
 from mimetypes import MimeTypes
 from pkgutil import get_data
 from io import StringIO
-import six
 
 from scrapy.http import Response
 from scrapy.utils.misc import load_object
@@ -36,7 +35,7 @@ class ResponseTypes(object):
         self.mimetypes = MimeTypes()
         mimedata = get_data('scrapy', 'mime.types').decode('utf8')
         self.mimetypes.readfp(StringIO(mimedata))
-        for mimetype, cls in six.iteritems(self.CLASSES):
+        for mimetype, cls in self.CLASSES.items():
             self.classes[mimetype] = load_object(cls)
 
     def from_mimetype(self, mimetype):

scrapy/settings/__init__.py

@@ -317,10 +317,10 @@ class BaseSettings(MutableMapping):
             values = json.loads(values)
         if values is not None:
             if isinstance(values, BaseSettings):
-                for name, value in six.iteritems(values):
+                for name, value in values.items():
                     self.set(name, value, values.getpriority(name))
             else:
-                for name, value in six.iteritems(values):
+                for name, value in values.items():
                     self.set(name, value, priority)
 
     def delete(self, name, priority='project'):
@@ -377,7 +377,7 @@ class BaseSettings(MutableMapping):
 
     def _to_dict(self):
         return {k: (v._to_dict() if isinstance(v, BaseSettings) else v)
-                for k, v in six.iteritems(self)}
+                for k, v in self.items()}
 
     def copy_to_dict(self):
         """
@@ -445,7 +445,7 @@ class Settings(BaseSettings):
         self.setmodule(default_settings, 'default')
         # Promote default dictionaries to BaseSettings instances for per-key
         # priorities
-        for name, val in six.iteritems(self):
+        for name, val in self.items():
             if isinstance(val, dict):
                 self.set(name, BaseSettings(val, 'default'), 'default')
         self.update(values, priority)

scrapy/utils/conf.py

@@ -1,11 +1,9 @@
-from configparser import ConfigParser
 import os
 import sys
 import numbers
+from configparser import ConfigParser
 from operator import itemgetter
 
-import six
-
 from scrapy.settings import BaseSettings
 from scrapy.utils.deprecate import update_classpath
 from scrapy.utils.python import without_none_values
@@ -22,7 +20,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
     def _map_keys(compdict):
         if isinstance(compdict, BaseSettings):
             compbs = BaseSettings()
-            for k, v in six.iteritems(compdict):
+            for k, v in compdict.items():
                 prio = compdict.getpriority(k)
                 if compbs.getpriority(convert(k)) == prio:
                     raise ValueError('Some paths in {!r} convert to the same '
@@ -33,11 +31,11 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
             return compbs
         else:
             _check_components(compdict)
-            return {convert(k): v for k, v in six.iteritems(compdict)}
+            return {convert(k): v for k, v in compdict.items()}
 
     def _validate_values(compdict):
         """Fail if a value in the components dict is not a real number or None."""
-        for name, value in six.iteritems(compdict):
+        for name, value in compdict.items():
             if value is not None and not isinstance(value, numbers.Real):
                 raise ValueError('Invalid value {} for component {}, please provide ' \
                                  'a real number or None instead'.format(value, name))
@@ -53,7 +51,7 @@ def build_component_list(compdict, custom=None, convert=update_classpath):
 
     _validate_values(compdict)
     compdict = without_none_values(_map_keys(compdict))
-    return [k for k, v in sorted(six.iteritems(compdict), key=itemgetter(1))]
+    return [k for k, v in sorted(compdict.items(), key=itemgetter(1))]
 
 
 def arglist_to_dict(arglist):

scrapy/utils/datatypes.py

@@ -7,11 +7,8 @@ This module must not depend on any module outside the Standard Library.
 
 import copy
 import collections
-from collections.abc import Mapping
 import warnings
 
-import six
-
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 
@@ -151,7 +148,7 @@ class MultiValueDict(dict):
                         self.setlistdefault(key, []).append(value)
                 except TypeError:
                     raise ValueError("MultiValueDict.update() takes either a MultiValueDict or dictionary")
-        for key, value in six.iteritems(kwargs):
+        for key, value in kwargs.items():
             self.setlistdefault(key, []).append(value)
 
 
@@ -226,7 +223,7 @@ class CaselessDict(dict):
         return dict.setdefault(self, self.normkey(key), self.normvalue(def_val))
 
     def update(self, seq):
-        seq = seq.items() if isinstance(seq, Mapping) else seq
+        seq = seq.items() if isinstance(seq, collections.abc.Mapping) else seq
         iseq = ((self.normkey(k), self.normvalue(v)) for k, v in seq)
         super(CaselessDict, self).update(iseq)
 

scrapy/utils/python.py

@@ -300,7 +300,7 @@ def stringify_dict(dct_or_tuples, encoding='utf-8', keys_only=True):
     dict or a list of tuples, like any dict constructor supports.
     """
     d = {}
-    for k, v in six.iteritems(dict(dct_or_tuples)):
+    for k, v in dict(dct_or_tuples).items():
         k = k.encode(encoding) if isinstance(k, six.text_type) else k
         if not keys_only:
             v = v.encode(encoding) if isinstance(v, six.text_type) else v
@@ -345,7 +345,7 @@ def without_none_values(iterable):
     value ``None`` have been removed.
     """
     try:
-        return {k: v for k, v in six.iteritems(iterable) if v is not None}
+        return {k: v for k, v in iterable.items() if v is not None}
     except AttributeError:
         return type(iterable)((v for v in iterable if v is not None))
 

scrapy/utils/spider.py

@@ -1,11 +1,10 @@
 import logging
 import inspect
 
-import six
-
 from scrapy.spiders import Spider
 from scrapy.utils.misc import  arg_to_iter
 
+
 logger = logging.getLogger(__name__)
 
 
@@ -21,7 +20,7 @@ def iter_spider_classes(module):
     # singleton in scrapy.spider.spiders
     from scrapy.spiders import Spider
 
-    for obj in six.itervalues(vars(module)):
+    for obj in vars(module).values():
         if inspect.isclass(obj) and \
            issubclass(obj, Spider) and \
            obj.__module__ == module.__name__ and \

scrapy/utils/trackref.py

@@ -13,7 +13,6 @@ import weakref
 from time import time
 from operator import itemgetter
 from collections import defaultdict
-import six
 
 
 NoneType = type(None)
@@ -36,13 +35,13 @@ def format_live_refs(ignore=NoneType):
     """Return a tabular representation of tracked objects"""
     s = "Live References\n\n"
     now = time()
-    for cls, wdict in sorted(six.iteritems(live_refs),
+    for cls, wdict in sorted(live_refs.items(),
                              key=lambda x: x[0].__name__):
         if not wdict:
             continue
         if issubclass(cls, ignore):
             continue
-        oldest = min(six.itervalues(wdict))
+        oldest = min(wdict.values())
         s += "%-30s %6d   oldest: %ds ago\n" % (
             cls.__name__, len(wdict), now - oldest
         )
@@ -56,15 +55,15 @@ def print_live_refs(*a, **kw):
 
 def get_oldest(class_name):
     """Get the oldest object for a specific class name"""
-    for cls, wdict in six.iteritems(live_refs):
+    for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
             if not wdict:
                 break
-            return min(six.iteritems(wdict), key=itemgetter(1))[0]
+            return min(wdict.items(), key=itemgetter(1))[0]
 
 
 def iter_all(class_name):
     """Iterate over all objects of the same class by its class name"""
-    for cls, wdict in six.iteritems(live_refs):
+    for cls, wdict in live_refs.items():
         if cls.__name__ == class_name:
-            return six.iterkeys(wdict)
+            return wdict.keys()

tests/test_settings/__init__.py

@@ -10,7 +10,7 @@ from . import default_settings
 class SettingsGlobalFuncsTest(unittest.TestCase):
 
     def test_get_settings_priority(self):
-        for prio_str, prio_num in six.iteritems(SETTINGS_PRIORITIES):
+        for prio_str, prio_num in SETTINGS_PRIORITIES.items():
             self.assertEqual(get_settings_priority(prio_str), prio_num)
         self.assertEqual(get_settings_priority(99), 99)
 
@@ -148,10 +148,10 @@ class BaseSettingsTest(unittest.TestCase):
         self.settings.setmodule(
             'tests.test_settings.default_settings', 10)
 
-        self.assertCountEqual(six.iterkeys(self.settings.attributes),
-                              six.iterkeys(ctrl_attributes))
+        self.assertCountEqual(self.settings.attributes.keys(),
+                              ctrl_attributes.keys())
 
-        for key in six.iterkeys(ctrl_attributes):
+        for key in ctrl_attributes.keys():
             attr = self.settings.attributes[key]
             ctrl_attr = ctrl_attributes[key]
             self.assertEqual(attr.value, ctrl_attr.value)
@@ -227,7 +227,7 @@ class BaseSettingsTest(unittest.TestCase):
         }
         settings = self.settings
         settings.attributes = {key: SettingsAttribute(value, 0) for key, value
-                               in six.iteritems(test_configuration)}
+                               in test_configuration.items()}
 
         self.assertTrue(settings.getbool('TEST_ENABLED1'))
         self.assertTrue(settings.getbool('TEST_ENABLED2'))

tests/test_webclient.py

@@ -318,7 +318,7 @@ class WebClientTestCase(unittest.TestCase):
         def cleanup(passthrough):
             # Clean up the server which is hanging around not doing
             # anything.
-            connected = list(six.iterkeys(self.wrapper.protocols))
+            connected = list(self.wrapper.protocols.keys())
             # There might be nothing here if the server managed to already see
             # that the connection was lost.
             if connected: