Merge branch 'master' into import-module

.travis.yml

@@ -5,8 +5,6 @@ matrix:
     exclude:
         - env: TRAVISBUG="#1027"
     include:
-        - python: "2.6"
-          env: BUILDENV=lucid
         - python: "2.7"
           env: BUILDENV=precise
         - python: "2.7"

.travis/requirements-lucid.txt

@@ -1,9 +0,0 @@
-# travis workers uses openssl 1.0 that fails for pyOpenSSL < 0.13
-pyOpenSSL==0.13
-lxml==2.2.4
-twisted==10.0.0
-boto==1.9b
-Pillow<2.0
-
-# testing requirements
-mock==1.0.1

README.rst

@@ -21,7 +21,7 @@ http://scrapy.org
 Requirements
 ============
 
-* Python 2.6 or up
+* Python 2.7
 * Works on Linux, Windows, Mac OSX, BSD
 
 Install

debian/control

@@ -2,7 +2,7 @@ Source: scrapy-SUFFIX
 Section: python
 Priority: optional
 Maintainer: Scrapinghub Team <info@scrapinghub.com>
-Build-Depends: debhelper (>= 7.0.50), python (>=2.6), python-twisted, python-w3lib, python-lxml
+Build-Depends: debhelper (>= 7.0.50), python (>=2.7), python-twisted, python-w3lib, python-lxml
 Standards-Version: 3.8.4
 Homepage: http://scrapy.org/
 

docs/faq.rst

@@ -29,13 +29,14 @@ comparing `jinja2`_ to `Django`_.
 What Python versions does Scrapy support?
 -----------------------------------------
 
-Scrapy runs in Python 2.6 and 2.7.
+Scrapy is supported under Python 2.7 only.
+Python 2.6 support was dropped starting at Scrapy 0.20.
 
-Does Scrapy work with Python 3.0?
+Does Scrapy work with Python 3?
 ---------------------------------
 
-No, and there are no plans to port Scrapy to Python 3.0 yet. At the moment,
-Scrapy works with Python 2.6 and 2.7.
+No, but there are plans to support Python 3.3+.
+At the moment, Scrapy works with Python 2.7.
 
 .. seealso:: :ref:`faq-python-versions`.
 

docs/intro/install.rst

@@ -9,7 +9,7 @@ Pre-requisites
 
 The installation steps assume that you have the following things installed:
 
-* `Python`_ 2.6 or 2.7
+* `Python`_ 2.7
 * `lxml`_. Most Linux distributions ships prepackaged versions of lxml. Otherwise refer to http://lxml.de/installation.html
 * `OpenSSL`_. This comes preinstalled in all operating systems except Windows (see :ref:`intro-install-platform-notes`)
 * `pip`_ or `easy_install`_ Python package managers

docs/news.rst

@@ -12,6 +12,7 @@ Release notes
 - Dropped libxml2 selectors backend
 - Dropped support for multiple selectors backends, sticking to lxml only
 - Selector Unified API with support for CSS expressions (:issue:`395` and :issue:`426`)
+- Dropped Python 2.6 support
 
 0.18.4 (released 2013-10-10)
 ----------------------------

docs/topics/ubuntu.rst

@@ -12,8 +12,8 @@ those in Ubuntu, and more stable too since they're continuously built from
 fixes.
 
 To use the packages, just add the following line to your
-``/etc/apt/sources.list``, and then run ``aptitude update`` and ``aptitude
-install scrapy-0.13``::
+``/etc/apt/sources.list``, and then run ``aptitude update`` and
+``apt-get install scrapy-0.18``::
 
     deb http://archive.scrapy.org/ubuntu DISTRO main
 
@@ -22,8 +22,7 @@ with command::
 
     lsb_release -cs
 
-Supported Ubuntu releases are: ``karmic``, ``lucid``, ``maverick``, ``natty``,
-``oneiric``, ``precise``, ``quantal``, ``raring``.
+Supported Ubuntu releases are: ``precise``, ``quantal``, ``raring``.
 
 For Ubuntu Raring (13.04)::
 
@@ -37,26 +36,6 @@ For Ubuntu Precise (12.04)::
 
     deb http://archive.scrapy.org/ubuntu precise main
 
-For Ubuntu Oneiric (11.10)::
-
-    deb http://archive.scrapy.org/ubuntu oneiric main
-
-For Ubuntu Natty (11.04)::
-
-    deb http://archive.scrapy.org/ubuntu natty main
-
-For Ubuntu Maverick (10.10)::
-
-    deb http://archive.scrapy.org/ubuntu maverick main
-
-For Ubuntu Lucid (10.04)::
-
-    deb http://archive.scrapy.org/ubuntu lucid main
-
-For Ubuntu Karmic (9.10)::
-
-    deb http://archive.scrapy.org/ubuntu karmic main
-
 .. warning:: Please note that these packages are updated frequently, and so if
    you find you can't download the packages, try updating your apt package
    lists first, e.g., with ``apt-get update`` or ``aptitude update``.

scrapy/__init__.py

@@ -8,8 +8,8 @@ version_info = tuple(__version__.split('.')[:3])
 
 import sys, os, warnings
 
-if sys.version_info < (2, 6):
-    print("Scrapy %s requires Python 2.6 or above" % __version__)
+if sys.version_info < (2, 7):
+    print("Scrapy %s requires Python 2.7" % __version__)
     sys.exit(1)
 
 # ignore noisy twisted deprecation warnings

scrapy/tests/test_downloadermiddleware_httpproxy.py

@@ -63,8 +63,6 @@ class TestDefaultHeadersMiddleware(TestCase):
 
 
     def test_no_proxy(self):
-        if sys.version_info < (2, 6):
-            raise SkipTest('no_proxy is not supported in python < 2.6')
         os.environ['http_proxy'] = http_proxy = 'https://proxy.for.http:3128'
         mw = HttpProxyMiddleware()
 

scrapy/utils/datatypes.py

@@ -6,8 +6,7 @@ This module must not depend on any module outside the Standard Library.
 """
 
 import copy
-
-from scrapy.utils.py27 import OrderedDict
+from collections import OrderedDict
 
 
 class MultiValueDictKeyError(KeyError):
@@ -57,7 +56,7 @@ class MultiValueDict(dict):
         return self.__class__(dict.items(self))
 
     def __deepcopy__(self, memo=None):
-        if memo is None: 
+        if memo is None:
             memo = {}
         result = self.__class__()
         memo[id(self)] = result

scrapy/utils/py26.py

@@ -1,10 +0,0 @@
-import json
-from shutil import copytree, ignore_patterns
-from multiprocessing import cpu_count
-from pkgutil import get_data
-
-from scrapy.exceptions import ScrapyDeprecationWarning
-
-import warnings
-warnings.warn("Module `scrapy.utils.py26` is deprecated and will be removed in Scrapy 0.17",
-    ScrapyDeprecationWarning, stacklevel=2)

scrapy/utils/py27.py

@@ -1,11 +0,0 @@
-"""
-This module provides functions added in Python 2.7, which weren't yet available
-in Python 2.6. The Python 2.7 function is used when available.
-"""
-
-__all__ = ['OrderedDict']
-
-try:
-    from collections import OrderedDict
-except ImportError:
-    from scrapy.xlib.ordereddict import OrderedDict

scrapy/xlib/ordereddict.py

@@ -1,127 +0,0 @@
-# Copyright (c) 2009 Raymond Hettinger
-#
-# Permission is hereby granted, free of charge, to any person
-# obtaining a copy of this software and associated documentation files
-# (the "Software"), to deal in the Software without restriction,
-# including without limitation the rights to use, copy, modify, merge,
-# publish, distribute, sublicense, and/or sell copies of the Software,
-# and to permit persons to whom the Software is furnished to do so,
-# subject to the following conditions:
-#
-#     The above copyright notice and this permission notice shall be
-#     included in all copies or substantial portions of the Software.
-#
-#     THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-#     EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
-#     OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-#     NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
-#     HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
-#     WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
-#     FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-#     OTHER DEALINGS IN THE SOFTWARE.
-
-from UserDict import DictMixin
-
-class OrderedDict(dict, DictMixin):
-
-    def __init__(self, *args, **kwds):
-        if len(args) > 1:
-            raise TypeError('expected at most 1 arguments, got %d' % len(args))
-        try:
-            self.__end
-        except AttributeError:
-            self.clear()
-        self.update(*args, **kwds)
-
-    def clear(self):
-        self.__end = end = []
-        end += [None, end, end]         # sentinel node for doubly linked list
-        self.__map = {}                 # key --> [key, prev, next]
-        dict.clear(self)
-
-    def __setitem__(self, key, value):
-        if key not in self:
-            end = self.__end
-            curr = end[1]
-            curr[2] = end[1] = self.__map[key] = [key, curr, end]
-        dict.__setitem__(self, key, value)
-
-    def __delitem__(self, key):
-        dict.__delitem__(self, key)
-        key, prev, next = self.__map.pop(key)
-        prev[2] = next
-        next[1] = prev
-
-    def __iter__(self):
-        end = self.__end
-        curr = end[2]
-        while curr is not end:
-            yield curr[0]
-            curr = curr[2]
-
-    def __reversed__(self):
-        end = self.__end
-        curr = end[1]
-        while curr is not end:
-            yield curr[0]
-            curr = curr[1]
-
-    def popitem(self, last=True):
-        if not self:
-            raise KeyError('dictionary is empty')
-        if last:
-            key = next(reversed(self))
-        else:
-            key = next(iter(self))
-        value = self.pop(key)
-        return key, value
-
-    def __reduce__(self):
-        items = [[k, self[k]] for k in self]
-        tmp = self.__map, self.__end
-        del self.__map, self.__end
-        inst_dict = vars(self).copy()
-        self.__map, self.__end = tmp
-        if inst_dict:
-            return (self.__class__, (items,), inst_dict)
-        return self.__class__, (items,)
-
-    def keys(self):
-        return list(self)
-
-    setdefault = DictMixin.setdefault
-    update = DictMixin.update
-    pop = DictMixin.pop
-    values = DictMixin.values
-    items = DictMixin.items
-    iterkeys = DictMixin.iterkeys
-    itervalues = DictMixin.itervalues
-    iteritems = DictMixin.iteritems
-
-    def __repr__(self):
-        if not self:
-            return '%s()' % (self.__class__.__name__,)
-        return '%s(%r)' % (self.__class__.__name__, self.items())
-
-    def copy(self):
-        return self.__class__(self)
-
-    @classmethod
-    def fromkeys(cls, iterable, value=None):
-        d = cls()
-        for key in iterable:
-            d[key] = value
-        return d
-
-    def __eq__(self, other):
-        if isinstance(other, OrderedDict):
-            if len(self) != len(other):
-                return False
-            for p, q in  zip(self.items(), other.items()):
-                if p != q:
-                    return False
-            return True
-        return dict.__eq__(self, other)
-
-    def __ne__(self, other):
-        return not self == other

setup.py

@@ -104,7 +104,6 @@ setup_args = {
     'scripts': scripts,
     'classifiers': [
         'Programming Language :: Python',
-        'Programming Language :: Python :: 2.6',
         'Programming Language :: Python :: 2.7',
         'License :: OSI Approved :: BSD License',
         'Operating System :: OS Independent',

tox.ini

@@ -4,7 +4,7 @@
 # and then run "tox" from this directory.
 
 [tox]
-envlist = py26, py27, lucid, precise, trunk
+envlist = py27, precise, trunk
 
 [testenv]
 deps =
@@ -12,11 +12,6 @@ deps =
 commands =
     {toxinidir}/bin/runtests.sh []
 
-[testenv:lucid]
-basepython = python2.6
-deps =
-    -r{toxinidir}/.travis/requirements-lucid.txt
-
 [testenv:precise]
 basepython = python2.7
 deps =