BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-21 04:13:26 +00:00
Code Releases Wiki Activity

Deprecation removals for Scrapy 1.7

Browse Source 
Removing deprecations of 2015 and prior (pre-1.1)
This commit is contained in:
nyov 2019-01-09 02:36:22 +00:00
parent 3d8f075b0a
commit 7a398b7086
27 changed files with 8 additions and 723 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.coveragerc
View File

@ -4,5 +4,3 @@ include = scrapy/*
omit =
tests/*
scrapy/xlib/*
scrapy/conf.py
scrapy/log.py

4
conftest.py
View File

@ -9,10 +9,6 @@ def _py_files(folder):
collect_ignore = [
# deprecated or moved modules
"scrapy/conf.py",
"scrapy/log.py",
# not a test, but looks like a test
"scrapy/utils/testsite.py",

16
scrapy/cmdline.py
View File

@ -99,13 +99,6 @@ def execute(argv=None, settings=None):
if argv is None:
argv = sys.argv
# --- backward compatibility for scrapy.conf.settings singleton ---
if settings is None and 'scrapy.conf' in sys.modules:
from scrapy import conf
if hasattr(conf, 'settings'):
settings = conf.settings
# ------------------------------------------------------------------
if settings is None:
settings = get_project_settings()
# set EDITOR from environment if available
@ -116,15 +109,6 @@ def execute(argv=None, settings=None):
settings['EDITOR'] = editor
check_deprecated_settings(settings)
# --- backward compatibility for scrapy.conf.settings singleton ---
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
with warnings.catch_warnings():
warnings.simplefilter("ignore", ScrapyDeprecationWarning)
from scrapy import conf
conf.settings = settings
# ------------------------------------------------------------------
inproject = inside_project()
cmds = _get_commands_dict(settings, inproject)
cmdname = _pop_command_name(argv)

13
scrapy/conf.py
View File

@ -1,13 +0,0 @@
# This module is kept for backward compatibility, so users can import
# scrapy.conf.settings and get the settings they expect
import sys
if 'scrapy.cmdline' not in sys.modules:
from scrapy.utils.project import get_project_settings
settings = get_project_settings()
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
warnings.warn("Module `scrapy.conf` is deprecated, use `crawler.settings` attribute instead",
ScrapyDeprecationWarning, stacklevel=2)

4
scrapy/core/downloader/__init__.py
View File

@ -60,10 +60,6 @@ class Slot(object):
def _get_concurrency_delay(concurrency, spider, settings):
delay = settings.getfloat('DOWNLOAD_DELAY')
if hasattr(spider, 'DOWNLOAD_DELAY'):
warnings.warn("%s.DOWNLOAD_DELAY attribute is deprecated, use %s.download_delay instead" %
(type(spider).__name__, type(spider).__name__))
delay = spider.DOWNLOAD_DELAY
if hasattr(spider, 'download_delay'):
delay = spider.download_delay

12
scrapy/core/downloader/handlers/http.py
View File

@ -1,15 +1,3 @@
from __future__ import absolute_import
from .http10 import HTTP10DownloadHandler
from .http11 import HTTP11DownloadHandler as HTTPDownloadHandler
# backward compatibility
class HttpDownloadHandler(HTTP10DownloadHandler):
def __init__(self, *args, **kwargs):
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
warnings.warn('HttpDownloadHandler is deprecated, import scrapy.core.downloader'
'.handlers.http10.HTTP10DownloadHandler instead',
category=ScrapyDeprecationWarning, stacklevel=1)
super(HttpDownloadHandler, self).__init__(*args, **kwargs)

9
scrapy/crawler.py
View File

@ -332,14 +332,7 @@ class CrawlerProcess(CrawlerRunner):
def _get_spider_loader(settings):
""" Get SpiderLoader instance from settings """
if settings.get('SPIDER_MANAGER_CLASS'):
warnings.warn(
'SPIDER_MANAGER_CLASS option is deprecated. '
'Please use SPIDER_LOADER_CLASS.',
category=ScrapyDeprecationWarning, stacklevel=2
)
cls_path = settings.get('SPIDER_MANAGER_CLASS',
settings.get('SPIDER_LOADER_CLASS'))
cls_path = settings.get('SPIDER_LOADER_CLASS')
loader_cls = load_object(cls_path)
try:
verifyClass(ISpiderLoader, loader_cls)

8
scrapy/loader/__init__.py
View File

@ -8,8 +8,6 @@ import six
from scrapy.item import Item
from scrapy.selector import Selector
from scrapy.utils.decorators import deprecated
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.utils.misc import arg_to_iter, extract_regex
from scrapy.utils.python import flatten
@ -191,10 +189,6 @@ class ItemLoader(object):
values = self._get_xpathvalues(xpath, **kw)
return self.get_value(values, *processors, **kw)
@deprecated(use_instead='._get_xpathvalues()')
def _get_values(self, xpaths, **kw):
return self._get_xpathvalues(xpaths, **kw)
def _get_xpathvalues(self, xpaths, **kw):
self._check_selector_method()
xpaths = arg_to_iter(xpaths)
@ -216,5 +210,3 @@ class ItemLoader(object):
self._check_selector_method()
csss = arg_to_iter(csss)
return flatten(self.selector.css(css).getall() for css in csss)
XPathItemLoader = create_deprecated_class('XPathItemLoader', ItemLoader)

60
scrapy/log.py
View File

@ -1,60 +0,0 @@
"""
This module is kept to provide a helpful warning about its removal.
"""
import logging
import warnings
from twisted.python.failure import Failure
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.log import failure_to_exc_info
logger = logging.getLogger(__name__)
warnings.warn("Module `scrapy.log` has been deprecated, Scrapy now relies on "
"the builtin Python library for logging. Read the updated "
"logging entry in the documentation to learn more.",
ScrapyDeprecationWarning, stacklevel=2)
# Imports and level_names variable kept for backward-compatibility
DEBUG = logging.DEBUG
INFO = logging.INFO
WARNING = logging.WARNING
ERROR = logging.ERROR
CRITICAL = logging.CRITICAL
SILENT = CRITICAL + 1
level_names = {
logging.DEBUG: "DEBUG",
logging.INFO: "INFO",
logging.WARNING: "WARNING",
logging.ERROR: "ERROR",
logging.CRITICAL: "CRITICAL",
SILENT: "SILENT",
}
def msg(message=None, _level=logging.INFO, **kw):
warnings.warn('log.msg has been deprecated, create a python logger and '
'log through it instead',
ScrapyDeprecationWarning, stacklevel=2)
level = kw.pop('level', _level)
message = kw.pop('format', message)
# NOTE: logger.log doesn't handle well passing empty dictionaries with format
# arguments because of some weird use-case:
# https://hg.python.org/cpython/file/648dcafa7e5f/Lib/logging/__init__.py#l269
logger.log(level, message, *[kw] if kw else [])
def err(_stuff=None, _why=None, **kw):
warnings.warn('log.err has been deprecated, create a python logger and '
'use its error method instead',
ScrapyDeprecationWarning, stacklevel=2)
level = kw.pop('level', logging.ERROR)
failure = kw.pop('failure', _stuff) or Failure()
message = kw.pop('why', _why) or failure.value
logger.log(level, message, *[kw] if kw else [], exc_info=failure_to_exc_info(failure))

31
scrapy/pipelines/files.py
View File

@ -458,33 +458,6 @@ class FilesPipeline(MediaPipeline):
return item
def file_path(self, request, response=None, info=None):
## start of deprecation warning block (can be removed in the future)
def _warn():
from scrapy.exceptions import ScrapyDeprecationWarning
import warnings
warnings.warn('FilesPipeline.file_key(url) method is deprecated, please use '
'file_path(request, response=None, info=None) instead',
category=ScrapyDeprecationWarning, stacklevel=1)
# check if called from file_key with url as first argument
if not isinstance(request, Request):
_warn()
url = request
else:
url = request.url
# detect if file_key() method has been overridden
if not hasattr(self.file_key, '_base'):
_warn()
return self.file_key(url)
## end of deprecation warning block
media_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation
media_ext = os.path.splitext(url)[1] # change to request.url after deprecation
media_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
media_ext = os.path.splitext(request.url)[1]
return 'full/%s%s' % (media_guid, media_ext)
# deprecated
def file_key(self, url):
return self.file_path(url)
file_key._base = True

64
scrapy/pipelines/images.py
View File

@ -165,69 +165,9 @@ class ImagesPipeline(FilesPipeline):
return item
def file_path(self, request, response=None, info=None):
## start of deprecation warning block (can be removed in the future)
def _warn():
from scrapy.exceptions import ScrapyDeprecationWarning
import warnings
warnings.warn('ImagesPipeline.image_key(url) and file_key(url) methods are deprecated, '
'please use file_path(request, response=None, info=None) instead',
category=ScrapyDeprecationWarning, stacklevel=1)
# check if called from image_key or file_key with url as first argument
if not isinstance(request, Request):
_warn()
url = request
else:
url = request.url
# detect if file_key() or image_key() methods have been overridden
if not hasattr(self.file_key, '_base'):
_warn()
return self.file_key(url)
elif not hasattr(self.image_key, '_base'):
_warn()
return self.image_key(url)
## end of deprecation warning block
image_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation
image_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return 'full/%s.jpg' % (image_guid)
def thumb_path(self, request, thumb_id, response=None, info=None):
## start of deprecation warning block (can be removed in the future)
def _warn():
from scrapy.exceptions import ScrapyDeprecationWarning
import warnings
warnings.warn('ImagesPipeline.thumb_key(url) method is deprecated, please use '
'thumb_path(request, thumb_id, response=None, info=None) instead',
category=ScrapyDeprecationWarning, stacklevel=1)
# check if called from thumb_key with url as first argument
if not isinstance(request, Request):
_warn()
url = request
else:
url = request.url
# detect if thumb_key() method has been overridden
if not hasattr(self.thumb_key, '_base'):
_warn()
return self.thumb_key(url, thumb_id)
## end of deprecation warning block
thumb_guid = hashlib.sha1(to_bytes(url)).hexdigest() # change to request.url after deprecation
thumb_guid = hashlib.sha1(to_bytes(request.url)).hexdigest()
return 'thumbs/%s/%s.jpg' % (thumb_id, thumb_guid)
# deprecated
def file_key(self, url):
return self.image_key(url)
file_key._base = True
# deprecated
def image_key(self, url):
return self.file_path(url)
image_key._base = True
# deprecated
def thumb_key(self, url, thumb_id):
return self.thumb_path(url, thumb_id)
thumb_key._base = True

1
scrapy/selector/__init__.py
View File

@ -2,4 +2,3 @@
Selectors
"""
from scrapy.selector.unified import *
from scrapy.selector.lxmlsel import *

15
scrapy/selector/csstranslator.py
View File

@ -1,15 +0,0 @@
from parsel.csstranslator import XPathExpr, GenericTranslator, HTMLTranslator
from scrapy.utils.deprecate import create_deprecated_class
ScrapyXPathExpr = create_deprecated_class(
'ScrapyXPathExpr', XPathExpr,
new_class_path='parsel.csstranslator.XPathExpr')
ScrapyGenericTranslator = create_deprecated_class(
'ScrapyGenericTranslator', GenericTranslator,
new_class_path='parsel.csstranslator.GenericTranslator')
ScrapyHTMLTranslator = create_deprecated_class(
'ScrapyHTMLTranslator', HTMLTranslator,
new_class_path='parsel.csstranslator.HTMLTranslator')

50
scrapy/selector/lxmlsel.py
View File

@ -1,50 +0,0 @@
"""
XPath selectors based on lxml
"""
from scrapy.utils.deprecate import create_deprecated_class
from .unified import Selector, SelectorList
__all__ = ['HtmlXPathSelector', 'XmlXPathSelector', 'XPathSelector',
'XPathSelectorList']
def _xpathselector_css(self, *a, **kw):
raise RuntimeError('.css() method not available for %s, '
'instantiate scrapy.Selector '
'instead' % type(self).__name__)
XPathSelector = create_deprecated_class(
'XPathSelector',
Selector,
{
'__slots__': (),
'_default_type': 'html',
'css': _xpathselector_css,
},
new_class_path='scrapy.Selector',
old_class_path='scrapy.selector.XPathSelector',
)
XmlXPathSelector = create_deprecated_class(
'XmlXPathSelector',
XPathSelector,
clsdict={
'__slots__': (),
'_default_type': 'xml',
},
new_class_path='scrapy.Selector',
old_class_path='scrapy.selector.XmlXPathSelector',
)
HtmlXPathSelector = create_deprecated_class(
'HtmlXPathSelector',
XPathSelector,
clsdict={
'__slots__': (),
'_default_type': 'html',
},
new_class_path='scrapy.Selector',
old_class_path='scrapy.selector.HtmlXPathSelector',
)
XPathSelectorList = create_deprecated_class('XPathSelectorList', SelectorList)

37
scrapy/selector/unified.py
View File

@ -8,7 +8,6 @@ from scrapy.utils.trackref import object_ref
from scrapy.utils.python import to_bytes
from scrapy.http import HtmlResponse, XmlResponse
from scrapy.utils.decorators import deprecated
from scrapy.exceptions import ScrapyDeprecationWarning
__all__ = ['Selector', 'SelectorList']
@ -31,17 +30,6 @@ class SelectorList(_ParselSelector.selectorlist_cls, object_ref):
The :class:`SelectorList` class is a subclass of the builtin ``list``
class, which provides a few additional methods.
"""
@deprecated(use_instead='.extract()')
def extract_unquoted(self):
return [x.extract_unquoted() for x in self]
@deprecated(use_instead='.xpath()')
def x(self, xpath):
return self.select(xpath)
@deprecated(use_instead='.xpath()')
def select(self, xpath):
return self.xpath(xpath)
class Selector(_ParselSelector, object_ref):
@ -78,21 +66,13 @@ class Selector(_ParselSelector, object_ref):
__slots__ = ['response']
selectorlist_cls = SelectorList
def __init__(self, response=None, text=None, type=None, root=None, _root=None, **kwargs):
def __init__(self, response=None, text=None, type=None, root=None, **kwargs):
if not(response is None or text is None):
raise ValueError('%s.__init__() received both response and text'
% self.__class__.__name__)
st = _st(response, type or self._default_type)
if _root is not None:
warnings.warn("Argument `_root` is deprecated, use `root` instead",
ScrapyDeprecationWarning, stacklevel=2)
if root is None:
root = _root
else:
warnings.warn("Ignoring deprecated `_root` argument, using provided `root`")
if text is not None:
response = _response_from_text(text, st)
@ -102,18 +82,3 @@ class Selector(_ParselSelector, object_ref):
self.response = response
super(Selector, self).__init__(text=text, type=st, root=root, **kwargs)
# Deprecated api
@property
def _root(self):
warnings.warn("Attribute `_root` is deprecated, use `root` instead",
ScrapyDeprecationWarning, stacklevel=2)
return self.root
@deprecated(use_instead='.xpath()')
def select(self, xpath):
return self.xpath(xpath)
@deprecated(use_instead='.extract()')
def extract_unquoted(self):
return self.extract()

26
scrapy/spiders/__init__.py
View File

@ -10,7 +10,6 @@ from scrapy import signals
from scrapy.http import Request
from scrapy.utils.trackref import object_ref
from scrapy.utils.url import url_is_from_spider
from scrapy.utils.deprecate import create_deprecated_class
from scrapy.exceptions import ScrapyDeprecationWarning
from scrapy.utils.deprecate import method_is_overridden
@ -52,15 +51,6 @@ class Spider(object_ref):
spider._set_crawler(crawler)
return spider
def set_crawler(self, crawler):
warnings.warn("set_crawler is deprecated, instantiate and bound the "
"spider to this crawler with from_crawler method "
"instead.",
category=ScrapyDeprecationWarning, stacklevel=2)
assert not hasattr(self, 'crawler'), "Spider already bounded to a " \
"crawler"
self._set_crawler(crawler)
def _set_crawler(self, crawler):
self.crawler = crawler
self.settings = crawler.settings
@ -109,22 +99,6 @@ class Spider(object_ref):
__repr__ = __str__
BaseSpider = create_deprecated_class('BaseSpider', Spider)
class ObsoleteClass(object):
def __init__(self, message):
self.message = message
def __getattr__(self, name):
raise AttributeError(self.message)
spiders = ObsoleteClass(
'"from scrapy.spider import spiders" no longer works - use '
'"from scrapy.spiderloader import SpiderLoader" and instantiate '
'it with your project settings"'
)
# Top-level imports
from scrapy.spiders.crawl import CrawlSpider, Rule
from scrapy.spiders.feed import XMLFeedSpider, CSVFeedSpider

4
scrapy/spiders/crawl.py
View File

@ -119,7 +119,3 @@ class CrawlSpider(Spider):
spider._follow_links = crawler.settings.getbool(
'CRAWLSPIDER_FOLLOW_LINKS', True)
return spider
def set_crawler(self, crawler):
super(CrawlSpider, self).set_crawler(crawler)
self._follow_links = crawler.settings.getbool('CRAWLSPIDER_FOLLOW_LINKS', True)

7
scrapy/telnet.py
View File

@ -1,7 +0,0 @@
import warnings
from scrapy.exceptions import ScrapyDeprecationWarning
warnings.warn("Module `scrapy.telnet` is deprecated, "
"use `scrapy.extensions.telnet` instead",
ScrapyDeprecationWarning, stacklevel=2)
from scrapy.extensions.telnet import *

13
scrapy/utils/python.py
View File

@ -84,19 +84,6 @@ def unique(list_, key=lambda x: x):
return result
@deprecated("scrapy.utils.python.to_unicode")
def str_to_unicode(text, encoding=None, errors='strict'):
""" This function is deprecated.
Please use scrapy.utils.python.to_unicode. """
return to_unicode(text, encoding, errors)
@deprecated("scrapy.utils.python.to_bytes")
def unicode_to_str(text, encoding=None, errors='strict'):
""" This function is deprecated. Please use scrapy.utils.python.to_bytes """
return to_bytes(text, encoding, errors)
def to_unicode(text, encoding=None, errors='strict'):
"""Return the unicode representation of a bytes object ``text``. If
``text`` is already an unicode object, return it as-is."""

8
scrapy/utils/response.py
View File

@ -11,14 +11,6 @@ from twisted.web import http
from scrapy.utils.python import to_bytes, to_native_str
from w3lib import html
from scrapy.utils.decorators import deprecated
@deprecated
def body_or_str(*a, **kw):
from scrapy.utils.iterators import _body_or_str
return _body_or_str(*a, **kw)
_baseurl_cache = weakref.WeakKeyDictionary()
def get_base_url(response):

21
tests/test_crawler.py
View File

@ -176,27 +176,6 @@ class CrawlerRunnerTestCase(BaseCrawlerTest):
sl_cls = load_object(runner.settings['SPIDER_LOADER_CLASS'])
self.assertIsInstance(spiders, sl_cls)
def test_spidermanager_deprecation(self):
with warnings.catch_warnings(record=True) as w:
runner = CrawlerRunner({
'SPIDER_MANAGER_CLASS': 'tests.test_crawler.CustomSpiderLoader'
})
self.assertIsInstance(runner.spider_loader, CustomSpiderLoader)
is_one_warning = len(w) == 1
if not is_one_warning:
for warning in w:
print(warning)
self.assertIn('Please use SPIDER_LOADER_CLASS', str(w[0].message))
self.assertTrue(is_one_warning)
def test_crawl_rejects_spider_objects(self):
with raises(ValueError):
CrawlerRunner().crawl(DefaultSpider())
def test_create_crawler_rejects_spider_objects(self):
with raises(ValueError):
CrawlerRunner().create_crawler(DefaultSpider())
class CrawlerProcessTest(BaseCrawlerTest):
def test_crawler_process_accepts_dict(self):

12
tests/test_downloader_handlers.py
View File

@ -24,7 +24,7 @@ from w3lib.url import path_to_file_uri
from scrapy.core.downloader.handlers import DownloadHandlers
from scrapy.core.downloader.handlers.datauri import DataURIDownloadHandler
from scrapy.core.downloader.handlers.file import FileDownloadHandler
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler, HttpDownloadHandler
from scrapy.core.downloader.handlers.http import HTTPDownloadHandler
from scrapy.core.downloader.handlers.http10 import HTTP10DownloadHandler
from scrapy.core.downloader.handlers.http11 import HTTP11DownloadHandler
from scrapy.core.downloader.handlers.s3 import S3DownloadHandler
@ -360,11 +360,6 @@ class HttpTestCase(unittest.TestCase):
return d
class DeprecatedHttpTestCase(HttpTestCase):
"""HTTP 1.0 test case"""
download_handler_cls = HttpDownloadHandler
class Http10TestCase(HttpTestCase):
"""HTTP 1.0 test case"""
download_handler_cls = HTTP10DownloadHandler
@ -656,11 +651,6 @@ class HttpProxyTestCase(unittest.TestCase):
return self.download_request(request, Spider('foo')).addCallback(_test)
class DeprecatedHttpProxyTestCase(unittest.TestCase):
"""Old deprecated reference to http10 downloader handler"""
download_handler_cls = HttpDownloadHandler
class Http10ProxyTestCase(HttpProxyTestCase):
download_handler_cls = HTTP10DownloadHandler

38
tests/test_pipeline_files.py
View File

@ -108,44 +108,6 @@ class FilesPipelineTestCase(unittest.TestCase):
p.stop()
class DeprecatedFilesPipeline(FilesPipeline):
def file_key(self, url):
media_guid = hashlib.sha1(to_bytes(url)).hexdigest()
media_ext = os.path.splitext(url)[1]
return 'empty/%s%s' % (media_guid, media_ext)
class DeprecatedFilesPipelineTestCase(unittest.TestCase):
def setUp(self):
self.tempdir = mkdtemp()
def init_pipeline(self, pipeline_class):
self.pipeline = pipeline_class.from_settings(Settings({'FILES_STORE': self.tempdir}))
self.pipeline.download_func = _mocked_download_func
self.pipeline.open_spider(None)
def test_default_file_key_method(self):
self.init_pipeline(FilesPipeline)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
self.assertEqual(self.pipeline.file_key("https://dev.mydeco.com/mydeco.pdf"),
'full/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
self.assertEqual(len(w), 1)
self.assertTrue('file_key(url) method is deprecated' in str(w[-1].message))
def test_overridden_file_key_method(self):
self.init_pipeline(DeprecatedFilesPipeline)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
self.assertEqual(self.pipeline.file_path(Request("https://dev.mydeco.com/mydeco.pdf")),
'empty/c9b564df929f4bc635bdd19fde4f3d4847c757c5.pdf')
self.assertEqual(len(w), 1)
self.assertTrue('file_key(url) method is deprecated' in str(w[-1].message))
def tearDown(self):
rmtree(self.tempdir)
class FilesPipelineTestCaseFields(unittest.TestCase):
def test_item_fields_default(self):

57
tests/test_pipeline_images.py
View File

@ -118,63 +118,6 @@ class DeprecatedImagesPipeline(ImagesPipeline):
return 'thumbsup/%s/%s.jpg' % (thumb_id, thumb_guid)
class DeprecatedImagesPipelineTestCase(unittest.TestCase):
def setUp(self):
self.tempdir = mkdtemp()
def init_pipeline(self, pipeline_class):
self.pipeline = pipeline_class(self.tempdir, download_func=_mocked_download_func)
self.pipeline.open_spider(None)
def test_default_file_key_method(self):
self.init_pipeline(ImagesPipeline)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
self.assertEqual(self.pipeline.file_key("https://dev.mydeco.com/mydeco.gif"),
'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
self.assertEqual(len(w), 1)
self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message))
def test_default_image_key_method(self):
self.init_pipeline(ImagesPipeline)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
self.assertEqual(self.pipeline.image_key("https://dev.mydeco.com/mydeco.gif"),
'full/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
self.assertEqual(len(w), 1)
self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message))
def test_overridden_file_key_method(self):
self.init_pipeline(DeprecatedImagesPipeline)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
self.assertEqual(self.pipeline.file_path(Request("https://dev.mydeco.com/mydeco.gif")),
'empty/3fd165099d8e71b8a48b2683946e64dbfad8b52d.jpg')
self.assertEqual(len(w), 1)
self.assertTrue('image_key(url) and file_key(url) methods are deprecated' in str(w[-1].message))
def test_default_thumb_key_method(self):
self.init_pipeline(ImagesPipeline)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
self.assertEqual(self.pipeline.thumb_key("file:///tmp/foo.jpg", 50),
'thumbs/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg')
self.assertEqual(len(w), 1)
self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message))
def test_overridden_thumb_key_method(self):
self.init_pipeline(DeprecatedImagesPipeline)
with warnings.catch_warnings(record=True) as w:
warnings.simplefilter('always')
self.assertEqual(self.pipeline.thumb_path(Request("file:///tmp/foo.jpg"), 50),
'thumbsup/50/38a86208c36e59d4404db9e37ce04be863ef0335.jpg')
self.assertEqual(len(w), 1)
self.assertTrue('thumb_key(url) method is deprecated' in str(w[-1].message))
def tearDown(self):
rmtree(self.tempdir)
class ImagesPipelineTestCaseFields(unittest.TestCase):
def test_item_fields_default(self):

122
tests/test_selector.py
View File

@ -3,7 +3,6 @@ import weakref
from twisted.trial import unittest
from scrapy.http import TextResponse, HtmlResponse, XmlResponse
from scrapy.selector import Selector
from scrapy.selector.lxmlsel import XmlXPathSelector, HtmlXPathSelector, XPathSelector
from lxml import etree
@ -40,22 +39,6 @@ class SelectorTestCase(unittest.TestCase):
sel = Selector(response)
self.assertEqual(url, sel.root.base)
def test_deprecated_root_argument(self):
with warnings.catch_warnings(record=True) as w:
root = etree.fromstring(u'<html/>')
sel = Selector(_root=root)
self.assertIs(root, sel.root)
self.assertEqual(str(w[-1].message),
'Argument `_root` is deprecated, use `root` instead')
def test_deprecated_root_argument_ambiguous(self):
with warnings.catch_warnings(record=True) as w:
_root = etree.fromstring(u'<xml/>')
root = etree.fromstring(u'<html/>')
sel = Selector(_root=_root, root=root)
self.assertIs(root, sel.root)
self.assertIn('Ignoring deprecated `_root` argument', str(w[-1].message))
def test_flavor_detection(self):
text = b'<div><img src="a.jpg"><p>Hello</div>'
sel = Selector(XmlResponse('http://example.com', body=text, encoding='utf-8'))
@ -101,111 +84,6 @@ class SelectorTestCase(unittest.TestCase):
assert not hasattr(x, '__dict__'), "%s does not use __slots__" % \
x.__class__.__name__
def test_deprecated_selector_methods(self):
sel = Selector(TextResponse(url="http://example.com", body=b'<p>some text</p>'))
with warnings.catch_warnings(record=True) as w:
sel.select('//p')
self.assertSubstring('Use .xpath() instead', str(w[-1].message))
with warnings.catch_warnings(record=True) as w:
sel.extract_unquoted()
self.assertSubstring('Use .extract() instead', str(w[-1].message))
def test_deprecated_selectorlist_methods(self):
sel = Selector(TextResponse(url="http://example.com", body=b'<p>some text</p>'))
with warnings.catch_warnings(record=True) as w:
sel.xpath('//p').select('.')
self.assertSubstring('Use .xpath() instead', str(w[-1].message))
with warnings.catch_warnings(record=True) as w:
sel.xpath('//p').extract_unquoted()
self.assertSubstring('Use .extract() instead', str(w[-1].message))
def test_selector_bad_args(self):
with self.assertRaisesRegexp(ValueError, 'received both response and text'):
Selector(TextResponse(url='http://example.com', body=b''), text=u'')
class DeprecatedXpathSelectorTest(unittest.TestCase):
text = '<div><img src="a.jpg"><p>Hello</div>'
def test_warnings_xpathselector(self):
cls = XPathSelector
with warnings.catch_warnings(record=True) as w:
class UserClass(cls):
pass
# subclassing must issue a warning
self.assertEqual(len(w), 1, str(cls))
self.assertIn('scrapy.Selector', str(w[0].message))
# subclass instance doesn't issue a warning
usel = UserClass(text=self.text)
self.assertEqual(len(w), 1)
# class instance must issue a warning
sel = cls(text=self.text)
self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
self.assertIn('scrapy.Selector', str(w[1].message))
# subclass and instance checks
self.assertTrue(issubclass(cls, Selector))
self.assertTrue(isinstance(sel, Selector))
self.assertTrue(isinstance(usel, Selector))
def test_warnings_xmlxpathselector(self):
cls = XmlXPathSelector
with warnings.catch_warnings(record=True) as w:
class UserClass(cls):
pass
# subclassing must issue a warning
self.assertEqual(len(w), 1, str(cls))
self.assertIn('scrapy.Selector', str(w[0].message))
# subclass instance doesn't issue a warning
usel = UserClass(text=self.text)
self.assertEqual(len(w), 1)
# class instance must issue a warning
sel = cls(text=self.text)
self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
self.assertIn('scrapy.Selector', str(w[1].message))
# subclass and instance checks
self.assertTrue(issubclass(cls, Selector))
self.assertTrue(issubclass(cls, XPathSelector))
self.assertTrue(isinstance(sel, Selector))
self.assertTrue(isinstance(usel, Selector))
self.assertTrue(isinstance(sel, XPathSelector))
self.assertTrue(isinstance(usel, XPathSelector))
def test_warnings_htmlxpathselector(self):
cls = HtmlXPathSelector
with warnings.catch_warnings(record=True) as w:
class UserClass(cls):
pass
# subclassing must issue a warning
self.assertEqual(len(w), 1, str(cls))
self.assertIn('scrapy.Selector', str(w[0].message))
# subclass instance doesn't issue a warning
usel = UserClass(text=self.text)
self.assertEqual(len(w), 1)
# class instance must issue a warning
sel = cls(text=self.text)
self.assertEqual(len(w), 2, str((cls, [x.message for x in w])))
self.assertIn('scrapy.Selector', str(w[1].message))
# subclass and instance checks
self.assertTrue(issubclass(cls, Selector))
self.assertTrue(issubclass(cls, XPathSelector))
self.assertTrue(isinstance(sel, Selector))
self.assertTrue(isinstance(usel, Selector))
self.assertTrue(isinstance(sel, XPathSelector))
self.assertTrue(isinstance(usel, XPathSelector))

22
tests/test_selector_csstranslator.py
View File

@ -1,22 +0,0 @@
"""
Selector tests for cssselect backend
"""
import warnings
from twisted.trial import unittest
from scrapy.selector.csstranslator import (
ScrapyHTMLTranslator,
ScrapyGenericTranslator,
ScrapyXPathExpr
)
class DeprecatedClassesTest(unittest.TestCase):
def test_deprecated_warnings(self):
for cls in [ScrapyHTMLTranslator, ScrapyGenericTranslator, ScrapyXPathExpr]:
with warnings.catch_warnings(record=True) as w:
obj = cls()
self.assertIn('%s is deprecated' % cls.__name__, str(w[-1].message),
'Missing deprecate warning for %s' % cls.__name__)

75
tests/test_spider.py
View File

@ -10,7 +10,7 @@ from scrapy import signals
from scrapy.settings import Settings
from scrapy.http import Request, Response, TextResponse, XmlResponse, HtmlResponse
from scrapy.spiders.init import InitSpider
from scrapy.spiders import Spider, BaseSpider, CrawlSpider, Rule, XMLFeedSpider, \
from scrapy.spiders import Spider, CrawlSpider, Rule, XMLFeedSpider, \
CSVFeedSpider, SitemapSpider
from scrapy.linkextractors import LinkExtractor
from scrapy.exceptions import ScrapyDeprecationWarning
@ -51,17 +51,6 @@ class SpiderTest(unittest.TestCase):
self.assertRaises(ValueError, self.spider_class)
self.assertRaises(ValueError, self.spider_class, somearg='foo')
def test_deprecated_set_crawler_method(self):
spider = self.spider_class('example.com')
crawler = get_crawler()
with warnings.catch_warnings(record=True) as w:
spider.set_crawler(crawler)
self.assertIn("set_crawler", str(w[0].message))
self.assertTrue(hasattr(spider, 'crawler'))
self.assertIs(spider.crawler, crawler)
self.assertTrue(hasattr(spider, 'settings'))
self.assertIs(spider.settings, crawler.settings)
def test_from_crawler_crawler_and_settings_population(self):
crawler = get_crawler()
spider = self.spider_class.from_crawler(crawler, 'example.com')
@ -377,20 +366,6 @@ class CrawlSpiderTest(SpiderTest):
self.assertTrue(hasattr(spider, '_follow_links'))
self.assertFalse(spider._follow_links)
def test_follow_links_attribute_deprecated_population(self):
spider = self.spider_class('example.com')
self.assertFalse(hasattr(spider, '_follow_links'))
spider.set_crawler(get_crawler())
self.assertTrue(hasattr(spider, '_follow_links'))
self.assertTrue(spider._follow_links)
spider = self.spider_class('example.com')
settings_dict = {'CRAWLSPIDER_FOLLOW_LINKS': False}
spider.set_crawler(get_crawler(settings_dict=settings_dict))
self.assertTrue(hasattr(spider, '_follow_links'))
self.assertFalse(spider._follow_links)
class SitemapSpiderTest(SpiderTest):
@ -578,57 +553,9 @@ Sitemap: /sitemap-relative-url.xml
class DeprecationTest(unittest.TestCase):
def test_basespider_is_deprecated(self):
with warnings.catch_warnings(record=True) as w:
class MySpider1(BaseSpider):
pass
self.assertEqual(len(w), 1)
self.assertEqual(w[0].category, ScrapyDeprecationWarning)
self.assertEqual(w[0].lineno, inspect.getsourcelines(MySpider1)[1])
def test_basespider_issubclass(self):
class MySpider2(Spider):
pass
class MySpider2a(MySpider2):
pass
class Foo(object):
pass
class Foo2(object_ref):
pass
assert issubclass(MySpider2, BaseSpider)
assert issubclass(MySpider2a, BaseSpider)
assert not issubclass(Foo, BaseSpider)
assert not issubclass(Foo2, BaseSpider)
def test_basespider_isinstance(self):
class MySpider3(Spider):
name = 'myspider3'
class MySpider3a(MySpider3):
pass
class Foo(object):
pass
class Foo2(object_ref):
pass
assert isinstance(MySpider3(), BaseSpider)
assert isinstance(MySpider3a(), BaseSpider)
assert not isinstance(Foo(), BaseSpider)
assert not isinstance(Foo2(), BaseSpider)
def test_crawl_spider(self):
assert issubclass(CrawlSpider, Spider)
assert issubclass(CrawlSpider, BaseSpider)
assert isinstance(CrawlSpider(name='foo'), Spider)
assert isinstance(CrawlSpider(name='foo'), BaseSpider)
def test_make_requests_from_url_deprecated(self):
class MySpider4(Spider):