BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-03-14 15:48:38 +00:00
Code Releases Wiki Activity

Merge branch 'master' into flake8-max-line-length

Browse Source
This commit is contained in:
Eugenio Lacuesta 2020-04-10 15:24:22 -03:00
commit 950a52468c
No known key found for this signature in database
GPG Key ID: DA3EF2D0913E9810
9 changed files with 31 additions and 39 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
pytest.ini
View File

@ -31,21 +31,20 @@ flake8-ignore =
# scrapy/commands
scrapy/commands/__init__.py E128
scrapy/commands/fetch.py E401 E128 E731
scrapy/commands/genspider.py E128 E502
scrapy/commands/genspider.py E128
scrapy/commands/parse.py E128 E731
scrapy/commands/settings.py E128
scrapy/commands/shell.py E128 E502
scrapy/commands/shell.py E128
scrapy/commands/startproject.py E127 E128
scrapy/commands/version.py E128
# scrapy/contracts
scrapy/contracts/__init__.py W504
scrapy/contracts/default.py E128
# scrapy/core
scrapy/core/engine.py E128 E127 E502
scrapy/core/engine.py E128 E127
scrapy/core/scraper.py E128 W504
scrapy/core/spidermw.py E731 E126
scrapy/core/downloader/contextfactory.py E128 E126
scrapy/core/downloader/middleware.py E502
scrapy/core/downloader/tls.py E241
scrapy/core/downloader/webclient.py E731 E128 E126
scrapy/core/downloader/handlers/ftp.py E128 E127
@ -85,7 +84,7 @@ flake8-ignore =
# scrapy/utils
scrapy/utils/conf.py E402
scrapy/utils/defer.py E128
scrapy/utils/deprecate.py E128 E127 E502
scrapy/utils/deprecate.py E128 E127
scrapy/utils/gz.py W504
scrapy/utils/http.py F403
scrapy/utils/log.py E128
@ -99,7 +98,7 @@ flake8-ignore =
scrapy/__init__.py E402
scrapy/dupefilters.py E202
scrapy/item.py E128
scrapy/mail.py E402 E128 E502
scrapy/mail.py E402 E128
scrapy/middleware.py E128
scrapy/responsetypes.py E128
scrapy/spiderloader.py F841 E126
@ -139,7 +138,7 @@ flake8-ignore =
tests/test_middleware.py E128
tests/test_pipeline_crawl.py E128 E126
tests/test_pipeline_images.py F841
tests/test_pipeline_media.py E741 E731 E128 E502
tests/test_pipeline_media.py E741 E731 E128
tests/test_proxy_connect.py E741
tests/test_scheduler.py E126 E123
tests/test_selector.py E127

7
scrapy/commands/genspider.py
View File

@ -90,8 +90,7 @@ class Command(ScrapyCommand):
'module': module,
'name': name,
'domain': domain,
'classname': '%sSpider' % ''.join(s.capitalize() \
for s in module.split('_'))
'classname': '%sSpider' % ''.join(s.capitalize() for s in module.split('_'))
}
if self.settings.get('NEWSPIDER_MODULE'):
spiders_module = import_module(self.settings['NEWSPIDER_MODULE'])
@ -102,8 +101,8 @@ class Command(ScrapyCommand):
spider_file = "%s.py" % join(spiders_dir, module)
shutil.copyfile(template_file, spider_file)
render_templatefile(spider_file, **tvars)
print("Created spider %r using template %r " % (name, \
template_name), end=('' if spiders_module else '\n'))
print("Created spider %r using template %r "
% (name, template_name), end=('' if spiders_module else '\n'))
if spiders_module:
print("in module:\n %s.%s" % (spiders_module.__name__, module))

2
scrapy/commands/shell.py
View File

@ -37,7 +37,7 @@ class Command(ScrapyCommand):
help="evaluate the code in the shell, print the result and exit")
parser.add_option("--spider", dest="spider",
help="use this spider")
parser.add_option("--no-redirect", dest="no_redirect", action="store_true", \
parser.add_option("--no-redirect", dest="no_redirect", action="store_true",
default=False, help="do not handle HTTP 3xx status codes and print response as-is")
def update_vars(self, vars):

14
scrapy/core/downloader/handlers/http11.py
View File

@ -341,20 +341,6 @@ class ScrapyAgent:
headers.removeHeader(b'Proxy-Authorization')
if request.body:
bodyproducer = _RequestBodyProducer(request.body)
elif method == b'POST':
# Setting Content-Length: 0 even for POST requests is not a
# MUST per HTTP RFCs, but it's common behavior, and some
# servers require this, otherwise returning HTTP 411 Length required
#
# RFC 7230#section-3.3.2:
# "a Content-Length header field is normally sent in a POST
# request even when the value is 0 (indicating an empty payload body)."
#
# Twisted < 17 will not add "Content-Length: 0" by itself;
# Twisted >= 17 fixes this;
# Using a producer with an empty-string sends `0` as Content-Length
# for all versions of Twisted.
bodyproducer = _RequestBodyProducer(b'')
else:
bodyproducer = None
start_time = time()

5
scrapy/core/engine.py
View File

@ -277,10 +277,9 @@ class ExecutionEngine:
next loop and this function is guaranteed to be called (at least) once
again for this spider.
"""
res = self.signals.send_catch_log(signal=signals.spider_idle, \
res = self.signals.send_catch_log(signal=signals.spider_idle,
spider=spider, dont_log=DontCloseSpider)
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) \
for _, x in res):
if any(isinstance(x, Failure) and isinstance(x.value, DontCloseSpider) for _, x in res):
return
if self.spider_is_idle(spider):

4
scrapy/mail.py
View File

@ -115,8 +115,8 @@ class MailSender:
from twisted.mail.smtp import ESMTPSenderFactory
msg = BytesIO(msg)
d = defer.Deferred()
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom, \
to_addrs, msg, d, heloFallback=True, requireAuthentication=False, \
factory = ESMTPSenderFactory(self.smtpuser, self.smtppass, self.mailfrom,
to_addrs, msg, d, heloFallback=True, requireAuthentication=False,
requireTransportSecurity=self.smtptls)
factory.noisy = False

15
scrapy/utils/deprecate.py
View File

@ -7,9 +7,12 @@ from scrapy.exceptions import ScrapyDeprecationWarning
def attribute(obj, oldattr, newattr, version='0.12'):
cname = obj.__class__.__name__
warnings.warn("%s.%s attribute is deprecated and will be no longer supported "
"in Scrapy %s, use %s.%s attribute instead" % \
(cname, oldattr, version, cname, newattr), ScrapyDeprecationWarning, stacklevel=3)
warnings.warn(
"%s.%s attribute is deprecated and will be no longer supported "
"in Scrapy %s, use %s.%s attribute instead"
% (cname, oldattr, version, cname, newattr),
ScrapyDeprecationWarning,
stacklevel=3)
def create_deprecated_class(name, new_class, clsdict=None,
@ -17,10 +20,10 @@ def create_deprecated_class(name, new_class, clsdict=None,
warn_once=True,
old_class_path=None,
new_class_path=None,
subclass_warn_message="{cls} inherits from "\
"deprecated class {old}, please inherit "\
subclass_warn_message="{cls} inherits from "
"deprecated class {old}, please inherit "
"from {new}.",
instance_warn_message="{cls} is deprecated, "\
instance_warn_message="{cls} is deprecated, "
"instantiate {new} instead."):
"""
Return a "deprecated" class that causes its subclasses to issue a warning.

5
setup.py
View File

@ -30,6 +30,11 @@ setup(
name='Scrapy',
version=version,
url='https://scrapy.org',
project_urls = {
'Documentation': 'https://docs.scrapy.org/',
'Source': 'https://github.com/scrapy/scrapy',
'Tracker': 'https://github.com/scrapy/scrapy/issues',
},
description='A high-level Web Crawling and Web Scraping framework',
long_description=open('README.rst').read(),
author='Scrapy developers',

5
tests/test_pipeline_media.py
View File

@ -325,8 +325,9 @@ class MediaPipelineTestCase(BaseMediaPipelineTestCase):
item = dict(requests=req)
new_item = yield self.pipe.process_item(item, self.spider)
self.assertEqual(new_item['results'], [(True, 'ITSME')])
self.assertEqual(self.pipe._mockcalled, \
['get_media_requests', 'media_to_download', 'item_completed'])
self.assertEqual(
self.pipe._mockcalled,
['get_media_requests', 'media_to_download', 'item_completed'])
class MediaPipelineAllowRedirectSettingsTestCase(unittest.TestCase):