BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-23 14:24:19 +00:00
Code Releases Wiki Activity

Rename setting to REFERRER_POLICY (with 2 Rs)

Browse Source
This commit is contained in:
Paul Tremberth 2017-02-21 17:08:39 +01:00
parent 537683f945
commit bc200d1155
4 changed files with 26 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
docs/topics/spider-middleware.rst
View File

@ -328,10 +328,10 @@ Default: ``True``
Whether to enable referer middleware. Whether to enable referer middleware.
.. setting:: REFERER_POLICY .. setting:: REFERRER_POLICY
REFERER_POLICY REFERRER_POLICY
^^^^^^^^^^^^^^ ^^^^^^^^^^^^^^^
.. versionadded:: 1.4 .. versionadded:: 1.4
@ -344,10 +344,10 @@ Default: ``'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'``
.. note:: .. note::
You can also set the Referrer Policy per request, You can also set the Referrer Policy per request,
using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key, using the special ``"referrer_policy"`` :ref:`Request.meta <topics-request-meta>` key,
with the same acceptable values as for the ``REFERER_POLICY`` setting. with the same acceptable values as for the ``REFERRER_POLICY`` setting.
Acceptable values for REFERER_POLICY Acceptable values for REFERRER_POLICY
************************************ *************************************
- either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy`` - either a path to a ``scrapy.spidermiddlewares.referer.ReferrerPolicy``
subclass — a custom policy or one of the built-in ones (see classes below), subclass — a custom policy or one of the built-in ones (see classes below),

2
scrapy/settings/default_settings.py
View File

@ -234,7 +234,7 @@ REDIRECT_MAX_TIMES = 20 # uses Firefox default setting
REDIRECT_PRIORITY_ADJUST = +2 REDIRECT_PRIORITY_ADJUST = +2
REFERER_ENABLED = True REFERER_ENABLED = True
REFERER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy' REFERRER_POLICY = 'scrapy.spidermiddlewares.referer.DefaultReferrerPolicy'
RETRY_ENABLED = True RETRY_ENABLED = True
RETRY_TIMES = 2 # initial response + 2 retries = 3 requests RETRY_TIMES = 2 # initial response + 2 retries = 3 requests

2
scrapy/spidermiddlewares/referer.py
View File

@ -266,7 +266,7 @@ class RefererMiddleware(object):
def __init__(self, settings=None): def __init__(self, settings=None):
self.default_policy = DefaultReferrerPolicy self.default_policy = DefaultReferrerPolicy
if settings is not None: if settings is not None:
policy = settings.get('REFERER_POLICY') policy = settings.get('REFERRER_POLICY')
if policy is not None: if policy is not None:
# expect a string for the path to the policy class # expect a string for the path to the policy class
try: try:

36
tests/test_spidermiddleware_referer.py
View File

@ -289,35 +289,35 @@ class TestRefererMiddlewareDefault(MixinDefault, TestRefererMiddleware):
# --- Tests using settings to set policy using class path # --- Tests using settings to set policy using class path
class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware): class TestRefererMiddlewareSettingsNoReferrer(MixinNoReferrer, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerPolicy'}
class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): class TestRefererMiddlewareSettingsNoReferrerWhenDowngrade(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware): class TestRefererMiddlewareSettingsSameOrigin(MixinSameOrigin, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware): class TestRefererMiddlewareSettingsOrigin(MixinOrigin, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginPolicy'}
class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware): class TestRefererMiddlewareSettingsStrictOrigin(MixinStrictOrigin, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginPolicy'}
class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware): class TestRefererMiddlewareSettingsOriginWhenCrossOrigin(MixinOriginWhenCrossOrigin, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware): class TestRefererMiddlewareSettingsStrictOriginWhenCrossOrigin(MixinStrictOriginWhenCrossOrigin, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.StrictOriginWhenCrossOriginPolicy'}
class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware): class TestRefererMiddlewareSettingsUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.UnsafeUrlPolicy'}
class CustomPythonOrgPolicy(ReferrerPolicy): class CustomPythonOrgPolicy(ReferrerPolicy):
@ -336,7 +336,7 @@ class CustomPythonOrgPolicy(ReferrerPolicy):
class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware): class TestRefererMiddlewareSettingsCustomPolicy(TestRefererMiddleware):
settings = {'REFERER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'} settings = {'REFERRER_POLICY': 'tests.test_spidermiddleware_referer.CustomPythonOrgPolicy'}
scenarii = [ scenarii = [
('https://example.com/', 'https://scrapy.org/', b'https://python.org/'), ('https://example.com/', 'https://scrapy.org/', b'https://python.org/'),
('http://example.com/', 'http://scrapy.org/', b'http://python.org/'), ('http://example.com/', 'http://scrapy.org/', b'http://python.org/'),
@ -385,17 +385,17 @@ class TestRefererMiddlewareUnsafeUrl(MixinUnsafeUrl, TestRefererMiddleware):
class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware): class TestRefererMiddlewareMetaPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
req_meta = {'referrer_policy': POLICY_UNSAFE_URL} req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware): class TestRefererMiddlewareMetaPredecence002(MixinNoReferrer, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
req_meta = {'referrer_policy': POLICY_NO_REFERRER} req_meta = {'referrer_policy': POLICY_NO_REFERRER}
class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware): class TestRefererMiddlewareMetaPredecence003(MixinUnsafeUrl, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
req_meta = {'referrer_policy': POLICY_UNSAFE_URL} req_meta = {'referrer_policy': POLICY_UNSAFE_URL}
@ -411,7 +411,7 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
(POLICY_UNSAFE_URL, UnsafeUrlPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
]: ]:
settings = Settings({'REFERER_POLICY': s}) settings = Settings({'REFERRER_POLICY': s})
mw = RefererMiddleware(settings) mw = RefererMiddleware(settings)
self.assertEquals(mw.default_policy, p) self.assertEquals(mw.default_policy, p)
@ -425,24 +425,24 @@ class TestRefererMiddlewareSettingsPolicyByName(TestCase):
(POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy), (POLICY_ORIGIN_WHEN_CROSS_ORIGIN, OriginWhenCrossOriginPolicy),
(POLICY_UNSAFE_URL, UnsafeUrlPolicy), (POLICY_UNSAFE_URL, UnsafeUrlPolicy),
]: ]:
settings = Settings({'REFERER_POLICY': s.upper()}) settings = Settings({'REFERRER_POLICY': s.upper()})
mw = RefererMiddleware(settings) mw = RefererMiddleware(settings)
self.assertEquals(mw.default_policy, p) self.assertEquals(mw.default_policy, p)
def test_invalid_name(self): def test_invalid_name(self):
settings = Settings({'REFERER_POLICY': 'some-custom-unknown-policy'}) settings = Settings({'REFERRER_POLICY': 'some-custom-unknown-policy'})
with self.assertRaises(NotConfigured): with self.assertRaises(NotConfigured):
mw = RefererMiddleware(settings) mw = RefererMiddleware(settings)
class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware): class TestRefererMiddlewarePolicyHeaderPredecence001(MixinUnsafeUrl, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.SameOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()} resp_headers = {'Referrer-Policy': POLICY_UNSAFE_URL.upper()}
class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware): class TestRefererMiddlewarePolicyHeaderPredecence002(MixinNoReferrer, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.NoReferrerWhenDowngradePolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER.swapcase()}
class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware): class TestRefererMiddlewarePolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
settings = {'REFERER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'} settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()} resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}