BackTrackZ/scrapy
BackTrackZ/scrapy
1
0
Fork 0
mirror of https://github.com/scrapy/scrapy.git synced 2025-02-24 13:23:59 +00:00
Code Releases Wiki Activity

Merge pull request #2627 from redapple/referrer-empty-string-header

Browse Source 
Fix referrer policy from response headers and support explicit empty string
This commit is contained in:
Mikhail Korobov 2017-03-06 21:44:08 +05:00 committed by GitHub
commit 20c4f9f43a
2 changed files with 13 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
scrapy/spidermiddlewares/referer.py
View File

@ -261,6 +261,9 @@ _policy_classes = {p.name: p for p in (
DefaultReferrerPolicy,
)}
# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
_policy_classes[''] = NoReferrerWhenDowngradePolicy
def _load_policy_class(policy, warning_only=False):
"""
@ -317,8 +320,9 @@ class RefererMiddleware(object):
policy_name = request.meta.get('referrer_policy')
if policy_name is None:
if isinstance(resp_or_url, Response):
policy_name = to_native_str(
resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))
policy_header = resp_or_url.headers.get('Referrer-Policy')
if policy_header is not None:
policy_name = to_native_str(policy_header.decode('latin1'))
if policy_name is None:
return self.default_policy()

7
tests/test_spidermiddleware_referer.py
View File

@ -526,6 +526,13 @@ class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMid
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
"""
The empty string means "no-referrer-when-downgrade"
"""
settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
resp_headers = {'Referrer-Policy': ''}
class TestReferrerOnRedirect(TestRefererMiddleware):