Merge pull request #2627 from redapple/referrer-empty-string-header

Fix referrer policy from response headers and support explicit empty string

scrapy/spidermiddlewares/referer.py

@@ -261,6 +261,9 @@ _policy_classes = {p.name: p for p in (
     DefaultReferrerPolicy,
 )}
 
+# Reference: https://www.w3.org/TR/referrer-policy/#referrer-policy-empty-string
+_policy_classes[''] = NoReferrerWhenDowngradePolicy
+
 
 def _load_policy_class(policy, warning_only=False):
     """
@@ -317,8 +320,9 @@ class RefererMiddleware(object):
         policy_name = request.meta.get('referrer_policy')
         if policy_name is None:
             if isinstance(resp_or_url, Response):
-                policy_name = to_native_str(
-                    resp_or_url.headers.get('Referrer-Policy', '').decode('latin1'))
+                policy_header = resp_or_url.headers.get('Referrer-Policy')
+                if policy_header is not None:
+                    policy_name = to_native_str(policy_header.decode('latin1'))
         if policy_name is None:
             return self.default_policy()
 

tests/test_spidermiddleware_referer.py

@@ -526,6 +526,13 @@ class TestPolicyHeaderPredecence003(MixinNoReferrerWhenDowngrade, TestRefererMid
     settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
     resp_headers = {'Referrer-Policy': POLICY_NO_REFERRER_WHEN_DOWNGRADE.title()}
 
+class TestPolicyHeaderPredecence004(MixinNoReferrerWhenDowngrade, TestRefererMiddleware):
+    """
+    The empty string means "no-referrer-when-downgrade"
+    """
+    settings = {'REFERRER_POLICY': 'scrapy.spidermiddlewares.referer.OriginWhenCrossOriginPolicy'}
+    resp_headers = {'Referrer-Policy': ''}
+
 
 class TestReferrerOnRedirect(TestRefererMiddleware):