Merge pull request #1786 from redapple/header-encoding

More liberal Content-Disposition header parsing

scrapy/responsetypes.py

@@ -59,7 +59,8 @@ class ResponseTypes(object):
 
     def from_content_disposition(self, content_disposition):
         try:
-            filename = to_native_str(content_disposition).split(';')[1].split('=')[1]
+            filename = to_native_str(content_disposition,
+                encoding='latin-1', errors='replace').split(';')[1].split('=')[1]
             filename = filename.strip('"\'')
             return self.from_filename(filename)
         except IndexError:

tests/test_responsetypes.py

+# -*- coding: utf-8 -*-
 import unittest
 from scrapy.responsetypes import responsetypes
 
@@ -20,8 +21,14 @@ class ResponseTypesTest(unittest.TestCase):
 
     def test_from_content_disposition(self):
         mappings = [
-            ('attachment; filename="data.xml"', XmlResponse),
-            ('attachment; filename=data.xml', XmlResponse),
+            (b'attachment; filename="data.xml"', XmlResponse),
+            (b'attachment; filename=data.xml', XmlResponse),
+            (u'attachment;filename=data£.tar.gz'.encode('utf-8'), Response),
+            (u'attachment;filename=dataµ.tar.gz'.encode('latin-1'), Response),
+            (u'attachment;filename=data高.doc'.encode('gbk'), Response),
+            (u'attachment;filename=دورهdata.html'.encode('cp720'), HtmlResponse),
+            (u'attachment;filename=日本語版Wikipedia.xml'.encode('iso2022_jp'), XmlResponse),
+
         ]
         for source, cls in mappings:
             retcls = responsetypes.from_content_disposition(source)