Backward compatibility for HTTP/10 context factory

New DOWNLOADER_CLIENT_TLS_METHOD setting to configure TLS method

scrapy/core/downloader/contextfactory.py

@@ -7,47 +7,79 @@ if twisted_version >= (14, 0, 0):
 
     from zope.interface.declarations import implementer
 
-    from twisted.internet.ssl import optionsForClientTLS
+    from twisted.internet.ssl import optionsForClientTLS, CertificateOptions, platformTrust
+    from twisted.internet._sslverify import ClientTLSOptions
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
 
     @implementer(IPolicyForHTTPS)
     class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
         """
-        Using Twisted recommended context factory for twisted.web.client.Agent
+        Non-peer-certificate verifying HTTPS context factory
 
-        Quoting:
-        "The default is to use a BrowserLikePolicyForHTTPS,
-        so unless you have special requirements you can leave this as-is."
+        Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
+        which allows TLS protocol negotiation
 
-        See http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html
+        'A TLS/SSL connection established with [this method] may
+         understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
         """
 
+        def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):
+            super(BrowserLikePolicyForHTTPS, self).__init__(*args, **kwargs)
+            self._ssl_method = method
+
+        def getCertificateOptions(self):
+            # setting verify=True will require you to provide CAs
+            # to verify against; in other words: it's not that simple
+            return CertificateOptions(verify=False, method=self._ssl_method)
+
+        # kept for old-style HTTP/1.0 downloader context twisted calls,
+        # e.g. connectSSL()
+        def getContext(self, hostname=None, port=None):
+            return self.getCertificateOptions().getContext()
+
+        def creatorForNetloc(self, hostname, port):
+            return ClientTLSOptions(hostname.decode("ascii"), self.getContext())
+
 
     @implementer(IPolicyForHTTPS)
-    class OpenSSLMethodContextFactory(ScrapyClientContextFactory):
+    class BrowserLikeContextFactory(ScrapyClientContextFactory):
+        """
+        Twisted-recommended context factory for web clients.
 
-        openssl_method = SSL.SSLv23_METHOD
+        Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
+        "The default is to use a BrowserLikePolicyForHTTPS,
+        so unless you have special requirements you can leave this as-is."
+
+        creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
+        except this context factory allows setting the TLS/SSL method to use.
 
+        Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
+        which allows TLS protocol negotiation.
+        """
         def creatorForNetloc(self, hostname, port):
+
+            # trustRoot set to platformTrust() will use the platform's root CAs.
+            #
+            # This means that a website like https://www.cacert.org will be rejected
+            # by default, since CAcert.org CA certificate is seldom shipped.
             return optionsForClientTLS(hostname.decode("ascii"),
-                                       trustRoot=self._trustRoot,
+                                       trustRoot=platformTrust(),
                                        extraCertificateOptions={
-                                            'method': self.openssl_method
+                                            'method': self._ssl_method,
                                        })
 
 
 else:
 
-    class OpenSSLMethodContextFactory(ClientContextFactory):
+    class ScrapyClientContextFactory(ClientContextFactory):
         "A SSL context factory which is more permissive against SSL bugs."
         # see https://github.com/scrapy/scrapy/issues/82
         # and https://github.com/scrapy/scrapy/issues/26
         # and https://github.com/scrapy/scrapy/issues/981
-        openssl_method = SSL.SSLv23_METHOD
 
-        def __init__(self):
-            self.method = self.openssl_method
+        def __init__(self, method=SSL.SSLv23_METHOD):
+            self.method = method
 
         def getContext(self, hostname=None, port=None):
             ctx = ClientContextFactory.getContext(self)
@@ -57,13 +89,3 @@ else:
             if hostname and ClientTLSOptions is not None: # workaround for TLS SNI
                 ClientTLSOptions(hostname, ctx)
             return ctx
-
-    ScrapyClientContextFactory = OpenSSLMethodContextFactory
-
-
-class SSLv3ContextFactory(OpenSSLMethodContextFactory):
-    openssl_method = SSL.SSLv3_METHOD
-
-
-class TLSv1ContextFactory(OpenSSLMethodContextFactory):
-    openssl_method = SSL.TLSv1_METHOD

scrapy/core/downloader/handlers/http11.py

@@ -18,6 +18,7 @@ from scrapy.xlib.tx import Agent, ProxyAgent, ResponseDone, \
 from scrapy.http import Headers
 from scrapy.responsetypes import responsetypes
 from scrapy.core.downloader.webclient import _parse
+from scrapy.core.downloader.tls import openssl_methods, METHOD_TLS
 from scrapy.utils.misc import load_object
 from scrapy.utils.python import to_bytes, to_unicode
 from scrapy import twisted_version
@@ -31,8 +32,16 @@ class HTTP11DownloadHandler(object):
         self._pool = HTTPConnectionPool(reactor, persistent=True)
         self._pool.maxPersistentPerHost = settings.getint('CONCURRENT_REQUESTS_PER_DOMAIN')
         self._pool._factory.noisy = False
+
+        self._sslMethod = openssl_methods[settings.get('DOWNLOADER_CLIENT_TLS_METHOD')]
         self._contextFactoryClass = load_object(settings['DOWNLOADER_CLIENTCONTEXTFACTORY'])
-        self._contextFactory = self._contextFactoryClass()
+        # try method-aware context factory
+        try:
+            self._contextFactory = self._contextFactoryClass(method=self._sslMethod)
+        except TypeError:
+            # use defaults
+            self._contextFactory = self._contextFactoryClass()
+
         self._default_maxsize = settings.getint('DOWNLOAD_MAXSIZE')
         self._default_warnsize = settings.getint('DOWNLOAD_WARNSIZE')
         self._disconnect_timeout = 1

scrapy/core/downloader/tls.py

+from OpenSSL import SSL
+
+
+METHOD_SSLv3 = 'SSLv3'
+METHOD_TLS = 'TLS'
+METHOD_TLSv10 = 'TLSv1.0'
+METHOD_TLSv11 = 'TLSv1.1'
+METHOD_TLSv12 = 'TLSv1.2'
+
+openssl_methods = {
+    METHOD_TLS: SSL.SSLv23_METHOD,                      # protocol negotiation (recommended)
+    METHOD_SSLv3: SSL.SSLv3_METHOD,                     # SSL 3 (NOT recommended)
+    METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
+    METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
+    METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
+}

scrapy/settings/default_settings.py

@@ -83,6 +83,8 @@ DOWNLOADER = 'scrapy.core.downloader.Downloader'
 
 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
+DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
+                                     # also allowing negotiation
 
 DOWNLOADER_MIDDLEWARES = {}
 

tests/keys/server.pem

+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDGnXh/GMCLpNNI
+AIvfBWlPrRCLFWxd2ICLYSUq3/jwh31CppGKlfaSmUYnrMxnT4hg2f6gBqlmq1gK
+jQqDkqQQtHsSljRQF58NRFtz99w45jRmrGs+F5zoggJuyv7/lKXy/BXOc40NodIl
+qRuo/uhoPjeXaUNziwpRj4rByRdLwQ6MfRcfdZ0TpVJ4J7apJ66pZt85L63u8TZi
+AZdBgcowX6giola7kUUMG66bSi2X6sIFxdXHwzrWlFNnSbKqOqKlGrjmBHUYqRFr
+gLALKkaMpF30olHn7QLqJ2592hMFVkOQAzNr8Xb15mF27BuWFYd7P6TirsxCEd+9
+BFFbhl0vAgMBAAECggEAf1ndN3GBlIi9SL/A7+GiYwpPPz8fWxVFZxmFIXa3QlM+
+CAyR6dC6Z8mL6EiuT9f5VFCzKZzb5g8bxrgk87SFKojvGT3ikTB0NaeNFFDrjjhd
+hTAtG4U8gQFL2gqjcvG3bpQgz13cJc+K1ccXC0dXce/i6Vz/eQjANwfZKuRr4Y/p
+Ml3ZtkarT7cZ2TZRDYd643U2nfrNmeLA9wwwO5mgNYu7HSpI4idpVM+rzoS9ZbU1
+Jxhrdq7GELvnd+Ko3WHTSrkQyb53GtJaLFOSAMxQlZAbRGJMv2VtXSPe+NyA0kxO
+V4O9CJ2QL1Bxqk4MQicKGtGoIsFbfR/qYQZgNEuHoQKBgQDlqEYSGpEcyKNvCjZ+
+Dv5hFUT7GZ24MNRvMBdMHPJEkmxvNpwaNkOGXL6zmhDk5Y0HOrzNv64Llmg623y0
+pn9Vh3CvMKDlq5t8910BJhXzZAM9/E9ui/YuQsFwCdWJQFYVMZUIAdylxbDkg6Mf
+WzHkx82edwWgb7hOFJhEW3h51wKBgQDdZaofTKeSLUmxIbF1yqm4iX67wyhHFFZL
+RZLEcu35ZTJqz3TRpr9KpHCtq9J2gZbqo2Dvqznwfk+yUqT2gkibtk3qvmv/qHQt
+FGX5joWLD2E22UgGc4bzonTSQcxPfPYjNZa6Iv7koPgJqO1k3aYLG3mDMgI/Yurz
+3j8udCI8aQKBgQCe8uNkfky3PkqXfNsQTnJTYTQ4EpettuYg+oj1Xgdz3F/+bS6H
+mc0Mfr5ichyFQDdmrImmYaclT/ld2zGpmp8A5FfieOazbx2T1WTieaixpyPzT+Du
+IyLFC+D0GWpYr1WlH4cFXryHY5c14cIR3r1emcc/iSM00n4eVHS6wxBUSwKBgQCQ
+fTAI20slgD7uxjF90XTwhXNbcONReDlVSKOfZ+5dbCj6QkhYJv4C5czP2yowwyRO
+H7A82j+m75htpbgZVS+zx8eUIxByumqPnRdsRhJrje8BD88MvsrdKDIbomuHmOM+
+WVP0SLCXX2JhU4kS4gDikNy9vZ5v5cu2ul37oDlTuQKBgBl4Qs6Q0xUotlvy/fMf
+L3c6GUIeRsLyQCKFfKYiUNLHqqutBSYmsAlx3XOhcK4DZ7/gFAXO6mpYJlwXOdrI
+IMcSXuJvZDPAYMXOyZUTgNjPcbQ8t0by+EKPwnTReIBf1Y17vPDfisf5IEKZEWvM
+YDXg6cfx9R5QePjvZohlItOu
+-----END PRIVATE KEY-----
+-----BEGIN CERTIFICATE-----
+MIID8DCCAtgCAws5AzANBgkqhkiG9w0BAQUFADCBuzELMAkGA1UEBhMCVFIxDjAM
+BgNVBAgUBcdvcnVtMRkwFwYDVQQHHhAAQgBhAV8AbQBhAGsA5wExMRIwEAYDVQQD
+Ewlsb2NhbGhvc3QxHDAaBgNVBAoTE1R3aXN0ZWQgTWF0cml4IExhYnMxJDAiBgNV
+BAsTG0F1dG9tYXRlZCBUZXN0aW5nIEF1dGhvcml0eTEpMCcGCSqGSIb3DQEJARYa
+c2VjdXJpdHlAdHdpc3RlZG1hdHJpeC5jb20wIBcNMTQwOTE2MDE0MjQ2WhgPMjEx
+NDA4MjMwMTQyNDZaMIG7MQswCQYDVQQGEwJUUjEOMAwGA1UECBQFx29ydW0xGTAX
+BgNVBAceEABCAGEBXwBtAGEAawDnATExEjAQBgNVBAMTCWxvY2FsaG9zdDEcMBoG
+A1UEChMTVHdpc3RlZCBNYXRyaXggTGFiczEkMCIGA1UECxMbQXV0b21hdGVkIFRl
+c3RpbmcgQXV0aG9yaXR5MSkwJwYJKoZIhvcNAQkBFhpzZWN1cml0eUB0d2lzdGVk
+bWF0cml4LmNvbTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMadeH8Y
+wIuk00gAi98FaU+tEIsVbF3YgIthJSrf+PCHfUKmkYqV9pKZRieszGdPiGDZ/qAG
+qWarWAqNCoOSpBC0exKWNFAXnw1EW3P33DjmNGasaz4XnOiCAm7K/v+UpfL8Fc5z
+jQ2h0iWpG6j+6Gg+N5dpQ3OLClGPisHJF0vBDox9Fx91nROlUngntqknrqlm3zkv
+re7xNmIBl0GByjBfqCKiVruRRQwbrptKLZfqwgXF1cfDOtaUU2dJsqo6oqUauOYE
+dRipEWuAsAsqRoykXfSiUeftAuonbn3aEwVWQ5ADM2vxdvXmYXbsG5YVh3s/pOKu
+zEIR370EUVuGXS8CAwEAATANBgkqhkiG9w0BAQUFAAOCAQEAK+HqbIUN6qHYYQZw
+7qqsTJWni8NOfK3aguyKAPcdCPP2DCZ6zlxxkUWL57gvsohyDu8Nr9iSI6wePjmI
+cN9eCZdc6mD9kYW4qBYhh2T48TOhEEW7zO6bWQUqWohHW+bG+GfrHnvxIx56OC2B
+eDS2djvvScYm45etlHprfrVEDIssh956O6qJCySax3D2w+i8YanXji1EbS61XTDw
+CMxDdWYmd2MDARRwlMcfcUIfKZUGl5NmqpnOx+H5MyAGwt86s647GMYZborQh+Mj
+tNHVpyKf/a8/HjqP1sCOrjCPZIjP6Qp5j4gifAjUStNmCgaBe7CpFtBSLnHqb4o/
+gU7u1w==
+-----END CERTIFICATE-----

tests/mockserver.py

@@ -206,8 +206,8 @@ class MockServer():
 
 def ssl_context_factory():
     return ssl.DefaultOpenSSLContextFactory(
-         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
-         os.path.join(os.path.dirname(__file__), 'keys/cert.pem'),
+         os.path.join(os.path.dirname(__file__), 'keys/server.pem'),
+         os.path.join(os.path.dirname(__file__), 'keys/server.pem'),
          )