Strip xlib.tx code of Twisted 10

scrapy/xlib/tx/__init__.py

@@ -15,9 +15,9 @@ else:
     client = endpoints = _Mock()
 
 
-Agent = client.Agent
-ProxyAgent = client.ProxyAgent
-ResponseDone = client.ResponseDone
-ResponseFailed = client.ResponseFailed
-HTTPConnectionPool = client.HTTPConnectionPool
-TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint
+Agent = client.Agent  # since < 11.1
+ProxyAgent = client.ProxyAgent  # since 11.1
+ResponseDone = client.ResponseDone  # since 11.1
+ResponseFailed = client.ResponseFailed  # since 11.1
+HTTPConnectionPool = client.HTTPConnectionPool  # since 12.1
+TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint  # since 10.1

scrapy/xlib/tx/_newclient.py

@@ -39,12 +39,25 @@ from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred
 from twisted.internet.defer import CancelledError
 from twisted.internet.protocol import Protocol
 from twisted.protocols.basic import LineReceiver
+from twisted.web.iweb import UNKNOWN_LENGTH
 from twisted.web.http_headers import Headers
 from twisted.web.http import NO_CONTENT, NOT_MODIFIED
 from twisted.web.http import _DataLoss, PotentialDataLoss
 from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder
 
-from .iweb import IResponse, UNKNOWN_LENGTH
+from twisted.web._newclient import (
+    BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException,
+    RequestGenerationFailed, RequestTransmissionFailed,
+    WrongBodyLength, ResponseDone, RequestNotSent,
+    LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder,
+    TransportProxyProducer,
+)
+# newer than 10.0.0
+#from twisted.web._newclient import (
+#    ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser,
+#    HTTPClientParser, Request, Response, HTTP11ClientProtocol,
+#)
+from .iweb import IResponse
 
 # States HTTPParser can be in
 STATUS = 'STATUS'
@@ -52,7 +65,7 @@ HEADER = 'HEADER'
 BODY = 'BODY'
 DONE = 'DONE'
 
-
+''' {{{
 class BadHeaders(Exception):
     """
     Headers passed to L{Request} were in some way invalid.
@@ -117,7 +130,7 @@ class RequestTransmissionFailed(_WrapperException):
     @ivar reasons: A C{list} of one or more L{Failure} instances giving the
         reasons the request transmission was considered to have failed.
     """
-
+}}} '''
 
 
 class ConnectionAborted(Exception):
@@ -126,7 +139,7 @@ class ConnectionAborted(Exception):
     """
 
 
-
+''' {{{
 class WrongBodyLength(Exception):
     """
     An L{IBodyProducer} declared the number of bytes it was going to
@@ -142,7 +155,7 @@ class ResponseDone(Exception):
     protocol passed to L{Response.deliverBody} and indicates that the entire
     response has been delivered.
     """
-
+}}} '''
 
 
 class ResponseFailed(_WrapperException):
@@ -169,7 +182,7 @@ class ResponseNeverReceived(ResponseFailed):
     """
 
 
-
+''' {{{
 class RequestNotSent(Exception):
     """
     L{RequestNotSent} indicates that an attempt was made to issue a request but
@@ -178,7 +191,7 @@ class RequestNotSent(Exception):
     to send a request using a protocol which is no longer connected to a
     server.
     """
-
+}}} '''
 
 
 def _callAppFunction(function):
@@ -764,7 +777,7 @@ class Request:
         _callAppFunction(self.bodyProducer.stopProducing)
 
 
-
+''' {{{
 class LengthEnforcingConsumer:
     """
     An L{IConsumer} proxy which enforces an exact length requirement on the
@@ -1188,7 +1201,7 @@ class TransportProxyProducer:
         """
         if self._producer is not None:
             self._producer.pauseProducing()
-
+}}} '''
 
 
 class HTTP11ClientProtocol(Protocol):

scrapy/xlib/tx/client.py

@@ -32,19 +32,29 @@ from twisted.internet.interfaces import IProtocol
 from twisted.python import failure
 from twisted.python.components import proxyForInterface
 from twisted.web import error
+from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer
 from twisted.web.http_headers import Headers
 
-from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
-from .iweb import IResponse, UNKNOWN_LENGTH, IBodyProducer
+from twisted.web.client import (
+    PartialDownloadError,
+)
+# newer than 10.0.0
+#from twisted.web.client import (
+#    CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer,
+#    HTTPConnectionPool, Agent, ProxyAgent,
+#)
 
+from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint
+from .iweb import IResponse
 
+''' {{{
 class PartialDownloadError(error.Error):
     """
     Page was only partially downloaded, we got disconnected in middle.
 
     @ivar response: All of the response body which was downloaded.
     """
-
+}}} '''
 
 class _URL(tuple):
     """
@@ -136,10 +146,13 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None,
 
 from twisted.web.error import SchemeNotSupported
 from ._newclient import Request, Response, HTTP11ClientProtocol
-from ._newclient import ResponseDone, ResponseFailed
-from ._newclient import RequestNotSent, RequestTransmissionFailed
+from twisted.web._newclient import ResponseDone
+from ._newclient import ResponseFailed
+from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed
+from twisted.web._newclient import (
+    PotentialDataLoss, _WrapperException)
 from ._newclient import (
-    ResponseNeverReceived, PotentialDataLoss, _WrapperException)
+    ResponseNeverReceived)
 
 try:
     from twisted.internet.ssl import ClientContextFactory
@@ -1161,8 +1174,7 @@ def readBody(response):
 
 
 __all__ = [
-    'PartialDownloadError', 'HTTPPageGetter', 'HTTPPageDownloader',
-    'HTTPClientFactory', 'HTTPDownloader', 'getPage', 'downloadPage',
+    'PartialDownloadError',
     'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent',
     'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent',
     'HTTPConnectionPool', 'readBody']

scrapy/xlib/tx/endpoints.py

@@ -15,23 +15,37 @@ parsed by the L{clientFromString} and L{serverFromString} functions.
 from __future__ import division, absolute_import
 
 import os
-import socket
+#import socket
 
 from zope.interface import implementer, directlyProvides
 import warnings
 
-from twisted.internet import interfaces, defer, error, fdesc, threads
+from twisted.internet import interfaces, defer, error, fdesc
 from twisted.internet.protocol import (
-        ClientFactory, Protocol, ProcessProtocol, Factory)
+        ClientFactory, Protocol, Factory)
+#from twisted.internet import threads, ProcessProtocol
 from twisted.internet.interfaces import IStreamServerEndpointStringParser
 from twisted.internet.interfaces import IStreamClientEndpointStringParser
 from twisted.python.filepath import FilePath
-from twisted.python.failure import Failure
-from twisted.python import log
+#from twisted.python.failure import Failure
+#from twisted.python import log
 from twisted.python.components import proxyForInterface
 
 from twisted.plugin import IPlugin, getPlugins
-from twisted.internet import stdio
+#from twisted.internet import stdio
+
+# newer than 10.0.0
+#from twisted.internet.endpoints import (
+#    TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint,
+#    UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol,
+#    quoteStringArgument,
+#    serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers
+#    clientFromString, #> using newer _clientParsers
+#    _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint,
+#    _parseTCP, _parseUNIX, _loadCAsFromDir,
+#    _parseSSL, _tokenize,
+#    _parseClientTCP, _parseClientSSL, _parseClientUNIX,
+#)
 
 from .interfaces import IFileDescriptorReceiver
 

scrapy/xlib/tx/interfaces.py

@@ -11,7 +11,28 @@ from __future__ import division, absolute_import
 
 from zope.interface import Interface, Attribute
 
-
+from twisted.internet.interfaces import (
+    IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL,
+    IReactorUDP, IReactorMulticast, IReactorProcess,
+    IReactorTime, IDelayedCall, IReactorThreads, IReactorCore,
+    IReactorPluggableResolver, IReactorFDSet,
+    IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor,
+    IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor,
+    ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer,
+    IProtocol, IProcessProtocol, IHalfCloseableProtocol,
+    IProtocolFactory, ITransport, IProcessTransport, IServiceCollection,
+    IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport,
+    IMulticastTransport,
+)
+# newer than 10.0.0
+#from twisted.internet.interfaces import (
+#    IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket,
+#    IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport,
+#    ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint,
+#    IStreamServerEndpointStringParser, IStreamClientEndpointStringParser,
+#)
+
+''' {{{
 class IAddress(Interface):
     """
     An address, e.g. a TCP C{(host, port)}.
@@ -74,7 +95,7 @@ class IResolverSimple(Interface):
         @raise twisted.internet.defer.TimeoutError: Raised (asynchronously)
         if the name cannot be resolved within the specified timeout period.
         """
-
+}}} '''
 
 
 class IResolver(IResolverSimple):
@@ -614,7 +635,7 @@ class IResolver(IResolverSimple):
         """
 
 
-
+''' {{{
 class IReactorTCP(Interface):
 
     def listenTCP(port, factory, backlog=50, interface=''):
@@ -701,7 +722,7 @@ class IReactorSSL(Interface):
 
         @param interface: the hostname to bind to, defaults to '' (all)
         """
-
+}}} '''
 
 
 class IReactorUNIX(Interface):
@@ -829,7 +850,7 @@ class IReactorWin32Events(Interface):
         """
 
 
-
+''' {{{
 class IReactorUDP(Interface):
     """
     UDP socket methods.
@@ -868,7 +889,7 @@ class IReactorMulticast(Interface):
         @see: L{twisted.internet.interfaces.IMulticastTransport}
         @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html}
         """
-
+}}} '''
 
 
 class IReactorSocket(Interface):
@@ -970,7 +991,7 @@ class IReactorSocket(Interface):
         """
 
 
-
+''' {{{
 class IReactorProcess(Interface):
 
     def spawnProcess(processProtocol, executable, args=(), env={}, path=None,
@@ -1347,7 +1368,7 @@ class IReactorPluggableResolver(Interface):
 
         @return: The previously installed resolver.
         """
-
+}}} '''
 
 class IReactorDaemonize(Interface):
     """
@@ -1379,7 +1400,7 @@ class IReactorDaemonize(Interface):
         """
 
 
-
+''' {{{
 class IReactorFDSet(Interface):
     """
     Implement me to be able to use L{IFileDescriptor} type resources.
@@ -1863,7 +1884,7 @@ class IHalfCloseableProtocol(Interface):
         This will never be called for TCP connections as TCP does not
         support notification of this type of half-close.
         """
-
+}}} '''
 
 
 class IFileDescriptorReceiver(Interface):
@@ -1884,7 +1905,7 @@ class IFileDescriptorReceiver(Interface):
         """
 
 
-
+''' {{{
 class IProtocolFactory(Interface):
     """
     Interface for protocol factories.
@@ -1974,7 +1995,7 @@ class ITransport(Interface):
 
         @return: An L{IAddress} provider.
         """
-
+}}} '''
 
 class ITCPTransport(ITransport):
     """
@@ -2095,7 +2116,7 @@ class ISSLTransport(ITCPTransport):
         Return an object with the peer's certificate info.
         """
 
-
+''' {{{
 class IProcessTransport(ITransport):
     """
     A process transport.
@@ -2324,7 +2345,7 @@ class IMulticastTransport(Interface):
         """
         Leave multicast group, return L{Deferred} of success.
         """
-
+}}} '''
 
 class IStreamClientEndpoint(Interface):
     """

scrapy/xlib/tx/iweb.py

@@ -12,8 +12,16 @@ Interface definitions for L{twisted.web}.
 
 from zope.interface import Interface, Attribute
 
-from twisted.internet.interfaces import IPushProducer
+#from twisted.internet.interfaces import IPushProducer
 
+from twisted.web.iweb import (
+    ICredentialFactory, IBodyProducer,
+    UNKNOWN_LENGTH,
+)
+# newer than 10.0.0
+#from twisted.web.iweb import (
+#    IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory,
+#)
 
 class IRequest(Interface):
     """
@@ -320,7 +328,7 @@ class IRequest(Interface):
         """
 
 
-
+''' {{{
 class ICredentialFactory(Interface):
     """
     A credential factory defines a way to generate a particular kind of
@@ -424,7 +432,7 @@ class IBodyProducer(IPushProducer):
         L{Deferred<twisted.internet.defer.Deferred>} returned by
         C{startProducing} is never fired.
         """
-
+}}} '''
 
 
 class IRenderable(Interface):
@@ -576,9 +584,9 @@ class _IRequestEncoderFactory(Interface):
         """
 
 
-
+''' {{{
 UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH"
-
+}}} '''
 __all__ = [
     "ICredentialFactory", "IRequest",
     "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder",