From e1ea0c433a96c2949b0a7c26cde65aea13fb79b4 Mon Sep 17 00:00:00 2001 From: nyov Date: Thu, 1 Dec 2016 22:02:10 +0000 Subject: [PATCH] Strip xlib.tx code of Twisted 10 --- scrapy/xlib/tx/__init__.py | 12 ++++----- scrapy/xlib/tx/_newclient.py | 31 ++++++++++++++++------- scrapy/xlib/tx/client.py | 28 +++++++++++++++------ scrapy/xlib/tx/endpoints.py | 26 ++++++++++++++----- scrapy/xlib/tx/interfaces.py | 49 +++++++++++++++++++++++++----------- scrapy/xlib/tx/iweb.py | 18 +++++++++---- 6 files changed, 116 insertions(+), 48 deletions(-) diff --git a/scrapy/xlib/tx/__init__.py b/scrapy/xlib/tx/__init__.py index 1ac4e0108..1c9bf09e5 100644 --- a/scrapy/xlib/tx/__init__.py +++ b/scrapy/xlib/tx/__init__.py @@ -15,9 +15,9 @@ else: client = endpoints = _Mock() -Agent = client.Agent -ProxyAgent = client.ProxyAgent -ResponseDone = client.ResponseDone -ResponseFailed = client.ResponseFailed -HTTPConnectionPool = client.HTTPConnectionPool -TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint +Agent = client.Agent # since < 11.1 +ProxyAgent = client.ProxyAgent # since 11.1 +ResponseDone = client.ResponseDone # since 11.1 +ResponseFailed = client.ResponseFailed # since 11.1 +HTTPConnectionPool = client.HTTPConnectionPool # since 12.1 +TCP4ClientEndpoint = endpoints.TCP4ClientEndpoint # since 10.1 diff --git a/scrapy/xlib/tx/_newclient.py b/scrapy/xlib/tx/_newclient.py index 16d0ca6b4..e902d6683 100644 --- a/scrapy/xlib/tx/_newclient.py +++ b/scrapy/xlib/tx/_newclient.py @@ -39,12 +39,25 @@ from twisted.internet.defer import Deferred, succeed, fail, maybeDeferred from twisted.internet.defer import CancelledError from twisted.internet.protocol import Protocol from twisted.protocols.basic import LineReceiver +from twisted.web.iweb import UNKNOWN_LENGTH from twisted.web.http_headers import Headers from twisted.web.http import NO_CONTENT, NOT_MODIFIED from twisted.web.http import _DataLoss, PotentialDataLoss from twisted.web.http import _IdentityTransferDecoder, _ChunkedTransferDecoder -from .iweb import IResponse, UNKNOWN_LENGTH +from twisted.web._newclient import ( + BadHeaders, ExcessWrite, ParseError, BadResponseVersion, _WrapperException, + RequestGenerationFailed, RequestTransmissionFailed, + WrongBodyLength, ResponseDone, RequestNotSent, + LengthEnforcingConsumer, makeStatefulDispatcher, ChunkedEncoder, + TransportProxyProducer, +) +# newer than 10.0.0 +#from twisted.web._newclient import ( +# ConnectionAborted, ResponseFailed, ResponseNeverReceived, HTTPParser, +# HTTPClientParser, Request, Response, HTTP11ClientProtocol, +#) +from .iweb import IResponse # States HTTPParser can be in STATUS = 'STATUS' @@ -52,7 +65,7 @@ HEADER = 'HEADER' BODY = 'BODY' DONE = 'DONE' - +''' {{{ class BadHeaders(Exception): """ Headers passed to L{Request} were in some way invalid. @@ -117,7 +130,7 @@ class RequestTransmissionFailed(_WrapperException): @ivar reasons: A C{list} of one or more L{Failure} instances giving the reasons the request transmission was considered to have failed. """ - +}}} ''' class ConnectionAborted(Exception): @@ -126,7 +139,7 @@ class ConnectionAborted(Exception): """ - +''' {{{ class WrongBodyLength(Exception): """ An L{IBodyProducer} declared the number of bytes it was going to @@ -142,7 +155,7 @@ class ResponseDone(Exception): protocol passed to L{Response.deliverBody} and indicates that the entire response has been delivered. """ - +}}} ''' class ResponseFailed(_WrapperException): @@ -169,7 +182,7 @@ class ResponseNeverReceived(ResponseFailed): """ - +''' {{{ class RequestNotSent(Exception): """ L{RequestNotSent} indicates that an attempt was made to issue a request but @@ -178,7 +191,7 @@ class RequestNotSent(Exception): to send a request using a protocol which is no longer connected to a server. """ - +}}} ''' def _callAppFunction(function): @@ -764,7 +777,7 @@ class Request: _callAppFunction(self.bodyProducer.stopProducing) - +''' {{{ class LengthEnforcingConsumer: """ An L{IConsumer} proxy which enforces an exact length requirement on the @@ -1188,7 +1201,7 @@ class TransportProxyProducer: """ if self._producer is not None: self._producer.pauseProducing() - +}}} ''' class HTTP11ClientProtocol(Protocol): diff --git a/scrapy/xlib/tx/client.py b/scrapy/xlib/tx/client.py index c3830dc47..396115985 100644 --- a/scrapy/xlib/tx/client.py +++ b/scrapy/xlib/tx/client.py @@ -32,19 +32,29 @@ from twisted.internet.interfaces import IProtocol from twisted.python import failure from twisted.python.components import proxyForInterface from twisted.web import error +from twisted.web.iweb import UNKNOWN_LENGTH, IBodyProducer from twisted.web.http_headers import Headers -from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint -from .iweb import IResponse, UNKNOWN_LENGTH, IBodyProducer +from twisted.web.client import ( + PartialDownloadError, +) +# newer than 10.0.0 +#from twisted.web.client import ( +# CookieAgent, GzipDecoder, ContentDecoderAgent, RedirectAgent, FileBodyProducer, +# HTTPConnectionPool, Agent, ProxyAgent, +#) +from .endpoints import TCP4ClientEndpoint, SSL4ClientEndpoint +from .iweb import IResponse +''' {{{ class PartialDownloadError(error.Error): """ Page was only partially downloaded, we got disconnected in middle. @ivar response: All of the response body which was downloaded. """ - +}}} ''' class _URL(tuple): """ @@ -136,10 +146,13 @@ def _makeGetterFactory(url, factoryFactory, contextFactory=None, from twisted.web.error import SchemeNotSupported from ._newclient import Request, Response, HTTP11ClientProtocol -from ._newclient import ResponseDone, ResponseFailed -from ._newclient import RequestNotSent, RequestTransmissionFailed +from twisted.web._newclient import ResponseDone +from ._newclient import ResponseFailed +from twisted.web._newclient import RequestNotSent, RequestTransmissionFailed +from twisted.web._newclient import ( + PotentialDataLoss, _WrapperException) from ._newclient import ( - ResponseNeverReceived, PotentialDataLoss, _WrapperException) + ResponseNeverReceived) try: from twisted.internet.ssl import ClientContextFactory @@ -1161,8 +1174,7 @@ def readBody(response): __all__ = [ - 'PartialDownloadError', 'HTTPPageGetter', 'HTTPPageDownloader', - 'HTTPClientFactory', 'HTTPDownloader', 'getPage', 'downloadPage', + 'PartialDownloadError', 'ResponseDone', 'Response', 'ResponseFailed', 'Agent', 'CookieAgent', 'ProxyAgent', 'ContentDecoderAgent', 'GzipDecoder', 'RedirectAgent', 'HTTPConnectionPool', 'readBody'] diff --git a/scrapy/xlib/tx/endpoints.py b/scrapy/xlib/tx/endpoints.py index d8a92ccd0..21a467433 100644 --- a/scrapy/xlib/tx/endpoints.py +++ b/scrapy/xlib/tx/endpoints.py @@ -15,23 +15,37 @@ parsed by the L{clientFromString} and L{serverFromString} functions. from __future__ import division, absolute_import import os -import socket +#import socket from zope.interface import implementer, directlyProvides import warnings -from twisted.internet import interfaces, defer, error, fdesc, threads +from twisted.internet import interfaces, defer, error, fdesc from twisted.internet.protocol import ( - ClientFactory, Protocol, ProcessProtocol, Factory) + ClientFactory, Protocol, Factory) +#from twisted.internet import threads, ProcessProtocol from twisted.internet.interfaces import IStreamServerEndpointStringParser from twisted.internet.interfaces import IStreamClientEndpointStringParser from twisted.python.filepath import FilePath -from twisted.python.failure import Failure -from twisted.python import log +#from twisted.python.failure import Failure +#from twisted.python import log from twisted.python.components import proxyForInterface from twisted.plugin import IPlugin, getPlugins -from twisted.internet import stdio +#from twisted.internet import stdio + +# newer than 10.0.0 +#from twisted.internet.endpoints import ( +# TCP4ServerEndpoint, TCP6ServerEndpoint, TCP4ClientEndpoint, SSL4ServerEndpoint, SSL4ClientEndpoint, +# UNIXServerEndpoint, UNIXClientEndpoint, AdoptedStreamServerEndpoint, connectProtocol, +# quoteStringArgument, +# serverFromString, #> using newer _parseSSL, _tokenize in _serverParsers +# clientFromString, #> using newer _clientParsers +# _WrappingProtocol, _WrappingFactory, _TCPServerEndpoint, +# _parseTCP, _parseUNIX, _loadCAsFromDir, +# _parseSSL, _tokenize, +# _parseClientTCP, _parseClientSSL, _parseClientUNIX, +#) from .interfaces import IFileDescriptorReceiver diff --git a/scrapy/xlib/tx/interfaces.py b/scrapy/xlib/tx/interfaces.py index f3e4ed5d8..a715d4a05 100644 --- a/scrapy/xlib/tx/interfaces.py +++ b/scrapy/xlib/tx/interfaces.py @@ -11,7 +11,28 @@ from __future__ import division, absolute_import from zope.interface import Interface, Attribute - +from twisted.internet.interfaces import ( + IAddress, IConnector, IResolverSimple, IReactorTCP, IReactorSSL, + IReactorUDP, IReactorMulticast, IReactorProcess, + IReactorTime, IDelayedCall, IReactorThreads, IReactorCore, + IReactorPluggableResolver, IReactorFDSet, + IListeningPort, ILoggingContext, IFileDescriptor, IReadDescriptor, + IWriteDescriptor, IReadWriteDescriptor, IHalfCloseableDescriptor, + ISystemHandle, IConsumer, IProducer, IPushProducer, IPullProducer, + IProtocol, IProcessProtocol, IHalfCloseableProtocol, + IProtocolFactory, ITransport, IProcessTransport, IServiceCollection, + IUDPTransport, IUNIXDatagramTransport, IUNIXDatagramConnectedTransport, + IMulticastTransport, +) +# newer than 10.0.0 +#from twisted.internet.interfaces import ( +# IResolver, IReactorUNIX, IReactorUNIXDatagram, IReactorWin32Events, IReactorSocket, +# IReactorDaemonize, IFileDescriptorReceiver, ITCPTransport, IUNIXTransport, +# ITLSTransport, ISSLTransport, IStreamClientEndpoint, IStreamServerEndpoint, +# IStreamServerEndpointStringParser, IStreamClientEndpointStringParser, +#) + +''' {{{ class IAddress(Interface): """ An address, e.g. a TCP C{(host, port)}. @@ -74,7 +95,7 @@ class IResolverSimple(Interface): @raise twisted.internet.defer.TimeoutError: Raised (asynchronously) if the name cannot be resolved within the specified timeout period. """ - +}}} ''' class IResolver(IResolverSimple): @@ -614,7 +635,7 @@ class IResolver(IResolverSimple): """ - +''' {{{ class IReactorTCP(Interface): def listenTCP(port, factory, backlog=50, interface=''): @@ -701,7 +722,7 @@ class IReactorSSL(Interface): @param interface: the hostname to bind to, defaults to '' (all) """ - +}}} ''' class IReactorUNIX(Interface): @@ -829,7 +850,7 @@ class IReactorWin32Events(Interface): """ - +''' {{{ class IReactorUDP(Interface): """ UDP socket methods. @@ -868,7 +889,7 @@ class IReactorMulticast(Interface): @see: L{twisted.internet.interfaces.IMulticastTransport} @see: U{http://twistedmatrix.com/documents/current/core/howto/udp.html} """ - +}}} ''' class IReactorSocket(Interface): @@ -970,7 +991,7 @@ class IReactorSocket(Interface): """ - +''' {{{ class IReactorProcess(Interface): def spawnProcess(processProtocol, executable, args=(), env={}, path=None, @@ -1347,7 +1368,7 @@ class IReactorPluggableResolver(Interface): @return: The previously installed resolver. """ - +}}} ''' class IReactorDaemonize(Interface): """ @@ -1379,7 +1400,7 @@ class IReactorDaemonize(Interface): """ - +''' {{{ class IReactorFDSet(Interface): """ Implement me to be able to use L{IFileDescriptor} type resources. @@ -1863,7 +1884,7 @@ class IHalfCloseableProtocol(Interface): This will never be called for TCP connections as TCP does not support notification of this type of half-close. """ - +}}} ''' class IFileDescriptorReceiver(Interface): @@ -1884,7 +1905,7 @@ class IFileDescriptorReceiver(Interface): """ - +''' {{{ class IProtocolFactory(Interface): """ Interface for protocol factories. @@ -1974,7 +1995,7 @@ class ITransport(Interface): @return: An L{IAddress} provider. """ - +}}} ''' class ITCPTransport(ITransport): """ @@ -2095,7 +2116,7 @@ class ISSLTransport(ITCPTransport): Return an object with the peer's certificate info. """ - +''' {{{ class IProcessTransport(ITransport): """ A process transport. @@ -2324,7 +2345,7 @@ class IMulticastTransport(Interface): """ Leave multicast group, return L{Deferred} of success. """ - +}}} ''' class IStreamClientEndpoint(Interface): """ diff --git a/scrapy/xlib/tx/iweb.py b/scrapy/xlib/tx/iweb.py index ddcb6ed7a..57a111411 100644 --- a/scrapy/xlib/tx/iweb.py +++ b/scrapy/xlib/tx/iweb.py @@ -12,8 +12,16 @@ Interface definitions for L{twisted.web}. from zope.interface import Interface, Attribute -from twisted.internet.interfaces import IPushProducer +#from twisted.internet.interfaces import IPushProducer +from twisted.web.iweb import ( + ICredentialFactory, IBodyProducer, + UNKNOWN_LENGTH, +) +# newer than 10.0.0 +#from twisted.web.iweb import ( +# IRequest, IRenderable, ITemplateLoader, IResponse, _IRequestEncoder, _IRequestEncoderFactory, +#) class IRequest(Interface): """ @@ -320,7 +328,7 @@ class IRequest(Interface): """ - +''' {{{ class ICredentialFactory(Interface): """ A credential factory defines a way to generate a particular kind of @@ -424,7 +432,7 @@ class IBodyProducer(IPushProducer): L{Deferred} returned by C{startProducing} is never fired. """ - +}}} ''' class IRenderable(Interface): @@ -576,9 +584,9 @@ class _IRequestEncoderFactory(Interface): """ - +''' {{{ UNKNOWN_LENGTH = u"twisted.web.iweb.UNKNOWN_LENGTH" - +}}} ''' __all__ = [ "ICredentialFactory", "IRequest", "IBodyProducer", "IRenderable", "IResponse", "_IRequestEncoder", -- GitLab