Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
looyolo
scrapy
提交
766f2d91
S
scrapy
项目概览
looyolo
/
scrapy
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
S
scrapy
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
766f2d91
编写于
9月 05, 2010
作者:
P
Pablo Hoffman
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Renamed Request Handlers to Download Handlers
上级
067ec65d
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
57 addition
and
56 deletion
+57
-56
docs/topics/settings.rst
docs/topics/settings.rst
+28
-28
scrapy/conf/default_settings.py
scrapy/conf/default_settings.py
+9
-8
scrapy/core/downloader/handlers/__init__.py
scrapy/core/downloader/handlers/__init__.py
+3
-3
scrapy/core/downloader/handlers/file.py
scrapy/core/downloader/handlers/file.py
+1
-1
scrapy/core/downloader/handlers/http.py
scrapy/core/downloader/handlers/http.py
+1
-1
scrapy/core/downloader/handlers/s3.py
scrapy/core/downloader/handlers/s3.py
+4
-4
scrapy/core/downloader/manager.py
scrapy/core/downloader/manager.py
+2
-2
scrapy/tests/test_downloader_handlers.py
scrapy/tests/test_downloader_handlers.py
+9
-9
未找到文件。
docs/topics/settings.rst
浏览文件 @
766f2d91
...
...
@@ -416,6 +416,34 @@ Another way to change the download delay (per spider, instead of globally) is
by using the ``download_delay`` spider attribute, which takes more precedence
than this setting.
.. setting:: DOWNLOAD_HANDLERS
DOWNLOAD_HANDLERS
-----------------
Default: ``{}``
A dict containing the request downloader handlers enabled in your project.
See `DOWNLOAD_HANDLERS_BASE` for example format.
.. setting:: DOWNLOAD_HANDLERS_BASE
DOWNLOAD_HANDLERS_BASE
----------------------
Default::
{
'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
'http': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler',
'https': 'scrapy.core.downloader.handlers.http.HttpDownloadHandler',
's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
}
A dict containing the request download handlers enabled by default in Scrapy.
You should never modify this setting in your project, modify
:setting:`DOWNLOAD_HANDLERS` instead.
.. setting:: DOWNLOAD_TIMEOUT
DOWNLOAD_TIMEOUT
...
...
@@ -756,34 +784,6 @@ Default: ``+2``
Adjust redirect request priority relative to original request.
A negative priority adjust means more priority.
.. setting:: REQUEST_HANDLERS
REQUEST_HANDLERS
----------------
Default: ``{}``
A dict containing the request downloader handlers enabled in your project.
See `REQUEST_HANDLERS_BASE` for example format.
.. setting:: REQUEST_HANDLERS_BASE
REQUEST_HANDLERS_BASE
---------------------
Default::
{
'file': 'scrapy.core.downloader.handlers.file.download_file',
'http': 'scrapy.core.downloader.handlers.http.download_http',
'https': 'scrapy.core.downloader.handlers.http.download_http',
's3': 'scrapy.core.downloader.handlers.s3.S3RequestHandler',
}
A dict containing the request download handlers enabled by default in Scrapy.
You should never modify this setting in your project, modify
:setting:`REQUEST_HANDLERS` instead.
.. setting:: REQUESTS_QUEUE_SIZE
REQUESTS_QUEUE_SIZE
...
...
scrapy/conf/default_settings.py
浏览文件 @
766f2d91
...
...
@@ -42,6 +42,15 @@ DEPTH_LIMIT = 0
DEPTH_STATS
=
True
DOWNLOAD_DELAY
=
0
DOWNLOAD_HANDLERS
=
{}
DOWNLOAD_HANDLERS_BASE
=
{
'file'
:
'scrapy.core.downloader.handlers.file.FileDownloadHandler'
,
'http'
:
'scrapy.core.downloader.handlers.http.HttpDownloadHandler'
,
'https'
:
'scrapy.core.downloader.handlers.http.HttpDownloadHandler'
,
's3'
:
'scrapy.core.downloader.handlers.s3.S3DownloadHandler'
,
}
DOWNLOAD_TIMEOUT
=
180
# 3mins
DOWNLOADER_DEBUG
=
False
...
...
@@ -183,14 +192,6 @@ REDIRECT_MAX_METAREFRESH_DELAY = 100
REDIRECT_MAX_TIMES
=
20
# uses Firefox default setting
REDIRECT_PRIORITY_ADJUST
=
+
2
REQUEST_HANDLERS
=
{}
REQUEST_HANDLERS_BASE
=
{
'file'
:
'scrapy.core.downloader.handlers.file.FileRequestHandler'
,
'http'
:
'scrapy.core.downloader.handlers.http.HttpRequestHandler'
,
'https'
:
'scrapy.core.downloader.handlers.http.HttpRequestHandler'
,
's3'
:
'scrapy.core.downloader.handlers.s3.S3RequestHandler'
,
}
REQUESTS_QUEUE_SIZE
=
0
# contrib.middleware.retry.RetryMiddleware default settings
...
...
scrapy/core/downloader/handlers/__init__.py
浏览文件 @
766f2d91
...
...
@@ -6,13 +6,13 @@ from scrapy.conf import settings
from
scrapy.utils.misc
import
load_object
class
Request
Handlers
(
object
):
class
Download
Handlers
(
object
):
def
__init__
(
self
):
self
.
_handlers
=
{}
self
.
_notconfigured
=
{}
handlers
=
settings
.
get
(
'
REQUEST
_HANDLERS_BASE'
)
handlers
.
update
(
settings
.
get
(
'
REQUEST
_HANDLERS'
,
{}))
handlers
=
settings
.
get
(
'
DOWNLOAD
_HANDLERS_BASE'
)
handlers
.
update
(
settings
.
get
(
'
DOWNLOAD
_HANDLERS'
,
{}))
for
scheme
,
clspath
in
handlers
.
iteritems
():
cls
=
load_object
(
clspath
)
try
:
...
...
scrapy/core/downloader/handlers/file.py
浏览文件 @
766f2d91
...
...
@@ -2,7 +2,7 @@ from scrapy.core.downloader.responsetypes import responsetypes
from
scrapy.utils.url
import
file_uri_to_path
from
scrapy.utils.decorator
import
defers
class
File
Request
Handler
(
object
):
class
File
Download
Handler
(
object
):
@
defers
def
download_request
(
self
,
request
,
spider
):
...
...
scrapy/core/downloader/handlers/http.py
浏览文件 @
766f2d91
...
...
@@ -17,7 +17,7 @@ HTTPClientFactory = load_object(settings['DOWNLOADER_HTTPCLIENTFACTORY'])
DOWNLOAD_TIMEOUT
=
settings
.
getint
(
'DOWNLOAD_TIMEOUT'
)
class
Http
Request
Handler
(
object
):
class
Http
Download
Handler
(
object
):
def
__init__
(
self
,
httpclientfactory
=
HTTPClientFactory
,
\
download_timeout
=
DOWNLOAD_TIMEOUT
):
...
...
scrapy/core/downloader/handlers/s3.py
浏览文件 @
766f2d91
...
...
@@ -2,13 +2,13 @@ from scrapy import optional_features
from
scrapy.exceptions
import
NotConfigured
from
scrapy.utils.httpobj
import
urlparse_cached
from
scrapy.conf
import
settings
from
.http
import
Http
Request
Handler
from
.http
import
Http
Download
Handler
class
S3
Request
Handler
(
object
):
class
S3
Download
Handler
(
object
):
def
__init__
(
self
,
aws_access_key_id
=
None
,
aws_secret_access_key
=
None
,
\
http
requesthandler
=
HttpRequest
Handler
):
http
downloadhandler
=
HttpDownload
Handler
):
if
'boto'
not
in
optional_features
:
raise
NotConfigured
(
"missing boto library"
)
...
...
@@ -22,7 +22,7 @@ class S3RequestHandler(object):
self
.
conn
=
connect_s3
(
aws_access_key_id
,
aws_secret_access_key
)
except
Exception
,
ex
:
raise
NotConfigured
(
str
(
ex
))
self
.
_download_http
=
http
request
handler
().
download_request
self
.
_download_http
=
http
download
handler
().
download_request
def
download_request
(
self
,
request
,
spider
):
p
=
urlparse_cached
(
request
)
...
...
scrapy/core/downloader/manager.py
浏览文件 @
766f2d91
...
...
@@ -13,7 +13,7 @@ from scrapy.conf import settings
from
scrapy.utils.defer
import
mustbe_deferred
from
scrapy
import
log
from
.middleware
import
DownloaderMiddlewareManager
from
.handlers
import
Request
Handlers
from
.handlers
import
Download
Handlers
class
SpiderInfo
(
object
):
...
...
@@ -71,7 +71,7 @@ class Downloader(object):
def
__init__
(
self
):
self
.
sites
=
{}
self
.
handlers
=
Request
Handlers
()
self
.
handlers
=
Download
Handlers
()
self
.
middleware
=
DownloaderMiddlewareManager
()
self
.
concurrent_spiders
=
settings
.
getint
(
'CONCURRENT_SPIDERS'
)
...
...
scrapy/tests/test_downloader_handlers.py
浏览文件 @
766f2d91
...
...
@@ -10,9 +10,9 @@ from twisted.web.test.test_webclient import ForeverTakingResource, \
PayloadResource
,
BrokenDownloadResource
from
scrapy.core.downloader.webclient
import
PartialDownloadError
from
scrapy.core.downloader.handlers.file
import
File
Request
Handler
from
scrapy.core.downloader.handlers.http
import
Http
Request
Handler
from
scrapy.core.downloader.handlers.s3
import
S3
Request
Handler
from
scrapy.core.downloader.handlers.file
import
File
Download
Handler
from
scrapy.core.downloader.handlers.http
import
Http
Download
Handler
from
scrapy.core.downloader.handlers.s3
import
S3
Download
Handler
from
scrapy.spider
import
BaseSpider
from
scrapy.http
import
Request
from
scrapy.utils.url
import
path_to_file_uri
...
...
@@ -26,7 +26,7 @@ class FileTestCase(unittest.TestCase):
fd
=
open
(
self
.
tmpname
+
'^'
,
'w'
)
fd
.
write
(
'0123456789'
)
fd
.
close
()
self
.
download_request
=
File
Request
Handler
().
download_request
self
.
download_request
=
File
Download
Handler
().
download_request
def
test_download
(
self
):
def
_test
(
response
):
...
...
@@ -61,7 +61,7 @@ class HttpTestCase(unittest.TestCase):
self
.
wrapper
=
WrappingFactory
(
self
.
site
)
self
.
port
=
reactor
.
listenTCP
(
0
,
self
.
wrapper
,
interface
=
'127.0.0.1'
)
self
.
portno
=
self
.
port
.
getHost
().
port
self
.
download_request
=
Http
Request
Handler
().
download_request
self
.
download_request
=
Http
Download
Handler
().
download_request
def
tearDown
(
self
):
return
self
.
port
.
stopListening
()
...
...
@@ -156,7 +156,7 @@ class HttpProxyTestCase(unittest.TestCase):
wrapper
=
WrappingFactory
(
site
)
self
.
port
=
reactor
.
listenTCP
(
0
,
wrapper
,
interface
=
'127.0.0.1'
)
self
.
portno
=
self
.
port
.
getHost
().
port
self
.
download_request
=
Http
Request
Handler
().
download_request
self
.
download_request
=
Http
Download
Handler
().
download_request
def
tearDown
(
self
):
return
self
.
port
.
stopListening
()
...
...
@@ -184,7 +184,7 @@ class HttpProxyTestCase(unittest.TestCase):
return
self
.
download_request
(
request
,
BaseSpider
(
'foo'
)).
addCallback
(
_test
)
class
Http
Request
HandlerMock
(
object
):
class
Http
Download
HandlerMock
(
object
):
def
download_request
(
self
,
request
,
spider
):
return
request
...
...
@@ -199,9 +199,9 @@ class S3TestCase(unittest.TestCase):
AWS_SECRET_ACCESS_KEY
=
'uV3F3YluFJax1cknvbcGwgjvx4QpvB+leU8dUj2o'
def
setUp
(
self
):
s3reqh
=
S3
Request
Handler
(
self
.
AWS_ACCESS_KEY_ID
,
\
s3reqh
=
S3
Download
Handler
(
self
.
AWS_ACCESS_KEY_ID
,
\
self
.
AWS_SECRET_ACCESS_KEY
,
\
http
requesthandler
=
HttpRequest
HandlerMock
)
http
downloadhandler
=
HttpDownload
HandlerMock
)
self
.
download_request
=
s3reqh
.
download_request
self
.
spider
=
BaseSpider
(
'foo'
)
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录