Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
looyolo
scrapy
提交
3df69462
S
scrapy
项目概览
looyolo
/
scrapy
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
S
scrapy
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
3df69462
编写于
3月 18, 2014
作者:
D
Daniel Graña
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #656 from kmike/httperror-logging
[MRG] Fix for #612 + small tests cleanup
上级
9edbb68b
60f9dc7a
变更
6
隐藏空白更改
内联
并排
Showing
6 changed file
with
108 addition
and
47 deletion
+108
-47
scrapy/contrib/spidermiddleware/httperror.py
scrapy/contrib/spidermiddleware/httperror.py
+7
-4
scrapy/tests/test_closespider.py
scrapy/tests/test_closespider.py
+1
-7
scrapy/tests/test_crawl.py
scrapy/tests/test_crawl.py
+1
-7
scrapy/tests/test_proxy_connect.py
scrapy/tests/test_proxy_connect.py
+1
-6
scrapy/tests/test_spidermiddleware_httperror.py
scrapy/tests/test_spidermiddleware_httperror.py
+85
-19
scrapy/utils/test.py
scrapy/utils/test.py
+13
-4
未找到文件。
scrapy/contrib/spidermiddleware/httperror.py
浏览文件 @
3df69462
...
...
@@ -25,7 +25,7 @@ class HttpErrorMiddleware(object):
self
.
handle_httpstatus_list
=
settings
.
getlist
(
'HTTPERROR_ALLOWED_CODES'
)
def
process_spider_input
(
self
,
response
,
spider
):
if
200
<=
response
.
status
<
300
:
# common case
if
200
<=
response
.
status
<
300
:
# common case
return
meta
=
response
.
meta
if
'handle_httpstatus_all'
in
meta
:
...
...
@@ -38,11 +38,14 @@ class HttpErrorMiddleware(object):
allowed_statuses
=
getattr
(
spider
,
'handle_httpstatus_list'
,
self
.
handle_httpstatus_list
)
if
response
.
status
in
allowed_statuses
:
return
log
.
msg
(
format
=
"Ignoring HTTP response code: not handled or not allowed: %(status_code)d"
,
level
=
log
.
DEBUG
,
spider
=
spider
,
status_code
=
response
.
status
)
raise
HttpError
(
response
,
'Ignoring non-200 response'
)
def
process_spider_exception
(
self
,
response
,
exception
,
spider
):
if
isinstance
(
exception
,
HttpError
):
log
.
msg
(
format
=
"Ignoring response %(response)r: HTTP status code is not handled or not allowed"
,
level
=
log
.
DEBUG
,
spider
=
spider
,
response
=
response
)
return
[]
scrapy/tests/test_closespider.py
浏览文件 @
3df69462
from
twisted.internet
import
defer
from
twisted.trial.unittest
import
TestCase
from
scrapy.utils.test
import
get_crawler
from
scrapy.utils.test
import
docrawl
from
scrapy.tests.spiders
import
FollowAllSpider
,
ItemSpider
,
ErrorSpider
from
scrapy.tests.mockserver
import
MockServer
def
docrawl
(
spider
,
settings
=
None
):
crawler
=
get_crawler
(
settings
)
crawler
.
configure
()
crawler
.
crawl
(
spider
)
return
crawler
.
start
()
class
TestCloseSpider
(
TestCase
):
def
setUp
(
self
):
...
...
scrapy/tests/test_crawl.py
浏览文件 @
3df69462
...
...
@@ -3,19 +3,13 @@ import socket
import
mock
from
twisted.internet
import
defer
from
twisted.trial.unittest
import
TestCase
from
scrapy.utils.test
import
get_crawler
,
get_testlog
from
scrapy.utils.test
import
docrawl
,
get_testlog
from
scrapy.tests.spiders
import
FollowAllSpider
,
DelaySpider
,
SimpleSpider
,
\
BrokenStartRequestsSpider
,
SingleRequestSpider
from
scrapy.tests.mockserver
import
MockServer
from
scrapy.http
import
Request
def
docrawl
(
spider
,
settings
=
None
):
crawler
=
get_crawler
(
settings
)
crawler
.
crawl
(
spider
)
return
crawler
.
start
()
class
CrawlTestCase
(
TestCase
):
def
setUp
(
self
):
...
...
scrapy/tests/test_proxy_connect.py
浏览文件 @
3df69462
...
...
@@ -8,16 +8,11 @@ from netlib import http_auth
from
twisted.internet
import
defer
from
twisted.trial.unittest
import
TestCase
from
scrapy.utils.test
import
get_
crawler
,
get_testlog
from
scrapy.utils.test
import
get_
testlog
,
docrawl
from
scrapy.tests.spiders
import
SimpleSpider
from
scrapy.tests.mockserver
import
MockServer
def
docrawl
(
spider
,
settings
=
None
):
crawler
=
get_crawler
(
settings
)
crawler
.
configure
()
crawler
.
crawl
(
spider
)
return
crawler
.
start
()
...
...
scrapy/tests/test_spidermiddleware_httperror.py
浏览文件 @
3df69462
from
unittest
import
TestCase
from
twisted.trial.unittest
import
TestCase
as
TrialTestCase
from
twisted.internet
import
defer
from
scrapy.utils.test
import
docrawl
,
get_testlog
from
scrapy.tests.mockserver
import
MockServer
from
scrapy.http
import
Response
,
Request
from
scrapy.spider
import
Spider
from
scrapy.contrib.spidermiddleware.httperror
import
HttpErrorMiddleware
,
HttpError
from
scrapy.settings
import
Settings
class
_HttpErrorSpider
(
Spider
):
name
=
'httperror'
start_urls
=
[
"http://localhost:8998/status?n=200"
,
"http://localhost:8998/status?n=404"
,
"http://localhost:8998/status?n=402"
,
"http://localhost:8998/status?n=500"
,
]
bypass_status_codes
=
set
()
def
__init__
(
self
,
*
args
,
**
kwargs
):
super
(
_HttpErrorSpider
,
self
).
__init__
(
*
args
,
**
kwargs
)
self
.
failed
=
set
()
self
.
skipped
=
set
()
self
.
parsed
=
set
()
def
start_requests
(
self
):
for
url
in
self
.
start_urls
:
yield
Request
(
url
,
self
.
parse
,
errback
=
self
.
on_error
)
def
parse
(
self
,
response
):
self
.
parsed
.
add
(
response
.
url
[
-
3
:])
def
on_error
(
self
,
failure
):
if
isinstance
(
failure
.
value
,
HttpError
):
response
=
failure
.
value
.
response
if
response
.
status
in
self
.
bypass_status_codes
:
self
.
skipped
.
add
(
response
.
url
[
-
3
:])
return
self
.
parse
(
response
)
# it assumes there is a response attached to failure
self
.
failed
.
add
(
failure
.
value
.
response
.
url
[
-
3
:])
return
failure
def
_responses
(
request
,
status_codes
):
responses
=
[]
for
code
in
status_codes
:
response
=
Response
(
request
.
url
,
status
=
code
)
response
.
request
=
request
responses
.
append
(
response
)
return
responses
class
TestHttpErrorMiddleware
(
TestCase
):
def
setUp
(
self
):
self
.
spider
=
Spider
(
'foo'
)
self
.
mw
=
HttpErrorMiddleware
(
Settings
({}))
self
.
req
=
Request
(
'http://scrapytest.org'
)
self
.
res200
=
Response
(
'http://scrapytest.org'
,
status
=
200
)
self
.
res200
.
request
=
self
.
req
self
.
res404
=
Response
(
'http://scrapytest.org'
,
status
=
404
)
self
.
res404
.
request
=
self
.
req
self
.
res200
,
self
.
res404
=
_responses
(
self
.
req
,
[
200
,
404
])
def
test_process_spider_input
(
self
):
self
.
assertEquals
(
None
,
...
...
@@ -43,6 +88,7 @@ class TestHttpErrorMiddleware(TestCase):
self
.
assertEquals
(
None
,
self
.
mw
.
process_spider_input
(
self
.
res404
,
self
.
spider
))
class
TestHttpErrorMiddlewareSettings
(
TestCase
):
"""Similar test, but with settings"""
...
...
@@ -50,13 +96,7 @@ class TestHttpErrorMiddlewareSettings(TestCase):
self
.
spider
=
Spider
(
'foo'
)
self
.
mw
=
HttpErrorMiddleware
(
Settings
({
'HTTPERROR_ALLOWED_CODES'
:
(
402
,)}))
self
.
req
=
Request
(
'http://scrapytest.org'
)
self
.
res200
=
Response
(
'http://scrapytest.org'
,
status
=
200
)
self
.
res200
.
request
=
self
.
req
self
.
res404
=
Response
(
'http://scrapytest.org'
,
status
=
404
)
self
.
res404
.
request
=
self
.
req
self
.
res402
=
Response
(
'http://scrapytest.org'
,
status
=
402
)
self
.
res402
.
request
=
self
.
req
self
.
res200
,
self
.
res404
,
self
.
res402
=
_responses
(
self
.
req
,
[
200
,
404
,
402
])
def
test_process_spider_input
(
self
):
self
.
assertEquals
(
None
,
...
...
@@ -86,19 +126,14 @@ class TestHttpErrorMiddlewareSettings(TestCase):
self
.
assertRaises
(
HttpError
,
self
.
mw
.
process_spider_input
,
self
.
res402
,
self
.
spider
)
class
TestHttpErrorMiddlewareHandleAll
(
TestCase
):
def
setUp
(
self
):
self
.
spider
=
Spider
(
'foo'
)
self
.
mw
=
HttpErrorMiddleware
(
Settings
({
'HTTPERROR_ALLOW_ALL'
:
True
}))
self
.
req
=
Request
(
'http://scrapytest.org'
)
self
.
res200
=
Response
(
'http://scrapytest.org'
,
status
=
200
)
self
.
res200
.
request
=
self
.
req
self
.
res404
=
Response
(
'http://scrapytest.org'
,
status
=
404
)
self
.
res404
.
request
=
self
.
req
self
.
res402
=
Response
(
'http://scrapytest.org'
,
status
=
402
)
self
.
res402
.
request
=
self
.
req
self
.
res200
,
self
.
res404
,
self
.
res402
=
_responses
(
self
.
req
,
[
200
,
404
,
402
])
def
test_process_spider_input
(
self
):
self
.
assertEquals
(
None
,
...
...
@@ -119,3 +154,34 @@ class TestHttpErrorMiddlewareHandleAll(TestCase):
self
.
assertRaises
(
HttpError
,
self
.
mw
.
process_spider_input
,
res402
,
self
.
spider
)
class
TestHttpErrorMiddlewareIntegrational
(
TrialTestCase
):
def
setUp
(
self
):
self
.
mockserver
=
MockServer
()
self
.
mockserver
.
__enter__
()
def
tearDown
(
self
):
self
.
mockserver
.
__exit__
(
None
,
None
,
None
)
@
defer
.
inlineCallbacks
def
test_middleware_works
(
self
):
spider
=
_HttpErrorSpider
()
yield
docrawl
(
spider
)
assert
not
spider
.
skipped
,
spider
.
skipped
self
.
assertEqual
(
spider
.
parsed
,
{
'200'
})
self
.
assertEqual
(
spider
.
failed
,
{
'404'
,
'402'
,
'500'
})
@
defer
.
inlineCallbacks
def
test_logging
(
self
):
spider
=
_HttpErrorSpider
(
bypass_status_codes
=
{
402
})
yield
docrawl
(
spider
)
# print(get_testlog())
self
.
assertEqual
(
spider
.
parsed
,
{
'200'
,
'402'
})
self
.
assertEqual
(
spider
.
skipped
,
{
'402'
})
self
.
assertEqual
(
spider
.
failed
,
{
'404'
,
'500'
})
log
=
get_testlog
()
self
.
assertIn
(
'Ignoring response <404'
,
log
)
self
.
assertIn
(
'Ignoring response <500'
,
log
)
self
.
assertNotIn
(
'Ignoring response <200'
,
log
)
self
.
assertNotIn
(
'Ignoring response <402'
,
log
)
scrapy/utils/test.py
浏览文件 @
3df69462
...
...
@@ -53,11 +53,13 @@ def get_testenv():
def
get_testlog
():
"""Get Scrapy log of current test, ignoring the rest"""
with
open
(
"test.log"
,
"rb"
)
as
fp
:
loglines
=
fp
.
readlines
()
thistest
=
[]
loglines
=
open
(
"test.log"
).
readlines
()
for
l
in
loglines
[::
-
1
]:
thistest
.
append
(
l
)
if
"[-] -->"
in
l
:
for
line
in
loglines
[::
-
1
]:
thistest
.
append
(
line
)
if
"[-] -->"
in
line
:
break
return
""
.
join
(
thistest
[::
-
1
])
...
...
@@ -67,3 +69,10 @@ def assert_samelines(testcase, text1, text2, msg=None):
line endings between platforms
"""
testcase
.
assertEqual
(
text1
.
splitlines
(),
text2
.
splitlines
(),
msg
)
def
docrawl
(
spider
,
settings
=
None
):
"""Configure and start Crawler; return the result of crawler.start()"""
crawler
=
get_crawler
(
settings
)
crawler
.
configure
()
crawler
.
crawl
(
spider
)
return
crawler
.
start
()
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录