Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
looyolo
scrapy
提交
89df18bd
S
scrapy
项目概览
looyolo
/
scrapy
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
S
scrapy
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
89df18bd
编写于
7月 31, 2014
作者:
J
Julia Medina
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Fix usage of crawler_process in ScrapyCommands
上级
d0edad4b
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
36 addition
and
41 deletion
+36
-41
scrapy/commands/bench.py
scrapy/commands/bench.py
+1
-3
scrapy/commands/check.py
scrapy/commands/check.py
+9
-23
scrapy/commands/crawl.py
scrapy/commands/crawl.py
+1
-3
scrapy/commands/edit.py
scrapy/commands/edit.py
+3
-4
scrapy/commands/genspider.py
scrapy/commands/genspider.py
+2
-3
scrapy/commands/list.py
scrapy/commands/list.py
+1
-2
scrapy/commands/runspider.py
scrapy/commands/runspider.py
+2
-3
scrapy/contracts/__init__.py
scrapy/contracts/__init__.py
+17
-0
未找到文件。
scrapy/commands/bench.py
浏览文件 @
89df18bd
...
...
@@ -22,9 +22,7 @@ class Command(ScrapyCommand):
def
run
(
self
,
args
,
opts
):
with
_BenchServer
():
spider
=
_BenchSpider
(
total
=
100000
)
crawler
=
self
.
crawler_process
.
create_crawler
()
crawler
.
crawl
(
spider
)
self
.
crawler_process
.
crawl
(
_BenchSpider
,
total
=
100000
)
self
.
crawler_process
.
start
()
...
...
scrapy/commands/check.py
浏览文件 @
89df18bd
...
...
@@ -69,20 +69,18 @@ class Command(ScrapyCommand):
# contract requests
contract_reqs
=
defaultdict
(
list
)
spman_cls
=
load_object
(
self
.
settings
[
'SPIDER_MANAGER_CLASS'
])
spiders
=
spman_cls
.
from_settings
(
self
.
settings
)
spiders
=
self
.
crawler_process
.
spiders
for
spider
in
args
or
spiders
.
list
():
spider
=
spiders
.
create
(
spider
)
requests
=
self
.
get_requests
(
spider
,
conman
,
result
)
contract_reqs
[
spider
.
name
]
=
[]
for
spidername
in
args
or
spiders
.
list
():
spidercls
=
spiders
.
load
(
spidername
)
spidercls
.
start_requests
=
lambda
s
:
conman
.
from_spider
(
s
,
result
)
tested_methods
=
conman
.
tested_methods_from_spidercls
(
spidercls
)
if
opts
.
list
:
for
req
in
requests
:
contract_reqs
[
spider
.
name
].
append
(
req
.
callback
.
__name__
)
elif
requests
:
crawler
=
self
.
crawler_process
.
create_crawler
(
spider
.
name
)
crawler
.
crawl
(
spider
,
requests
)
for
method
in
tested_methods
:
contract_reqs
[
spidercls
.
name
].
append
(
method
)
elif
tested_methods
:
self
.
crawler_process
.
crawl
(
spidercls
)
# start checks
if
opts
.
list
:
...
...
@@ -101,15 +99,3 @@ class Command(ScrapyCommand):
result
.
printSummary
(
start
,
stop
)
self
.
exitcode
=
int
(
not
result
.
wasSuccessful
())
def
get_requests
(
self
,
spider
,
conman
,
result
):
requests
=
[]
for
key
,
value
in
vars
(
type
(
spider
)).
items
():
if
callable
(
value
)
and
value
.
__doc__
:
bound_method
=
value
.
__get__
(
spider
,
type
(
spider
))
request
=
conman
.
from_method
(
bound_method
,
result
)
if
request
:
requests
.
append
(
request
)
return
requests
scrapy/commands/crawl.py
浏览文件 @
89df18bd
...
...
@@ -54,7 +54,5 @@ class Command(ScrapyCommand):
raise
UsageError
(
"running 'scrapy crawl' with more than one spider is no longer supported"
)
spname
=
args
[
0
]
crawler
=
self
.
crawler_process
.
create_crawler
()
spider
=
crawler
.
spiders
.
create
(
spname
,
**
opts
.
spargs
)
crawler
.
crawl
(
spider
)
self
.
crawler_process
.
crawl
(
spname
,
**
opts
.
spargs
)
self
.
crawler_process
.
start
()
scrapy/commands/edit.py
浏览文件 @
89df18bd
...
...
@@ -25,13 +25,12 @@ class Command(ScrapyCommand):
if
len
(
args
)
!=
1
:
raise
UsageError
()
crawler
=
self
.
crawler_process
.
create_crawler
()
editor
=
crawler
.
settings
[
'EDITOR'
]
editor
=
self
.
settings
[
'EDITOR'
]
try
:
spider
=
crawler
.
spiders
.
create
(
args
[
0
])
spider
cls
=
self
.
crawler_process
.
spiders
.
load
(
args
[
0
])
except
KeyError
:
return
self
.
_err
(
"Spider not found: %s"
%
args
[
0
])
sfile
=
sys
.
modules
[
spider
.
__module__
].
__file__
sfile
=
sys
.
modules
[
spider
cls
.
__module__
].
__file__
sfile
=
sfile
.
replace
(
'.pyc'
,
'.py'
)
self
.
exitcode
=
os
.
system
(
'%s "%s"'
%
(
editor
,
sfile
))
scrapy/commands/genspider.py
浏览文件 @
89df18bd
...
...
@@ -65,15 +65,14 @@ class Command(ScrapyCommand):
return
try
:
crawler
=
self
.
crawler_process
.
create_crawler
()
spider
=
crawler
.
spiders
.
create
(
name
)
spidercls
=
self
.
crawler_process
.
spiders
.
load
(
name
)
except
KeyError
:
pass
else
:
# if spider already exists and not --force then halt
if
not
opts
.
force
:
print
(
"Spider %r already exists in module:"
%
name
)
print
(
" %s"
%
spider
.
__module__
)
print
(
" %s"
%
spider
cls
.
__module__
)
return
template_file
=
self
.
_find_template
(
opts
.
template
)
if
template_file
:
...
...
scrapy/commands/list.py
浏览文件 @
89df18bd
...
...
@@ -10,6 +10,5 @@ class Command(ScrapyCommand):
return
"List available spiders"
def
run
(
self
,
args
,
opts
):
crawler
=
self
.
crawler_process
.
create_crawler
()
for
s
in
sorted
(
crawler
.
spiders
.
list
()):
for
s
in
sorted
(
self
.
crawler_process
.
spiders
.
list
()):
print
(
s
)
scrapy/commands/runspider.py
浏览文件 @
89df18bd
...
...
@@ -83,8 +83,7 @@ class Command(ScrapyCommand):
spclasses
=
list
(
iter_spider_classes
(
module
))
if
not
spclasses
:
raise
UsageError
(
"No spider found in file: %s
\n
"
%
filename
)
spider
=
spclasses
.
pop
()(
**
opts
.
spargs
)
spider
cls
=
spclasses
.
pop
(
)
crawler
=
self
.
crawler_process
.
create_crawler
()
crawler
.
crawl
(
spider
)
self
.
crawler_process
.
crawl
(
spidercls
,
**
opts
.
spargs
)
self
.
crawler_process
.
start
()
scrapy/contracts/__init__.py
浏览文件 @
89df18bd
...
...
@@ -15,6 +15,15 @@ class ContractsManager(object):
for
contract
in
contracts
:
self
.
contracts
[
contract
.
name
]
=
contract
def
tested_methods_from_spidercls
(
self
,
spidercls
):
methods
=
[]
for
key
,
value
in
vars
(
spidercls
).
items
():
if
(
callable
(
value
)
and
value
.
__doc__
and
re
.
search
(
r
'^\s*@'
,
value
.
__doc__
,
re
.
MULTILINE
)):
methods
.
append
(
key
)
return
methods
def
extract_contracts
(
self
,
method
):
contracts
=
[]
for
line
in
method
.
__doc__
.
split
(
'
\n
'
):
...
...
@@ -28,6 +37,14 @@ class ContractsManager(object):
return
contracts
def
from_spider
(
self
,
spider
,
results
):
requests
=
[]
for
method
in
self
.
tested_methods_from_spidercls
(
type
(
spider
)):
bound_method
=
spider
.
__getattribute__
(
method
)
requests
.
append
(
self
.
from_method
(
bound_method
,
results
))
return
requests
def
from_method
(
self
,
method
,
results
):
contracts
=
self
.
extract_contracts
(
method
)
if
contracts
:
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录