Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
looyolo
scrapy
提交
9cbbfd8b
S
scrapy
项目概览
looyolo
/
scrapy
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
S
scrapy
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
9cbbfd8b
编写于
8月 06, 2014
作者:
J
Julia Medina
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
Adjust spiders' utils to new SpiderManager API
上级
900a4876
变更
5
隐藏空白更改
内联
并排
Showing
5 changed file
with
65 addition
and
51 deletion
+65
-51
scrapy/commands/fetch.py
scrapy/commands/fetch.py
+6
-8
scrapy/commands/parse.py
scrapy/commands/parse.py
+26
-23
scrapy/commands/shell.py
scrapy/commands/shell.py
+19
-7
scrapy/shell.py
scrapy/shell.py
+3
-6
scrapy/utils/spider.py
scrapy/utils/spider.py
+11
-7
未找到文件。
scrapy/commands/fetch.py
浏览文件 @
9cbbfd8b
...
...
@@ -3,9 +3,8 @@ from w3lib.url import is_url
from
scrapy.command
import
ScrapyCommand
from
scrapy.http
import
Request
from
scrapy.spider
import
Spider
from
scrapy.exceptions
import
UsageError
from
scrapy.utils.spider
import
create_spider_for_request
from
scrapy.utils.spider
import
spidercls_for_request
,
DefaultSpider
class
Command
(
ScrapyCommand
):
...
...
@@ -48,12 +47,11 @@ class Command(ScrapyCommand):
request
=
Request
(
args
[
0
],
callback
=
cb
,
dont_filter
=
True
)
request
.
meta
[
'handle_httpstatus_all'
]
=
True
crawler
=
self
.
crawler_process
.
create_crawler
()
spider
=
None
spidercls
=
DefaultSpider
spider
s
=
self
.
crawler_process
.
spiders
if
opts
.
spider
:
spider
=
crawler
.
spiders
.
create
(
opts
.
spider
)
spider
cls
=
spiders
.
load
(
opts
.
spider
)
else
:
spider
=
create_spider_for_request
(
crawler
.
spiders
,
request
,
\
default_spider
=
Spider
(
'default'
))
crawler
.
crawl
(
spider
,
[
request
])
spidercls
=
spidercls_for_request
(
spiders
,
request
,
spidercls
)
self
.
crawler_process
.
crawl
(
spidercls
,
start_requests
=
lambda
:
[
request
])
self
.
crawler_process
.
start
()
scrapy/commands/parse.py
浏览文件 @
9cbbfd8b
...
...
@@ -5,7 +5,7 @@ from scrapy.http import Request
from
scrapy.item
import
BaseItem
from
scrapy.utils
import
display
from
scrapy.utils.conf
import
arglist_to_dict
from
scrapy.utils.spider
import
iterate_spider_output
,
create_spider
_for_request
from
scrapy.utils.spider
import
iterate_spider_output
,
spidercls
_for_request
from
scrapy.exceptions
import
UsageError
from
scrapy
import
log
...
...
@@ -113,41 +113,45 @@ class Command(ScrapyCommand):
requests
.
append
(
x
)
return
items
,
requests
def
get_callback_from_rules
(
self
,
response
):
if
getattr
(
s
elf
.
s
pider
,
'rules'
,
None
):
for
rule
in
s
elf
.
s
pider
.
rules
:
def
get_callback_from_rules
(
self
,
spider
,
response
):
if
getattr
(
spider
,
'rules'
,
None
):
for
rule
in
spider
.
rules
:
if
rule
.
link_extractor
.
matches
(
response
.
url
)
and
rule
.
callback
:
return
rule
.
callback
else
:
log
.
msg
(
format
=
'No CrawlSpider rules found in spider %(spider)r, '
'please specify a callback to use for parsing'
,
level
=
log
.
ERROR
,
spider
=
s
elf
.
s
pider
.
name
)
level
=
log
.
ERROR
,
spider
=
spider
.
name
)
def
set_spider
(
self
,
url
,
opts
):
def
set_spidercls
(
self
,
url
,
opts
):
spiders
=
self
.
crawler_process
.
spiders
if
opts
.
spider
:
try
:
self
.
spider
=
self
.
pcrawler
.
spiders
.
create
(
opts
.
spider
,
**
opts
.
spargs
)
self
.
spider
cls
=
spiders
.
load
(
opts
.
spider
)
except
KeyError
:
log
.
msg
(
format
=
'Unable to find spider: %(spider)s'
,
level
=
log
.
ERROR
,
spider
=
opts
.
spider
)
else
:
self
.
spider
=
create_spider_for_request
(
self
.
pcrawler
.
spiders
,
Request
(
url
),
**
opts
.
spargs
)
if
not
self
.
spider
:
self
.
spider
cls
=
spidercls_for_request
(
spiders
,
Request
(
url
)
)
if
not
self
.
spider
cls
:
log
.
msg
(
format
=
'Unable to find spider for: %(url)s'
,
level
=
log
.
ERROR
,
url
=
url
)
def
start_parsing
(
self
,
url
,
opts
):
request
=
Request
(
url
,
opts
.
callback
)
request
=
self
.
prepare_request
(
request
,
opts
)
_start_requests
=
lambda
s
:
[
self
.
prepare_request
(
s
,
request
,
opts
)]
self
.
spidercls
.
start_requests
=
_start_requests
self
.
pcrawler
.
crawl
(
self
.
spider
,
[
request
])
def
start_parsing
(
self
,
url
,
opts
):
self
.
crawler_process
.
crawl
(
self
.
spidercls
,
**
opts
.
spargs
)
self
.
pcrawler
=
list
(
self
.
crawler_process
.
crawlers
)[
0
]
self
.
crawler_process
.
start
()
if
not
self
.
first_response
:
log
.
msg
(
format
=
'No response downloaded for: %(
request
)s'
,
level
=
log
.
ERROR
,
request
=
request
)
log
.
msg
(
format
=
'No response downloaded for: %(
url
)s'
,
level
=
log
.
ERROR
,
url
=
url
)
def
prepare_request
(
self
,
request
,
opts
):
def
prepare_request
(
self
,
spider
,
request
,
opts
):
def
callback
(
response
):
# memorize first request
if
not
self
.
first_response
:
...
...
@@ -157,17 +161,17 @@ class Command(ScrapyCommand):
cb
=
response
.
meta
[
'_callback'
]
if
not
cb
:
if
opts
.
rules
and
self
.
first_response
==
response
:
cb
=
self
.
get_callback_from_rules
(
response
)
cb
=
self
.
get_callback_from_rules
(
spider
,
response
)
else
:
cb
=
'parse'
if
not
callable
(
cb
):
cb_method
=
getattr
(
s
elf
.
s
pider
,
cb
,
None
)
cb_method
=
getattr
(
spider
,
cb
,
None
)
if
callable
(
cb_method
):
cb
=
cb_method
else
:
log
.
msg
(
format
=
'Cannot find callback %(callback)r in spider: %(spider)s'
,
callback
=
callback
,
spider
=
s
elf
.
s
pider
.
name
,
level
=
log
.
ERROR
)
callback
=
callback
,
spider
=
spider
.
name
,
level
=
log
.
ERROR
)
return
# parse items and requests
...
...
@@ -177,7 +181,7 @@ class Command(ScrapyCommand):
if
opts
.
pipelines
:
itemproc
=
self
.
pcrawler
.
engine
.
scraper
.
itemproc
for
item
in
items
:
itemproc
.
process_item
(
item
,
s
elf
.
s
pider
)
itemproc
.
process_item
(
item
,
spider
)
self
.
add_items
(
depth
,
items
)
self
.
add_requests
(
depth
,
requests
)
...
...
@@ -207,10 +211,9 @@ class Command(ScrapyCommand):
else
:
url
=
args
[
0
]
# prepare spider
self
.
pcrawler
=
self
.
crawler_process
.
create_crawler
()
self
.
set_spider
(
url
,
opts
)
# prepare spidercls
self
.
set_spidercls
(
url
,
opts
)
if
self
.
spider
and
opts
.
depth
>
0
:
if
self
.
spider
cls
and
opts
.
depth
>
0
:
self
.
start_parsing
(
url
,
opts
)
self
.
print_results
(
opts
)
scrapy/commands/shell.py
浏览文件 @
9cbbfd8b
...
...
@@ -8,6 +8,9 @@ from threading import Thread
from
scrapy.command
import
ScrapyCommand
from
scrapy.shell
import
Shell
from
scrapy.http
import
Request
from
scrapy
import
log
from
scrapy.utils.spider
import
spidercls_for_request
,
DefaultSpider
class
Command
(
ScrapyCommand
):
...
...
@@ -38,18 +41,27 @@ class Command(ScrapyCommand):
pass
def
run
(
self
,
args
,
opts
):
crawler
=
self
.
crawler_process
.
create_crawler
()
url
=
args
[
0
]
if
args
else
None
spider
=
crawler
.
spiders
.
create
(
opts
.
spider
)
if
opts
.
spider
else
None
self
.
crawler_process
.
start_crawling
()
spiders
=
self
.
crawler_process
.
spiders
spidercls
=
DefaultSpider
if
opts
.
spider
:
spidercls
=
spiders
.
load
(
opts
.
spider
)
elif
url
:
spidercls
=
spidercls_for_request
(
spiders
,
Request
(
url
),
spidercls
,
log_multiple
=
True
)
crawler
=
self
.
crawler_process
.
_create_logged_crawler
(
spidercls
)
crawler
.
engine
=
crawler
.
_create_engine
()
crawler
.
engine
.
start
()
self
.
crawler_process
.
_start_logging
()
self
.
_start_crawler_thread
()
shell
=
Shell
(
crawler
,
update_vars
=
self
.
update_vars
,
code
=
opts
.
code
)
shell
.
start
(
url
=
url
,
spider
=
spider
)
shell
.
start
(
url
=
url
)
def
_start_crawler_thread
(
self
):
t
=
Thread
(
target
=
self
.
crawler_process
.
start_reactor
)
t
=
Thread
(
target
=
self
.
crawler_process
.
_start_reactor
,
kwargs
=
{
'stop_after_crawl'
:
False
})
t
.
daemon
=
True
t
.
start
()
scrapy/shell.py
浏览文件 @
9cbbfd8b
...
...
@@ -21,7 +21,6 @@ from scrapy.spider import Spider
from
scrapy.utils.console
import
start_python_console
from
scrapy.utils.misc
import
load_object
from
scrapy.utils.response
import
open_in_browser
from
scrapy.utils.spider
import
create_spider_for_request
class
Shell
(
object
):
...
...
@@ -67,11 +66,9 @@ class Shell(object):
return
self
.
spider
if
spider
is
None
:
spider
=
create_spider_for_request
(
self
.
crawler
.
spiders
,
request
,
Spider
(
'default'
),
log_multiple
=
True
)
spider
.
set_crawler
(
self
.
crawler
)
spider
=
self
.
crawler
.
spider
or
self
.
crawler
.
_create_spider
()
self
.
crawler
.
spider
=
spider
self
.
crawler
.
engine
.
open_spider
(
spider
,
close_if_idle
=
False
)
self
.
spider
=
spider
return
spider
...
...
scrapy/utils/spider.py
浏览文件 @
9cbbfd8b
...
...
@@ -4,6 +4,7 @@ import six
from
scrapy
import
log
from
scrapy.item
import
BaseItem
from
scrapy.spider
import
Spider
from
scrapy.utils.misc
import
arg_to_iter
...
...
@@ -25,21 +26,21 @@ def iter_spider_classes(module):
getattr
(
obj
,
'name'
,
None
):
yield
obj
def
create_spider_for_request
(
spidermanager
,
request
,
default_spider
=
None
,
\
log_none
=
False
,
log_multiple
=
False
,
**
spider_kwargs
):
"""
Create a spider to handle
the given Request.
def
spidercls_for_request
(
spidermanager
,
request
,
default_spidercls
=
None
,
log_none
=
False
,
log_multiple
=
False
):
"""
Return a spider class that handles
the given Request.
This will look for the spiders that can handle the given request (using
the spider manager) and return a
(new) Spider
if (and only if) there is
the spider manager) and return a
Spider class
if (and only if) there is
only one Spider able to handle the Request.
If multiple spiders (or no spider) are found, it will return the
default_spider passed. It can optionally log if multiple or no spiders
default_spider
cls
passed. It can optionally log if multiple or no spiders
are found.
"""
snames
=
spidermanager
.
find_by_request
(
request
)
if
len
(
snames
)
==
1
:
return
spidermanager
.
create
(
snames
[
0
],
**
spider_kwargs
)
return
spidermanager
.
load
(
snames
[
0
]
)
if
len
(
snames
)
>
1
and
log_multiple
:
log
.
msg
(
format
=
'More than one spider can handle: %(request)s - %(snames)s'
,
...
...
@@ -49,5 +50,8 @@ def create_spider_for_request(spidermanager, request, default_spider=None, \
log
.
msg
(
format
=
'Unable to find spider that handles: %(request)s'
,
level
=
log
.
ERROR
,
request
=
request
)
return
default_spider
return
default_spider
cls
class
DefaultSpider
(
Spider
):
name
=
'default'
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录