Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
looyolo
scrapy
提交
dd5ab6c5
S
scrapy
项目概览
looyolo
/
scrapy
与 Fork 源项目一致
从无法访问的项目Fork
通知
2
Star
0
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
S
scrapy
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
dd5ab6c5
编写于
7月 03, 2017
作者:
M
Mikhail Korobov
提交者:
GitHub
7月 03, 2017
浏览文件
操作
浏览文件
下载
差异文件
Merge pull request #2793 from lopuhin/pypy2
Fix PyPy test failures
上级
5aebdac4
b0a92363
变更
8
隐藏空白更改
内联
并排
Showing
8 changed file
with
82 addition
and
23 deletion
+82
-23
.travis.yml
.travis.yml
+2
-5
scrapy/cmdline.py
scrapy/cmdline.py
+7
-1
scrapy/extensions/httpcache.py
scrapy/extensions/httpcache.py
+2
-1
scrapy/utils/python.py
scrapy/utils/python.py
+13
-0
setup.py
setup.py
+22
-1
tests/test_loader.py
tests/test_loader.py
+1
-1
tests/test_squeues.py
tests/test_squeues.py
+20
-10
tests/test_utils_python.py
tests/test_utils_python.py
+15
-4
未找到文件。
.travis.yml
浏览文件 @
dd5ab6c5
...
...
@@ -11,19 +11,16 @@ matrix:
env
:
TOXENV=py27
-
python
:
2.7
env
:
TOXENV=jessie
-
python
:
2.7
env
:
TOXENV=pypy
-
python
:
3.3
env
:
TOXENV=py33
-
python
:
3.5
env
:
TOXENV=py35
-
python
:
3.6
env
:
TOXENV=py36
-
python
:
2.7
env
:
TOXENV=pypy
-
python
:
3.6
env
:
TOXENV=docs
allow_failures
:
-
python
:
2.7
env
:
TOXENV=pypy
install
:
-
|
if [ "$TOXENV" = "pypy" ]; then
...
...
scrapy/cmdline.py
浏览文件 @
dd5ab6c5
...
...
@@ -11,6 +11,7 @@ from scrapy.commands import ScrapyCommand
from
scrapy.exceptions
import
UsageError
from
scrapy.utils.misc
import
walk_modules
from
scrapy.utils.project
import
inside_project
,
get_project_settings
from
scrapy.utils.python
import
garbage_collect
from
scrapy.settings.deprecated
import
check_deprecated_settings
def
_iter_command_classes
(
module_name
):
...
...
@@ -165,4 +166,9 @@ def _run_command_profiled(cmd, args, opts):
p
.
dump_stats
(
opts
.
profile
)
if
__name__
==
'__main__'
:
execute
()
try
:
execute
()
finally
:
# Twisted prints errors in DebugInfo.__del__, but PyPy does not run gc.collect()
# on exit: http://doc.pypy.org/en/latest/cpython_differences.html?highlight=gc.collect#differences-related-to-garbage-collection-strategies
garbage_collect
()
scrapy/extensions/httpcache.py
浏览文件 @
dd5ab6c5
...
...
@@ -13,7 +13,7 @@ from scrapy.responsetypes import responsetypes
from
scrapy.utils.request
import
request_fingerprint
from
scrapy.utils.project
import
data_path
from
scrapy.utils.httpobj
import
urlparse_cached
from
scrapy.utils.python
import
to_bytes
,
to_unicode
from
scrapy.utils.python
import
to_bytes
,
to_unicode
,
garbage_collect
logger
=
logging
.
getLogger
(
__name__
)
...
...
@@ -362,6 +362,7 @@ class LeveldbCacheStorage(object):
# avoid them being removed in storages with timestamp-based autoremoval.
self
.
db
.
CompactRange
()
del
self
.
db
garbage_collect
()
def
retrieve_response
(
self
,
spider
,
request
):
data
=
self
.
_read_data
(
spider
,
request
)
...
...
scrapy/utils/python.py
浏览文件 @
dd5ab6c5
"""
This module contains essential stuff that should've come with Python itself ;)
"""
import
gc
import
os
import
re
import
inspect
...
...
@@ -8,6 +9,8 @@ import weakref
import
errno
import
six
from
functools
import
partial
,
wraps
import
sys
import
time
from
scrapy.utils.decorators
import
deprecated
...
...
@@ -355,3 +358,13 @@ def global_object_name(obj):
'scrapy.http.request.Request'
"""
return
"%s.%s"
%
(
obj
.
__module__
,
obj
.
__name__
)
if
hasattr
(
sys
,
"pypy_version_info"
):
def
garbage_collect
():
# Collecting weakreferences can take two collections on PyPy.
gc
.
collect
()
gc
.
collect
()
else
:
def
garbage_collect
():
gc
.
collect
()
setup.py
浏览文件 @
dd5ab6c5
from
os.path
import
dirname
,
join
from
setuptools
import
setup
,
find_packages
from
pkg_resources
import
parse_version
from
setuptools
import
setup
,
find_packages
,
__version__
as
setuptools_version
with
open
(
join
(
dirname
(
__file__
),
'scrapy/VERSION'
),
'rb'
)
as
f
:
version
=
f
.
read
().
decode
(
'ascii'
).
strip
()
def
has_environment_marker_platform_impl_support
():
"""Code extracted from 'pytest/setup.py'
https://github.com/pytest-dev/pytest/blob/7538680c/setup.py#L31
The first known release to support environment marker with range operators
it is 18.5, see:
https://setuptools.readthedocs.io/en/latest/history.html#id235
"""
return
parse_version
(
setuptools_version
)
>=
parse_version
(
'18.5'
)
extras_require
=
{}
if
has_environment_marker_platform_impl_support
():
extras_require
[
':platform_python_implementation == "PyPy"'
]
=
[
'PyPyDispatcher>=2.1.0'
,
]
setup
(
name
=
'Scrapy'
,
version
=
version
,
...
...
@@ -53,4 +73,5 @@ setup(
'PyDispatcher>=2.0.5'
,
'service_identity'
,
],
extras_require
=
extras_require
,
)
tests/test_loader.py
浏览文件 @
dd5ab6c5
...
...
@@ -290,7 +290,7 @@ class BasicItemLoaderTest(unittest.TestCase):
il
=
TestItemLoader
()
il
.
add_value
(
'name'
,
[
u
'$10'
])
try
:
float
(
'$10'
)
float
(
u
'$10'
)
except
Exception
as
e
:
expected_exc_str
=
str
(
e
)
...
...
tests/test_squeues.py
浏览文件 @
dd5ab6c5
import
pickle
from
queuelib.tests
import
test_queue
as
t
from
scrapy.squeues
import
MarshalFifoDiskQueue
,
MarshalLifoDiskQueue
,
PickleFifoDiskQueue
,
PickleLifoDiskQueue
from
scrapy.item
import
Item
,
Field
...
...
@@ -14,6 +16,22 @@ class TestLoader(ItemLoader):
default_item_class
=
TestItem
name_out
=
staticmethod
(
_test_procesor
)
def
nonserializable_object_test
(
self
):
try
:
pickle
.
dumps
(
lambda
x
:
x
)
except
Exception
:
# Trigger Twisted bug #7989
import
twisted.persisted.styles
# NOQA
q
=
self
.
queue
()
self
.
assertRaises
(
ValueError
,
q
.
push
,
lambda
x
:
x
)
else
:
# Use a different unpickleable object
class
A
(
object
):
pass
a
=
A
()
a
.
__reduce__
=
a
.
__reduce_ex__
=
None
q
=
self
.
queue
()
self
.
assertRaises
(
ValueError
,
q
.
push
,
a
)
class
MarshalFifoDiskQueueTest
(
t
.
FifoDiskQueueTest
):
chunksize
=
100000
...
...
@@ -30,11 +48,7 @@ class MarshalFifoDiskQueueTest(t.FifoDiskQueueTest):
self
.
assertEqual
(
q
.
pop
(),
123
)
self
.
assertEqual
(
q
.
pop
(),
{
'a'
:
'dict'
})
def
test_nonserializable_object
(
self
):
# Trigger Twisted bug #7989
import
twisted.persisted.styles
# NOQA
q
=
self
.
queue
()
self
.
assertRaises
(
ValueError
,
q
.
push
,
lambda
x
:
x
)
test_nonserializable_object
=
nonserializable_object_test
class
ChunkSize1MarshalFifoDiskQueueTest
(
MarshalFifoDiskQueueTest
):
chunksize
=
1
...
...
@@ -110,11 +124,7 @@ class MarshalLifoDiskQueueTest(t.LifoDiskQueueTest):
self
.
assertEqual
(
q
.
pop
(),
123
)
self
.
assertEqual
(
q
.
pop
(),
'a'
)
def
test_nonserializable_object
(
self
):
# Trigger Twisted bug #7989
import
twisted.persisted.styles
# NOQA
q
=
self
.
queue
()
self
.
assertRaises
(
ValueError
,
q
.
push
,
lambda
x
:
x
)
test_nonserializable_object
=
nonserializable_object_test
class
PickleLifoDiskQueueTest
(
MarshalLifoDiskQueueTest
):
...
...
tests/test_utils_python.py
浏览文件 @
dd5ab6c5
import
gc
import
functools
import
operator
import
unittest
from
itertools
import
count
import
platform
import
six
from
scrapy.utils.python
import
(
...
...
@@ -144,6 +146,9 @@ class UtilsPythonTestCase(unittest.TestCase):
self
.
assertNotEqual
(
v
,
wk
[
_Weakme
()])
self
.
assertEqual
(
v
,
wk
[
k
])
del
k
for
_
in
range
(
100
):
if
wk
.
_weakdict
:
gc
.
collect
()
self
.
assertFalse
(
len
(
wk
.
_weakdict
))
@
unittest
.
skipUnless
(
six
.
PY2
,
"deprecated function"
)
...
...
@@ -208,10 +213,16 @@ class UtilsPythonTestCase(unittest.TestCase):
self
.
assertEqual
(
get_func_args
(
cal
),
[
'a'
,
'b'
,
'c'
])
self
.
assertEqual
(
get_func_args
(
object
),
[])
# TODO: how do we fix this to return the actual argument names?
self
.
assertEqual
(
get_func_args
(
six
.
text_type
.
split
),
[])
self
.
assertEqual
(
get_func_args
(
" "
.
join
),
[])
self
.
assertEqual
(
get_func_args
(
operator
.
itemgetter
(
2
)),
[])
if
platform
.
python_implementation
()
==
'CPython'
:
# TODO: how do we fix this to return the actual argument names?
self
.
assertEqual
(
get_func_args
(
six
.
text_type
.
split
),
[])
self
.
assertEqual
(
get_func_args
(
" "
.
join
),
[])
self
.
assertEqual
(
get_func_args
(
operator
.
itemgetter
(
2
)),
[])
else
:
self
.
assertEqual
(
get_func_args
(
six
.
text_type
.
split
),
[
'sep'
,
'maxsplit'
])
self
.
assertEqual
(
get_func_args
(
" "
.
join
),
[
'list'
])
self
.
assertEqual
(
get_func_args
(
operator
.
itemgetter
(
2
)),
[
'obj'
])
def
test_without_none_values
(
self
):
self
.
assertEqual
(
without_none_values
([
1
,
None
,
3
,
4
]),
[
1
,
3
,
4
])
...
...
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录