Skip to content
体验新版
项目
组织
正在加载...
登录
切换导航
打开侧边栏
moyewuji
Python-100-Days
提交
452b6f14
P
Python-100-Days
项目概览
moyewuji
/
Python-100-Days
与 Fork 源项目一致
从无法访问的项目Fork
通知
1
Star
1
Fork
0
代码
文件
提交
分支
Tags
贡献者
分支图
Diff
Issue
0
列表
看板
标记
里程碑
合并请求
0
Wiki
0
Wiki
分析
仓库
DevOps
项目成员
Pages
P
Python-100-Days
项目概览
项目概览
详情
发布
仓库
仓库
文件
提交
分支
标签
贡献者
分支图
比较
Issue
0
Issue
0
列表
看板
标记
里程碑
合并请求
0
合并请求
0
Pages
分析
分析
仓库分析
DevOps
Wiki
0
Wiki
成员
成员
收起侧边栏
关闭侧边栏
动态
分支图
创建新Issue
提交
Issue看板
体验新版 GitCode,发现更多精彩内容 >>
提交
452b6f14
编写于
5月 28, 2018
作者:
骆昊的技术专栏
浏览文件
操作
浏览文件
下载
电子邮件补丁
差异文件
更新了爬虫第1天代码
上级
402e0564
变更
3
隐藏空白更改
内联
并排
Showing
3 changed file
with
137 addition
and
0 deletion
+137
-0
Day66-75/code/example01.py
Day66-75/code/example01.py
+60
-0
Day66-75/code/example02.py
Day66-75/code/example02.py
+50
-0
Day66-75/code/example03.py
Day66-75/code/example03.py
+27
-0
未找到文件。
Day66-75/code/example01.py
0 → 100644
浏览文件 @
452b6f14
from
urllib.error
import
URLError
from
urllib.request
import
urlopen
import
re
import
pymysql
def
get_page_code
(
start_url
,
*
,
retry_times
=
3
,
charsets
=
(
'utf-8'
,
)):
try
:
for
charset
in
charsets
:
try
:
html
=
urlopen
(
start_url
).
read
().
decode
(
charset
)
break
except
UnicodeDecodeError
:
html
=
None
except
URLError
as
ex
:
print
(
'Error:'
,
ex
)
return
get_page_code
(
start_url
,
retry_times
=
retry_times
-
1
,
charsets
=
charsets
)
if
\
retry_times
>
0
else
None
return
html
def
main
():
url_list
=
[
'http://sports.sohu.com/nba_a.shtml'
]
visited_list
=
set
({})
while
len
(
url_list
)
>
0
:
current_url
=
url_list
.
pop
(
0
)
visited_list
.
add
(
current_url
)
print
(
current_url
)
html
=
get_page_code
(
current_url
,
charsets
=
(
'utf-8'
,
'gbk'
,
'gb2312'
))
if
html
:
link_regex
=
re
.
compile
(
r
'<a[^>]+href=["\'](.*?)["\']'
,
re
.
IGNORECASE
)
link_list
=
re
.
findall
(
link_regex
,
html
)
url_list
+=
link_list
conn
=
pymysql
.
connect
(
host
=
'localhost'
,
port
=
3306
,
db
=
'crawler'
,
user
=
'root'
,
passwd
=
'123456'
,
charset
=
'utf8'
)
try
:
for
link
in
link_list
:
if
link
not
in
visited_list
:
visited_list
.
add
(
link
)
print
(
link
)
html
=
get_page_code
(
link
,
charsets
=
(
'utf-8'
,
'gbk'
,
'gb2312'
))
if
html
:
title_regex
=
re
.
compile
(
r
'<h1>(.*)<span'
,
re
.
IGNORECASE
)
match_list
=
title_regex
.
findall
(
html
)
if
len
(
match_list
)
>
0
:
title
=
match_list
[
0
]
with
conn
.
cursor
()
as
cursor
:
cursor
.
execute
(
'insert into tb_result (rtitle, rurl) values (%s, %s)'
,
(
title
,
link
))
conn
.
commit
()
finally
:
conn
.
close
()
print
(
'执行完成!'
)
if
__name__
==
'__main__'
:
main
()
Day66-75/code/example02.py
0 → 100644
浏览文件 @
452b6f14
from
bs4
import
BeautifulSoup
import
re
def
main
():
html
=
"""
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<title>首页</title>
</head>
<body>
<h1>Hello, world!</h1>
<p>Good!!!</p>
<hr>
<div>
<h2>这是一个例子程序</h2>
<p>静夜思</p>
<p class="foo">床前明月光</p>
<p id="bar">疑似地上霜</p>
<p class="foo">举头望明月</p>
<div><a href="http://www.baidu.com"><p>低头思故乡</p></a></div>
</div>
<a class="foo" href="http://www.qq.com">腾讯网</a>
<img src="./img/pretty-girl.png" alt="美女">
<img src="./img/hellokitty.png" alt="凯蒂猫">
<img src="./static/img/pretty-girl.png" alt="美女">
<goup>Hello, Goup!</goup>
</body>
</html>
"""
# resp = requests.get('http://sports.sohu.com/nba_a.shtml')
# html = resp.content.decode('gbk')
soup
=
BeautifulSoup
(
html
,
'lxml'
)
print
(
soup
.
title
)
# JavaScript: document.body.h1
# JavaScript: document.forms[0]
print
(
soup
.
body
.
h1
)
print
(
soup
.
find_all
(
re
.
compile
(
r
'p$'
)))
print
(
soup
.
find_all
(
'img'
,
{
'src'
:
re
.
compile
(
r
'\./img/\w+.png'
)}))
print
(
soup
.
find_all
(
lambda
x
:
len
(
x
.
attrs
)
==
2
))
print
(
soup
.
find_all
(
'p'
,
{
'class'
:
'foo'
}))
for
elem
in
soup
.
select
(
'a[href]'
):
print
(
elem
.
attrs
[
'href'
])
if
__name__
==
'__main__'
:
main
()
Day66-75/code/example03.py
0 → 100644
浏览文件 @
452b6f14
from
bs4
import
BeautifulSoup
import
requests
import
re
def
main
():
# 通过requests第三方库的get方法获取页面
resp
=
requests
.
get
(
'http://sports.sohu.com/nba_a.shtml'
)
# 对响应的字节串(bytes)进行解码操作(搜狐的部分页面使用了GBK编码)
html
=
resp
.
content
.
decode
(
'gbk'
)
# 创建BeautifulSoup对象来解析页面(相当于JavaScript的DOM)
bs
=
BeautifulSoup
(
html
,
'lxml'
)
# 通过CSS选择器语法查找元素并通过循环进行处理
# for elem in bs.find_all(lambda x: 'test' in x.attrs):
for
elem
in
bs
.
select
(
'a[test]'
):
# 通过attrs属性(字典)获取元素的属性值
link_url
=
elem
.
attrs
[
'href'
]
resp
=
requests
.
get
(
link_url
)
bs_sub
=
BeautifulSoup
(
resp
.
text
,
'lxml'
)
# 使用正则表达式对获取的数据做进一步的处理
print
(
re
.
sub
(
r
'[\r\n]'
,
''
,
bs_sub
.
find
(
'h1'
).
text
))
if
__name__
==
'__main__'
:
main
()
编辑
预览
Markdown
is supported
0%
请重试
或
添加新附件
.
添加附件
取消
You are about to add
0
people
to the discussion. Proceed with caution.
先完成此消息的编辑!
取消
想要评论请
注册
或
登录