lambda

NO39/中少绘本网爬虫.py

+import asyncio
+import time
+import requests
+from bs4 import BeautifulSoup
+import lxml
+
+BASE_URL = "http://banan.huiben.61read.com"
+
+
+async def requests_get(url):
+    headers = {
+        "Referer": "http://banan.huiben.61read.com/",
+        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.54 Safari/537.36"
+    }
+    try:
+        res = requests.get(url, headers=headers)
+        return res
+    except Exception as e:
+        print(e)
+        return None
+
+
+async def get_video(name, url):
+    res = await requests_get(url)
+    if res is not None:
+        with open(f'./mp4/{name}.mp4', "wb") as f:
+            f.write(res.content)
+        return (name, url, "success")
+    else:
+        return None
+
+
+async def get_list_url():
+    """获取列表页地址"""
+    res = await requests_get("http://banan.huiben.61read.com/")
+    soup = BeautifulSoup(res.text, "lxml")
+    all_a = []
+    for ul in soup.find_all(attrs={'class', 'inline'}):
+        all_a.extend(BASE_URL + _['href'] for _ in ul.find_all('a'))
+    return all_a
+
+
+async def get_mp4_url(url):
+    """获取MP4地址"""
+    res = await requests_get(url)
+    soup = BeautifulSoup(res.text, "lxml")
+    mp4s = []
+    for div_tag in soup.find_all(attrs={'class', 'item_list'}):
+        # 获取图片缩略图
+        src = div_tag.a.img['src']
+        # 将缩略图地址替换为 mp4 视频地址
+        src = src.replace('cover.jpg', 'web/1.mp4').replace('cover.png', 'web/1.mp4')
+        name = div_tag.div.a.text.strip()
+        mp4s.append((src, name))
+
+    return mp4s
+
+
+async def main():
+    # 获取列表页地址任务
+    task_list_url = asyncio.create_task(get_list_url())
+    all_a = await task_list_url
+    # 创建任务列表
+    tasks = [asyncio.ensure_future(get_mp4_url(url)) for url in all_a]
+    # 添加回调函数
+    # ret = map(lambda x: x.add_done_callback(callback), tasks)
+    # 异步执行
+    dones, pendings = await asyncio.wait(tasks)
+    all_mp4 = []
+    for task in dones:
+        all_mp4.extend(task.result())
+    # 获取到所有的MP4地址
+
+    totle = len(all_mp4)
+    print("累计获取到【", totle, "】个视频")
+    print("_" * 100)
+    print("准备下载视频")
+
+    # 每次下载10个
+    totle_page = totle // 10 if totle % 10 == 0 else totle // 10 + 1
+    # print(totle_page)
+    for page in range(0, totle_page):
+        print("正在下载第{}页视频".format(page + 1))
+        start_page = 0 if page == 0 else page * 10
+        end_page = (page + 1) * 10
+        print("待下载地址")
+        print(all_mp4[start_page:end_page])
+        mp4_download_tasks = [asyncio.ensure_future(get_video(name, url)) for url, name in all_mp4[start_page:end_page]]
+        mp4_dones, mp4_pendings = await asyncio.wait(mp4_download_tasks)
+        for task in mp4_dones:
+            print(task.result())
+
+
+if __name__ == '__main__':
+    asyncio.run(main())