bensound.com音频爬虫.py

import time

import asyncio
import aiohttp

from bs4 import BeautifulSoup
import lxml


async def get_html(client, url):
    print("正在采集", url)
    async with client.get(url, timeout=5) as resp:
        html = await resp.text()
        soup = BeautifulSoup(html, 'lxml')
        divs = soup.find_all(attrs={'class': 'img_mini'})
        mp3_urls = [get_mp3_url("https://www.bensound.com/" + div.a.img["src"]) for div in divs]
        return mp3_urls


def get_mp3_url(img_url):
    img_url = img_url
    name = img_url[img_url.rfind("/") + 1:img_url.rfind(".")]

    mp3_url = f"https://www.bensound.com/bensound-music/bensound-{name}.mp3"
    return mp3_url


async def get_mp3_file(client, url):
    print("正在采集 mp3 文件", url)
    headers = {
        "User-Agent": "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/94.0.4606.61 Safari/537.36",
        "Referer": "https://www.bensound.com/royalty-free-music"
    }
    mp3_file_name = url[url.rfind('-') + 1:url.rfind('.')]
    print(mp3_file_name)
    async with client.get(url, headers=headers) as resp:
        content = await resp.read()
        with open(f'./mp3/{mp3_file_name}.mp3', "wb") as f:
            f.write(content)
        return (url, "success")


async def main(urls):
    timeout = aiohttp.ClientTimeout(total=600)  # 将超时时间设置为600秒
    connector = aiohttp.TCPConnector(limit=50)  # 将并发数量降低
    async with aiohttp.ClientSession(connector=connector, timeout=timeout) as client:
        tasks = [asyncio.ensure_future(get_html(client, urls[i])) for i in range(len(urls))]

        dones, pendings = await asyncio.wait(tasks)
        print("异步执行完毕，开始输出对应结果：")
        all_mp3 = []
        for task in dones:
            all_mp3.extend(task.result())

        totle = len(all_mp3)
        print("累计获取到【", totle, "】个 MP3 文件")
        print("_" * 100)
        print("准备下载 MP3 文件")

        # 每次下载10个
        totle_page = totle // 10 if totle % 10 == 0 else totle // 10 + 1

        for page in range(0, totle_page):
            print("正在下载第{}页 MP3 文件".format(page + 1))
            start_page = 0 if page == 0 else page * 10
            end_page = (page + 1) * 10
            print("待下载地址")
            print(all_mp3[start_page:end_page])
            mp3_download_tasks = [asyncio.ensure_future(get_mp3_file(client, url)) for url in
                                  all_mp3[start_page:end_page]]
            mp3_dones, mp3_pendings = await asyncio.wait(mp3_download_tasks)
            for task in mp3_dones:
                print(task.result())


if __name__ == '__main__':
    url_format = "https://www.bensound.com/royalty-free-music/{}"
    urls = [url_format.format(i) for i in range(1, 5)]
    start_time = time.perf_counter()
    asyncio.run(main(urls))
    print("aiohttp 异步采集消耗时间为：", time.perf_counter() - start_time)