Python视频爬虫案例如何抓取视频链接

wen python案例 2026-06-30 239

本文目录导读：

基础方法：解析HTML页面
处理动态加载的视频（使用Selenium）
抓取M3U8流媒体视频
实战案例：爬取B站视频
通用视频链接提取工具
注意事项

我来分享几个常见的Python视频爬虫案例,帮助你理解如何抓取视频链接。

基础方法：解析HTML页面

import requests
from bs4 import BeautifulSoup
import re
def get_video_links_from_page(url):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')
    video_links = []
    # 方法1：查找video标签
    for video in soup.find_all('video'):
        src = video.get('src')
        if src:
            video_links.append(src)
    # 方法2：查找包含视频链接的src属性
    for source in soup.find_all('source'):
        src = source.get('src')
        if src:
            video_links.append(src)
    # 方法3：正则查找视频链接
    pattern = r'(https?://[^\s"\']+\.(mp4|avi|mov|mkv|flv))'
    links = re.findall(pattern, response.text)
    video_links.extend([link[0] for link in links])
    return video_links

处理动态加载的视频（使用Selenium）

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
def get_video_links_dynamic(url):
    driver = webdriver.Chrome()  # 需要安装chromedriver
    driver.get(url)
    # 等待页面加载
    time.sleep(3)
    video_links = set()
    # 查找所有video元素
    videos = driver.find_elements(By.TAG_NAME, 'video')
    for video in videos:
        src = video.get_attribute('src')
        if src and src.startswith('http'):
            video_links.add(src)
    # 查找iframe中的视频（如YouTube等）
    iframes = driver.find_elements(By.TAG_NAME, 'iframe')
    for iframe in iframes:
        src = iframe.get_attribute('src')
        if src and 'youtube' in src:
            # 提取YouTube视频ID
            video_id = src.split('/')[-1].split('?')[0]
            video_links.add(f'https://www.youtube.com/watch?v={video_id}')
    driver.quit()
    return list(video_links)

抓取M3U8流媒体视频

import requests
import m3u8
from urllib.parse import urljoin
def get_m3u8_video_links(url):
    try:
        # 下载m3u8文件
        response = requests.get(url)
        response.raise_for_status()
        # 解析m3u8文件
        m3u8_obj = m3u8.loads(response.text)
        video_links = []
        base_url = url.rsplit('/', 1)[0] + '/'
        # 获取所有分片链接
        for segment in m3u8_obj.segments:
            full_url = urljoin(url, segment.uri)
            video_links.append(full_url)
        # 如果有备用播放列表
        for playlist in m3u8_obj.playlists:
            full_url = urljoin(url, playlist.uri)
            video_links.append(full_url)
        return video_links
    except Exception as e:
        print(f"解析m3u8失败: {e}")
        return []

实战案例：爬取B站视频

import requests
import json
import re
class BilibiliVideoCrawler:
    def __init__(self):
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            'Referer': 'https://www.bilibili.com'
        }
    def get_video_info(self, bv_id):
        """获取B站视频信息"""
        api_url = f'https://api.bilibili.com/x/web-interface/view?bvid={bv_id}'
        response = requests.get(api_url, headers=self.headers)
        data = response.json()
        if data['code'] != 0:
            print(f"获取视频信息失败: {data['message']}")
            return None
        return data['data']
    def get_video_links(self, bv_id, quality=80):
        """获取视频下载链接"""
        # 获取视频信息
        video_info = self.get_video_info(bv_id)
        if not video_info:
            return []
        # 获取cid
        cid = video_info['cid']
        # 获取播放地址
        play_url = f'https://api.bilibili.com/x/player/playurl?bvid={bv_id}&cid={cid}&qn={quality}'
        response = requests.get(play_url, headers=self.headers)
        data = response.json()
        if data['code'] != 0:
            print(f"获取播放地址失败: {data['message']}")
            return []
        # 解析视频链接
        video_links = []
        for durl in data['data']['durl']:
            video_links.append({
                'url': durl['url'],
                'size': durl['size'],
                'length': durl['length']
            })
        return video_links
    def download_video(self, url, filename):
        """下载视频文件"""
        response = requests.get(url, headers=self.headers, stream=True)
        total_size = int(response.headers.get('content-length', 0))
        with open(filename, 'wb') as f:
            downloaded = 0
            for chunk in response.iter_content(chunk_size=8192):
                if chunk:
                    f.write(chunk)
                    downloaded += len(chunk)
                    progress = (downloaded / total_size) * 100
                    print(f"\r下载进度: {progress:.2f}%", end='')
# 使用示例
def main():
    crawler = BilibiliVideoCrawler()
    # 替换为你要下载的视频BV号
    bv_id = 'BV1GJ411x7w4'
    # 获取视频链接
    links = crawler.get_video_links(bv_id)
    print(f"找到 {len(links)} 个视频链接")
    for i, link in enumerate(links):
        print(f"视频 {i+1}: {link['url'][:50]}... (大小: {link['size']/1024/1024:.2f}MB)")
    # 下载第一个视频
    if links:
        crawler.download_video(links[0]['url'], 'video.mp4')
        print("\n下载完成！")
if __name__ == '__main__':
    main()

通用视频链接提取工具

import re
import requests
from urllib.parse import urlparse, urljoin
class VideoLinkExtractor:
    def __init__(self):
        self.video_extensions = ['.mp4', '.avi', '.mov', '.mkv', '.flv', '.wmv', '.webm']
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
    def extract_from_html(self, url):
        """从HTML中提取视频链接"""
        links = set()
        try:
            response = requests.get(url, headers=self.headers, timeout=10)
            response.raise_for_status()
            # 正则匹配视频链接
            patterns = [
                r'https?://[^\s"\']+\.(?:mp4|avi|mov|mkv|flv|wmv|webm)',
                r'//[^\s"\']+\.(?:mp4|avi|mov|mkv|flv|wmv|webm)',
                r'src=["\']([^"\']+)["\']',
                r'href=["\']([^"\']+)["\']'
            ]
            for pattern in patterns:
                matches = re.findall(pattern, response.text, re.IGNORECASE)
                for match in matches:
                    if any(ext in match.lower() for ext in self.video_extensions):
                        if match.startswith('//'):
                            match = 'https:' + match
                        full_url = urljoin(url, match)
                        links.add(full_url)
            # 检查script标签中的JSON数据
            script_pattern = r'<script[^>]*>([^<]+)</script>'
            scripts = re.findall(script_pattern, response.text, re.DOTALL)
            for script in scripts:
                # 查找可能的视频配置
                json_pattern = r'\{[^{}]*"url"[^{}]*\}'
                json_matches = re.findall(json_pattern, script)
                for json_str in json_matches:
                    try:
                        data = json.loads(json_str)
                        if isinstance(data, dict):
                            for key in ['url', 'src', 'video_url', 'play_url']:
                                if key in data:
                                    links.add(data[key])
                    except:
                        pass
        except Exception as e:
            print(f"提取失败: {e}")
        return list(links)
# 使用示例
def main():
    extractor = VideoLinkExtractor()
    urls = [
        'https://example.com/video-page',
        # 添加更多URL
    ]
    for url in urls:
        print(f"\n正在分析: {url}")
        links = extractor.extract_from_html(url)
        print(f"找到 {len(links)} 个视频链接:")
        for link in links:
            print(f"  - {link}")
if __name__ == '__main__':
    main()