本文目录导读：
我来分享几个常见的Python视频爬虫案例,帮助你理解如何抓取视频链接。
基础方法：解析HTML页面
import requests
from bs4 import BeautifulSoup
import re
def get_video_links_from_page(url):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
response = requests.get(url, headers=headers)
soup = BeautifulSoup(response.text, 'html.parser')
video_links = []
# 方法1：查找video标签
for video in soup.find_all('video'):
src = video.get('src')
if src:
video_links.append(src)
# 方法2：查找包含视频链接的src属性
for source in soup.find_all('source'):
src = source.get('src')
if src:
video_links.append(src)
# 方法3：正则查找视频链接
pattern = r'(https?://[^\s"\']+\.(mp4|avi|mov|mkv|flv))'
links = re.findall(pattern, response.text)
video_links.extend([link[0] for link in links])
return video_links
处理动态加载的视频（使用Selenium）
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
import time
def get_video_links_dynamic(url):
driver = webdriver.Chrome() # 需要安装chromedriver
driver.get(url)
# 等待页面加载
time.sleep(3)
video_links = set()
# 查找所有video元素
videos = driver.find_elements(By.TAG_NAME, 'video')
for video in videos:
src = video.get_attribute('src')
if src and src.startswith('http'):
video_links.add(src)
# 查找iframe中的视频（如YouTube等）
iframes = driver.find_elements(By.TAG_NAME, 'iframe')
for iframe in iframes:
src = iframe.get_attribute('src')
if src and 'youtube' in src:
# 提取YouTube视频ID
video_id = src.split('/')[-1].split('?')[0]
video_links.add(f'https://www.youtube.com/watch?v={video_id}')
driver.quit()
return list(video_links)
抓取M3U8流媒体视频
import requests
import m3u8
from urllib.parse import urljoin
def get_m3u8_video_links(url):
try:
# 下载m3u8文件
response = requests.get(url)
response.raise_for_status()
# 解析m3u8文件
m3u8_obj = m3u8.loads(response.text)
video_links = []
base_url = url.rsplit('/', 1)[0] + '/'
# 获取所有分片链接
for segment in m3u8_obj.segments:
full_url = urljoin(url, segment.uri)
video_links.append(full_url)
# 如果有备用播放列表
for playlist in m3u8_obj.playlists:
full_url = urljoin(url, playlist.uri)
video_links.append(full_url)
return video_links
except Exception as e:
print(f"解析m3u8失败: {e}")
return []
实战案例：爬取B站视频
import requests
import json
import re
class BilibiliVideoCrawler:
def __init__(self):
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
'Referer': 'https://www.bilibili.com'
}
def get_video_info(self, bv_id):
"""获取B站视频信息"""
api_url = f'https://api.bilibili.com/x/web-interface/view?bvid={bv_id}'
response = requests.get(api_url, headers=self.headers)
data = response.json()
if data['code'] != 0:
print(f"获取视频信息失败: {data['message']}")
return None
return data['data']
def get_video_links(self, bv_id, quality=80):
"""获取视频下载链接"""
# 获取视频信息
video_info = self.get_video_info(bv_id)
if not video_info:
return []
# 获取cid
cid = video_info['cid']
# 获取播放地址
play_url = f'https://api.bilibili.com/x/player/playurl?bvid={bv_id}&cid={cid}&qn={quality}'
response = requests.get(play_url, headers=self.headers)
data = response.json()
if data['code'] != 0:
print(f"获取播放地址失败: {data['message']}")
return []
# 解析视频链接
video_links = []
for durl in data['data']['durl']:
video_links.append({
'url': durl['url'],
'size': durl['size'],
'length': durl['length']
})
return video_links
def download_video(self, url, filename):
"""下载视频文件"""
response = requests.get(url, headers=self.headers, stream=True)
total_size = int(response.headers.get('content-length', 0))
with open(filename, 'wb') as f:
downloaded = 0
for chunk in response.iter_content(chunk_size=8192):
if chunk:
f.write(chunk)
downloaded += len(chunk)
progress = (downloaded / total_size) * 100
print(f"\r下载进度: {progress:.2f}%", end='')
# 使用示例
def main():
crawler = BilibiliVideoCrawler()
# 替换为你要下载的视频BV号
bv_id = 'BV1GJ411x7w4'
# 获取视频链接
links = crawler.get_video_links(bv_id)
print(f"找到 {len(links)} 个视频链接")
for i, link in enumerate(links):
print(f"视频 {i+1}: {link['url'][:50]}... (大小: {link['size']/1024/1024:.2f}MB)")
# 下载第一个视频
if links:
crawler.download_video(links[0]['url'], 'video.mp4')
print("\n下载完成！")
if __name__ == '__main__':
main()
通用视频链接提取工具
import re
import requests
from urllib.parse import urlparse, urljoin
class VideoLinkExtractor:
def __init__(self):
self.video_extensions = ['.mp4', '.avi', '.mov', '.mkv', '.flv', '.wmv', '.webm']
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
def extract_from_html(self, url):
"""从HTML中提取视频链接"""
links = set()
try:
response = requests.get(url, headers=self.headers, timeout=10)
response.raise_for_status()
# 正则匹配视频链接
patterns = [
r'https?://[^\s"\']+\.(?:mp4|avi|mov|mkv|flv|wmv|webm)',
r'//[^\s"\']+\.(?:mp4|avi|mov|mkv|flv|wmv|webm)',
r'src=["\']([^"\']+)["\']',
r'href=["\']([^"\']+)["\']'
]
for pattern in patterns:
matches = re.findall(pattern, response.text, re.IGNORECASE)
for match in matches:
if any(ext in match.lower() for ext in self.video_extensions):
if match.startswith('//'):
match = 'https:' + match
full_url = urljoin(url, match)
links.add(full_url)
# 检查script标签中的JSON数据
script_pattern = r'<script[^>]*>([^<]+)</script>'
scripts = re.findall(script_pattern, response.text, re.DOTALL)
for script in scripts:
# 查找可能的视频配置
json_pattern = r'\{[^{}]*"url"[^{}]*\}'
json_matches = re.findall(json_pattern, script)
for json_str in json_matches:
try:
data = json.loads(json_str)
if isinstance(data, dict):
for key in ['url', 'src', 'video_url', 'play_url']:
if key in data:
links.add(data[key])
except:
pass
except Exception as e:
print(f"提取失败: {e}")
return list(links)
# 使用示例
def main():
extractor = VideoLinkExtractor()
urls = [
'https://example.com/video-page',
# 添加更多URL
]
for url in urls:
print(f"\n正在分析: {url}")
links = extractor.extract_from_html(url)
print(f"找到 {len(links)} 个视频链接:")
for link in links:
print(f" - {link}")
if __name__ == '__main__':
main()
注意事项
- 遵守法律法规：确保你的爬取行为符合相关法律和使用条款
- 尊重robots.txt：检查网站的爬取规则
- 控制请求频率：添加适当的延时，避免对服务器造成压力
- 处理反爬机制：注意验证码、IP封锁等问题
- 版权问题：不要下载受版权保护的视频用于商业用途
这些案例覆盖了常见的视频爬取场景,根据具体需求，你可能需要调整代码以适应不同的网站结构和反爬机制。
