本文目录导读：
我来给你介绍几个Python网页解析的实际案例,从基础到进阶。
基础案例：使用BeautifulSoup提取文章内容
import requests
from bs4 import BeautifulSoup
# 发送请求
url = "https://example.com/article"
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
response = requests.get(url, headers=headers)
response.encoding = 'utf-8'
# 解析HTML
soup = BeautifulSoup(response.text, 'html.parser')
title = soup.find('h1').text.strip()
# 提取文章正文
article = soup.find('article')
paragraphs = article.find_all('p')
content = '\n'.join([p.text.strip() for p in paragraphs])
# 提取发布日期
date = soup.find('time', class_='post-date')
if date:
publish_date = date.get('datetime')
print(f"标题: {title}")
print(f"内容: {content[:100]}...")
提取链接和图片
def extract_links_and_images(soup):
# 提取所有链接
links = []
for a in soup.find_all('a', href=True):
href = a['href']
text = a.text.strip()
if href and not href.startswith('#'):
links.append({
'text': text,
'url': href
})
# 提取所有图片
images = []
for img in soup.find_all('img', src=True):
images.append({
'src': img['src'],
'alt': img.get('alt', ''),
'width': img.get('width', 'auto'),
'height': img.get('height', 'auto')
})
return links, images
表格数据提取
import pandas as pd
def extract_table_data(soup):
tables = soup.find_all('table')
all_tables = []
for table in tables:
# 提取表格头
headers = []
th_elements = table.find_all('th')
if th_elements:
headers = [th.text.strip() for th in th_elements]
# 提取表格数据
rows = []
for row in table.find_all('tr'):
cells = row.find_all('td')
if cells:
row_data = [cell.text.strip() for cell in cells]
rows.append(row_data)
# 转换为DataFrame
if rows:
if headers:
df = pd.DataFrame(rows, columns=headers)
else:
df = pd.DataFrame(rows)
all_tables.append(df)
return all_tables
动态页面解析（Selenium示例）
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
def scrape_dynamic_page(url):
# 设置无头浏览器
options = webdriver.ChromeOptions()
options.add_argument('--headless')
driver = webdriver.Chrome(options=options)
try:
driver.get(url)
# 等待页面加载
wait = WebDriverWait(driver, 10)
element = wait.until(
EC.presence_of_element_located((By.CLASS_NAME, "content"))
)
# 滚动页面加载更多内容
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(2)
# 提取数据
items = driver.find_elements(By.CLASS_NAME, "item")
data = []
for item in items:
title = item.find_element(By.TAG_NAME, "h2").text
description = item.find_element(By.CLASS_NAME, "description").text
data.append({"title": title, "description": description})
return data
finally:
driver.quit()
实用案例：新闻网站爬取
import json
from datetime import datetime
class NewsScraper:
def __init__(self):
self.session = requests.Session()
self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
})
def scrape_news_list(self, url):
"""爬取新闻列表"""
response = self.session.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
news_items = []
articles = soup.find_all('article', class_='news-item')
for article in articles:
item = {
'title': article.find('h2').text.strip(),
'link': article.find('a')['href'],
'summary': article.find('p', class_='summary').text.strip(),
'date': article.find('time').get('datetime') if article.find('time') else None
}
news_items.append(item)
return news_items
def scrape_news_detail(self, url):
"""爬取新闻详情"""
response = self.session.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
news_detail = {
'title': soup.find('h1', class_='title').text.strip(),
'author': soup.find('span', class_='author').text.strip(),
'publish_date': soup.find('time', class_='publish-date').get('datetime'),
'content': self._extract_content(soup),
'tags': [tag.text for tag in soup.find_all('a', class_='tag')]
}
return news_detail
def _extract_content(self, soup):
"""提取文章正文"""
article = soup.find('article') or soup.find('div', class_='content')
if article:
# 移除广告和无关元素
for unwanted in article.find_all(['script', 'style', 'advertisement']):
unwanted.decompose()
paragraphs = article.find_all('p')
return '\n'.join([p.text.strip() for p in paragraphs if p.text.strip()])
return ''
def save_to_json(self, data, filename):
"""保存为JSON文件"""
with open(filename, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2)
# 使用示例
scraper = NewsScraper()
# news_list = scraper.scrape_news_list('https://example.com/news')
# news_detail = scraper.scrape_news_detail(news_list[0]['link'])
处理常见问题
def robust_scrape(url, max_retries=3):
"""健壮的网页爬取"""
for attempt in range(max_retries):
try:
# 添加延迟避免被封
time.sleep(1)
# 使用代理
proxies = {
'http': 'http://proxy.example.com:8080',
'https': 'https://proxy.example.com:8080'
}
response = requests.get(
url,
headers=headers,
proxies=proxies,
timeout=10,
verify=False # 跳过SSL验证
)
response.raise_for_status() # 检查HTTP错误
# 自动检测编码
response.encoding = response.apparent_encoding
return response.text
except requests.RequestException as e:
print(f"尝试 {attempt + 1} 失败: {e}")
if attempt == max_retries - 1:
raise
安装依赖
pip install requests beautifulsoup4 lxml selenium pandas
使用建议
- 遵守robots.txt：查看网站的robots.txt文件
- 控制访问频率：添加延时，避免对服务器造成压力
- 异常处理：妥善处理网络错误和解析错误
- 数据验证：验证提取的数据完整性
这些案例涵盖了常见的网页解析场景,你可以根据实际需求选择合适的方案。
