本文目录导读：
- 使用 Python + BeautifulSoup（最常用）
- 使用 Python + lxml（性能更好）
- 使用 Node.js + Cheerio
- 使用 JavaScript（浏览器端）
- 完整的 Python 解析脚本示例
- 实用技巧
- 选择建议
我来介绍几种常用的HTML页面解析脚本方法：
使用 Python + BeautifulSoup（最常用）
from bs4 import BeautifulSoup
import requests
# 获取网页内容
url = "https://example.com"
response = requests.get(url)
response.encoding = 'utf-8'
# 解析HTML
soup = BeautifulSoup(response.text, 'html.parser')
# 查找元素= soup.find('title').text
all_links = soup.find_all('a')
first_paragraph = soup.find('p')
# 使用CSS选择器
items = soup.select('.class-name')
specific_div = soup.select_one('#id-name')
# 提取数据
for link in all_links:
href = link.get('href')
text = link.text.strip()
print(f"链接: {href}, 文本: {text}")
使用 Python + lxml（性能更好）
from lxml import html
import requests
page = requests.get('https://example.com')
tree = html.fromstring(page.content)
# XPath查询= tree.xpath('//title/text()')
links = tree.xpath('//a/@href')
paragraphs = tree.xpath('//p/text()')
# 复杂查询
data = tree.xpath('//div[@class="content"]//a[@href]')
使用 Node.js + Cheerio
const axios = require('axios');
const cheerio = require('cheerio');
async function parseHTML() {
const response = await axios.get('https://example.com');
const $ = cheerio.load(response.data);
// 查找元素
const title = $('title').text();
const links = [];
$('a').each((i, el) => {
links.push({
href: $(el).attr('href'),
text: $(el).text().trim()
});
});
// CSS选择器
const items = $('.class-name').map((i, el) => $(el).text()).get();
return { title, links, items };
}
使用 JavaScript（浏览器端）
// 在浏览器控制台运行
function parsePageHTML() {
// 获取元素
const title = document.querySelector('title').textContent;
const links = Array.from(document.querySelectorAll('a')).map(a => ({
href: a.href,
text: a.textContent.trim()
}));
// 使用更复杂的选择器
const articles = document.querySelectorAll('article');
const data = Array.from(articles).map(article => ({
title: article.querySelector('h2')?.textContent,
content: article.querySelector('.content')?.innerHTML
}));
return { title, links, data };
}
完整的 Python 解析脚本示例
import requests
from bs4 import BeautifulSoup
import csv
import json
from urllib.parse import urljoin
class HTMLParser:
def __init__(self, base_url):
self.base_url = base_url
self.session = requests.Session()
self.session.headers.update({
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
})
def fetch_page(self, url):
"""获取页面内容"""
try:
response = self.session.get(url, timeout=10)
response.raise_for_status()
return response.text
except requests.RequestException as e:
print(f"请求失败: {e}")
return None
def parse_links(self, html):
"""解析页面链接"""
soup = BeautifulSoup(html, 'html.parser')
links = []
for a in soup.find_all('a', href=True):
href = urljoin(self.base_url, a['href'])
text = a.get_text(strip=True)
if text and href.startswith('http'):
links.append({'text': text, 'href': href})
return links
def parse_table(self, html):
"""解析表格数据"""
soup = BeautifulSoup(html, 'html.parser')
tables = []
for table in soup.find_all('table'):
rows = []
for tr in table.find_all('tr'):
cells = tr.find_all(['td', 'th'])
row = [cell.get_text(strip=True) for cell in cells]
if row:
rows.append(row)
if rows:
tables.append(rows)
return tables
def save_to_csv(self, data, filename):
"""保存为CSV文件"""
if not data:
return
with open(filename, 'w', newline='', encoding='utf-8') as f:
writer = csv.DictWriter(f, fieldnames=data[0].keys())
writer.writeheader()
writer.writerows(data)
def save_to_json(self, data, filename):
"""保存为JSON文件"""
with open(filename, 'w', encoding='utf-8') as f:
json.dump(data, f, ensure_ascii=False, indent=2)
# 使用示例
parser = HTMLParser('https://example.com')
html = parser.fetch_page('/some-page')
if html:
# 解析链接
links = parser.parse_links(html)
parser.save_to_json(links, 'links.json')
# 解析表格
tables = parser.parse_table(html)
if tables:
parser.save_to_csv(tables[0], 'table_data.csv')
实用技巧
# 处理动态加载的页面
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
driver = webdriver.Chrome()
driver.get('https://example.com')
# 等待页面加载
time.sleep(3)
# 获取完整HTML
html = driver.page_source
# 滚动加载更多内容
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(2)
# 点击"加载更多"按钮
load_more = driver.find_element(By.CLASS_NAME, 'load-more')
load_more.click()
# 提取数据
elements = driver.find_elements(By.CSS_SELECTOR, '.item')
for element in elements:
print(element.text)
driver.quit()
选择建议
- 简单静态页面：BeautifulSoup + requests
- 性能要求高：lxml
- JavaScript渲染：Selenium 或 Playwright
- Node.js环境：Cheerio
- 简单数据提取：正则表达式（不推荐复杂HTML）
根据你的具体需求和运行环境选择合适的工具即可。
