如何写HTML页面解析脚本

本文目录导读：

1. 使用 Python + BeautifulSoup（最常用）
2. 使用 Python + lxml（性能更好）
3. 使用 Node.js + Cheerio
4. 使用 JavaScript（浏览器端）
5. 完整的 Python 解析脚本示例
6. 实用技巧
7. 选择建议

我来介绍几种常用的HTML页面解析脚本方法：

使用 Python + BeautifulSoup（最常用）

from bs4 import BeautifulSoup
import requests
# 获取网页内容
url = "https://example.com"
response = requests.get(url)
response.encoding = 'utf-8'
# 解析HTML
soup = BeautifulSoup(response.text, 'html.parser')
# 查找元素= soup.find('title').text
all_links = soup.find_all('a')
first_paragraph = soup.find('p')
# 使用CSS选择器
items = soup.select('.class-name')
specific_div = soup.select_one('#id-name')
# 提取数据
for link in all_links:
    href = link.get('href')
    text = link.text.strip()
    print(f"链接: {href}, 文本: {text}")

使用 Python + lxml（性能更好）

from lxml import html
import requests
page = requests.get('https://example.com')
tree = html.fromstring(page.content)
# XPath查询= tree.xpath('//title/text()')
links = tree.xpath('//a/@href')
paragraphs = tree.xpath('//p/text()')
# 复杂查询
data = tree.xpath('//div[@class="content"]//a[@href]')

使用 Node.js + Cheerio

const axios = require('axios');
const cheerio = require('cheerio');
async function parseHTML() {
    const response = await axios.get('https://example.com');
    const $ = cheerio.load(response.data);
    // 查找元素
    const title = $('title').text();
    const links = [];
    $('a').each((i, el) => {
        links.push({
            href: $(el).attr('href'),
            text: $(el).text().trim()
        });
    });
    // CSS选择器
    const items = $('.class-name').map((i, el) => $(el).text()).get();
    return { title, links, items };
}

使用 JavaScript（浏览器端）

// 在浏览器控制台运行
function parsePageHTML() {
    // 获取元素
    const title = document.querySelector('title').textContent;
    const links = Array.from(document.querySelectorAll('a')).map(a => ({
        href: a.href,
        text: a.textContent.trim()
    }));
    // 使用更复杂的选择器
    const articles = document.querySelectorAll('article');
    const data = Array.from(articles).map(article => ({
        title: article.querySelector('h2')?.textContent,
        content: article.querySelector('.content')?.innerHTML
    }));
    return { title, links, data };
}

完整的 Python 解析脚本示例

import requests
from bs4 import BeautifulSoup
import csv
import json
from urllib.parse import urljoin
class HTMLParser:
    def __init__(self, base_url):
        self.base_url = base_url
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })
    def fetch_page(self, url):
        """获取页面内容"""
        try:
            response = self.session.get(url, timeout=10)
            response.raise_for_status()
            return response.text
        except requests.RequestException as e:
            print(f"请求失败: {e}")
            return None
    def parse_links(self, html):
        """解析页面链接"""
        soup = BeautifulSoup(html, 'html.parser')
        links = []
        for a in soup.find_all('a', href=True):
            href = urljoin(self.base_url, a['href'])
            text = a.get_text(strip=True)
            if text and href.startswith('http'):
                links.append({'text': text, 'href': href})
        return links
    def parse_table(self, html):
        """解析表格数据"""
        soup = BeautifulSoup(html, 'html.parser')
        tables = []
        for table in soup.find_all('table'):
            rows = []
            for tr in table.find_all('tr'):
                cells = tr.find_all(['td', 'th'])
                row = [cell.get_text(strip=True) for cell in cells]
                if row:
                    rows.append(row)
            if rows:
                tables.append(rows)
        return tables
    def save_to_csv(self, data, filename):
        """保存为CSV文件"""
        if not data:
            return
        with open(filename, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=data[0].keys())
            writer.writeheader()
            writer.writerows(data)
    def save_to_json(self, data, filename):
        """保存为JSON文件"""
        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
# 使用示例
parser = HTMLParser('https://example.com')
html = parser.fetch_page('/some-page')
if html:
    # 解析链接
    links = parser.parse_links(html)
    parser.save_to_json(links, 'links.json')
    # 解析表格
    tables = parser.parse_table(html)
    if tables:
        parser.save_to_csv(tables[0], 'table_data.csv')

实用技巧

# 处理动态加载的页面
from selenium import webdriver
from selenium.webdriver.common.by import By
import time
driver = webdriver.Chrome()
driver.get('https://example.com')
# 等待页面加载
time.sleep(3)
# 获取完整HTML
html = driver.page_source
# 滚动加载更多内容
driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
time.sleep(2)
# 点击"加载更多"按钮
load_more = driver.find_element(By.CLASS_NAME, 'load-more')
load_more.click()
# 提取数据
elements = driver.find_elements(By.CSS_SELECTOR, '.item')
for element in elements:
    print(element.text)
driver.quit()

选择建议

• 简单静态页面：BeautifulSoup + requests
• 性能要求高：lxml
• JavaScript渲染：Selenium 或 Playwright
• Node.js环境：Cheerio
• 简单数据提取：正则表达式（不推荐复杂HTML）

根据你的具体需求和运行环境选择合适的工具即可。