Python xpath案例如何解析网页数据

wen python案例 2026-06-30 229

本文目录导读：

环境准备
基础XPath案例
高级XPath技巧
最佳实践与注意事项
XPath常用表达式速查

我来为你详细介绍Python中使用XPath解析网页数据的完整案例。

环境准备

首先安装必要的库：

pip install lxml requests

基础XPath案例

示例1：解析静态HTML

from lxml import etree
import requests
# 准备示例HTML
html_content = """
<html>
    <body>
        <div class="container">
            <h1>文章列表</h1>
            <div class="article">
                <h2 class="title">Python入门教程</h2>
                <p class="author">作者：张三</p>
                <p class="date">2024-01-15</p>
                <p class="content">这是Python入门教程的内容...</p>
                <a href="https://example.com/article/1">阅读更多</a>
            </div>
            <div class="article">
                <h2 class="title">Web开发实战</h2>
                <p class="author">作者：李四</p>
                <p class="date">2024-01-16</p>
                <p class="content">这是Web开发实战的内容...</p>
                <a href="https://example.com/article/2">阅读更多</a>
            </div>
        </div>
        <div class="footer">
            <p>© 2024 版权所有</p>
        </div>
    </body>
</html>
"""
# 解析HTML
root = etree.HTML(html_content)
# 1. 获取所有文章标题s = root.xpath('//h2[@class="title"]/text()')
print("文章标题:", titles)
# 2. 获取所有文章作者
authors = root.xpath('//p[@class="author"]/text()')
print("作者:", authors)
# 3. 获取所有文章链接
links = root.xpath('//div[@class="article"]/a/@href')
print("链接:", links)
# 4. 获取第一篇文章的完整信息
first_article = root.xpath('//div[@class="article"][1]')
if first_article:= first_article[0].xpath('.//h2/text()')[0]
    author = first_article[0].xpath('.//p[@class="author"]/text()')[0]
    date = first_article[0].xpath('.//p[@class="date"]/text()')[0]
    print(f"第一篇文章: {title} - {author} - {date}")
# 5. 复杂的筛选条件
python_articles = root.xpath('//div[@class="article" and contains(.//h2/text(), "Python")]')
print(f"包含Python的文章数: {len(python_articles)}")

示例2：真实网站数据抓取

import requests
from lxml import etree
import time
def fetch_quotes():
    """爬取名言警句网站"""
    url = "http://quotes.toscrape.com/"
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }
    try:
        response = requests.get(url, headers=headers)
        response.encoding = 'utf-8'
        if response.status_code == 200:
            root = etree.HTML(response.text)
            # 提取所有名言
            quotes = []
            # 获取每个quote块
            quote_divs = root.xpath('//div[@class="quote"]')
            for quote_div in quote_divs:
                # 名言内容
                text = quote_div.xpath('.//span[@class="text"]/text()')
                # 作者
                author = quote_div.xpath('.//small[@class="author"]/text()')
                # 标签
                tags = quote_div.xpath('.//a[@class="tag"]/text()')
                if text:
                    quote_data = {
                        'text': text[0],
                        'author': author[0] if author else 'Unknown',
                        'tags': tags
                    }
                    quotes.append(quote_data)
            return quotes
        else:
            print(f"请求失败，状态码: {response.status_code}")
            return []
    except Exception as e:
        print(f"发生错误: {e}")
        return []
# 执行爬取
quotes = fetch_quotes()
for i, quote in enumerate(quotes[:5], 1):  # 只显示前5条
    print(f"{i}. {quote['text']}")
    print(f"   作者: {quote['author']}")
    print(f"   标签: {', '.join(quote['tags'])}")
    print("-" * 50)
    time.sleep(0.5)  # 礼貌性延时

示例3：动态内容处理（带翻页）

import requests
from lxml import etree
import time
def scrape_books(base_url, max_pages=3):
    """爬取书籍信息（支持翻页）"""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }
    all_books = []
    for page in range(1, max_pages + 1):
        url = f"{base_url}catalogue/page-{page}.html"
        print(f"正在爬取第 {page} 页: {url}")
        try:
            response = requests.get(url, headers=headers, timeout=10)
            if response.status_code != 200:
                print(f"第 {page} 页请求失败")
                continue
            root = etree.HTML(response.text)
            # 提取书籍信息
            book_articles = root.xpath('//article[@class="product_pod"]')
            for book in book_articles:
                # 书名
                title = book.xpath('.//h3/a/@title')
                # 价格
                price = book.xpath('.//p[@class="price_color"]/text()')
                # 链接
                link = book.xpath('.//h3/a/@href')
                # 评分（通过CSS类判断）
                rating_class = book.xpath('.//p[contains(@class, "star-rating")]/@class')
                book_data = {
                    'title': title[0] if title else 'N/A',
                    'price': price[0] if price else 'N/A',
                    'link': f"{base_url}catalogue/{link[0]}" if link else 'N/A',
                    'rating': rating_class[0].replace('star-rating ', '') if rating_class else 'N/A'
                }
                all_books.append(book_data)
            time.sleep(1)  # 翻页间隔
        except Exception as e:
            print(f"爬取第 {page} 页时出错: {e}")
    return all_books
# 使用示例
base_url = "http://books.toscrape.com/"
books = scrape_books(base_url, max_pages=2)
print(f"\n共爬取 {len(books)} 本书")
for i, book in enumerate(books[:5], 1):
    print(f"{i}. {book['title']} - {book['price']} - 评分: {book['rating']}")

高级XPath技巧

示例4：复杂条件匹配

from lxml import etree
html = """
<table>
    <tr class="even">
        <td>商品A</td>
        <td class="price">100</td>
        <td class="stock">有货</td>
    </tr>
    <tr class="odd">
        <td>商品B</td>
        <td class="price">200</td>
        <td class="stock">缺货</td>
    </tr>
    <tr class="even">
        <td>商品C</td>
        <td class="price">150</td>
        <td class="stock">有货</td>
    </tr>
</table>
"""
root = etree.HTML(html)
# 1. 获取价格大于100的商品
expensive_items = root.xpath('//tr[td[@class="price" and number(text()) > 100]]')
print("高价商品:")
for item in expensive_items:
    name = item.xpath('./td[1]/text()')[0]
    price = item.xpath('./td[@class="price"]/text()')[0]
    print(f"  {name}: {price}")
# 2. 获取有货的商品（string()函数）
in_stock = root.xpath('//tr[string(td[@class="stock"]) = "有货"]')
print("\n有货商品:")
for item in in_stock:
    name = item.xpath('./td[1]/text()')[0]
    print(f"  {name}")
# 3. 使用not()函数
out_of_stock = root.xpath('//tr[not(td[@class="stock"]/text() = "有货")]')
print("\n缺货商品:")
for item in out_of_stock:
    name = item.xpath('./td[1]/text()')[0]
    print(f"  {name}")

示例5：XML命名空间处理

from lxml import etree
# 包含命名空间的XML
xml_ns = """<?xml version="1.0"?>
<ns:books xmlns:ns="http://example.com/books">
    <ns:book id="1">
        <ns:title>Python编程</ns:title>
        <ns:author>张三</ns:author>
        <ns:price currency="CNY">59.00</ns:price>
    </ns:book>
    <ns:book id="2">
        <ns:title>数据结构</ns:title>
        <ns:author>李四</ns:author>
        <ns:price currency="CNY">49.00</ns:price>
    </ns:book>
</ns:books>
"""
# 定义命名空间映射
ns_map = {'ns': 'http://example.com/books'}
root = etree.fromstring(xml_ns)
# 使用命名空间查询s = root.xpath('//ns:title/text()', namespaces=ns_map)
print("书名:", titles)
# 获取所有价格
prices = root.xpath('//ns:price/text()', namespaces=ns_map)
print("价格:", prices)

最佳实践与注意事项

通用函数封装

import requests
from lxml import etree
import logging
# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class XPathScraper:
    def __init__(self, base_url, headers=None):
        self.base_url = base_url
        self.headers = headers or {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
        self.session = requests.Session()
    def fetch_page(self, url):
        """获取页面内容"""
        try:
            response = self.session.get(url, headers=self.headers, timeout=10)
            response.raise_for_status()
            response.encoding = response.apparent_encoding
            return etree.HTML(response.text)
        except requests.RequestException as e:
            logger.error(f"请求失败: {e}")
            return None
    def extract_data(self, html, xpath_expression):
        """通用数据提取方法"""
        if html is None:
            return []
        return html.xpath(xpath_expression)
# 使用示例
if __name__ == "__main__":
    scraper = XPathScraper("http://quotes.toscrape.com/")
    # 提取数据
    root = scraper.fetch_page("http://quotes.toscrape.com/")
    if root:
        quotes = scraper.extract_data(root, '//span[@class="text"]/text()')
        authors = scraper.extract_data(root, '//small[@class="author"]/text()')
        for quote, author in zip(quotes, authors):
            print(f"{quote} - {author}")

注意事项

错误处理：始终使用try-except处理网络请求
反爬策略：添加User-Agent、请求间隔、代理等
性能优化：使用Session复用连接
编码问题：正确设置页面编码
：对于JavaScript渲染的内容，考虑使用Selenium

XPath常用表达式速查

表达式	说明	示例
选择任意位置的节点	`//div`
从根节点选择	`/html/body`
当前节点	`./a`
父节点	`..//span`
选择属性	`//@href`
`text()`	选择文本	`//h1/text()`
`[N]`	选择第N个	`//div[2]`
`[last()]`	选择最后一个	`//div[last()]`
`[position()]`	按位置筛选	`//div[position()<3]`
`contains()`	包含指定文本	`//p[contains(text(),"Python")]`
`starts-with()`	以指定文本开头	`//@class[starts-with(.,"article")]`

通过这些案例,你应该能够掌握使用Python XPath解析网页数据的基本方法和技巧，实际应用中要根据具体网页结构灵活调整XPath表达式，并做好异常处理。

本文地址： https://www.bmjup.com/post/12592.html