IT技术社区_专业开发者平台_最新编程教程与解决方案 -沐辰社区

Python xpath案例如何解析网页数据

wen python案例 28

本文目录导读：

Python xpath案例如何解析网页数据

  1. 环境准备
  2. 基础XPath案例
  3. 高级XPath技巧
  4. 最佳实践与注意事项
  5. XPath常用表达式速查

我来为你详细介绍Python中使用XPath解析网页数据的完整案例。

环境准备

首先安装必要的库：

pip install lxml requests

基础XPath案例

示例1：解析静态HTML

from lxml import etree
import requests
# 准备示例HTML
html_content = """
<html>
    <body>
        <div class="container">
            <h1>文章列表</h1>
            <div class="article">
                <h2 class="title">Python入门教程</h2>
                <p class="author">作者：张三</p>
                <p class="date">2024-01-15</p>
                <p class="content">这是Python入门教程的内容...</p>
                <a href="https://example.com/article/1">阅读更多</a>
            </div>
            <div class="article">
                <h2 class="title">Web开发实战</h2>
                <p class="author">作者：李四</p>
                <p class="date">2024-01-16</p>
                <p class="content">这是Web开发实战的内容...</p>
                <a href="https://example.com/article/2">阅读更多</a>
            </div>
        </div>
        <div class="footer">
            <p>© 2024 版权所有</p>
        </div>
    </body>
</html>
"""
# 解析HTML
root = etree.HTML(html_content)
# 1. 获取所有文章标题s = root.xpath('//h2[@class="title"]/text()')
print("文章标题:", titles)
# 2. 获取所有文章作者
authors = root.xpath('//p[@class="author"]/text()')
print("作者:", authors)
# 3. 获取所有文章链接
links = root.xpath('//div[@class="article"]/a/@href')
print("链接:", links)
# 4. 获取第一篇文章的完整信息
first_article = root.xpath('//div[@class="article"][1]')
if first_article:= first_article[0].xpath('.//h2/text()')[0]
    author = first_article[0].xpath('.//p[@class="author"]/text()')[0]
    date = first_article[0].xpath('.//p[@class="date"]/text()')[0]
    print(f"第一篇文章: {title} - {author} - {date}")
# 5. 复杂的筛选条件
python_articles = root.xpath('//div[@class="article" and contains(.//h2/text(), "Python")]')
print(f"包含Python的文章数: {len(python_articles)}")

示例2：真实网站数据抓取

import requests
from lxml import etree
import time
def fetch_quotes():
    """爬取名言警句网站"""
    url = "http://quotes.toscrape.com/"
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }
    try:
        response = requests.get(url, headers=headers)
        response.encoding = 'utf-8'
        if response.status_code == 200:
            root = etree.HTML(response.text)
            # 提取所有名言
            quotes = []
            # 获取每个quote块
            quote_divs = root.xpath('//div[@class="quote"]')
            for quote_div in quote_divs:
                # 名言内容
                text = quote_div.xpath('.//span[@class="text"]/text()')
                # 作者
                author = quote_div.xpath('.//small[@class="author"]/text()')
                # 标签
                tags = quote_div.xpath('.//a[@class="tag"]/text()')
                if text:
                    quote_data = {
                        'text': text[0],
                        'author': author[0] if author else 'Unknown',
                        'tags': tags
                    }
                    quotes.append(quote_data)
            return quotes
        else:
            print(f"请求失败，状态码: {response.status_code}")
            return []
    except Exception as e:
        print(f"发生错误: {e}")
        return []
# 执行爬取
quotes = fetch_quotes()
for i, quote in enumerate(quotes[:5], 1):  # 只显示前5条
    print(f"{i}. {quote['text']}")
    print(f"   作者: {quote['author']}")
    print(f"   标签: {', '.join(quote['tags'])}")
    print("-" * 50)
    time.sleep(0.5)  # 礼貌性延时

示例3：动态内容处理（带翻页）

import requests
from lxml import etree
import time
def scrape_books(base_url, max_pages=3):
    """爬取书籍信息（支持翻页）"""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
    }
    all_books = []
    for page in range(1, max_pages + 1):
        url = f"{base_url}catalogue/page-{page}.html"
        print(f"正在爬取第 {page} 页: {url}")
        try:
            response = requests.get(url, headers=headers, timeout=10)
            if response.status_code != 200:
                print(f"第 {page} 页请求失败")
                continue
            root = etree.HTML(response.text)
            # 提取书籍信息
            book_articles = root.xpath('//article[@class="product_pod"]')
            for book in book_articles:
                # 书名
                title = book.xpath('.//h3/a/@title')
                # 价格
                price = book.xpath('.//p[@class="price_color"]/text()')
                # 链接
                link = book.xpath('.//h3/a/@href')
                # 评分（通过CSS类判断）
                rating_class = book.xpath('.//p[contains(@class, "star-rating")]/@class')
                book_data = {
                    'title': title[0] if title else 'N/A',
                    'price': price[0] if price else 'N/A',
                    'link': f"{base_url}catalogue/{link[0]}" if link else 'N/A',
                    'rating': rating_class[0].replace('star-rating ', '') if rating_class else 'N/A'
                }
                all_books.append(book_data)
            time.sleep(1)  # 翻页间隔
        except Exception as e:
            print(f"爬取第 {page} 页时出错: {e}")
    return all_books
# 使用示例
base_url = "http://books.toscrape.com/"
books = scrape_books(base_url, max_pages=2)
print(f"\n共爬取 {len(books)} 本书")
for i, book in enumerate(books[:5], 1):
    print(f"{i}. {book['title']} - {book['price']} - 评分: {book['rating']}")

高级XPath技巧

示例4：复杂条件匹配

from lxml import etree
html = """
<table>
    <tr class="even">
        <td>商品A</td>
        <td class="price">100</td>
        <td class="stock">有货</td>
    </tr>
    <tr class="odd">
        <td>商品B</td>
        <td class="price">200</td>
        <td class="stock">缺货</td>
    </tr>
    <tr class="even">
        <td>商品C</td>
        <td class="price">150</td>
        <td class="stock">有货</td>
    </tr>
</table>
"""
root = etree.HTML(html)
# 1. 获取价格大于100的商品
expensive_items = root.xpath('//tr[td[@class="price" and number(text()) > 100]]')
print("高价商品:")
for item in expensive_items:
    name = item.xpath('./td[1]/text()')[0]
    price = item.xpath('./td[@class="price"]/text()')[0]
    print(f"  {name}: {price}")
# 2. 获取有货的商品（string()函数）
in_stock = root.xpath('//tr[string(td[@class="stock"]) = "有货"]')
print("\n有货商品:")
for item in in_stock:
    name = item.xpath('./td[1]/text()')[0]
    print(f"  {name}")
# 3. 使用not()函数
out_of_stock = root.xpath('//tr[not(td[@class="stock"]/text() = "有货")]')
print("\n缺货商品:")
for item in out_of_stock:
    name = item.xpath('./td[1]/text()')[0]
    print(f"  {name}")

示例5：XML命名空间处理

from lxml import etree
# 包含命名空间的XML
xml_ns = """<?xml version="1.0"?>
<ns:books xmlns:ns="http://example.com/books">
    <ns:book id="1">
        <ns:title>Python编程</ns:title>
        <ns:author>张三</ns:author>
        <ns:price currency="CNY">59.00</ns:price>
    </ns:book>
    <ns:book id="2">
        <ns:title>数据结构</ns:title>
        <ns:author>李四</ns:author>
        <ns:price currency="CNY">49.00</ns:price>
    </ns:book>
</ns:books>
"""
# 定义命名空间映射
ns_map = {'ns': 'http://example.com/books'}
root = etree.fromstring(xml_ns)
# 使用命名空间查询s = root.xpath('//ns:title/text()', namespaces=ns_map)
print("书名:", titles)
# 获取所有价格
prices = root.xpath('//ns:price/text()', namespaces=ns_map)
print("价格:", prices)

最佳实践与注意事项

通用函数封装

import requests
from lxml import etree
import logging
# 配置日志
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
class XPathScraper:
    def __init__(self, base_url, headers=None):
        self.base_url = base_url
        self.headers = headers or {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
        self.session = requests.Session()
    def fetch_page(self, url):
        """获取页面内容"""
        try:
            response = self.session.get(url, headers=self.headers, timeout=10)
            response.raise_for_status()
            response.encoding = response.apparent_encoding
            return etree.HTML(response.text)
        except requests.RequestException as e:
            logger.error(f"请求失败: {e}")
            return None
    def extract_data(self, html, xpath_expression):
        """通用数据提取方法"""
        if html is None:
            return []
        return html.xpath(xpath_expression)
# 使用示例
if __name__ == "__main__":
    scraper = XPathScraper("http://quotes.toscrape.com/")
    # 提取数据
    root = scraper.fetch_page("http://quotes.toscrape.com/")
    if root:
        quotes = scraper.extract_data(root, '//span[@class="text"]/text()')
        authors = scraper.extract_data(root, '//small[@class="author"]/text()')
        for quote, author in zip(quotes, authors):
            print(f"{quote} - {author}")

注意事项

  1. 错误处理：始终使用try-except处理网络请求
  2. 反爬策略：添加User-Agent、请求间隔、代理等
  3. 性能优化：使用Session复用连接
  4. 编码问题：正确设置页面编码
  5. ：对于JavaScript渲染的内容，考虑使用Selenium

XPath常用表达式速查

表达式 说明 示例
选择任意位置的节点 //div
从根节点选择 /html/body
当前节点 ./a
父节点 ..//span
选择属性 //@href
text() 选择文本 //h1/text()
[N] 选择第N个 //div[2]
[last()] 选择最后一个 //div[last()]
[position()] 按位置筛选 //div[position()<3]
contains() 包含指定文本 //p[contains(text(),"Python")]
starts-with() 以指定文本开头 //@class[starts-with(.,"article")]

通过这些案例,你应该能够掌握使用Python XPath解析网页数据的基本方法和技巧，实际应用中要根据具体网页结构灵活调整XPath表达式，并做好异常处理。

上一篇Python bs4案例如何解析网页标签

下一篇Python爬虫翻页案例如何实现翻页爬取

相关文章

抱歉，评论功能暂时关闭!