Python网页解析案例如何提取网页内容

wen python案例 2026-06-30 209

本文目录导读：

基础案例：使用BeautifulSoup提取文章内容
提取链接和图片
表格数据提取
动态页面解析（Selenium示例）
实用案例：新闻网站爬取
处理常见问题
安装依赖
使用建议

我来给你介绍几个Python网页解析的实际案例,从基础到进阶。

基础案例：使用BeautifulSoup提取文章内容

import requests
from bs4 import BeautifulSoup
# 发送请求
url = "https://example.com/article"
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
response = requests.get(url, headers=headers)
response.encoding = 'utf-8'
# 解析HTML
soup = BeautifulSoup(response.text, 'html.parser')
title = soup.find('h1').text.strip()
# 提取文章正文
article = soup.find('article')
paragraphs = article.find_all('p')
content = '\n'.join([p.text.strip() for p in paragraphs])
# 提取发布日期
date = soup.find('time', class_='post-date')
if date:
    publish_date = date.get('datetime')
print(f"标题: {title}")
print(f"内容: {content[:100]}...")

提取链接和图片

def extract_links_and_images(soup):
    # 提取所有链接
    links = []
    for a in soup.find_all('a', href=True):
        href = a['href']
        text = a.text.strip()
        if href and not href.startswith('#'):
            links.append({
                'text': text,
                'url': href
            })
    # 提取所有图片
    images = []
    for img in soup.find_all('img', src=True):
        images.append({
            'src': img['src'],
            'alt': img.get('alt', ''),
            'width': img.get('width', 'auto'),
            'height': img.get('height', 'auto')
        })
    return links, images

表格数据提取

import pandas as pd
def extract_table_data(soup):
    tables = soup.find_all('table')
    all_tables = []
    for table in tables:
        # 提取表格头
        headers = []
        th_elements = table.find_all('th')
        if th_elements:
            headers = [th.text.strip() for th in th_elements]
        # 提取表格数据
        rows = []
        for row in table.find_all('tr'):
            cells = row.find_all('td')
            if cells:
                row_data = [cell.text.strip() for cell in cells]
                rows.append(row_data)
        # 转换为DataFrame
        if rows:
            if headers:
                df = pd.DataFrame(rows, columns=headers)
            else:
                df = pd.DataFrame(rows)
            all_tables.append(df)
    return all_tables

动态页面解析（Selenium示例）

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
def scrape_dynamic_page(url):
    # 设置无头浏览器
    options = webdriver.ChromeOptions()
    options.add_argument('--headless')
    driver = webdriver.Chrome(options=options)
    try:
        driver.get(url)
        # 等待页面加载
        wait = WebDriverWait(driver, 10)
        element = wait.until(
            EC.presence_of_element_located((By.CLASS_NAME, "content"))
        )
        # 滚动页面加载更多内容
        driver.execute_script("window.scrollTo(0, document.body.scrollHeight);")
        time.sleep(2)
        # 提取数据
        items = driver.find_elements(By.CLASS_NAME, "item")
        data = []
        for item in items:
            title = item.find_element(By.TAG_NAME, "h2").text
            description = item.find_element(By.CLASS_NAME, "description").text
            data.append({"title": title, "description": description})
        return data
    finally:
        driver.quit()

实用案例：新闻网站爬取

import json
from datetime import datetime
class NewsScraper:
    def __init__(self):
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        })
    def scrape_news_list(self, url):
        """爬取新闻列表"""
        response = self.session.get(url)
        soup = BeautifulSoup(response.text, 'html.parser')
        news_items = []
        articles = soup.find_all('article', class_='news-item')
        for article in articles:
            item = {
                'title': article.find('h2').text.strip(),
                'link': article.find('a')['href'],
                'summary': article.find('p', class_='summary').text.strip(),
                'date': article.find('time').get('datetime') if article.find('time') else None
            }
            news_items.append(item)
        return news_items
    def scrape_news_detail(self, url):
        """爬取新闻详情"""
        response = self.session.get(url)
        soup = BeautifulSoup(response.text, 'html.parser')
        news_detail = {
            'title': soup.find('h1', class_='title').text.strip(),
            'author': soup.find('span', class_='author').text.strip(),
            'publish_date': soup.find('time', class_='publish-date').get('datetime'),
            'content': self._extract_content(soup),
            'tags': [tag.text for tag in soup.find_all('a', class_='tag')]
        }
        return news_detail
    def _extract_content(self, soup):
        """提取文章正文"""
        article = soup.find('article') or soup.find('div', class_='content')
        if article:
            # 移除广告和无关元素
            for unwanted in article.find_all(['script', 'style', 'advertisement']):
                unwanted.decompose()
            paragraphs = article.find_all('p')
            return '\n'.join([p.text.strip() for p in paragraphs if p.text.strip()])
        return ''
    def save_to_json(self, data, filename):
        """保存为JSON文件"""
        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
# 使用示例
scraper = NewsScraper()
# news_list = scraper.scrape_news_list('https://example.com/news')
# news_detail = scraper.scrape_news_detail(news_list[0]['link'])

处理常见问题

def robust_scrape(url, max_retries=3):
    """健壮的网页爬取"""
    for attempt in range(max_retries):
        try:
            # 添加延迟避免被封
            time.sleep(1)
            # 使用代理
            proxies = {
                'http': 'http://proxy.example.com:8080',
                'https': 'https://proxy.example.com:8080'
            }
            response = requests.get(
                url, 
                headers=headers,
                proxies=proxies,
                timeout=10,
                verify=False  # 跳过SSL验证
            )
            response.raise_for_status()  # 检查HTTP错误
            # 自动检测编码
            response.encoding = response.apparent_encoding
            return response.text
        except requests.RequestException as e:
            print(f"尝试 {attempt + 1} 失败: {e}")
            if attempt == max_retries - 1:
                raise

安装依赖

pip install requests beautifulsoup4 lxml selenium pandas

使用建议

遵守robots.txt：查看网站的robots.txt文件
控制访问频率：添加延时，避免对服务器造成压力
异常处理：妥善处理网络错误和解析错误
数据验证：验证提取的数据完整性

这些案例涵盖了常见的网页解析场景,你可以根据实际需求选择合适的方案。

本文地址： https://www.bmjup.com/post/12613.html