Python小说爬虫案例如何爬取文本内容

wen python案例 2026-06-30 174

本文目录导读：

准备工作
基础爬虫示例
进阶版：爬取整本小说
重要提醒 ⚠️

我来分享一个Python小说爬虫的基础案例,以爬取免费小说网站为例：

准备工作

pip install requests beautifulsoup4 lxml

基础爬虫示例

import requests
from bs4 import BeautifulSoup
import time
import re
def crawl_novel(url, output_file='novel.txt'):
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
    }
    try:
        # 发送请求
        response = requests.get(url, headers=headers, timeout=10)
        response.encoding = 'utf-8'
        # 解析HTML
        soup = BeautifulSoup(response.text, 'lxml')
        # 获取小说标题
        title = soup.find('h1') or soup.find('h2')
        if title:
            novel_title = title.text.strip()
            print(f'正在爬取: {novel_title}')
        # 获取正文内容，根据实际网站调整选择器
        # 示例：class="content" 或 id="content"
        content_div = soup.find('div', class_='content') or soup.find('div', id='content')
        if content_div:
            # 清理HTML标签，保留纯文本
            text = content_div.get_text(separator='\n', strip=True)
            # 写入文件
            with open(output_file, 'a', encoding='utf-8') as f:
                f.write(f'{novel_title}\n\n')
                f.write(text)
                f.write('\n\n' + '='*50 + '\n\n')
            print(f'成功保存内容到: {output_file}')
        else:
            print('未找到内容区域，请检查选择器')
    except Exception as e:
        print(f'爬取失败: {e}')
# 使用示例
if __name__ == '__main__':
    # 替换为实际的小说章节URL
    chapter_urls = [
        'https://example-novel-site.com/chapter1',
        'https://example-novel-site.com/chapter2'
    ]
    for url in chapter_urls:
        crawl_novel(url)
        time.sleep(2)  # 避免请求过频繁

进阶版：爬取整本小说

import requests
from bs4 import BeautifulSoup
import time
import random
class NovelCrawler:
    def __init__(self, base_url, start_chapter=1, end_chapter=10):
        self.base_url = base_url
        self.start_chapter = start_chapter
        self.end_chapter = end_chapter
        self.headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
        }
    def get_chapter_list(self):
        """获取章节列表"""
        response = requests.get(self.base_url, headers=self.headers)
        response.encoding = 'utf-8'
        soup = BeautifulSoup(response.text, 'lxml')
        # 根据实际网站调整选择器
        chapters = soup.find_all('a', class_='chapter-link') or soup.find_all('a', href=re.compile(r'.*chapter.*'))
        chapter_list = []
        for chapter in chapters:
            href = chapter.get('href')
            title = chapter.text.strip()
            if href and not href.startswith('http'):
                href = self.base_url + href
            chapter_list.append((title, href))
        return chapter_list[self.start_chapter-1:self.end_chapter]
    def crawl_chapter(self, url):
        """爬取单个章节"""
        try:
            response = requests.get(url, headers=self.headers, timeout=10)
            response.encoding = 'utf-8'
            soup = BeautifulSoup(response.text, 'lxml')
            # 获取正文
            content = soup.find('div', class_='content') or soup.find('div', id='content')
            if content:
                # 移除广告等无关元素
                for ad in content.find_all(['script', 'ins', 'div'], class_=re.compile(r'ad|recommend')):
                    ad.decompose()
                text = content.get_text(separator='\n', strip=True)
                return text
            return None
        except Exception as e:
            print(f'爬取章节失败: {url}, 错误: {e}')
            return None
    def save_novel(self, filename='novel.txt'):
        """保存整本小说"""
        with open(filename, 'w', encoding='utf-8') as f:
            for title, url in self.get_chapter_list():
                print(f'正在爬取: {title}')
                content = self.crawl_chapter(url)
                if content:
                    f.write(f'{title}\n\n')
                    f.write(content)
                    f.write('\n\n')
                    print(f'完成: {title}')
                else:
                    print(f'跳过: {title}')
                # 随机延迟，模拟人类行为
                time.sleep(random.uniform(1, 3))
        print(f'小说保存完成: {filename}')
# 使用示例
if __name__ == '__main__':
    # 替换为实际的小说首页URL
    novel_url = 'https://example-novel-site.com/novel-id'
    crawler = NovelCrawler(novel_url, start_chapter=1, end_chapter=5)
    crawler.save_novel('我的小说.txt')