本文目录导读：
我来分享一个Python小说爬虫的基础案例,以爬取免费小说网站为例：
准备工作
pip install requests beautifulsoup4 lxml
基础爬虫示例
import requests
from bs4 import BeautifulSoup
import time
import re
def crawl_novel(url, output_file='novel.txt'):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8'
}
try:
# 发送请求
response = requests.get(url, headers=headers, timeout=10)
response.encoding = 'utf-8'
# 解析HTML
soup = BeautifulSoup(response.text, 'lxml')
# 获取小说标题
title = soup.find('h1') or soup.find('h2')
if title:
novel_title = title.text.strip()
print(f'正在爬取: {novel_title}')
# 获取正文内容，根据实际网站调整选择器
# 示例：class="content" 或 id="content"
content_div = soup.find('div', class_='content') or soup.find('div', id='content')
if content_div:
# 清理HTML标签，保留纯文本
text = content_div.get_text(separator='\n', strip=True)
# 写入文件
with open(output_file, 'a', encoding='utf-8') as f:
f.write(f'{novel_title}\n\n')
f.write(text)
f.write('\n\n' + '='*50 + '\n\n')
print(f'成功保存内容到: {output_file}')
else:
print('未找到内容区域，请检查选择器')
except Exception as e:
print(f'爬取失败: {e}')
# 使用示例
if __name__ == '__main__':
# 替换为实际的小说章节URL
chapter_urls = [
'https://example-novel-site.com/chapter1',
'https://example-novel-site.com/chapter2'
]
for url in chapter_urls:
crawl_novel(url)
time.sleep(2) # 避免请求过频繁
进阶版：爬取整本小说
import requests
from bs4 import BeautifulSoup
import time
import random
class NovelCrawler:
def __init__(self, base_url, start_chapter=1, end_chapter=10):
self.base_url = base_url
self.start_chapter = start_chapter
self.end_chapter = end_chapter
self.headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}
def get_chapter_list(self):
"""获取章节列表"""
response = requests.get(self.base_url, headers=self.headers)
response.encoding = 'utf-8'
soup = BeautifulSoup(response.text, 'lxml')
# 根据实际网站调整选择器
chapters = soup.find_all('a', class_='chapter-link') or soup.find_all('a', href=re.compile(r'.*chapter.*'))
chapter_list = []
for chapter in chapters:
href = chapter.get('href')
title = chapter.text.strip()
if href and not href.startswith('http'):
href = self.base_url + href
chapter_list.append((title, href))
return chapter_list[self.start_chapter-1:self.end_chapter]
def crawl_chapter(self, url):
"""爬取单个章节"""
try:
response = requests.get(url, headers=self.headers, timeout=10)
response.encoding = 'utf-8'
soup = BeautifulSoup(response.text, 'lxml')
# 获取正文
content = soup.find('div', class_='content') or soup.find('div', id='content')
if content:
# 移除广告等无关元素
for ad in content.find_all(['script', 'ins', 'div'], class_=re.compile(r'ad|recommend')):
ad.decompose()
text = content.get_text(separator='\n', strip=True)
return text
return None
except Exception as e:
print(f'爬取章节失败: {url}, 错误: {e}')
return None
def save_novel(self, filename='novel.txt'):
"""保存整本小说"""
with open(filename, 'w', encoding='utf-8') as f:
for title, url in self.get_chapter_list():
print(f'正在爬取: {title}')
content = self.crawl_chapter(url)
if content:
f.write(f'{title}\n\n')
f.write(content)
f.write('\n\n')
print(f'完成: {title}')
else:
print(f'跳过: {title}')
# 随机延迟，模拟人类行为
time.sleep(random.uniform(1, 3))
print(f'小说保存完成: {filename}')
# 使用示例
if __name__ == '__main__':
# 替换为实际的小说首页URL
novel_url = 'https://example-novel-site.com/novel-id'
crawler = NovelCrawler(novel_url, start_chapter=1, end_chapter=5)
crawler.save_novel('我的小说.txt')
重要提醒 ⚠️
-
合法性：
- 仅爬取免费、公开的小说
- 遵守网站的robots.txt规则
- 不要爬取付费内容或侵权作品
-
频率控制：
- 设置合理的请求间隔（建议2-5秒）
- 不要对服务器造成压力
-
处理反爬：
- 模拟浏览器User-Agent
- 使用代理IP池
- 添加随机延迟
-
数据保存：
- 建议分段保存，避免单文件过大
- 考虑使用数据库存储
-
网站规则：
- 实际网站需要根据具体HTML结构调整选择器
- 检查是否需要登录或验证码
这个案例提供基础框架，实际使用时需要根据目标网站的HTML结构调整解析逻辑，建议先测试1-2个页面,确认解析正确后再批量爬取。
