本文目录导读：
- Python + BeautifulSoup（最常用）
- Python + lxml（性能更好）
- JavaScript + DOM（浏览器环境）
- 完整的Python脚本示例
- 高级功能：异步爬取多个页面
- 常用提取模式
- 注意事项
Python + BeautifulSoup（最常用）
from bs4 import BeautifulSoup
import requests
# 获取网页内容
url = "https://example.com"
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
# 提取特定标签
# 提取所有链接
links = soup.find_all('a')
for link in links:
print(link.get('href'), link.text)
# 提取所有图片
images = soup.find_all('img')
for img in images:
print(img.get('src'))
# 提取特定class的标签
divs = soup.find_all('div', class_='content')
for div in divs:
print(div.text)
# 提取特定id的标签
element = soup.find('div', id='main-content')
if element:
print(element.text)
Python + lxml（性能更好）
from lxml import html
import requests
page = requests.get('https://example.com')
tree = html.fromstring(page.content)
# 使用XPath提取
links = tree.xpath('//a/@href')s = tree.xpath('//h1/text()')
images = tree.xpath('//img/@src')
# 提取包含特定文本的元素
items = tree.xpath("//div[contains(text(), '关键词')]")
JavaScript + DOM（浏览器环境）
// 基础选择器
// 提取所有链接
const links = document.querySelectorAll('a');
links.forEach(link => {
console.log(link.href, link.textContent);
});
// 提取特定class的元素
const items = document.querySelectorAll('.item');
items.forEach(item => {
console.log(item.innerHTML);
});
// 提取表格数据
const rows = document.querySelectorAll('table tr');
rows.forEach(row => {
const cells = row.querySelectorAll('td');
cells.forEach(cell => {
console.log(cell.textContent);
});
});
完整的Python脚本示例
import requests
from bs4 import BeautifulSoup
import re
import json
class WebScraper:
def __init__(self, url):
self.url = url
self.soup = None
def fetch_page(self):
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
response = requests.get(self.url, headers=headers)
response.encoding = 'utf-8'
self.soup = BeautifulSoup(response.text, 'html.parser')
def extract_by_tag(self, tag_name, attrs=None):
"""按标签名和属性提取"""
if attrs:
return self.soup.find_all(tag_name, attrs=attrs)
return self.soup.find_all(tag_name)
def extract_by_css(self, css_selector):
"""按CSS选择器提取"""
return self.soup.select(css_selector)
def extract_by_regex(self, pattern):
"""按正则表达式匹配文本"""
return re.findall(pattern, self.soup.get_text())
def extract_metadata(self):
"""提取页面元数据"""
data = {
'title': self.soup.title.text if self.soup.title else None,
'links': [a.get('href') for a in self.soup.find_all('a')],
'images': [img.get('src') for img in self.soup.find_all('img')],
'text': self.soup.get_text(strip=True)
}
return data
# 使用示例
scraper = WebScraper('https://example.com')
scraper.fetch_page()
# 提取所有段落
paragraphs = scraper.extract_by_tag('p')
for p in paragraphs:
print(p.text)
# 提取特定class的元素
articles = scraper.extract_by_css('.article')
for article in articles:
print(article.text)
# 保存为JSON
with open('output.json', 'w', encoding='utf-8') as f:
json.dump(scraper.extract_metadata(), f, ensure_ascii=False)
高级功能：异步爬取多个页面
import asyncio
import aiohttp
from bs4 import BeautifulSoup
async def fetch_page(session, url):
async with session.get(url) as response:
return await response.text()
async def extract_multiple_pages(urls):
async with aiohttp.ClientSession() as session:
tasks = []
for url in urls:
task = asyncio.create_task(fetch_page(session, url))
tasks.append(task)
pages = await asyncio.gather(*tasks)
results = []
for html_content, url in zip(pages, urls):
soup = BeautifulSoup(html_content, 'html.parser')
# 提取需要的标签
titles = [h.text for h in soup.find_all('h2')]
results.append({'url': url, 'titles': titles})
return results
# 运行异步爬虫
urls = ['https://example1.com', 'https://example2.com']
result = asyncio.run(extract_multiple_pages(urls))
常用提取模式
提取表格数据
def extract_table(soup):
table = soup.find('table')
headers = [th.text for th in table.find_all('th')]
rows = []
for tr in table.find_all('tr')[1:]:
row = [td.text for td in tr.find_all('td')]
rows.append(row)
return {'headers': headers, 'rows': rows}
def extract_article(soup):
article = soup.find('article')
if article:
return {
'title': article.find('h1').text if article.find('h1') else None,
'content': ' '.join([p.text for p in article.find_all('p')]),
'date': article.find('time').text if article.find('time') else None
}
注意事项
- 遵守robots.txt：爬取前检查网站的爬虫规则
- 设置请求延迟：避免对服务器造成压力
- 处理异常：添加try/except处理网络错误
- 伪装请求头：添加User-Agent等headers
- 编码处理：正确设置字符编码
选择哪种方法取决于你的具体需求：
- 简单提取：BeautifulSoup
- 性能要求高：lxml
- 浏览器环境：原生JavaScript
- 大规模爬取：异步框架
