如何编写解析提取网页标签脚本

本文目录导读：

1. Python + BeautifulSoup（最常用）
2. Python + lxml（性能更好）
3. JavaScript + DOM（浏览器环境）
4. 完整的Python脚本示例
5. 高级功能：异步爬取多个页面
6. 常用提取模式
7. 注意事项

Python + BeautifulSoup（最常用）

from bs4 import BeautifulSoup
import requests
# 获取网页内容
url = "https://example.com"
response = requests.get(url)
soup = BeautifulSoup(response.text, 'html.parser')
# 提取特定标签
# 提取所有链接
links = soup.find_all('a')
for link in links:
    print(link.get('href'), link.text)
# 提取所有图片
images = soup.find_all('img')
for img in images:
    print(img.get('src'))
# 提取特定class的标签
divs = soup.find_all('div', class_='content')
for div in divs:
    print(div.text)
# 提取特定id的标签
element = soup.find('div', id='main-content')
if element:
    print(element.text)

Python + lxml（性能更好）

from lxml import html
import requests
page = requests.get('https://example.com')
tree = html.fromstring(page.content)
# 使用XPath提取
links = tree.xpath('//a/@href')s = tree.xpath('//h1/text()')
images = tree.xpath('//img/@src')
# 提取包含特定文本的元素
items = tree.xpath("//div[contains(text(), '关键词')]")

JavaScript + DOM（浏览器环境）

// 基础选择器
// 提取所有链接
const links = document.querySelectorAll('a');
links.forEach(link => {
    console.log(link.href, link.textContent);
});
// 提取特定class的元素
const items = document.querySelectorAll('.item');
items.forEach(item => {
    console.log(item.innerHTML);
});
// 提取表格数据
const rows = document.querySelectorAll('table tr');
rows.forEach(row => {
    const cells = row.querySelectorAll('td');
    cells.forEach(cell => {
        console.log(cell.textContent);
    });
});

完整的Python脚本示例

import requests
from bs4 import BeautifulSoup
import re
import json
class WebScraper:
    def __init__(self, url):
        self.url = url
        self.soup = None
    def fetch_page(self):
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
        response = requests.get(self.url, headers=headers)
        response.encoding = 'utf-8'
        self.soup = BeautifulSoup(response.text, 'html.parser')
    def extract_by_tag(self, tag_name, attrs=None):
        """按标签名和属性提取"""
        if attrs:
            return self.soup.find_all(tag_name, attrs=attrs)
        return self.soup.find_all(tag_name)
    def extract_by_css(self, css_selector):
        """按CSS选择器提取"""
        return self.soup.select(css_selector)
    def extract_by_regex(self, pattern):
        """按正则表达式匹配文本"""
        return re.findall(pattern, self.soup.get_text())
    def extract_metadata(self):
        """提取页面元数据"""
        data = {
            'title': self.soup.title.text if self.soup.title else None,
            'links': [a.get('href') for a in self.soup.find_all('a')],
            'images': [img.get('src') for img in self.soup.find_all('img')],
            'text': self.soup.get_text(strip=True)
        }
        return data
# 使用示例
scraper = WebScraper('https://example.com')
scraper.fetch_page()
# 提取所有段落
paragraphs = scraper.extract_by_tag('p')
for p in paragraphs:
    print(p.text)
# 提取特定class的元素
articles = scraper.extract_by_css('.article')
for article in articles:
    print(article.text)
# 保存为JSON
with open('output.json', 'w', encoding='utf-8') as f:
    json.dump(scraper.extract_metadata(), f, ensure_ascii=False)

高级功能：异步爬取多个页面

import asyncio
import aiohttp
from bs4 import BeautifulSoup
async def fetch_page(session, url):
    async with session.get(url) as response:
        return await response.text()
async def extract_multiple_pages(urls):
    async with aiohttp.ClientSession() as session:
        tasks = []
        for url in urls:
            task = asyncio.create_task(fetch_page(session, url))
            tasks.append(task)
        pages = await asyncio.gather(*tasks)
        results = []
        for html_content, url in zip(pages, urls):
            soup = BeautifulSoup(html_content, 'html.parser')
            # 提取需要的标签
            titles = [h.text for h in soup.find_all('h2')]
            results.append({'url': url, 'titles': titles})
        return results
# 运行异步爬虫
urls = ['https://example1.com', 'https://example2.com']
result = asyncio.run(extract_multiple_pages(urls))

常用提取模式

提取表格数据

def extract_table(soup):
    table = soup.find('table')
    headers = [th.text for th in table.find_all('th')]
    rows = []
    for tr in table.find_all('tr')[1:]:
        row = [td.text for td in tr.find_all('td')]
        rows.append(row)
    return {'headers': headers, 'rows': rows}

def extract_article(soup):
    article = soup.find('article')
    if article:
        return {
            'title': article.find('h1').text if article.find('h1') else None,
            'content': ' '.join([p.text for p in article.find_all('p')]),
            'date': article.find('time').text if article.find('time') else None
        }

注意事项

1. 遵守robots.txt：爬取前检查网站的爬虫规则
2. 设置请求延迟：避免对服务器造成压力
3. 处理异常：添加try/except处理网络错误
4. 伪装请求头：添加User-Agent等headers
5. 编码处理：正确设置字符编码

选择哪种方法取决于你的具体需求：

• 简单提取：BeautifulSoup
• 性能要求高：lxml
• 浏览器环境：原生JavaScript
• 大规模爬取：异步框架