Python网页源码案例如何获取网页代码

wen python案例 2026-06-30 33

本文目录导读：

使用 requests 库（推荐）
使用 urllib 库（内置库）
带 Session 处理的方法
完整的工具类示例
处理动态网页（需要渲染JavaScript）
注意事项

我来为您介绍几种获取网页源码的Python方法：

使用 requests 库（推荐）

import requests
# 基本用法
url = "https://www.example.com"
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
try:
    response = requests.get(url, headers=headers, timeout=10)
    response.encoding = 'utf-8'  # 设置编码
    html_content = response.text
    print(html_content)
except requests.RequestException as e:
    print(f"请求失败: {e}")

使用 urllib 库（内置库）

from urllib import request
url = "https://www.example.com"
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
try:
    req = request.Request(url, headers=headers)
    with request.urlopen(req, timeout=10) as response:
        html_content = response.read().decode('utf-8')
        print(html_content)
except Exception as e:
    print(f"请求失败: {e}")

带 Session 处理的方法

import requests
from bs4 import BeautifulSoup
# 带 Session 的请求
session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.8,en-US;q=0.5,en;q=0.3'
})
try:
    response = session.get("https://www.example.com", timeout=10)
    response.encoding = 'utf-8'
    # 保存到文件
    with open('webpage.html', 'w', encoding='utf-8') as f:
        f.write(response.text)
    # 使用 BeautifulSoup 解析
    soup = BeautifulSoup(response.text, 'html.parser')= soup.title.string if soup.title else "无标题"
    print(f"页面标题: {title}")
except requests.RequestException as e:
    print(f"请求失败: {e}")

完整的工具类示例

import requests
from typing import Optional, Dict
class WebPageFetcher:
    def __init__(self):
        self.session = requests.Session()
        self.session.headers.update({
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8'
        })
    def fetch(self, url: str, encoding: str = 'utf-8', timeout: int = 10) -> Optional[str]:
        """
        获取网页源码
        Args:
            url: 网页URL
            encoding: 编码方式
            timeout: 超时时间（秒）
        Returns:
            网页源码字符串，失败返回None
        """
        try:
            response = self.session.get(url, timeout=timeout)
            response.raise_for_status()  # 检查HTTP状态码
            response.encoding = encoding
            return response.text
        except requests.RequestException as e:
            print(f"请求失败: {e}")
            return None
    def fetch_with_cookies(self, url: str, cookies: Dict[str, str]) -> Optional[str]:
        """带Cookie的请求"""
        try:
            response = self.session.get(url, cookies=cookies)
            response.encoding = 'utf-8'
            return response.text
        except requests.RequestException as e:
            print(f"请求失败: {e}")
            return None
# 使用示例
fetcher = WebPageFetcher()
html = fetcher.fetch("https://www.example.com")
if html:
    print(html[:500])  # 打印前500个字符

处理动态网页（需要渲染JavaScript）

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
def get_dynamic_page(url: str) -> str:
    """获取动态渲染的网页源码"""
    chrome_options = Options()
    chrome_options.add_argument('--headless')  # 无头模式
    chrome_options.add_argument('--no-sandbox')
    driver = webdriver.Chrome(options=chrome_options)
    try:
        driver.get(url)
        # 等待页面加载
        driver.implicitly_wait(10)
        html = driver.page_source
        return html
    finally:
        driver.quit()
# 使用示例
# html = get_dynamic_page("https://www.example.com")

注意事项

遵守robots.txt：查看网站的爬取规则
控制请求频率：添加延迟避免被封
处理反爬：可能需要设置更多headers或使用代理
异常处理：总是添加try-catch

选择哪种方法取决于您的具体需求：

静态页面：使用requests即可
需要JS渲染：使用selenium或playwright
需要处理Session：使用带Session的requests

需要我详细解释某个特定场景的实现吗？