Python爬虫伪装案例如何伪装浏览器

wen python案例 2026-06-30 23

本文目录导读：

Python爬虫伪装案例如何伪装浏览器

基础伪装方法
高级伪装技巧
使用Selenium完整伪装
完整伪装案例
反反爬虫最佳实践

我来为你详细介绍Python爬虫伪装浏览器的几种常用方法和完整案例。

基础伪装方法

1 设置User-Agent

import requests
# 常见的浏览器User-Agent
user_agents = [
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
    'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:89.0) Gecko/20100101 Firefox/89.0',
    'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/14.1.1 Safari/605.1.15'
]
# 添加User-Agent到请求头
headers = {
    'User-Agent': user_agents[0],
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
    'Accept-Encoding': 'gzip, deflate, br',
    'Connection': 'keep-alive',
    'Upgrade-Insecure-Requests': '1'
}
response = requests.get('https://example.com', headers=headers)

2 使用fake_useragent库

# 安装: pip install fake-useragent
from fake_useragent import UserAgent
import requests
# 随机生成User-Agent
ua = UserAgent()
headers = {
    'User-Agent': ua.random,  # 随机UA
    # 或者指定浏览器类型
    # 'User-Agent': ua.chrome,
    # 'User-Agent': ua.firefox,
    # 'User-Agent': ua.safari
}
response = requests.get('https://example.com', headers=headers)
print(f"使用的UA: {headers['User-Agent']}")

高级伪装技巧

1 完整的请求头伪装

import requests
from fake_useragent import UserAgent
def get_complete_headers():
    ua = UserAgent()
    headers = {
        'User-Agent': ua.random,
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8,en-US;q=0.7',
        'Accept-Encoding': 'gzip, deflate, br',
        'Cache-Control': 'no-cache',
        'Pragma': 'no-cache',
        'Connection': 'keep-alive',
        'DNT': '1',  # Do Not Track
        'Upgrade-Insecure-Requests': '1',
        'Sec-Fetch-Dest': 'document',
        'Sec-Fetch-Mode': 'navigate',
        'Sec-Fetch-Site': 'none',
        'Sec-Fetch-User': '?1',
    }
    return headers
# 使用
headers = get_complete_headers()
session = requests.Session()
session.headers.update(headers)
response = session.get('https://example.com')

2 使用请求会话保持Cookie

import requests
from fake_useragent import UserAgent
def create_authenticated_session():
    session = requests.Session()
    # 设置初始headers
    ua = UserAgent()
    session.headers.update({
        'User-Agent': ua.random,
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
        'Accept-Language': 'en-US,en;q=0.5',
    })
    # 访问首页获取Cookie
    session.get('https://example.com')
    # 模拟登录（如果需要）
    login_data = {
        'username': 'your_username',
        'password': 'your_password',
        'csrf_token': ''  # 从页面获取
    }
    session.post('https://example.com/login', data=login_data)
    return session
session = create_authenticated_session()
response = session.get('https://example.com/protected-page')

使用Selenium完整伪装

1 基础Selenium伪装

from selenium import webdriver
from selenium.webdriver.chrome.options import Options
from fake_useragent import UserAgent
import undetected_chromedriver as uc
def create_stealth_driver():
    options = Options()
    # 禁用自动化标识
    options.add_experimental_option('excludeSwitches', ['enable-automation'])
    options.add_experimental_option('useAutomationExtension', False)
    # 设置User-Agent
    ua = UserAgent()
    options.add_argument(f'user-agent={ua.chrome}')
    # 窗口大小
    options.add_argument('--window-size=1920,1080')
    # 禁用GPU加速
    options.add_argument('--disable-gpu')
    # 忽略证书错误
    options.add_argument('--ignore-certificate-errors')
    # 禁用通知
    options.add_argument('--disable-notifications')
    # 设置语言
    options.add_argument('--lang=zh-CN')
    # 创建driver
    driver = webdriver.Chrome(options=options)
    # 隐藏WebDriver特征
    driver.execute_script("Object.defineProperty(navigator, 'webdriver', {get: () => undefined})")
    return driver
driver = create_stealth_driver()
driver.get('https://www.baidu.com')

2 使用undetected_chromedriver（推荐）

import undetected_chromedriver as uc
from fake_useragent import UserAgent
def create_stealth_driver():
    ua = UserAgent()
    # 使用undetected_chromedriver
    driver = uc.Chrome(
        version_main=90,  # 指定Chrome版本
        options=webdriver.ChromeOptions()
    )
    # 设置自定义User-Agent
    driver.execute_cdp_cmd('Network.setUserAgentOverride', {
        "userAgent": ua.chrome
    })
    return driver
driver = create_stealth_driver()
driver.get('https://www.baidu.com')

完整伪装案例

1 伪装成浏览器访问API

import requests
import time
import random
from fake_useragent import UserAgent
class BrowserSimulator:
    def __init__(self):
        self.session = requests.Session()
        self.ua = UserAgent()
        self.update_headers()
    def update_headers(self):
        """更新请求头，模拟真实浏览器"""
        self.session.headers.update({
            'User-Agent': self.ua.random,
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
            'Accept-Encoding': 'gzip, deflate, br',
            'Connection': 'keep-alive',
            'DNT': '1',
            'Upgrade-Insecure-Requests': '1'
        })
    def add_random_delay(self, min_delay=1, max_delay=3):
        """添加随机延迟，模拟人类行为"""
        time.sleep(random.uniform(min_delay, max_delay))
    def get(self, url, **kwargs):
        """GET请求"""
        self.update_headers()
        self.add_random_delay()
        # 随机更换UA
        self.session.headers['User-Agent'] = self.ua.random
        return self.session.get(url, **kwargs)
    def post(self, url, data=None, json=None, **kwargs):
        """POST请求"""
        self.update_headers()
        self.add_random_delay()
        # 添加Referer
        self.session.headers['Referer'] = 'https://www.google.com/'
        return self.session.post(url, data=data, json=json, **kwargs)
# 使用示例
simulator = BrowserSimulator()
response = simulator.get('https://api.example.com/data')
print(response.status_code)

2 使用代理IP

import requests
from fake_useragent import UserAgent
def get_with_proxy(url):
    ua = UserAgent()
    # 代理配置
    proxies = {
        'http': 'http://user:password@proxy_host:port',
        'https': 'https://user:password@proxy_host:port'
    }
    headers = {
        'User-Agent': ua.random,
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
    }
    try:
        response = requests.get(
            url, 
            headers=headers, 
            proxies=proxies,
            timeout=10,
            verify=False  # 忽略SSL验证
        )
        return response
    except Exception as e:
        print(f"请求失败: {e}")
        return None
# 使用
url = 'https://httpbin.org/ip'
response = get_with_proxy(url)
if response:
    print(response.json())

反反爬虫最佳实践

1 请求频率控制

import time
import random
from collections import deque
class RateLimiter:
    def __init__(self, max_requests=10, time_window=60):
        self.max_requests = max_requests
        self.time_window = time_window
        self.requests_timestamps = deque()
    def can_request(self):
        now = time.time()
        # 移除过期的记录
        while self.requests_timestamps and self.requests_timestamps[0] < now - self.time_window:
            self.requests_timestamps.popleft()
        # 检查是否超过限制
        if len(self.requests_timestamps) >= self.max_requests:
            wait_time = self.requests_timestamps[0] + self.time_window - now
            if wait_time > 0:
                print(f"请求限制，等待 {wait_time:.2f} 秒")
                time.sleep(wait_time)
        self.requests_timestamps.append(now)
        return True
    def add_random_delay(self, min_delay=0.5, max_delay=2.0):
        delay = random.uniform(min_delay, max_delay)
        time.sleep(delay)
# 使用
limiter = RateLimiter(max_requests=5, time_window=10)
for i in range(10):
    if limiter.can_request():
        # 执行请求
        print(f"执行请求 {i+1}")
        limiter.add_random_delay(1, 3)

2 完整的反反爬虫策略

class AntiAntiSpider:
    def __init__(self):
        self.session = requests.Session()
        self.ua = UserAgent()
        self.limiter = RateLimiter()
        self.proxy_pool = self.init_proxy_pool()
    def init_proxy_pool(self):
        """初始化代理池"""
        # 从代理API获取代理列表
        return ['http://proxy1.com', 'http://proxy2.com']
    def get_random_proxy(self):
        """获取随机代理"""
        if self.proxy_pool:
            return random.choice(self.proxy_pool)
        return None
    def request(self, url, method='GET', **kwargs):
        """统一的请求方法"""
        # 1. 检查频率限制
        self.limiter.can_request()
        # 2. 生成随机延迟
        self.limiter.add_random_delay(1, 3)
        # 3. 更新请求头
        headers = kwargs.pop('headers', {})
        headers.update({
            'User-Agent': self.ua.random,
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'zh-CN,zh;q=0.9',
            'Cache-Control': 'max-age=0',
        })
        # 4. 设置代理
        proxy = self.get_random_proxy()
        if proxy:
            kwargs['proxies'] = {'http': proxy, 'https': proxy}
        # 5. 添加Cookie
        if 'cookies' not in kwargs:
            kwargs['cookies'] = self.session.cookies.get_dict()
        # 6. 执行请求
        try:
            if method == 'GET':
                response = self.session.get(url, headers=headers, **kwargs)
            else:
                response = self.session.post(url, headers=headers, **kwargs)
            # 7. 检查响应
            if response.status_code == 200:
                return response
            elif response.status_code == 403:
                print("被禁止访问，更换IP/UA")
                return None
            elif response.status_code == 429:
                print("请求过频，等待重试...")
                time.sleep(60)
                return self.request(url, method, **kwargs)
        except requests.exceptions.RequestException as e:
            print(f"请求异常: {e}")
            return None
# 使用
spider = AntiAntiSpider()
response = spider.request('https://example.com')

伪装浏览器的关键点：

User-Agent：必须设置且要经常更换
请求头完整性：模拟真实浏览器的所有请求头
请求频率控制：添加随机延迟
Cookie管理：使用会话保持状态
代理IP：避免IP被封
JavaScript支持：使用Selenium处理需要JS的页面

建议根据目标网站的防护程度,选择合适的伪装策略组合。