Python项目性能怎么持续优化

wen python案例 2026-07-04 79

Python项目性能持续优化指南

建立性能基线

在开始优化之前,必须先建立性能基线：

Python项目性能怎么持续优化

# 使用timeit进行微基准测试
import timeit
def measure_function(func, *args, **kwargs):
    def wrapper():
        return func(*args, **kwargs)
    return timeit.timeit(wrapper, number=1000)
# 使用cProfile进行性能分析
import cProfile
import pstats
def profile_function(func, *args, **kwargs):
    profiler = cProfile.Profile()
    profiler.enable()
    result = func(*args, **kwargs)
    profiler.disable()
    stats = pstats.Stats(profiler)
    stats.sort_stats('cumulative')
    stats.print_stats(20)
# 使用memory_profiler分析内存
# pip install memory_profiler
from memory_profiler import profile
@profile
def memory_intensive_function():
    # 你的代码
    pass

代码层面的优化策略

数据结构选择

# ❌ 低效：频繁查找
items = ['a', 'b', 'c', 'd']
if 'b' in items:  # O(n) 时间复杂度
    pass
# ✅ 高效：使用集合
items_set = {'a', 'b', 'c', 'd'}
if 'b' in items_set:  # O(1) 时间复杂度
    pass
# ❌ 低效：使用列表作为队列
queue = []
queue.append('item')
queue.pop(0)  # O(n) 操作
# ✅ 高效：使用collections.deque
from collections import deque
queue = deque()
queue.append('item')
queue.popleft()  # O(1) 操作

字符串优化

# ❌ 低效：字符串拼接
result = ''
for i in range(1000):
    result += str(i)  # 每次创建新字符串
# ✅ 高效：使用join
result = ''.join(str(i) for i in range(1000))
# ❌ 低效：格式化字符串
name = "World"
greeting = "Hello, " + name + "!"
# ✅ 高效：使用f-string
greeting = f"Hello, {name}!"

循环优化

# ❌ 低效：在循环中重复计算
for i in range(len(items)):
    print(items[i])
# ✅ 高效：直接迭代
for item in items:
    print(item)
# ❌ 低效：在循环中调用函数
for i in range(1000):
    result = expensive_function(i)
# ✅ 高效：预计算或使用缓存
results = [expensive_function(i) for i in range(1000)]
# 使用局部变量加速访问
def process_items(items):
    # 将全局变量变为局部变量
    len_items = len(items)
    result = []
    for i in range(len_items):
        result.append(items[i] * 2)
    return result

函数调用优化

# ❌ 低效：频繁访问属性
def process_data(data):
    for item in data:
        # 每次访问属性
        result = item.method1()
        result += item.method2()
        item.method3(result)
# ✅ 高效：缓存对象方法
def process_data_optimized(data):
    for item in data:
        method1 = item.method1
        method2 = item.method2
        method3 = item.method3
        result = method1()
        result += method2()
        method3(result)

使用性能分析工具

py-spy（无侵入式分析）

# 安装
pip install py-spy
# 实时查看调用栈
py-spy top --pid <PID>
# 生成火焰图
py-spy record -o profile.svg --pid <PID>

scalene（高级性能分析器）

# 安装
pip install scalene
# 运行分析
scalene your_script.py

使用优化编译器

# Numba JIT编译
from numba import jit
@jit(nopython=True)
def sum_of_squares(n):
    total = 0
    for i in range(n):
        total += i * i
    return total
# Cython优化
# 创建 setup.py
"""
from setuptools import setup
from Cython.Build import cythonize
setup(
    ext_modules = cythonize("optimized_module.pyx")
)
"""

并发与并行优化

多线程（IO密集型）

from concurrent.futures import ThreadPoolExecutor
import requests
def fetch_url(url):
    response = requests.get(url)
    return response.status_code
urls = ['https://example.com' for _ in range(100)]
# ✅ 使用线程池
with ThreadPoolExecutor(max_workers=10) as executor:
    results = list(executor.map(fetch_url, urls))

多进程（CPU密集型）

from multiprocessing import Pool
def cpu_intensive_task(n):
    # 计算密集型任务
    result = 0
    for i in range(n):
        result += i ** 2
    return result
# ✅ 使用进程池
with Pool(processes=4) as pool:
    results = pool.map(cpu_intensive_task, range(1000))

异步编程

import asyncio
import aiohttp
async def fetch_url_async(session, url):
    async with session.get(url) as response:
        return await response.text()
async def main():
    async with aiohttp.ClientSession() as session:
        tasks = [fetch_url_async(session, url) for url in urls]
        results = await asyncio.gather(*tasks)
asyncio.run(main())

缓存策略

内置缓存

from functools import lru_cache
@lru_cache(maxsize=1000)
def fibonacci(n):
    if n < 2:
        return n
    return fibonacci(n-1) + fibonacci(n-2)

Redis缓存

import redis
import json
class RedisCache:
    def __init__(self, host='localhost', port=6379):
        self.redis = redis.Redis(host=host, port=port)
    def get(self, key):
        data = self.redis.get(key)
        return json.loads(data) if data else None
    def set(self, key, value, expire=3600):
        self.redis.setex(key, expire, json.dumps(value))

数据库优化

查询优化

# ❌ N+1查询问题
for user in User.objects.all():
    print(user.profile.email)
# ✅ 使用select_related或prefetch_related
users = User.objects.select_related('profile').all()
for user in users:
    print(user.profile.email)

批量操作

# ❌ 单条插入
for user_data in user_list:
    User.objects.create(**user_data)
# ✅ 批量插入
User.objects.bulk_create([
    User(**data) for data in user_list
])

监控与告警

性能监控

import time
from functools import wraps
def performance_monitor(func):
    @wraps(func)
    def wrapper(*args, **kwargs):
        start = time.perf_counter()
        result = func(*args, **kwargs)
        elapsed = time.perf_counter() - start
        if elapsed > 1.0:  # 耗时超过1秒的告警
            print(f"WARNING: {func.__name__} took {elapsed:.2f}s")
        return result
    return wrapper
# 使用
@performance_monitor
def slow_function():
    time.sleep(2)

关键指标追踪

class PerformanceMetrics:
    def __init__(self):
        self.metrics = {}
    def record(self, name, value):
        if name not in self.metrics:
            self.metrics[name] = []
        self.metrics[name].append(value)
    def get_statistics(self, name):
        values = self.metrics.get(name, [])
        if not values:
            return {}
        return {
            'avg': sum(values) / len(values),
            'max': max(values),
            'min': min(values),
            'p95': sorted(values)[int(len(values) * 0.95)]
        }

代码质量与性能

代码审查清单

[ ] 避免过度的对象创建
[ ] 使用生成器代替列表推导式（大数据集）
[ ] 避免重复计算
[ ] 正确使用异常处理（不要用异常控制流程）
[ ] 避免过多的函数调用层级

持续集成中的性能测试

# .github/workflows/performance.yml
name: Performance Tests
on: [push, pull_request]
jobs:
  benchmark:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - uses: actions/setup-python@v2
      - name: Run benchmarks
        run: |
          pip install pytest-benchmark
          pytest tests/benchmarks/ -o "testpaths=tests/benchmarks"

实践案例

优化一个数据处理管道

# 优化前
def process_data_old(data):
    result = []
    for item in data:
        temp = []
        for i in range(len(item)):
            temp.append(clean(item[i]))
        result.append(filter(temp))
    return result
# 优化后
def process_data_new(data):
    # 使用生成器节省内存
    def generate_clean_items(items):
        for item in items:
            yield clean(item)
    return [
        filter(list(generate_clean_items(item)))
        for item in data
    ]
# 进一步优化：使用并行处理
from multiprocessing import Pool
def process_item(item):
    return filter([clean(x) for x in item])
def process_data_parallel(data, workers=4):
    with Pool(workers) as pool:
        return list(pool.map(process_item, data))