Python案例如何用Scikit-learn做验证曲线

本文目录导读：

1. 验证曲线的基本概念
2. 基础示例：SVM的RBF核参数gamma验证曲线
3. 更完整的示例：随机森林的树数量验证
4. 优化计算：使用学习曲线对比
5. 批量分析多个参数
6. 实际应用案例：房价预测
7. 注意事项和最佳实践

我来详细介绍如何使用Scikit-learn绘制验证曲线（Validation Curve）来分析模型参数的影响。

验证曲线的基本概念

验证曲线展示不同参数值下模型的训练集和验证集性能,帮助我们：

• 判断模型是否过拟合或欠拟合
• 选择合适的超参数值
• 分析参数对模型性能的影响

基础示例：SVM的RBF核参数gamma验证曲线

import numpy as np
import matplotlib.pyplot as plt
from sklearn.datasets import make_classification
from sklearn.model_selection import validation_curve
from sklearn.svm import SVC
from sklearn.preprocessing import StandardScaler
from sklearn.pipeline import Pipeline
# 1. 生成数据
X, y = make_classification(
    n_samples=1000,
    n_features=20,
    n_informative=10,
    n_redundant=5,
    random_state=42
)
# 2. 创建管道（包含标准化和SVM）
pipeline = Pipeline([
    ('scaler', StandardScaler()),
    ('svm', SVC(kernel='rbf'))
])
# 3. 定义要测试的参数范围
param_range = np.logspace(-3, 3, 7)  # 从0.001到1000，7个值
# 4. 计算验证曲线
train_scores, test_scores = validation_curve(
    pipeline,           # 估计器
    X, y,               # 数据
    param_name='svm__gamma',  # 参数名称（注意管道中的前缀）
    param_range=param_range,  # 参数范围
    cv=5,               # 5折交叉验证
    scoring='accuracy', # 评估指标
    n_jobs=-1           # 使用所有CPU核心
)
# 5. 计算平均分和标准差
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
# 6. 绘制验证曲线
plt.figure(figsize=(10, 6))
plt.semilogx(param_range, train_mean, 'o-', color='blue', label='Training score')
plt.fill_between(param_range, 
                 train_mean - train_std, 
                 train_mean + train_std, 
                 alpha=0.25, color='blue')
plt.semilogx(param_range, test_mean, 's-', color='red', label='Cross-validation score')
plt.fill_between(param_range, 
                 test_mean - test_std, 
                 test_mean + test_std, 
                 alpha=0.25, color='red')
plt.xlabel('Parameter gamma')
plt.ylabel('Score')
plt.legend(loc='best')'Validation Curve with SVM (RBF kernel)')
plt.grid(True, alpha=0.3)
plt.show()
# 7. 打印结果
print("Gamma值验证结果:")
for i, gamma in enumerate(param_range):
    print(f"gamma={gamma:.4f}: Train={train_mean[i]:.4f} (+/- {train_std[i]:.4f}), "
          f"Test={test_mean[i]:.4f} (+/- {test_std[i]:.4f})")

更完整的示例：随机森林的树数量验证

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import StratifiedKFold
# 1. 使用更复杂的数据集
from sklearn.datasets import load_digits
data = load_digits()
X, y = data.data, data.target
# 2. 创建随机森林模型
rf = RandomForestClassifier(random_state=42)
# 3. 测试不同数量的树
param_range = [10, 50, 100, 200, 500]
# 4. 使用分层K折交叉验证
cv = StratifiedKFold(n_splits=5, shuffle=True, random_state=42)
# 5. 计算验证曲线（使用更多评估指标）
from sklearn.metrics import make_scorer, accuracy_score, f1_score
# 同时计算准确率和F1分数
train_scores_acc, test_scores_acc = validation_curve(
    rf, X, y,
    param_name='n_estimators',
    param_range=param_range,
    cv=cv,
    scoring='accuracy',
    n_jobs=-1
)
train_scores_f1, test_scores_f1 = validation_curve(
    rf, X, y,
    param_name='n_estimators',
    param_range=param_range,
    cv=cv,
    scoring='f1_macro',
    n_jobs=-1
)
# 6. 绘制双指标对比图
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
# 准确率曲线
for ax, scores, metric in [(ax1, (train_scores_acc, test_scores_acc), 'Accuracy'),
                           (ax2, (train_scores_f1, test_scores_f1), 'F1 Score')]:
    train_mean = np.mean(scores[0], axis=1)
    train_std = np.std(scores[0], axis=1)
    test_mean = np.mean(scores[1], axis=1)
    test_std = np.std(scores[1], axis=1)
    ax.plot(param_range, train_mean, 'o-', color='blue', label='Training')
    ax.fill_between(param_range, 
                     train_mean - train_std, 
                     train_mean + train_std, 
                     alpha=0.25, color='blue')
    ax.plot(param_range, test_mean, 's-', color='red', label='Cross-validation')
    ax.fill_between(param_range, 
                     test_mean - test_std, 
                     test_mean + test_std, 
                     alpha=0.25, color='red')
    ax.set_xlabel('Number of trees')
    ax.set_ylabel(metric)
    ax.set_title(f'Validation Curve ({metric})')
    ax.legend(loc='best')
    ax.grid(True, alpha=0.3)
    ax.set_xscale('log')
plt.tight_layout()
plt.show()

优化计算：使用学习曲线对比

from sklearn.model_selection import learning_curve
# 有时候验证曲线不够直观，可以结合学习曲线
def plot_validation_and_learning_curves(estimator, X, y, param_name, param_range):
    """绘制验证曲线和学习曲线的组合图"""
    fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(14, 5))
    # 1. 验证曲线
    train_sizes, train_scores, test_scores = validation_curve(
        estimator, X, y,
        param_name=param_name,
        param_range=param_range,
        cv=5,
        scoring='accuracy',
        n_jobs=-1
    )
    # 绘制验证曲线
    train_mean = np.mean(train_scores, axis=1)
    train_std = np.std(train_scores, axis=1)
    test_mean = np.mean(test_scores, axis=1)
    test_std = np.std(test_scores, axis=1)
    ax1.plot(param_range, train_mean, 'o-', color='blue', label='Training score')
    ax1.fill_between(param_range, train_mean - train_std, 
                     train_mean + train_std, alpha=0.1, color='blue')
    ax1.plot(param_range, test_mean, 's-', color='red', label='Cross-validation score')
    ax1.fill_between(param_range, test_mean - test_std, 
                     test_mean + test_std, alpha=0.1, color='red')
    ax1.set_xlabel('Parameter value')
    ax1.set_ylabel('Score')
    ax1.set_title('Validation Curve')
    ax1.legend(loc='best')
    ax1.grid(True, alpha=0.3)
    # 2. 学习曲线（使用最佳参数）
    best_idx = np.argmax(test_mean)
    estimator.set_params(**{param_name: param_range[best_idx]})
    train_sizes_lc, train_scores_lc, test_scores_lc = learning_curve(
        estimator, X, y,
        train_sizes=np.linspace(0.1, 1.0, 10),
        cv=5,
        scoring='accuracy',
        n_jobs=-1
    )
    train_mean_lc = np.mean(train_scores_lc, axis=1)
    train_std_lc = np.std(train_scores_lc, axis=1)
    test_mean_lc = np.mean(test_scores_lc, axis=1)
    test_std_lc = np.std(test_scores_lc, axis=1)
    ax2.plot(train_sizes_lc, train_mean_lc, 'o-', color='blue', label='Training score')
    ax2.fill_between(train_sizes_lc, train_mean_lc - train_std_lc, 
                     train_mean_lc + train_std_lc, alpha=0.1, color='blue')
    ax2.plot(train_sizes_lc, test_mean_lc, 's-', color='red', label='Cross-validation score')
    ax2.fill_between(train_sizes_lc, test_mean_lc - test_std_lc, 
                     test_mean_lc + test_std_lc, alpha=0.1, color='red')
    ax2.set_xlabel('Training examples')
    ax2.set_ylabel('Score')
    ax2.set_title(f'Learning Curve (best {param_name}={param_range[best_idx]:.4f})')
    ax2.legend(loc='best')
    ax2.grid(True, alpha=0.3)
    plt.tight_layout()
    plt.show()
    return best_idx, param_range[best_idx]
# 使用示例
print("分析SVM的gamma参数...")
best_idx, best_param = plot_validation_and_learning_curves(
    pipeline, X, y, 'svm__gamma', param_range
)
print(f"最佳gamma值: {best_param:.4f}")

批量分析多个参数

from sklearn.model_selection import ParameterGrid
def analyze_multiple_params(estimator, X, y, param_grid, scoring='accuracy'):
    """批量分析多个参数的验证曲线"""
    results = {}
    for param_name, param_range in param_grid.items():
        print(f"分析参数: {param_name}")
        train_scores, test_scores = validation_curve(
            estimator, X, y,
            param_name=param_name,
            param_range=param_range,
            cv=5,
            scoring=scoring,
            n_jobs=-1
        )
        results[param_name] = {
            'param_range': param_range,
            'train_mean': np.mean(train_scores, axis=1),
            'train_std': np.std(train_scores, axis=1),
            'test_mean': np.mean(test_scores, axis=1),
            'test_std': np.std(test_scores, axis=1),
            'best_param': param_range[np.argmax(np.mean(test_scores, axis=1))]
        }
    # 绘制多个参数的验证曲线
    n_params = len(results)
    fig, axes = plt.subplots(1, n_params, figsize=(5*n_params, 5))
    if n_params == 1:
        axes = [axes]
    for ax, (param_name, result) in zip(axes, results.items()):
        ax.plot(result['param_range'], result['train_mean'], 'o-', label='Training')
        ax.fill_between(result['param_range'], 
                       result['train_mean'] - result['train_std'],
                       result['train_mean'] + result['train_std'], 
                       alpha=0.2)
        ax.plot(result['param_range'], result['test_mean'], 's-', label='Cross-validation')
        ax.fill_between(result['param_range'],
                       result['test_mean'] - result['test_std'],
                       result['test_mean'] + result['test_std'], 
                       alpha=0.2)
        ax.set_xlabel(param_name)
        ax.set_ylabel(scoring)
        ax.set_title(f'Validation Curve: {param_name}')
        ax.legend()
        ax.grid(True, alpha=0.3)
        # 标记最佳值
        best_val = result['best_param']
        ax.axvline(x=best_val, color='green', linestyle='--', alpha=0.5)
        ax.annotate(f'Best: {best_val:.4f}', 
                   xy=(best_val, ax.get_ylim()[1]),
                   xytext=(best_val, ax.get_ylim()[1]*0.9),
                   arrowprops=dict(arrowstyle='->'))
    plt.tight_layout()
    plt.show()
    return results
# 使用示例
param_grid = {
    'svc__C': [0.001, 0.01, 0.1, 1, 10, 100],
    'svc__gamma': [0.001, 0.01, 0.1, 1, 10, 100]
}
# 注意：需要调整管道定义
pipeline2 = Pipeline([
    ('scaler', StandardScaler()),
    ('svc', SVC(kernel='rbf'))
])
# 由于验证曲线一次只能分析一个参数，我们分别分析
analysis_results = analyze_multiple_params(
    pipeline2, X, y, 
    {'svc__C': [0.001, 0.01, 0.1, 1, 10, 100]}
)

实际应用案例：房价预测

from sklearn.datasets import fetch_california_housing
from sklearn.ensemble import GradientBoostingRegressor
from sklearn.metrics import mean_squared_error, r2_score
# 加载数据
housing = fetch_california_housing()
X, y = housing.data, housing.target
# 创建GBDT模型
gbr = GradientBoostingRegressor(random_state=42)
# 分析学习率参数
param_range = [0.01, 0.05, 0.1, 0.2, 0.5, 1.0]
train_scores, test_scores = validation_curve(
    gbr, X, y,
    param_name='learning_rate',
    param_range=param_range,
    cv=5,
    scoring='r2',
    n_jobs=-1
)
# 绘制详细的结果图
plt.figure(figsize=(12, 5))
plt.subplot(1, 2, 1)
train_mean = np.mean(train_scores, axis=1)
train_std = np.std(train_scores, axis=1)
test_mean = np.mean(test_scores, axis=1)
test_std = np.std(test_scores, axis=1)
plt.semilogx(param_range, train_mean, 'o-', color='blue', label='Training R²')
plt.fill_between(param_range, 
                 train_mean - train_std, 
                 train_mean + train_std, 
                 alpha=0.2, color='blue')
plt.semilogx(param_range, test_mean, 's-', color='red', label='Cross-validation R²')
plt.fill_between(param_range, 
                 test_mean - test_std, 
                 test_mean + test_std, 
                 alpha=0.2, color='red')
plt.xlabel('Learning Rate')
plt.ylabel('R² Score')'Validation Curve: Learning Rate Effect')
plt.legend(loc='best')
plt.grid(True, alpha=0.3)
plt.subplot(1, 2, 2)
# 计算过拟合程度
overfitting = train_mean - test_mean
plt.semilogx(param_range, overfitting, 'd-', color='purple', linewidth=2)
plt.axhline(y=0, color='gray', linestyle='--', alpha=0.5)
plt.xlabel('Learning Rate')
plt.ylabel('Train - Test Score (Overfitting)')'Overfitting Analysis')
plt.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()
print(f"最佳学习率: {param_range[np.argmax(test_mean)]:.3f}")
print(f"最佳R²分数: {np.max(test_mean):.4f}")

注意事项和最佳实践

# 1. 参数范围选择
def suggest_param_range(estimator, X, y, param_name):
    """自动建议参数范围"""
    # 这里只是一个示例，实际需要根据模型类型调整
    base_params = estimator.get_params()
    default_val = base_params.get(param_name, 1.0)
    # 建议从默认值的1/10到10倍
    if isinstance(default_val, (int, float)):
        if default_val > 0:
            return np.logspace(
                np.log10(default_val) - 1,
                np.log10(default_val) + 1,
                10
            )
    return np.linspace(0.1, 10, 10)
# 2. 并行计算优化
from joblib import parallel_backend
def optimized_validation_curve(estimator, X, y, param_name, param_range):
    """使用并行优化的验证曲线"""
    with parallel_backend('threading', n_jobs=-1):
        train_scores, test_scores = validation_curve(
            estimator, X, y,
            param_name=param_name,
            param_range=param_range,
            cv=5,
            scoring='accuracy',
            n_jobs=-1
        )
    return train_scores, test_scores
# 3. 结果保存和比较
def save_validation_results(results, filename='validation_results.npy'):
    """保存验证曲线结果"""
    np.save(filename, results)
    print(f"Results saved to {filename}")
def load_validation_results(filename='validation_results.npy'):
    """加载验证曲线结果"""
    return np.load(filename, allow_pickle=True).item()
# 4. 自动选择最佳参数
def find_optimal_param(estimator, X, y, param_name, param_range, 
                      metric='accuracy', verbose=True):
    """自动找到使验证分数最高的参数"""
    train_scores, test_scores = validation_curve(
        estimator, X, y,
        param_name=param_name,
        param_range=param_range,
        cv=5,
        scoring=metric,
        n_jobs=-1
    )
    test_mean = np.mean(test_scores, axis=1)
    best_idx = np.argmax(test_mean)
    best_param = param_range[best_idx]
    if verbose:
        print(f"Best {param_name}: {best_param}")
        print(f"Best {metric}: {test_mean[best_idx]:.4f}")
        print(f"Training {metric}: {np.mean(train_scores, axis=1)[best_idx]:.4f}")
        # 检查过拟合
        overfit = np.mean(train_scores, axis=1)[best_idx] - test_mean[best_idx]
        if overfit > 0.1:
            print(f"Warning: Possible overfitting (difference: {overfit:.4f})")
    return best_param, test_mean[best_idx]

通过以上案例,你可以：

1. 快速理解验证曲线的作用和用法
2. 掌握不同模型的参数调优方法
3. 学会分析过拟合和欠拟合情况
4. 自动化参数选择过程

验证曲线是机器学习模型优化的重要工具,建议在实际项目中经常使用！