超参数调优自动化

超参数调优自动化

超参数调优自动化是机器学习工作流程中的关键环节,目的是通过算法自动找到最优的超参数组合，提高模型性能，减少人工试错成本。

主要自动化调优方法

网格搜索

from sklearn.model_selection import GridSearchCV
from sklearn.ensemble import RandomForestClassifier
# 定义超参数网格
param_grid = {
    'n_estimators': [100, 200, 300],
    'max_depth': [10, 20, None],
    'min_samples_split': [2, 5, 10]
}
# 网格搜索
grid_search = GridSearchCV(
    RandomForestClassifier(),
    param_grid,
    cv=5,
    scoring='accuracy',
    n_jobs=-1
)
grid_search.fit(X_train, y_train)
print(f"Best parameters: {grid_search.best_params_}")

缺点：维度灾难，计算成本随参数数量指数增长

随机搜索

from sklearn.model_selection import RandomizedSearchCV
from scipy.stats import randint, uniform
param_dist = {
    'n_estimators': randint(100, 1000),
    'max_depth': randint(5, 50),
    'learning_rate': uniform(0.01, 0.3)
}
random_search = RandomizedSearchCV(
    GradientBoostingClassifier(),
    param_dist,
    n_iter=100,  # 迭代次数
    cv=5,
    random_state=42
)
random_search.fit(X_train, y_train)

优点：比网格搜索更高效，适合高维参数空间

贝叶斯优化

基于概率模型的序贯优化方法,是目前最高效的方法之一。

!pip install scikit-optimize
from skopt import BayesSearchCV
from skopt.space import Real, Integer
search_space = {
    'n_estimators': Integer(100, 1000),
    'max_depth': Integer(5, 50),
    'learning_rate': Real(0.01, 0.3, 'log-uniform')
}
bayes_search = BayesSearchCV(
    GradientBoostingClassifier(),
    search_space,
    n_iter=50,
    cv=5,
    n_jobs=-1
)
bayes_search.fit(X_train, y_train)

核心思想：通过高斯过程或TPE（Tree-structured Parzen Estimator）构建代理模型，平衡探索与利用。

Hyperopt

from hyperopt import fmin, tpe, hp, Trials
# 定义搜索空间
space = {
    'n_estimators': hp.choice('n_estimators', [100, 200, 300, 500, 800]),
    'max_depth': hp.quniform('max_depth', 5, 50, 1),
    'learning_rate': hp.loguniform('learning_rate', -4.6, -1.2)
}
def objective(params):
    model = GradientBoostingClassifier(
        n_estimators=params['n_estimators'],
        max_depth=int(params['max_depth']),
        learning_rate=params['learning_rate']
    )
    score = cross_val_score(model, X_train, y_train, cv=5).mean()
    return -score  # Hyperopt 默认最小化
# 运行优化
best = fmin(fn=objective, space=space, algo=tpe.suggest, max_evals=100)

Optuna

现代框架,支持剪枝和分布式优化。

import optuna
def objective(trial):
    # 定义超参数
    params = {
        'n_estimators': trial.suggest_int('n_estimators', 100, 1000),
        'max_depth': trial.suggest_int('max_depth', 5, 50),
        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),
        'min_samples_split': trial.suggest_int('min_samples_split', 2, 20)
    }
    model = GradientBoostingClassifier(**params)
    score = cross_val_score(model, X_train, y_train, cv=5).mean()
    return score
# 创建研究
study = optuna.create_study(direction='maximize')
study.optimize(objective, n_trials=100)
print(f"Best trial: {study.best_trial.params}")
print(f"Best score: {study.best_trial.value}")

遗传算法

# 使用 DEAP 或 PyGAD 库
from deap import base, creator, tools, algorithms
# 定义适应度函数
def fitness_function(individual):
    params = decode_individual(individual)
    model = create_model(params)
    score = cross_val_score(model, X_train, y_train, cv=5).mean()
    return score,
# 遗传算法配置
# ... (使用选择、交叉、变异操作)

高级技术

多阶段调优

# 第一阶段：粗粒度搜索
coarse_search = ...
# 第二阶段：细粒度搜索（基于第一阶段结果）
fine_search = ...

早停与剪枝

# Optuna 中的剪枝示例
from optuna.pruners import MedianPruner
study = optuna.create_study(
    pruner=MedianPruner(n_startup_trials=5, n_warmup_steps=30)
)

多目标优化

# 同时优化准确率和推理速度
def multi_objective(trial):
    params = ...
    accuracy = ...
    inference_time = ...
    return accuracy, -inference_time  # 最大化准确率，最小化推理时间
study = optuna.create_study(directions=['maximize', 'maximize'])

约束优化

处理某些超参数组合不可行的情况：

• 学习率 × 迭代次数 ≤ 某个上限
• 模型大小限制

平台与工具

实践建议

1. 先随机搜索，再贝叶斯优化：快速了解参数空间
2. 使用交叉验证：避免过拟合
3. 并行化：利用多核/GPU
4. 设置早停条件：避免浪费计算资源
5. 记录实验：使用 MLflow/W&B 跟踪每次实验
6. 考虑成本约束：在某些场景下，模型大小/推理时间也很重要

完整工作流示例

# 结合 Optuna + 早停 + 交叉验证
import optuna
from sklearn.model_selection import cross_val_score
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.datasets import make_classification
from sklearn.model_selection import train_test_split
# 准备数据
X, y = make_classification(n_samples=1000, n_features=20, random_state=42)
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
def objective(trial):
    params = {
        'n_estimators': trial.suggest_int('n_estimators', 50, 500),
        'max_depth': trial.suggest_int('max_depth', 3, 30),
        'learning_rate': trial.suggest_float('learning_rate', 0.01, 0.3, log=True),
        'subsample': trial.suggest_float('subsample', 0.5, 1.0),
        'min_samples_split': trial.suggest_int('min_samples_split', 2, 20)
    }
    model = GradientBoostingClassifier(
        **params,
        early_stopping=True,
        validation_fraction=0.1,
        n_iter_no_change=10,
        random_state=42
    )
    score = cross_val_score(model, X_train, y_train, cv=5).mean()
    return score
# 创建带剪枝的研究
study = optuna.create_study(
    direction='maximize',
    pruner=optuna.pruners.MedianPruner(
        n_startup_trials=5,
        n_warmup_steps=10,
        interval_steps=1
    )
)
# 优化
study.optimize(objective, n_trials=50)
# 最终模型
best_params = study.best_params
final_model = GradientBoostingClassifier(**best_params, random_state=42)
final_model.fit(X_train, y_train)
print(f"Best AUC: {study.best_value:.4f}")
test_score = final_model.score(X_test, y_test)
print(f"Test Accuracy: {test_score:.4f}")

选择合适的超参数自动化调优方法可以显著提高模型性能,同时节省大量开发时间，建议根据问题规模、计算资源和时间约束选择最适合的方法。