综合python案例，最终判断的置信度有多高？

wen python案例 2026-08-06 194

本文目录导读：

综合Python机器学习案例：信用评分预测
置信度分析总结

我可以帮你设计一个综合的Python机器学习案例，并详细分析最终判断的置信度，下面是一个包含数据预处理、模型训练、集成学习和置信度评估的完整案例。

综合Python机器学习案例：信用评分预测

数据准备与预处理

import numpy as np
import pandas as pd
from sklearn.model_selection import train_test_split, cross_val_score
from sklearn.preprocessing import StandardScaler, LabelEncoder
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier, VotingClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.metrics import accuracy_score, classification_report, confusion_matrix
from sklearn.calibration import CalibratedClassifierCV
import warnings
warnings.filterwarnings('ignore')
# 生成模拟信用数据
np.random.seed(42)
n_samples = 1000
# 特征生成
age = np.random.randint(18, 70, n_samples)
income = np.random.normal(50000, 20000, n_samples)
credit_score = np.random.normal(650, 100, n_samples)
debt_ratio = np.random.beta(2, 5, n_samples) * 100
years_employed = np.random.randint(0, 30, n_samples)
num_loans = np.random.randint(0, 10, n_samples)
default_history = np.random.choice([0, 1], n_samples, p=[0.8, 0.2])
education = np.random.choice(['High School', 'Bachelor', 'Master', 'PhD'], n_samples)
marital_status = np.random.choice(['Single', 'Married', 'Divorced', 'Widowed'], n_samples)
# 构造目标变量（信用良好与否）
prob_creditworthy = 1 / (1 + np.exp(-(0.01 * credit_score - 0.02 * debt_ratio + 0.01 * income/10000 - 0.5)))
target = np.random.binomial(1, prob_creditworthy)
# 创建DataFrame
data = pd.DataFrame({
    'age': age,
    'income': income,
    'credit_score': credit_score,
    'debt_ratio': debt_ratio,
    'years_employed': years_employed,
    'num_loans': num_loans,
    'default_history': default_history,
    'education': education,
    'marital_status': marital_status
})
data['creditworthy'] = target
print("数据形状:", data.shape)
print("\n目标变量分布:")
print(data['creditworthy'].value_counts(normalize=True))

特征工程与数据划分

# 编码分类变量
le_education = LabelEncoder()
le_marital = LabelEncoder()
data['education_encoded'] = le_education.fit_transform(data['education'])
data['marital_encoded'] = le_marital.fit_transform(data['marital_status'])
# 特征选择
features = ['age', 'income', 'credit_score', 'debt_ratio', 'years_employed', 
            'num_loans', 'default_history', 'education_encoded', 'marital_encoded']
X = data[features]
y = data['creditworthy']
# 数据标准化
scaler = StandardScaler()
X_scaled = scaler.fit_transform(X)
# 训练测试分割
X_train, X_test, y_train, y_test = train_test_split(
    X_scaled, y, test_size=0.3, random_state=42, stratify=y
)
print(f"训练集大小: {X_train.shape[0]}, 测试集大小: {X_test.shape[0]}")

多模型训练与集成

# 建立基础模型
models = {
    'Logistic Regression': LogisticRegression(max_iter=1000, random_state=42),
    'Random Forest': RandomForestClassifier(n_estimators=100, random_state=42),
    'Gradient Boosting': GradientBoostingClassifier(n_estimators=100, random_state=42),
    'SVM': SVC(probability=True, random_state=42)
}
# 训练并评估单个模型
print("=== 各模型性能 ===")
model_results = {}
for name, model in models.items():
    model.fit(X_train, y_train)
    y_pred = model.predict(X_test)
    y_proba = model.predict_proba(X_test)[:, 1]
    accuracy = accuracy_score(y_test, y_pred)
    model_results[name] = {
        'model': model,
        'accuracy': accuracy,
        'y_proba': y_proba
    }
    print(f"{name}: 准确率 = {accuracy:.4f}, AUC = {np.mean(y_proba[y_test==1] > 0.5):.4f}")
# 集成模型
voting_clf = VotingClassifier(
    estimators=[
        ('lr', models['Logistic Regression']),
        ('rf', models['Random Forest']),
        ('gb', models['Gradient Boosting']),
        ('svm', models['SVM'])
    ],
    voting='soft'
)
voting_clf.fit(X_train, y_train)

置信度分析与概率校准

from sklearn.calibration import calibration_curve
import matplotlib.pyplot as plt
# 获取集成模型的预测概率
voting_proba = voting_clf.predict_proba(X_test)[:, 1]
# 校准概率（使用Platt缩放）
calibrated_voting = CalibratedClassifierCV(voting_clf, cv=5, method='sigmoid')
calibrated_voting.fit(X_train, y_train)
calibrated_proba = calibrated_voting.predict_proba(X_test)[:, 1]
# 置信度分析
def analyze_confidence(y_true, y_proba, thresholds=[0.5, 0.6, 0.7, 0.8, 0.9]):
    """分析不同置信度阈值下的性能"""
    results = []
    for threshold in thresholds:
        y_pred_high_conf = (y_proba >= threshold).astype(int)
        y_pred_low_conf = (y_proba < threshold).astype(int)
        # 高置信度样本
        high_conf_mask = y_proba >= threshold
        if np.sum(high_conf_mask) > 0:
            high_conf_acc = accuracy_score(y_true[high_conf_mask], y_pred_high_conf[high_conf_mask])
        else:
            high_conf_acc = 0
        # 样本覆盖率和准确率
        coverage = np.mean(high_conf_mask)
        results.append({
            'threshold': threshold,
            'coverage': coverage,
            'high_conf_accuracy': high_conf_acc,
            'num_samples': np.sum(high_conf_mask)
        })
    return pd.DataFrame(results)
# 分析不同阈值下的置信度
print("\n=== 集成模型置信度分析 ===")
conf_analysis = analyze_confidence(y_test, voting_proba)
print(conf_analysis)
# 获取置信度区间
def get_confidence_intervals(y_true, y_proba, n_bins=10):
    """计算不同概率区间的准确率"""
    bins = np.linspace(0, 1, n_bins+1)
    bin_centers = (bins[:-1] + bins[1:]) / 2
    accuracies = []
    counts = []
    for i in range(n_bins):
        mask = (y_proba >= bins[i]) & (y_proba < bins[i+1])
        if np.sum(mask) > 0:
            acc = accuracy_score(y_true[mask], (y_proba[mask] >= 0.5).astype(int))
            accuracies.append(acc)
            counts.append(np.sum(mask))
        else:
            accuracies.append(0)
            counts.append(0)
    return bin_centers, accuracies, counts
# 计算置信度区间
bin_centers, accuracies, counts = get_confidence_intervals(y_test, voting_proba)
print("\n=== 概率分箱分析 ===")
for center, acc, count in zip(bin_centers, accuracies, counts):
    if count > 0:
        print(f"预测概率 {center:.1f}: 准确率 = {acc:.4f}, 样本数 = {count}")

可视化置信度

import matplotlib.pyplot as plt
# 创建子图
fig, axes = plt.subplots(2, 2, figsize=(15, 12))
# 1. 校准曲线
ax1 = axes[0, 0]
prob_true, prob_pred = calibration_curve(y_test, voting_proba, n_bins=10)
ax1.plot(prob_pred, prob_true, marker='o', label='Voting Classifier')
ax1.plot(prob_pred, calibrated_proba[y_test.argsort()[:len(prob_pred)]], marker='s', label='Calibrated')
ax1.plot([0, 1], [0, 1], 'k--', label='Perfect')
ax1.set_xlabel('Predicted Probability')
ax1.set_ylabel('Actual Probability')
ax1.set_title('Calibration Curve')
ax1.legend()
# 2. 置信度分布
ax2 = axes[0, 1]
ax2.hist(voting_proba[y_test==0], bins=20, alpha=0.5, label='Negative', color='red')
ax2.hist(voting_proba[y_test==1], bins=20, alpha=0.5, label='Positive', color='green')
ax2.set_xlabel('Predicted Probability')
ax2.set_ylabel('Frequency')
ax2.set_title('Confidence Distribution')
ax2.legend()
# 3. 准确率 vs 置信度
ax3 = axes[1, 0]
ax3.plot(bin_centers[counts>0], accuracies[counts>0], 'o-')
ax3.set_xlabel('Predicted Probability Bin')
ax3.set_ylabel('Accuracy')
ax3.set_title('Accuracy vs Confidence')
ax3.axhline(y=0.5, color='r', linestyle='--', alpha=0.5, label='Random')
ax3.legend()
# 4. 覆盖度 vs 准确率权衡
ax4 = axes[1, 1]
thresholds = np.linspace(0.5, 0.95, 20)
coverages = []
accs = []
for t in thresholds:
    mask = voting_proba >= t
    if np.sum(mask) > 0:
        coverages.append(np.mean(mask))
        accs.append(accuracy_score(y_test[mask], (voting_proba[mask] >= 0.5).astype(int)))
    else:
        coverages.append(0)
        accs.append(0)
ax4.plot(coverages, accs, 'o-')
ax4.set_xlabel('Coverage')
ax4.set_ylabel('Accuracy')
ax4.set_title('Coverage vs Accuracy Trade-off')
ax4.grid(True, alpha=0.3)
plt.tight_layout()
plt.show()

最终置信度评估

# 综合置信度评估函数
def assess_model_confidence(model, X_test, y_test, feature_importance=None):
    """
    评估模型最终判断的置信度
    返回多个置信度指标
    """
    y_proba = model.predict_proba(X_test)[:, 1]
    # 1. 平均置信度
    avg_confidence = np.mean(np.maximum(y_proba, 1-y_proba))
    # 2. 高置信度样本比例
    high_conf_threshold = 0.8
    high_conf_ratio = np.mean(np.maximum(y_proba, 1-y_proba) >= high_conf_threshold)
    # 3. 预测概率的校准度
    prob_true, prob_pred = calibration_curve(y_test, y_proba, n_bins=10)
    calibration_error = np.mean(np.abs(prob_true - prob_pred))
    # 4. 交叉验证稳定性
    cv_scores = cross_val_score(model, X_train, y_train, cv=5)
    cv_stability = 1 - np.std(cv_scores)
    # 5. 模型多样性得分（如果是集成模型）
    if hasattr(model, 'estimators_'):
        predictions = np.column_stack([
            est.predict_proba(X_test)[:, 1] for est in model.estimators_
        ])
        model_diversity = 1 - np.mean(np.std(predictions, axis=1))
    else:
        model_diversity = 0.5
    return {
        'average_confidence': avg_confidence,
        'high_confidence_ratio': high_conf_ratio,
        'calibration_error': calibration_error,
        'cv_stability': cv_stability,
        'model_diversity': model_diversity
    }
# 最终评估
print("\n=== 最终置信度评估 ===")
confidence_metrics = assess_model_confidence(voting_clf, X_test, y_test)
for metric, value in confidence_metrics.items():
    print(f"{metric}: {value:.4f}")
# 综合置信度得分
final_confidence = (
    confidence_metrics['average_confidence'] * 0.3 +
    confidence_metrics['high_confidence_ratio'] * 0.2 +
    (1 - confidence_metrics['calibration_error']) * 0.3 +
    confidence_metrics['cv_stability'] * 0.1 +
    confidence_metrics['model_diversity'] * 0.1
)
print(f"\n综合置信度得分: {final_confidence:.4f} ({final_confidence*100:.2f}%)")
print("\n置信度等级: ", 
      "极高" if final_confidence > 0.8 else 
      "高" if final_confidence > 0.7 else 
      "中等" if final_confidence > 0.6 else 
      "需要谨慎使用")