Python案例如何用Pandas做数据分组相似度

wen python案例 2026-07-14 262

本文目录导读：

基础准备
计算组内相似度
组间相似度比较
综合相似度分析案例
可视化相似度结果
实际应用案例：客户分组相似度
进阶：时间序列分组相似度

我将为您详细介绍如何使用Pandas进行数据分组相似度分析,这里提供几个实用案例：

基础准备

import pandas as pd
import numpy as np
from scipy.spatial.distance import pdist, squareform
from scipy.stats import pearsonr
from sklearn.metrics.pairwise import cosine_similarity
import warnings
warnings.filterwarnings('ignore')
# 创建示例数据
np.random.seed(42)
data = pd.DataFrame({
    'group': ['A']*4 + ['B']*4 + ['C']*4,
    'feature1': np.random.randn(12) * 10,
    'feature2': np.random.randn(12) * 10,
    'feature3': np.random.randn(12) * 10,
    'feature4': np.random.randn(12) * 10
})
print("示例数据预览：")
print(data.head(8))

计算组内相似度

1 欧氏距离相似度

def euclidean_similarity(matrix):
    """将欧氏距离转换为相似度"""
    distances = pdist(matrix, metric='euclidean')
    distances_matrix = squareform(distances)
    # 归一化为相似度（0-1之间）
    max_dist = distances_matrix.max()
    similarity = 1 - (distances_matrix / max_dist)
    return similarity
def group_euclidean_similarity(df, group_col, feature_cols):
    """计算每个组内的欧氏距离相似度"""
    results = {}
    for group, group_data in df.groupby(group_col):
        features = group_data[feature_cols].values
        if len(features) > 1:
            sim_matrix = euclidean_similarity(features)
            avg_similarity = (sim_matrix.sum() - len(features)) / (len(features) * (len(features) - 1))
            results[group] = {
                'avg_similarity': avg_similarity,
                'similarity_matrix': pd.DataFrame(
                    sim_matrix,
                    index=group_data.index,
                    columns=group_data.index
                )
            }
    return results
# 计算组内欧氏距离相似度
euclidean_results = group_euclidean_similarity(data, 'group', ['feature1', 'feature2', 'feature3', 'feature4'])
print("\n各组欧氏距离平均相似度：")
for group, result in euclidean_results.items():
    print(f"组 {group}: {result['avg_similarity']:.4f}")

2 余弦相似度

def group_cosine_similarity(df, group_col, feature_cols):
    """计算每个组内的余弦相似度"""
    results = {}
    for group, group_data in df.groupby(group_col):
        features = group_data[feature_cols].values
        if len(features) > 1:
            cos_sim = cosine_similarity(features)
            avg_similarity = (cos_sim.sum() - len(features)) / (len(features) * (len(features) - 1))
            results[group] = {
                'avg_similarity': avg_similarity,
                'similarity_matrix': pd.DataFrame(
                    cos_sim,
                    index=group_data.index,
                    columns=group_data.index
                )
            }
    return results
cosine_results = group_cosine_similarity(data, 'group', ['feature1', 'feature2', 'feature3', 'feature4'])
print("\n各组余弦相似度结果：")
for group, result in cosine_results.items():
    print(f"组 {group}: 平均余弦相似度 = {result['avg_similarity']:.4f}")

组间相似度比较

def between_group_similarity(df, group_col, feature_cols):
    """计算不同组之间的平均相似度"""
    groups = df.groupby(group_col)
    group_means = groups[feature_cols].mean()
    # 计算各组均值之间的余弦相似度
    similarity_matrix = cosine_similarity(group_means.values)
    similarity_df = pd.DataFrame(
        similarity_matrix,
        index=group_means.index,
        columns=group_means.index
    )
    return similarity_df
between_group_sim = between_group_similarity(data, 'group', ['feature1', 'feature2', 'feature3', 'feature4'])
print("\n组间相似度矩阵（基于均值）：")
print(between_group_sim)

综合相似度分析案例

def comprehensive_similarity_analysis(df, group_col, feature_cols, method='cosine'):
    """综合相似度分析"""
    # 1. 组内相似度
    within_group_sim = {}
    for group, group_data in df.groupby(group_col):
        features = group_data[feature_cols].values
        if len(features) > 1:
            if method == 'cosine':
                sim_matrix = cosine_similarity(features)
            else:  # euclidean
                sim_matrix = euclidean_similarity(features)
            # 统计信息
            upper_tri = sim_matrix[np.triu_indices_from(sim_matrix, k=1)]
            within_group_sim[group] = {
                'mean': np.mean(upper_tri),
                'std': np.std(upper_tri),
                'min': np.min(upper_tri),
                'max': np.max(upper_tri),
                'median': np.median(upper_tri)
            }
    within_df = pd.DataFrame(within_group_sim).T
    within_df.index.name = group_col
    # 2. 组间相似度
    group_means = df.groupby(group_col)[feature_cols].mean()
    if method == 'cosine':
        between_sim = cosine_similarity(group_means.values)
    else:
        between_sim = euclidean_similarity(group_means.values)
    between_df = pd.DataFrame(
        between_sim,
        index=group_means.index,
        columns=group_means.index
    )
    return {
        'within_group': within_df,
        'between_group': between_df,
        'group_means': group_means
    }
# 运行综合分析
results = comprehensive_similarity_analysis(data, 'group', 
                                            ['feature1', 'feature2', 'feature3', 'feature4'],
                                            method='cosine')
print("\n=== 组内相似度统计 ===")
print(results['within_group'])
print("\n=== 组间相似度矩阵 ===")
print(results['between_group'])
print("\n=== 各组特征均值 ===")
print(results['group_means'])

可视化相似度结果

import matplotlib.pyplot as plt
import seaborn as sns
def plot_similarity_analysis(results):
    """可视化相似度分析结果"""
    fig, axes = plt.subplots(1, 3, figsize=(15, 5))
    # 1. 组内相似度分布
    within_data = results['within_group']
    axes[0].bar(within_data.index, within_data['mean'], yerr=within_data['std'])
    axes[0].set_title('组内平均相似度')
    axes[0].set_xlabel('Group')
    axes[0].set_ylabel('相似度')
    # 2. 组间相似度热力图
    sns.heatmap(results['between_group'], annot=True, fmt='.3f', 
                cmap='YlOrRd', ax=axes[1])
    axes[1].set_title('组间相似度热力图')
    # 3. 特征均值雷达图（简化版）
    means = results['group_means'].T
    for i, group in enumerate(means.columns):
        axes[2].plot(means.index, means[group], marker='o', label=group)
    axes[2].set_title('各组特征均值比较')
    axes[2].set_xlabel('Features')
    axes[2].set_ylabel('Mean Value')
    axes[2].legend()
    plt.tight_layout()
    plt.show()
plot_similarity_analysis(results)

实际应用案例：客户分组相似度

# 创建客户数据示例
customers = pd.DataFrame({
    'customer_id': range(1, 13),
    'segment': ['高端']*4 + ['中端']*4 + ['低端']*4,
    'annual_spending': np.random.randint(1000, 50000, 12),
    'purchase_frequency': np.random.randint(1, 20, 12),
    'avg_order_value': np.random.randint(50, 500, 12),
    'tenure_months': np.random.randint(1, 36, 12)
})
print("客户数据预览：")
print(customers)
# 标准化处理
from sklearn.preprocessing import StandardScaler
features = ['annual_spending', 'purchase_frequency', 'avg_order_value', 'tenure_months']
scaler = StandardScaler()
customers_scaled = customers.copy()
customers_scaled[features] = scaler.fit_transform(customers[features])
# 分析各客户分组的相似度
segments_analysis = comprehensive_similarity_analysis(
    customers_scaled, 'segment', features, method='cosine'
)
print("\n不同客户分组的相似度分析：")
print("="*50)
print(f"\n组内相似度统计：")
print(segments_analysis['within_group'])
print(f"\n不同分组间的相似度：")
print(segments_analysis['between_group'])
# 找出最相似和最不相似的客户对
def find_most_similar_customers(df, feature_cols, top_n=3):
    """找出最相似的客户对"""
    features = df[feature_cols].values
    sim_matrix = cosine_similarity(features)
    np.fill_diagonal(sim_matrix, 0)  # 忽略自身
    similar_pairs = []
    for i in range(len(sim_matrix)):
        for j in range(i+1, len(sim_matrix)):
            similar_pairs.append({
                'customer1': df.iloc[i]['customer_id'],
                'customer2': df.iloc[j]['customer_id'],
                'similarity': sim_matrix[i][j],
                'segment1': df.iloc[i]['segment'],
                'segment2': df.iloc[j]['segment']
            })
    similar_df = pd.DataFrame(similar_pairs)
    return similar_df.sort_values('similarity', ascending=False).head(top_n)
top_similar = find_most_similar_customers(customers_scaled, features)
print("\n最相似的前3对客户：")
print(top_similar)

进阶：时间序列分组相似度

# 创建时间序列数据
dates = pd.date_range('2024-01-01', periods=30, freq='D')
time_series_data = []
for group in ['A', 'B', 'C']:
    for date in dates:
        time_series_data.append({
            'date': date,
            'group': group,
            'value': np.random.randn() * 10 + 50 + (ord(group) - ord('A')) * 5
        })
ts_df = pd.DataFrame(time_series_data)
ts_df.set_index('date', inplace=True)
def time_series_group_similarity(df, group_col, value_col, window=7):
    """计算时间序列分组相似度"""
    # 计算滚动窗口内的统计量
    results = {}
    for group in df[group_col].unique():
        group_data = df[df[group_col] == group][value_col]
        results[group] = {
            'rolling_mean': group_data.rolling(window=window).mean(),
            'rolling_std': group_data.rolling(window=window).std()
        }
    # 计算皮尔逊相关系数
    mean_series = pd.DataFrame({
        group: results[group]['rolling_mean'] 
        for group in results.keys()
    }).dropna()
    corr_matrix = mean_series.corr(method='pearson')
    return corr_matrix, mean_series
# 运行时间序列分析
time_corr, time_means = time_series_group_similarity(ts_df, 'group', 'value')
print("\n时间序列分组相关性：")
print(time_corr)
# 可视化时间序列相似度
plt.figure(figsize=(12, 5))
plt.plot(time_means.index, time_means.values)'各组的滚动均值趋势')
plt.xlabel('Date')
plt.ylabel('Rolling Mean')
plt.legend(time_means.columns)
plt.xticks(rotation=45)
plt.show()

这些案例展示了Pandas在数据分组相似度分析中的主要应用：