脚本如何实现文件内容模糊C均值

本文目录导读：

1. 基本实现方案
2. 高级实现：使用文本向量化
3. 优化建议

我来为您介绍如何用Python实现文件内容的模糊C均值（FCM）聚类算法。

基本实现方案

import numpy as np
import os
from typing import List, Tuple
import re
class FuzzyCMeans:
    def __init__(self, n_clusters=3, m=2, max_iter=100, epsilon=1e-5):
        """
        模糊C均值聚类
        Parameters:
        - n_clusters: 聚类数量
        - m: 模糊指数（通常为2）
        - max_iter: 最大迭代次数
        - epsilon: 收敛阈值
        """
        self.n_clusters = n_clusters
        self.m = m
        self.max_iter = max_iter
        self.epsilon = epsilon
    def _initialize_membership(self, n_samples):
        """初始化隶属度矩阵"""
        membership = np.random.dirichlet(np.ones(self.n_clusters), size=n_samples)
        return membership.T
    def _compute_centers(self, data, membership):
        """计算聚类中心"""
        um = membership ** self.m
        centers = np.dot(um, data) / np.sum(um, axis=1, keepdims=True)
        return centers
    def _update_membership(self, data, centers):
        """更新隶属度矩阵"""
        n_samples = data.shape[0]
        membership = np.zeros((self.n_clusters, n_samples))
        for i in range(self.n_clusters):
            for j in range(n_samples):
                # 计算样本到各中心的距离
                distances = np.linalg.norm(data[j] - centers, axis=1)
                # 避免除零
                distances = np.maximum(distances, 1e-10)
                # 计算隶属度
                membership[i, j] = 1.0 / np.sum((distances[i] / distances) ** (2/(self.m-1)))
        return membership
    def fit(self, data):
        """执行FCM聚类"""
        n_samples = data.shape[0]
        # 初始化隶属度矩阵
        membership = self._initialize_membership(n_samples)
        # 迭代优化
        for iteration in range(self.max_iter):
            # 计算聚类中心
            centers = self._compute_centers(data, membership)
            # 更新隶属度
            new_membership = self._update_membership(data, centers)
            # 检查收敛
            if np.max(np.abs(new_membership - membership)) < self.epsilon:
                print(f"迭代 {iteration+1} 次后收敛")
                break
            membership = new_membership
        self.membership_ = membership
        self.cluster_centers_ = centers
        self.labels_ = np.argmax(membership, axis=0)
        return self
class FileContentCluster:
    def __init__(self, fcm_params=None):
        """文件内容聚类器"""
        if fcm_params is None:
            fcm_params = {}
        self.fcm = FuzzyCMeans(**fcm_params)
        self.vectorizer = None
    def _extract_features(self, content: str) -> np.ndarray:
        """从文本内容提取特征向量"""
        # 简单词频统计
        words = re.findall(r'\w+', content.lower())
        word_freq = {}
        for word in words:
            word_freq[word] = word_freq.get(word, 0) + 1
        # 提取一些简单特征
        features = [
            len(words),  # 词数
            len(set(words)),  # 不同词数
            len(content),  # 字符数
            len(re.findall(r'[.!?]', content)),  # 句子数
            np.mean([len(w) for w in words]) if words else 0,  # 平均词长
        ]
        return np.array(features)
    def cluster_files(self, file_contents: List[str]) -> Tuple[np.ndarray, np.ndarray]:
        """
        聚类文件内容
        Parameters:
        - file_contents: 文件内容列表
        Returns:
        - cluster_labels: 聚类标签
        - membership: 隶属度矩阵
        """
        # 提取特征
        features = np.array([self._extract_features(content) for content in file_contents])
        # 标准化特征
        features = (features - features.mean(axis=0)) / (features.std(axis=0) + 1e-10)
        # 执行聚类
        self.fcm.fit(features)
        return self.fcm.labels_, self.fcm.membership_
    def cluster_files_with_paths(self, file_paths: List[str]) -> dict:
        """
        从文件路径读取并聚类
        Parameters:
        - file_paths: 文件路径列表
        Returns:
        - result: 聚类结果字典
        """
        file_contents = []
        valid_files = []
        for file_path in file_paths:
            try:
                with open(file_path, 'r', encoding='utf-8') as f:
                    content = f.read()
                file_contents.append(content)
                valid_files.append(file_path)
            except Exception as e:
                print(f"读取文件 {file_path} 失败: {e}")
        if not file_contents:
            return {}
        # 聚类
        labels, membership = self.cluster_files(file_contents)
        # 组织结果
        result = {}
        for i, (file_path, label) in enumerate(zip(valid_files, labels)):
            cluster_key = f"Cluster_{label}"
            if cluster_key not in result:
                result[cluster_key] = {
                    'files': [],
                    'membership': []
                }
            result[cluster_key]['files'].append(file_path)
            result[cluster_key]['membership'].append(membership[label, i])
        return result
# 使用示例
if __name__ == "__main__":
    # 示例文件内容
    contents = [
        "Python is a programming language. It is easy to learn.",
        "Machine learning uses algorithms to learn from data.",
        "The weather is nice today. Let's go outside.",
        "Deep learning is a subset of machine learning."
    ]
    # 创建聚类器
    clusterer = FileContentCluster(n_clusters=2, m=2)
    # 聚类
    labels, membership = clusterer.cluster_files(contents)
    # 打印结果
    for i, (content, label) in enumerate(zip(contents, labels)):
        print(f"文档 {i+1}: 聚类 {label}, 隶属度: {membership[label, i]:.3f}")
        print(f"   内容: {content[:50]}...")
        print()
    # 文件路径示例
    file_paths = ['file1.txt', 'file2.txt', 'file3.txt']
    if all(os.path.exists(p) for p in file_paths):
        result = clusterer.cluster_files_with_paths(file_paths)
        for cluster_key, cluster_info in result.items():
            print(f"\n{cluster_key}:")
            for file_path, mem in zip(cluster_info['files'], cluster_info['membership']):
                print(f"  - {file_path} (隶属度: {mem:.3f})")

高级实现：使用文本向量化

from sklearn.feature_extraction.text import TfidfVectorizer
import numpy as np
class AdvancedFileCluster:
    def __init__(self, n_clusters=3, m=2):
        self.n_clusters = n_clusters
        self.m = m
        self.vectorizer = TfidfVectorizer(max_features=100, stop_words='english')
        self.fcm = FuzzyCMeans(n_clusters=n_clusters, m=m)
    def cluster_with_tfidf(self, file_contents):
        """
        使用TF-IDF向量化进行聚类
        """
        # TF-IDF向量化
        tfidf_matrix = self.vectorizer.fit_transform(file_contents)
        data = tfidf_matrix.toarray()
        # 聚类
        self.fcm.fit(data)
        return self.fcm.labels_, self.fcm.membership_
# 使用示例
contents = [
    "Python programming machine learning data science",
    "JavaScript web development frontend backend",
    "Deep learning neural networks AI artificial intelligence",
    "CSS HTML web design responsive layout"
]
clusterer = AdvancedFileCluster(n_clusters=2)
labels, membership = clusterer.cluster_with_tfidf(contents)
for i, (content, label) in enumerate(zip(contents, labels)):
    print(f"文档 {i+1} 聚类: {label}, 隶属度: {membership[label, i]:.3f}")

优化建议

1. 特征工程：根据具体文件类型（代码、文档、日志等）设计更合适的特征
2. 并行处理：处理大量文件时使用多线程/进程
3. 增量学习：对新增文件进行增量聚类
4. 结果评估：添加轮廓系数等评估指标
5. 可视化：使用PCA/t-SNE降维可视化聚类结果

这个实现可以根据您的具体需求进行调整和优化。