脚本如何实现文件内容模糊粗糙分类

本文目录导读：

1. 基于关键词匹配的简单分类
2. 基于TF-IDF的文本分类
3. 基于规则的模糊分类器
4. 完整文件分类脚本
5. 使用建议

模糊分类的方法,从简单到复杂：

基于关键词匹配的简单分类

import os
import re
def classify_by_keywords(file_path):
    """基于关键词的简单分类"""
    with open(file_path, 'r', encoding='utf-8', errors='ignore') as f:
        content = f.read().lower()
    # 定义分类规则（关键词到类别的映射）
    categories = {
        '编程': ['python', 'java', 'javascript', 'code', 'programming'],
        '金融': ['money', 'stock', 'finance', 'investment'],
        '科技': ['tech', 'AI', 'machine learning', 'technology'],
        '教育': ['education', 'school', 'university', 'learn']
    }
    scores = {}
    for category, keywords in categories.items():
        score = sum(content.count(keyword) for keyword in keywords)
        if score > 0:
            scores[category] = score
    if not scores:
        return "未分类"
    # 返回得分最高的类别
    return max(scores, key=scores.get)

基于TF-IDF的文本分类

from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.cluster import KMeans
import numpy as np
def tfidf_classify(texts, n_clusters=5):
    """使用TF-IDF和KMeans进行模糊分类"""
    # 文本向量化
    vectorizer = TfidfVectorizer(max_features=1000, stop_words='english')
    tfidf_matrix = vectorizer.fit_transform(texts)
    # KMeans聚类
    kmeans = KMeans(n_clusters=n_clusters, random_state=42)
    clusters = kmeans.fit_predict(tfidf_matrix)
    return clusters
# 获取每个聚类的关键词
def get_cluster_keywords(vectorizer, kmeans, n_words=10):
    feature_names = vectorizer.get_feature_names_out()
    cluster_keywords = {}
    for i in range(kmeans.n_clusters):
        center = kmeans.cluster_centers_[i]
        top_indices = center.argsort()[-n_words:][::-1]
        cluster_keywords[i] = [feature_names[idx] for idx in top_indices]
    return cluster_keywords

基于规则的模糊分类器

import re
from collections import Counter
class FuzzyClassifier:
    def __init__(self):
        self.rules = {
            'technical': {
                'keywords': ['algorithm', 'database', 'system', 'code'],
                'patterns': [r'\b[A-Z]{2,}\b', r'\d+\.\d+'],
                'weights': {'keywords': 0.6, 'patterns': 0.4}
            },
            'business': {
                'keywords': ['market', 'strategy', 'revenue', 'profit'],
                'patterns': [r'\$\d+', r'\d+%', r'quarter[ly]?\s+\d'],
                'weights': {'keywords': 0.7, 'patterns': 0.3}
            },
            'academic': {
                'keywords': ['research', 'study', 'analysis', 'methodology'],
                'patterns': [r'\[\d+\]', r'Figure\s+\d+', r'Table\s+\d+'],
                'weights': {'keywords': 0.5, 'patterns': 0.5}
            }
        }
    def classify(self, text):
        """模糊分类主方法"""
        text = text.lower()
        scores = {}
        for category, rule in self.rules.items():
            keyword_score = self._score_keywords(text, rule['keywords'])
            pattern_score = self._score_patterns(text, rule['patterns'])
            total_score = (
                keyword_score * rule['weights']['keywords'] +
                pattern_score * rule['weights']['patterns']
            )
            scores[category] = total_score
        # 归一化并返回结果
        total = sum(scores.values())
        if total == 0:
            return {'category': 'unknown', 'confidence': 0}
        best_category = max(scores, key=scores.get)
        confidence = scores[best_category] / total
        return {
            'category': best_category,
            'confidence': confidence,
            'all_scores': scores
        }
    def _score_keywords(self, text, keywords):
        """计算关键词得分"""
        return sum(1 for kw in keywords if kw in text) / len(keywords)
    def _score_patterns(self, text, patterns):
        """计算模式匹配得分"""
        matches = sum(1 for pattern in patterns if re.search(pattern, text))
        return matches / len(patterns)

完整文件分类脚本

#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import re
import argparse
from collections import defaultdict
class FileClassifier:
    def __init__(self):
        self.category_rules = {
            '技术文档': {
                'keywords': ['API', 'SDK', '代码', '程序', '开发'],
                'extensions': ['.py', '.java', '.js', '.cpp', '.h'],
                'min_content_words': 100
            },
            '项目计划': {
                'keywords': ['里程碑', 'deadline', '预算', '资源', '进度'],
                'patterns': [r'\d{4}-\d{2}-\d{2}', r'Sprint\d+']
            },
            '报告分析': {
                'keywords': ['分析', '#39;, '#39;, '建议', '趋势'],
                'patterns': [r'\d+\.\d+%', r'同比增长']
            }
        }
    def classify_file(self, filepath):
        """对单个文件进行模糊分类"""
        if not os.path.exists(filepath):
            return None
        # 获取文件信息
        filename = os.path.basename(filepath)
        ext = os.path.splitext(filename)[1].lower()
        # 读取文件内容
        try:
            with open(filepath, 'r', encoding='utf-8', errors='ignore') as f:
                content = f.read()
        except:
            return None
        # 特征提取
        features = self._extract_features(content, filename, ext)
        # 分类匹配
        results = {}
        for category, rules in self.category_rules.items():
            score = self._calculate_similarity(features, rules)
            if score > 0.3:  # 阈值过滤
                results[category] = score
        if not results:
            return {'category': '其他', 'confidence': 0, 'features': features}
        best_match = max(results, key=results.get)
        return {
            'category': best_match,
            'confidence': results[best_match],
            'all_matches': results,
            'features': features
        }
    def _extract_features(self, content, filename, ext):
        """提取文件特征"""
        features = {
            'filename': filename,
            'extension': ext,
            'word_count': len(content.split()),
            'char_count': len(content),
            'lines': content.count('\n'),
            'capital_words': len(re.findall(r'\b[A-Z]{2,}\b', content)),
            'numbers': len(re.findall(r'\d+', content)),
            'special_chars': len(re.findall(r'[!@#$%^&*()]', content)),
        }
        # 提取关键词频率
        all_keywords = set()
        for rules in self.category_rules.values():
            all_keywords.update(rules.get('keywords', []))
        keyword_freq = {}
        for kw in all_keywords:
            keyword_freq[kw] = content.count(kw)
        features['keyword_freq'] = keyword_freq
        return features
    def _calculate_similarity(self, features, rules):
        """计算相似度分数"""
        score = 0
        # 关键词匹配
        if 'keywords' in rules:
            keyword_matches = sum(
                1 for kw in rules['keywords'] 
                if features['keyword_freq'].get(kw, 0) > 0
            )
            score += keyword_matches / len(rules['keywords']) * 0.5
        # 文件扩展名匹配
        if 'extensions' in rules:
            if features['extension'] in rules['extensions']:
                score += 0.3
        # 内容长度检查
        if 'min_content_words' in rules:
            if features['word_count'] >= rules['min_content_words']:
                score += 0.2
        return score
# 使用示例
def batch_classify(directory):
    """批量分类目录中的文件"""
    classifier = FileClassifier()
    results = defaultdict(list)
    for root, dirs, files in os.walk(directory):
        for file in files:
            filepath = os.path.join(root, file)
            result = classifier.classify_file(filepath)
            if result:
                results[result['category']].append({
                    'file': filepath,
                    'confidence': result['confidence']
                })
    return results
# 主函数
if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='文件模糊分类工具')
    parser.add_argument('path', help='文件或目录路径')
    parser.add_argument('--verbose', '-v', action='store_true', help='显示详细信息')
    args = parser.parse_args()
    if os.path.isfile(args.path):
        classifier = FileClassifier()
        result = classifier.classify_file(args.path)
        if result:
            print(f"文件: {args.path}")
            print(f"分类: {result['category']}")
            print(f"置信度: {result['confidence']:.2%}")
    elif os.path.isdir(args.path):
        results = batch_classify(args.path)
        for category, files in results.items():
            print(f"\n{category} (共{len(files)}个文件):")
            for f in files:
                print(f"  - {f['file']} (置信度: {f['confidence']:.2%})")

使用建议

1. 根据需求定制分类规则：调整关键词、阈值和权重
2. 渐进式优化：从简单规则开始，逐步添加复杂特征
3. 处理中文文本：可能需要分词工具（如jieba）
4. 性能优化：对大文件进行采样分析

这个脚本可以根据你的具体需求进行调整和扩展。