脚本如何实现文件内容模糊粗糙对比学习

本文目录导读：

1. 基于字符相似度的模糊对比
2. 基于行级别的模糊匹配
3. 使用N-gram进行模糊对比
4. 综合模糊对比工具
5. 实用的模糊对比脚本
6. 使用建议

基于字符相似度的模糊对比

import difflib
def fuzzy_compare(text1, text2, threshold=0.6):
    """
    模糊对比两个文本内容
    """
    # 计算相似度
    similarity = difflib.SequenceMatcher(None, text1, text2).ratio()
    # 获取不同之处
    diff = list(difflib.unified_diff(
        text1.splitlines(keepends=True),
        text2.splitlines(keepends=True),
        n=0
    ))
    return {
        'similarity': similarity,
        'is_match': similarity >= threshold,
        'diffs': diff
    }
# 使用示例
with open('file1.txt', 'r') as f1, open('file2.txt', 'r') as f2:
    result = fuzzy_compare(f1.read(), f2.read())
    print(f"相似度: {result['similarity']*100:.2f}%")

基于行级别的模糊匹配

def line_based_fuzzy_match(file1, file2, threshold=0.8):
    """
    逐行进行模糊匹配
    """
    with open(file1, 'r', encoding='utf-8') as f1, \
         open(file2, 'r', encoding='utf-8') as f2:
        lines1 = f1.read().splitlines()
        lines2 = f2.read().splitlines()
    matches = []
    for i, line1 in enumerate(lines1):
        for j, line2 in enumerate(lines2):
            similarity = difflib.SequenceMatcher(None, line1, line2).ratio()
            if similarity >= threshold:
                matches.append({
                    'file1_line': i + 1,
                    'file2_line': j + 1,
                    'similarity': similarity,
                    'content1': line1,
                    'content2': line2
                })
    return matches

使用N-gram进行模糊对比

from collections import defaultdict
class NGramMatcher:
    def __init__(self, n=3):
        self.n = n
    def extract_ngrams(self, text):
        """提取N-gram特征"""
        words = text.lower().split()
        ngrams = set()
        for i in range(len(words) - self.n + 1):
            ngram = ' '.join(words[i:i+self.n])
            ngrams.add(ngram)
        return ngrams
    def compare(self, text1, text2):
        """基于N-gram的相似度对比"""
        ngrams1 = self.extract_ngrams(text1)
        ngrams2 = self.extract_ngrams(text2)
        if not ngrams1 or not ngrams2:
            return 0
        intersection = ngrams1 & ngrams2
        union = ngrams1 | ngrams2
        return len(intersection) / len(union)
# 使用示例
matcher = NGramMatcher(n=3)
with open('file1.txt') as f1, open('file2.txt') as f2:
    similarity = matcher.compare(f1.read(), f2.read())
    print(f"N-gram相似度: {similarity*100:.2f}%")

综合模糊对比工具

import re
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.metrics.pairwise import cosine_similarity
class ComprehensiveFuzzyComparator:
    def __init__(self):
        self.methods = {
            'exact': self.exact_match,
            'word': self.word_similarity,
            'ngram': self.ngram_similarity,
            'cosine': self.cosine_similarity
        }
    def preprocess(self, text):
        """文本预处理"""
        # 去除标点符号和空格
        text = re.sub(r'[^\w\s]', ' ', text)
        text = re.sub(r'\s+', ' ', text)
        return text.strip().lower()
    def exact_match(self, text1, text2):
        """精确匹配"""
        return text1 == text2
    def word_similarity(self, text1, text2):
        """基于词集的相似度"""
        words1 = set(text1.split())
        words2 = set(text2.split())
        if not words1 or not words2:
            return 0
        intersection = len(words1 & words2)
        union = len(words1 | words2)
        return intersection / union
    def ngram_similarity(self, text1, text2, n=3):
        """N-gram相似度"""
        def get_ngrams(text, n):
            chars = list(text.replace(' ', ''))
            return set(''.join(chars[i:i+n]) for i in range(len(chars)-n+1))
        ngrams1 = get_ngrams(text1, n)
        ngrams2 = get_ngrams(text2, n)
        if not ngrams1 or not ngrams2:
            return 0
        intersection = len(ngrams1 & ngrams2)
        union = len(ngrams1 | ngrams2)
        return intersection / union
    def cosine_similarity(self, text1, text2):
        """TF-IDF余弦相似度"""
        vectorizer = TfidfVectorizer()
        try:
            tfidf_matrix = vectorizer.fit_transform([text1, text2])
            similarity = (tfidf_matrix * tfidf_matrix.T).A[0, 1]
            return similarity
        except:
            return 0
    def compare_files(self, file1_path, file2_path, methods=None):
        """对比两个文件"""
        with open(file1_path, 'r', encoding='utf-8') as f1, \
             open(file2_path, 'r', encoding='utf-8') as f2:
            text1 = self.preprocess(f1.read())
            text2 = self.preprocess(f2.read())
        results = {}
        methods = methods or ['word', 'ngram', 'cosine']
        for method in methods:
            if method in self.methods:
                results[method] = self.methods[method](text1, text2)
        # 计算综合分数
        if results:
            results['combined'] = sum(results.values()) / len(results)
        return results
# 使用示例
comparator = ComprehensiveFuzzyComparator()
results = comparator.compare_files('file1.txt', 'file2.txt')
for method, score in results.items():
    if isinstance(score, float):
        print(f"{method}: {score*100:.2f}%")
    else:
        print(f"{method}: {score}")

实用的模糊对比脚本

#!/usr/bin/env python3
"""模糊对比工具
"""
import argparse
import sys
from pathlib import Path
def main():
    parser = argparse.ArgumentParser(description='文件内容模糊对比工具')
    parser.add_argument('file1', help='第一个文件')
    parser.add_argument('file2', help='第二个文件')
    parser.add_argument('--method', choices=['exact', 'fuzzy', 'line', 'ngram', 'all'],
                       default='fuzzy', help='对比方法')
    parser.add_argument('--threshold', type=float, default=0.6, help='相似度阈值')
    args = parser.parse_args()
    # 检查文件是否存在
    if not Path(args.file1).exists() or not Path(args.file2).exists():
        print("文件不存在！")
        sys.exit(1)
    # 读取文件
    with open(args.file1, 'r', encoding='utf-8') as f1, \
         open(args.file2, 'r', encoding='utf-8') as f2:
        text1 = f1.read()
        text2 = f2.read()
    # 执行对比
    if args.method == 'exact':
        result = text1 == text2
        print(f"精确匹配结果: {result}")
    elif args.method == 'fuzzy':
        from difflib import SequenceMatcher
        similarity = SequenceMatcher(None, text1, text2).ratio()
        print(f"模糊匹配相似度: {similarity*100:.2f}%")
    elif args.method == 'line':
        from itertools import zip_longest
        lines1 = text1.splitlines()
        lines2 = text2.splitlines()
        match_count = 0
        total = max(len(lines1), len(lines2))
        for l1, l2 in zip_longest(lines1, lines2, fillvalue=''):
            if l1 == l2:
                match_count += 1
        print(f"行匹配率: {match_count/total*100:.2f}%")
    elif args.method == 'ngram':
        from sklearn.feature_extraction.text import CountVectorizer
        vect = CountVectorizer(analyzer='char', ngram_range=(3, 3))
        try:
            matrix = vect.fit_transform([text1, text2])
            similarity = (matrix * matrix.T).A[0, 1]
            print(f"N-gram相似度: {similarity*100:.2f}%")
        except:
            print("N-gram对比失败")
    elif args.method == 'all':
        # 综合对比
        comparator = ComprehensiveFuzzyComparator()
        results = comparator.compare_files(args.file1, args.file2)
        for method, score in results.items():
            if isinstance(score, float):
                print(f"{method}: {score*100:.2f}%")
if __name__ == '__main__':
    main()

使用建议

1. 选择合适的算法：

   • 短文本：用编辑距离或字符级N-gram
   • 长文本：用TF-IDF或文档级相似度
   • 代码文件：用行级别的精确匹配

2. 性能优化：

   # 对于大文件，使用流式处理
   def compare_large_files(file1, file2, chunk_size=8192):
       with open(file1, 'rb') as f1, open(file2, 'rb') as f2:
           while True:
               chunk1 = f1.read(chunk_size)
               chunk2 = f2.read(chunk_size)
               if not chunk1 and not chunk2:
                   break
               # 处理chunk

3. 安装依赖：

   pip install difflib scikit-learn  # 根据需要安装

这些方法可以根据你的具体需求选择使用,也可以组合使用以达到更好的对比效果。