Python文本去重案例如何剔除重复行

本文目录导读：

1. 基础去重（保持顺序）
2. 忽略空格和大小写去重
3. 统计重复次数
4. 高级去重（模糊匹配）
5. 完整的命令行工具
6. 使用示例
7. 实际应用案例

我来介绍几种Python文本去重的方法,从简单到复杂：

基础去重（保持顺序）

def remove_duplicates_by_order(file_path):
    """
    去除重复行，保持首次出现顺序
    """
    seen = set()
    unique_lines = []
    with open(file_path, 'r', encoding='utf-8') as f:
        for line in f:
            # 去除行末换行符，保留空行
            stripped_line = line.rstrip('\n')
            if stripped_line not in seen:
                seen.add(stripped_line)
                unique_lines.append(line)
    return unique_lines
# 使用示例
lines = remove_duplicates_by_order('input.txt')
with open('output.txt', 'w', encoding='utf-8') as f:
    f.writelines(lines)

忽略空格和大小写去重

def remove_duplicates_case_insensitive(file_path):
    """
    忽略大小写和首尾空格的去重
    """
    seen = set()
    unique_lines = []
    with open(file_path, 'r', encoding='utf-8') as f:
        for line in f:
            # 标准化处理：去除首尾空格并转小写
            normalized = line.strip().lower()
            if normalized and normalized not in seen:
                seen.add(normalized)
                unique_lines.append(line)
    return unique_lines
# 使用示例
lines = remove_duplicates_case_insensitive('input.txt')
with open('output.txt', 'w', encoding='utf-8') as f:
    f.writelines(lines)

统计重复次数

from collections import Counter
def count_and_remove_duplicates(file_path):
    """
    统计每行出现次数，可选择保留所有出现或只保留一次
    """
    with open(file_path, 'r', encoding='utf-8') as f:
        lines = [line.rstrip('\n') for line in f]
    # 统计行数
    line_counts = Counter(lines)
    # 输出统计报告
    print("=== 重复行统计 ===")
    for line, count in line_counts.most_common():
        if count > 1:
            print(f"出现{count}次: {line[:50]}...")  # 只显示前50字符
    # 去重结果（保持顺序）
    seen = set()
    unique_with_count = []
    for line in lines:
        if line not in seen:
            seen.add(line)
            unique_with_count.append((line, line_counts[line]))
    return unique_with_count
# 使用示例
result = count_and_remove_duplicates('input.txt')
with open('output.txt', 'w', encoding='utf-8') as f:
    for line, count in result:
        f.write(f"{line}\n")

高级去重（模糊匹配）

from difflib import SequenceMatcher
def similar(a, b, threshold=0.8):
    """
    判断两行文本是否相似
    """
    return SequenceMatcher(None, a, b).ratio() > threshold
def fuzzy_remove_duplicates(file_path, threshold=0.8):
    """
    模糊去重：去除相似度超过阈值的重复行
    """
    with open(file_path, 'r', encoding='utf-8') as f:
        lines = [line.rstrip('\n') for line in f]
    unique_lines = []
    for line in lines:
        is_duplicate = False
        for existing in unique_lines:
            if similar(line, existing, threshold):
                is_duplicate = True
                break
        if not is_duplicate:
            unique_lines.append(line)
    return unique_lines
# 使用示例
lines = fuzzy_remove_duplicates('input.txt', threshold=0.85)
with open('output.txt', 'w', encoding='utf-8') as f:
    for line in lines:
        f.write(f"{line}\n")

完整的命令行工具

#!/usr/bin/env python3
"""
文本去重工具
用法: python dedup.py input.txt -o output.txt [选项]
"""
import argparse
import sys
def dedup_text(input_file, output_file=None, ignore_case=False, 
               strip_whitespace=False, show_stats=False):
    """
    完整的文本去重函数
    """
    with open(input_file, 'r', encoding='utf-8') as f:
        lines = f.readlines()
    original_count = len(lines)
    seen = set()
    unique_lines = []
    for line in lines:
        # 处理选项
        processed = line
        if strip_whitespace:
            processed = processed.strip()
        if ignore_case:
            processed = processed.lower()
        if processed and processed not in seen:
            seen.add(processed)
            unique_lines.append(line)
    # 输出结果
    if output_file:
        with open(output_file, 'w', encoding='utf-8') as f:
            f.writelines(unique_lines)
    else:
        for line in unique_lines:
            print(line, end='')
    # 显示统计信息
    if show_stats:
        removed = original_count - len(unique_lines)
        print(f"\n=== 统计信息 ===", file=sys.stderr)
        print(f"原始行数: {original_count}", file=sys.stderr)
        print(f"去重后行数: {len(unique_lines)}", file=sys.stderr)
        print(f"移除重复行: {removed}", file=sys.stderr)
def main():
    parser = argparse.ArgumentParser(description='文本去重工具')
    parser.add_argument('input', help='输入文件')
    parser.add_argument('-o', '--output', help='输出文件')
    parser.add_argument('-i', '--ignore-case', action='store_true', 
                       help='忽略大小写')
    parser.add_argument('-s', '--strip', action='store_true',
                       help='去除首尾空白字符')
    parser.add_argument('-v', '--verbose', action='store_true',
                       help='显示统计信息')
    args = parser.parse_args()
    dedup_text(
        args.input,
        args.output,
        ignore_case=args.ignore_case,
        strip_whitespace=args.strip,
        show_stats=args.verbose
    )
if __name__ == '__main__':
    main()

使用示例

# 基础去重
python dedup.py input.txt -o output.txt
# 忽略大小写去重
python dedup.py input.txt -o output.txt -i
# 去除空格后去重并显示统计
python dedup.py input.txt -o output.txt -s -v

实际应用案例

# 案例：清洗日志文件中的重复错误
def clean_error_logs(file_path):
    """
    清洗错误日志，去除重复的错误信息
    """
    import re
    seen_errors = set()
    unique_errors = []
    with open(file_path, 'r', encoding='utf-8') as f:
        for line in f:
            # 提取错误信息（假设格式：时间戳 - ERROR - 错误信息）
            match = re.search(r'ERROR - (.+)$', line)
            if match:
                error_msg = match.group(1).strip()
                if error_msg not in seen_errors:
                    seen_errors.add(error_msg)
                    unique_errors.append(line)
            else:
                # 非错误行直接保留
                unique_errors.append(line)
    return unique_errors
# 使用示例
cleaned_logs = clean_error_logs('app.log')
with open('cleaned_app.log', 'w', encoding='utf-8') as f:
    f.writelines(cleaned_logs)

这些方法可以根据你的具体需求选择使用，如果需要处理大文件,建议使用迭代器方式以避免内存溢出。