本文目录导读：
我来介绍几种Python文本去重的方法,从简单到复杂：
基础去重（保持顺序）
def remove_duplicates_by_order(file_path):
"""
去除重复行，保持首次出现顺序
"""
seen = set()
unique_lines = []
with open(file_path, 'r', encoding='utf-8') as f:
for line in f:
# 去除行末换行符，保留空行
stripped_line = line.rstrip('\n')
if stripped_line not in seen:
seen.add(stripped_line)
unique_lines.append(line)
return unique_lines
# 使用示例
lines = remove_duplicates_by_order('input.txt')
with open('output.txt', 'w', encoding='utf-8') as f:
f.writelines(lines)
忽略空格和大小写去重
def remove_duplicates_case_insensitive(file_path):
"""
忽略大小写和首尾空格的去重
"""
seen = set()
unique_lines = []
with open(file_path, 'r', encoding='utf-8') as f:
for line in f:
# 标准化处理：去除首尾空格并转小写
normalized = line.strip().lower()
if normalized and normalized not in seen:
seen.add(normalized)
unique_lines.append(line)
return unique_lines
# 使用示例
lines = remove_duplicates_case_insensitive('input.txt')
with open('output.txt', 'w', encoding='utf-8') as f:
f.writelines(lines)
统计重复次数
from collections import Counter
def count_and_remove_duplicates(file_path):
"""
统计每行出现次数，可选择保留所有出现或只保留一次
"""
with open(file_path, 'r', encoding='utf-8') as f:
lines = [line.rstrip('\n') for line in f]
# 统计行数
line_counts = Counter(lines)
# 输出统计报告
print("=== 重复行统计 ===")
for line, count in line_counts.most_common():
if count > 1:
print(f"出现{count}次: {line[:50]}...") # 只显示前50字符
# 去重结果（保持顺序）
seen = set()
unique_with_count = []
for line in lines:
if line not in seen:
seen.add(line)
unique_with_count.append((line, line_counts[line]))
return unique_with_count
# 使用示例
result = count_and_remove_duplicates('input.txt')
with open('output.txt', 'w', encoding='utf-8') as f:
for line, count in result:
f.write(f"{line}\n")
高级去重（模糊匹配）
from difflib import SequenceMatcher
def similar(a, b, threshold=0.8):
"""
判断两行文本是否相似
"""
return SequenceMatcher(None, a, b).ratio() > threshold
def fuzzy_remove_duplicates(file_path, threshold=0.8):
"""
模糊去重：去除相似度超过阈值的重复行
"""
with open(file_path, 'r', encoding='utf-8') as f:
lines = [line.rstrip('\n') for line in f]
unique_lines = []
for line in lines:
is_duplicate = False
for existing in unique_lines:
if similar(line, existing, threshold):
is_duplicate = True
break
if not is_duplicate:
unique_lines.append(line)
return unique_lines
# 使用示例
lines = fuzzy_remove_duplicates('input.txt', threshold=0.85)
with open('output.txt', 'w', encoding='utf-8') as f:
for line in lines:
f.write(f"{line}\n")
完整的命令行工具
#!/usr/bin/env python3
"""
文本去重工具
用法: python dedup.py input.txt -o output.txt [选项]
"""
import argparse
import sys
def dedup_text(input_file, output_file=None, ignore_case=False,
strip_whitespace=False, show_stats=False):
"""
完整的文本去重函数
"""
with open(input_file, 'r', encoding='utf-8') as f:
lines = f.readlines()
original_count = len(lines)
seen = set()
unique_lines = []
for line in lines:
# 处理选项
processed = line
if strip_whitespace:
processed = processed.strip()
if ignore_case:
processed = processed.lower()
if processed and processed not in seen:
seen.add(processed)
unique_lines.append(line)
# 输出结果
if output_file:
with open(output_file, 'w', encoding='utf-8') as f:
f.writelines(unique_lines)
else:
for line in unique_lines:
print(line, end='')
# 显示统计信息
if show_stats:
removed = original_count - len(unique_lines)
print(f"\n=== 统计信息 ===", file=sys.stderr)
print(f"原始行数: {original_count}", file=sys.stderr)
print(f"去重后行数: {len(unique_lines)}", file=sys.stderr)
print(f"移除重复行: {removed}", file=sys.stderr)
def main():
parser = argparse.ArgumentParser(description='文本去重工具')
parser.add_argument('input', help='输入文件')
parser.add_argument('-o', '--output', help='输出文件')
parser.add_argument('-i', '--ignore-case', action='store_true',
help='忽略大小写')
parser.add_argument('-s', '--strip', action='store_true',
help='去除首尾空白字符')
parser.add_argument('-v', '--verbose', action='store_true',
help='显示统计信息')
args = parser.parse_args()
dedup_text(
args.input,
args.output,
ignore_case=args.ignore_case,
strip_whitespace=args.strip,
show_stats=args.verbose
)
if __name__ == '__main__':
main()
使用示例
# 基础去重 python dedup.py input.txt -o output.txt # 忽略大小写去重 python dedup.py input.txt -o output.txt -i # 去除空格后去重并显示统计 python dedup.py input.txt -o output.txt -s -v
实际应用案例
# 案例：清洗日志文件中的重复错误
def clean_error_logs(file_path):
"""
清洗错误日志，去除重复的错误信息
"""
import re
seen_errors = set()
unique_errors = []
with open(file_path, 'r', encoding='utf-8') as f:
for line in f:
# 提取错误信息（假设格式：时间戳 - ERROR - 错误信息）
match = re.search(r'ERROR - (.+)$', line)
if match:
error_msg = match.group(1).strip()
if error_msg not in seen_errors:
seen_errors.add(error_msg)
unique_errors.append(line)
else:
# 非错误行直接保留
unique_errors.append(line)
return unique_errors
# 使用示例
cleaned_logs = clean_error_logs('app.log')
with open('cleaned_app.log', 'w', encoding='utf-8') as f:
f.writelines(cleaned_logs)
这些方法可以根据你的具体需求选择使用，如果需要处理大文件,建议使用迭代器方式以避免内存溢出。
