本文目录导读：
我来介绍几种检测重复文件夹的方法,包括不同的实现方案：
基于文件内容哈希的精确检测
Python 实现
import os
import hashlib
from collections import defaultdict
def get_folder_hash(folder_path):
"""计算文件夹的哈希值（基于所有文件内容）"""
sha256_hash = hashlib.sha256()
# 遍历所有文件（按排序保持一致性）
for root, dirs, files in os.walk(folder_path):
dirs.sort()
files.sort()
for file_name in files:
file_path = os.path.join(root, file_name)
# 添加相对路径信息
rel_path = os.path.relpath(file_path, folder_path)
sha256_hash.update(rel_path.encode())
# 添加文件内容
with open(file_path, "rb") as f:
for chunk in iter(lambda: f.read(4096), b""):
sha256_hash.update(chunk)
return sha256_hash.hexdigest()
def find_duplicate_folders(root_path):
"""查找重复文件夹"""
folder_hashes = defaultdict(list)
# 遍历所有子文件夹
for item in os.listdir(root_path):
item_path = os.path.join(root_path, item)
if os.path.isdir(item_path):
try:
folder_hash = get_folder_hash(item_path)
folder_hashes[folder_hash].append(item_path)
except Exception as e:
print(f"Error processing {item_path}: {e}")
# 返回重复的文件夹
duplicates = {hash_val: paths for hash_val, paths in folder_hashes.items()
if len(paths) > 1}
return duplicates
# 使用示例
root = "/path/to/search"
duplicates = find_duplicate_folders(root)
for hash_val, paths in duplicates.items():
print(f"Duplicate folders ({len(paths)} copies):")
for path in paths:
print(f" - {path}")
基于文件元数据的快速检测
import os
from collections import defaultdict
def get_folder_signature(folder_path):
"""快速获取文件夹特征（文件数量、大小等）"""
total_files = 0
total_size = 0
structure = []
for root, dirs, files in os.walk(folder_path):
dirs.sort()
files.sort()
# 记录相对路径结构
rel_path = os.path.relpath(root, folder_path)
structure.append({
'path': rel_path,
'files': files,
'dirs': dirs
})
total_files += len(files)
for file_name in files:
file_path = os.path.join(root, file_name)
try:
total_size += os.path.getsize(file_path)
except:
pass
return {
'total_files': total_files,
'total_size': total_size,
'structure_hash': hash(str(structure))
}
def find_duplicate_folders_fast(root_path):
"""快速查找可能的重复文件夹"""
folder_signatures = defaultdict(list)
for item in os.listdir(root_path):
item_path = os.path.join(root_path, item)
if os.path.isdir(item_path):
sig = get_folder_signature(item_path)
folder_signatures[sig['structure_hash']].append(item_path)
# 只返回可能的重复项
potential_dupes = {k: v for k, v in folder_signatures.items() if len(v) > 1}
return potential_dupes
Shell 脚本检测
#!/bin/bash
# 基于文件列表的重复检测
function find_duplicate_dirs() {
local search_path="$1"
# 获取所有子目录的文件列表
for dir in "$search_path"/*/; do
if [ -d "$dir" ]; then
dir_name=$(basename "$dir")
# 生成文件列表（排序后）
file_list=$(find "$dir" -type f -exec basename {} \; | sort | md5sum)
# 使用关联数组存储
echo "$dir_name|$file_list"
fi
done | sort -t'|' -k2 | awk -F'|' '
{
if ($2 == prev_hash) {
print "DUPLICATE: " prev_dir " <=> " $1
} else {
prev_hash = $2
prev_dir = $1
}
}
'
}
# 使用示例
find_duplicate_dirs "/path/to/check"
使用 fdupes 工具（快速）
import subprocess
import json
def find_duplicates_with_fdupes(directory, recurse=True):
"""使用 fdupes 工具检测重复文件，然后分析文件夹"""
cmd = ['fdupes', '-r', '-S', directory]
result = subprocess.run(cmd, capture_output=True, text=True)
# 解析输出，分组重复文件
duplicate_groups = []
current_group = []
for line in result.stdout.split('\n'):
if line.strip() == '':
if current_group:
duplicate_groups.append(current_group)
current_group = []
else:
current_group.append(line.strip())
# 分析哪些文件夹包含重复文件
folder_duplicates = defaultdict(set)
for group in duplicate_groups:
if len(group) > 1:
for file_path in group:
folder = os.path.dirname(file_path)
# 记录重复文件对
for other_file in group:
if other_file != file_path:
folder_duplicates[folder].add(other_file)
return folder_duplicates
Windows PowerShell 脚本
# 查找重复文件夹的PowerShell脚本
function Find-DuplicateFolders {
param(
[string]$Path,
[switch]$Recurse
)
$folderHashes = @{}
# 获取所有子文件夹
$folders = Get-ChildItem -Path $Path -Directory -Recurse:$Recurse
foreach ($folder in $folders) {
$hash = New-Object System.Security.Cryptography.SHA256Managed
# 获取文件夹中所有文件
$files = Get-ChildItem -Path $folder.FullName -File -Recurse | Sort-Object Name
foreach ($file in $files) {
$stream = [System.IO.File]::OpenRead($file.FullName)
$hash.ComputeHash($stream) | Out-Null
$stream.Close()
}
$hashString = [System.BitConverter]::ToString($hash.Hash) -replace '-', ''
if (-not $folderHashes.ContainsKey($hashString)) {
$folderHashes[$hashString] = @()
}
$folderHashes[$hashString] += $folder.FullName
}
# 只返回重复项
$folderHashes.GetEnumerator() | Where-Object { $_.Value.Count -gt 1 }
}
# 使用示例
Find-DuplicateFolders -Path "C:\MyFolder" -Recurse
性能优化版本
import os
import hashlib
from concurrent.futures import ThreadPoolExecutor, as_completed
def hash_file_quick(file_path):
"""快速哈希单个文件"""
h = hashlib.md5()
with open(file_path, 'rb') as f:
# 只读取前4KB和最后4KB作为快速比较
first_chunk = f.read(4096)
h.update(first_chunk)
f.seek(-4096, 2)
last_chunk = f.read(4096)
h.update(last_chunk)
return h.hexdigest()
def compare_folders_parallel(folder1, folder2):
"""并行比较两个文件夹"""
def get_files_structure(path):
"""获取文件结构"""
files = []
for root, dirs, fnames in os.walk(path):
for fname in fnames:
fpath = os.path.join(root, fname)
rel_path = os.path.relpath(fpath, path)
files.append((rel_path, fpath))
return sorted(files, key=lambda x: x[0])
files1 = get_files_structure(folder1)
files2 = get_files_structure(folder2)
# 快速大小和数量检查
if len(files1) != len(files2):
return False
# 并行哈希比较
with ThreadPoolExecutor(max_workers=4) as executor:
futures = []
for i, ((rel1, path1), (rel2, path2)) in enumerate(zip(files1, files2)):
if rel1 != rel2:
return False
futures.append(executor.submit(hash_file_quick, path1))
futures.append(executor.submit(hash_file_quick, path2))
# 验证哈希
hashes = []
for future in as_completed(futures):
hashes.append(future.result())
# 分组比较
for i in range(0, len(hashes), 2):
if hashes[i] != hashes[i+1]:
return False
return True
使用建议
-
选择合适的方法：
- 精确检测：使用完整的文件哈希
- 快速检测：使用文件数量和大小
- 大规模检测：使用并行处理
-
性能优化：
- 先比较文件数量和总大小
- 使用快速哈希（MD5而非SHA256）
- 只读取部分文件内容
-
注意事项：
- 处理权限问题
- 考虑符号链接
- 排除系统文件
- 处理大文件时的内存管理
根据具体需求选择合适的实现方案即可。
