如何编写重复文件清理脚本

本文目录导读：

1. Python 脚本（推荐）
2. Bash 脚本（Linux系统）
3. PowerShell 脚本（Windows系统）
4. 使用建议

Python 脚本（推荐）

基础版本（MD5校验）

import os
import hashlib
from pathlib import Path
def get_file_hash(filepath):
    """计算文件MD5哈希值"""
    hash_md5 = hashlib.md5()
    with open(filepath, "rb") as f:
        for chunk in iter(lambda: f.read(4096), b""):
            hash_md5.update(chunk)
    return hash_md5.hexdigest()
def find_duplicates(directory):
    """查找重复文件"""
    files = {}
    duplicates = []
    for filepath in Path(directory).rglob("*"):
        if filepath.is_file():
            try:
                file_hash = get_file_hash(filepath)
                file_size = filepath.stat().st_size
                key = (file_hash, file_size)
                if key in files:
                    duplicates.append((files[key], filepath))
                else:
                    files[key] = filepath
            except Exception as e:
                print(f"Error processing {filepath}: {e}")
    return duplicates
def clean_duplicates(duplicates, delete=False):
    """清理重复文件"""
    print(f"Found {len(duplicates)} duplicate groups:")
    for original, duplicate in duplicates:
        print(f"  Original: {original}")
        print(f"  Duplicate: {duplicate}")
        print("-" * 50)
        if delete:
            try:
                os.remove(duplicate)
                print(f"  Deleted: {duplicate}")
            except Exception as e:
                print(f"  Error deleting: {e}")
if __name__ == "__main__":
    import argparse
    parser = argparse.ArgumentParser(description="Find and clean duplicate files")
    parser.add_argument("directory", help="Directory to scan")
    parser.add_argument("--delete", action="store_true", help="Actually delete duplicates")
    parser.add_argument("--dry-run", action="store_true", help="Show what would be deleted")
    args = parser.parse_args()
    duplicates = find_duplicates(args.directory)
    clean_duplicates(duplicates, delete=args.delete or args.dry_run)

增强版本（多线程+进度条）

import os
import hashlib
from pathlib import Path
from concurrent.futures import ThreadPoolExecutor
import tqdm
def get_file_hash(filepath):
    """带进度显示的文件哈希计算"""
    hash_md5 = hashlib.md5()
    file_size = filepath.stat().st_size
    with open(filepath, "rb") as f:
        with tqdm.tqdm(total=file_size, unit='B', unit_scale=True, 
                      desc=filepath.name, leave=False) as pbar:
            for chunk in iter(lambda: f.read(8192), b""):
                hash_md5.update(chunk)
                pbar.update(len(chunk))
    return hash_md5.hexdigest()
def find_duplicates_parallel(directory, workers=4):
    """并行查找重复文件"""
    files = []
    for filepath in Path(directory).rglob("*"):
        if filepath.is_file():
            files.append(filepath)
    print(f"Scanning {len(files)} files...")
    with ThreadPoolExecutor(max_workers=workers) as executor:
        file_hashes = list(tqdm(
            executor.map(get_file_hash, files),
            total=len(files),
            desc="Calculating hashes"
        ))
    # 按文件大小分组，先快速比较大小
    size_groups = {}
    for filepath, file_hash in zip(files, file_hashes):
        size = filepath.stat().st_size
        if size not in size_groups:
            size_groups[size] = []
        size_groups[size].append((file_hash, filepath))
    # 找出重复文件
    duplicates = []
    for size, group in size_groups.items():
        if len(group) > 1:
            hash_groups = {}
            for file_hash, filepath in group:
                if file_hash not in hash_groups:
                    hash_groups[file_hash] = []
                hash_groups[file_hash].append(filepath)
            for hash_val, file_list in hash_groups.items():
                if len(file_list) > 1:
                    duplicates.append(file_list)
    return duplicates

Bash 脚本（Linux系统）

#!/bin/bash
# 重复文件清理脚本
# 使用方法: ./dedup.sh [目录路径]
# 设置颜色输出
RED='\033[0;31m'
GREEN='\033[0;32m'
NC='\033[0m'
find_duplicates() {
    local search_dir="${1:-.}"
    echo -e "${GREEN}扫描目录: $search_dir${NC}"
    # 按文件大小分组
    find "$search_dir" -type f -exec stat -c '%s %n' {} \; | \
        sort -n | \
        awk '{
            if ($1 == size) {
                print $0
            }
            size = $1
        }' > /tmp/same_size.txt
    # 计算MD5并找出重复
    while IFS= read -r line; do
        size=$(echo "$line" | cut -d' ' -f1)
        file=$(echo "$line" | cut -d' ' -f2-)
        md5=$(md5sum "$file" | cut -d' ' -f1)
        echo "$size $md5 $file"
    done < /tmp/same_size.txt | \
        sort -k1,2 | \
        awk '{
            if ($1 == size && $2 == md5) {
                print $0
            }
            size = $1
            md5 = $2
        }' > /tmp/duplicates.txt
    echo -e "${RED}找到重复文件:${NC}"
    cat /tmp/duplicates.txt
    # 询问是否删除
    read -p "是否删除重复文件? (y/n): " answer
    if [[ "$answer" == "y" ]]; then
        while IFS= read -r line; do
            file=$(echo "$line" | cut -d' ' -f3-)
            echo "删除: $file"
            # rm "$file"  # 取消注释以实际删除
        done < /tmp/duplicates.txt
    fi
    # 清理临时文件
    rm -f /tmp/same_size.txt /tmp/duplicates.txt
}
# 主程序
main() {
    local dir="${1:-.}"
    if [ ! -d "$dir" ]; then
        echo "错误: 目录不存在"
        exit 1
    fi
    find_duplicates "$dir"
}
main "$@"

PowerShell 脚本（Windows系统）

# 重复文件清理脚本
# 使用方法: .\dedup.ps1 -Path "C:\MyFiles"
param(
    [Parameter(Mandatory=$true)]
    [string]$Path,
    [switch]$Delete,
    [switch]$DryRun
)
function Get-FileHash {
    param([string]$FilePath)
    try {
        $hash = Get-FileHash -Path $FilePath -Algorithm MD5
        return $hash.Hash
    }
    catch {
        Write-Warning "无法读取文件: $FilePath"
        return $null
    }
}
function Find-Duplicates {
    param([string]$Directory)
    $files = Get-ChildItem -Path $Directory -Recurse -File
    $groups = @{}
    foreach ($file in $files) {
        $key = "$($file.Length)|$($file.Name)"  # 先用大小和文件名分组
        if (-not $groups.ContainsKey($key)) {
            $groups[$key] = @()
        }
        $groups[$key] += $file
    }
    $duplicates = @()
    foreach ($group in $groups.Values) {
        if ($group.Count -gt 1) {
            # 进一步用哈希值确认
            $hashGroups = @{}
            foreach ($file in $group) {
                $hash = Get-FileHash -FilePath $file.FullName
                if ($hash) {
                    if (-not $hashGroups.ContainsKey($hash)) {
                        $hashGroups[$hash] = @()
                    }
                    $hashGroups[$hash] += $file
                }
            }
            foreach ($hashGroup in $hashGroups.Values) {
                if ($hashGroup.Count -gt 1) {
                    $duplicates += $hashGroup
                }
            }
        }
    }
    return $duplicates
}
function Clean-Duplicates {
    param(
        [array]$Duplicates,
        [bool]$Delete = $false,
        [bool]$DryRun = $true
    )
    Write-Host "找到 $($Duplicates.Count) 个重复文件" -ForegroundColor Yellow
    # 保留第一个，删除其余的
    for ($i = 0; $i -lt $Duplicates.Count; $i += 2) {
        $original = $Duplicates[$i]
        $duplicate = $Duplicates[$i + 1]
        Write-Host "原始文件: $($original.FullName)" -ForegroundColor Green
        Write-Host "重复文件: $($duplicate.FullName)" -ForegroundColor Red
        if ($Delete -and -not $DryRun) {
            try {
                Remove-Item -Path $duplicate.FullName -Force
                Write-Host "已删除" -ForegroundColor Green
            }
            catch {
                Write-Host "删除失败: $_" -ForegroundColor Red
            }
        }
        else {
            Write-Host "预览模式，未实际删除" -ForegroundColor Yellow
        }
    }
}
# 主程序
Write-Host "开始扫描: $Path" -ForegroundColor Cyan
$duplicates = Find-Duplicates -Directory $Path
Clean-Duplicates -Duplicates $duplicates -Delete $Delete.IsPresent -DryRun $DryRun.IsPresent
Write-Host "完成！" -ForegroundColor Cyan

使用建议

安全第一

• 始终先用 --dry-run 或预览模式查看结果
• 建议先复制到测试目录再运行
• 重要文件先备份

优化技巧

1. 先按大小过滤：相同大小的文件才可能是重复
2. 分批处理：大文件建议分批处理
3. 使用增量哈希：先计算部分哈希，匹配再计算完整哈希

安装依赖（Python版本）

pip install tqdm  # 进度条

选择适合你系统的脚本,首先在测试目录中运行，确保一切正常后再应用于重要文件。